[
  {
    "path": ".gitattributes",
    "content": "*.t/run.t linguist-vendored\n"
  },
  {
    "path": ".github/workflows/deploy.yml",
    "content": "name: Deploy on release\n\non:\n  push:\n    tags:\n      - \"[0-9]*\"\n      - \"test*\"\n    branches:\n      - \"ci-test\"\n\njobs:\n  build:\n    strategy:\n      fail-fast: false\n      matrix:\n        os:\n          - ubuntu-22.04\n          - ubuntu-22.04-arm\n          - macos-latest\n\n    runs-on: ${{ matrix.os }}\n\n    steps:\n      - name: Checkout code\n        uses: actions/checkout@v4\n\n      - run: echo \"GITHUB_TAG=$(git describe --always --tags)\" >> $GITHUB_ENV\n\n      - if: ${{ startsWith(matrix.os, 'ubuntu') && !endsWith(matrix.os, 'arm') }}\n        run: echo \"OS_SHORT_NAME=linux\" >> $GITHUB_ENV\n\n      - if: ${{ startsWith(matrix.os, 'ubuntu') && endsWith(matrix.os, 'arm') }}\n        run: echo \"OS_SHORT_NAME=linux-arm\" >> $GITHUB_ENV\n\n      - if: ${{ startsWith(matrix.os, 'macos') }}\n        run: echo \"OS_SHORT_NAME=macos\" >> $GITHUB_ENV\n\n      - if: ${{ startsWith(matrix.os, 'windows') }}\n        run: echo \"OS_SHORT_NAME=windows\" >> $GITHUB_ENV\n\n      - name: Set up OCaml for Linux\n        uses: ocaml/setup-ocaml@v3\n        with:\n          ocaml-compiler: \"5.2.1\"\n\n      - run: opam install dune\n\n      - run: opam install . --deps-only --with-test\n\n      - name: Use commit hash as version if on ci-test branch\n        if: ${{ github.ref_name == 'ci-test' }}\n        run: |\n          echo \"DOCFD_VERSION_OVERRIDE=${{ env.GITHUB_TAG }}\" >> $GITHUB_ENV\n\n      - name: Create build for macOS\n        if: ${{ env.OS_SHORT_NAME == 'macos' }}\n        run: |\n          export DOCFD_VERSION_OVERRIDE=${{ env.GITHUB_TAG }}\n          opam exec -- make release-build\n\n      - name: Create static build for Linux\n        if: ${{ env.OS_SHORT_NAME == 'linux' }}\n        run: |\n          export DOCFD_VERSION_OVERRIDE=${{ env.GITHUB_TAG }}\n          opam exec -- make release-static-build\n\n      - name: Create static build for Linux ARM\n        if: ${{ env.OS_SHORT_NAME == 'linux-arm' }}\n        run: |\n          export DOCFD_VERSION_OVERRIDE=${{ env.GITHUB_TAG }}\n          opam exec -- make release-static-build-arm\n\n      - name: Package into tar.gz\n        run: |\n          mv release/docfd docfd\n          tar -cvzf docfd-${{ env.GITHUB_TAG }}-${{ env.OS_SHORT_NAME }}.tar.gz docfd\n\n      - name: Upload artifacts\n        if: ${{ github.ref_name == 'ci-test' }}\n        uses: actions/upload-artifact@v4\n        with:\n          name: docfd-${{ env.GITHUB_TAG }}-${{ env.OS_SHORT_NAME }}.tar.gz\n          path: docfd-${{ env.GITHUB_TAG }}-${{ env.OS_SHORT_NAME }}.tar.gz\n\n      - name: Release\n        if: ${{ github.ref_name != 'ci-test' }}\n        uses: softprops/action-gh-release@v2\n        with:\n          files: |\n            docfd-${{ env.GITHUB_TAG }}-${{ env.OS_SHORT_NAME }}.tar.gz\n\n      - name: Release preview\n        if: ${{ github.ref_name == 'ci-test' }}\n        uses: softprops/action-gh-release@v2\n        with:\n          tag_name: preview\n          name: \"Preview Build\"\n          body: \"Automated preview build from commit ${{ github.sha }}. This release is updated on every push to ci-test.\"\n          prerelease: true\n          files: |\n            docfd-${{ env.GITHUB_TAG }}-${{ env.OS_SHORT_NAME }}.tar.gz\n"
  },
  {
    "path": ".gitignore",
    "content": "_build/\n_coverage/\n.merlin\n*.rst~\n*.install\nbisect*.out\nbisect*.coverage\nfuzz-*-input\nfuzz-*-output\nfuzz-logs/\n/test*.md\n/test*.txt\ntest*.pdf\ntest*.docx\n*.tar.gz\n/release/\nperf.data*\n*.pdf\n.cache\n/*.log\n*.mp4\ndummy.gif\n*.db\n*.db-journal\n"
  },
  {
    "path": "CHANGELOG.md",
    "content": "# Changelog\n\n## 13.0.0\n\n- Removed fzf dependency entirely\n    - Switched from fzf to a built-in implementation for fuzzy selection\n      menus\n    - Replaced \"sort by fzf\" functionality with built-in PATH-FUZZY-RANK mode\n- Renamed OPEN-SCRIPT to SCRIPTS\n- Moved DELETE-SCRIPT functionality to `Ctrl`+`X` under SCRIPTS\n- Added `y` key binding to copy in LINKS\n- Changed key binding for rotation of key binding info grid from `?` to `<` and `>`\n- Added \"focus content\" via `z` which hides the bottom right pane\n- Added key binding info pane toggle via `?`\n- Made the UX of various input fields slightly more polished\n    - This includes adding `Esc` for cancelling during SAVE-SCRIPT\n- Added request handling debouncing to reduce pointless workload triggered during\n  fast typing\n- Improved link extraction\n- Minor UI fixes and polishes\n\n## 12.3.2\n\n- Fixed missing stop signal passing for `content:\"...\"` filter expression handling\n    - Previously, filter expressions with `content:\"...\"` were not\n      cancelled properly\n- Added additional guards against potential freezing due to DB pool\n  exhaustion or DB connection issues\n\n## 12.3.1\n\n- Fixed key binding info grid\n    - Added the missing `l` key binding info\n    - Fixed some other key binding labels\n\n## 12.3.0\n\n**Note**: This update contains a feature that requires removing the existing index DB to take effect.\n\n- Docfd script comment support improvement\n    - Added `;` prefix for system comments, which are not preserved after editing of command history\n    - `#` now denotes user comment and is preserved after editing of command history\n    - Ordering of `#` is also preserved during saving a session as script\n- Added link opening support via LINKS mode\n    - This will **only** work after recreating the index DB\n    - `l` opens LINKS mode with the same navigation keybinds\n        - `Enter` to open link\n        - `o` to open link and remain in LINKS\n        - Links which are closest to the selected search result will be prioritized first\n- Added key binding `xh` to clear command history quickly\n\n## 12.2.0\n\n- Dependencies adjustment for CI build\n- Internal refactoring\n    - Refactored document store module into session state module to\n      better reflect that the data structure is capturing not just\n      documents and search results, but also some UI states, etc\n- Moved screen split handling to the level of session state instead of\n  plain UI state\n    - This allows Docfd script to better capture the view on screen\n- Added `Shift`+`Tab` for changing screen split ratio in the other\n  direction, and removed the \"rotating\" behaviour of `Tab`\n\n## 12.1.0\n\n- Added missing sorting based on paths when path dates are the same\n- Added PDF viewer integration for Zathura on Linux\n- Moved sorting handling to the level of document store command instead of just plain UI update\n    - This allows Docfd script to better capture the view on screen\n- Fixed default cache directory location on macOS\n    - Changed from `~/Library/Application Support` to `~/Library/Caches`\n- Added `--reverse` to fzf invocation for better UX\n- Added `Ctrl`+`D` for script deletion\n\n## 12.0.0\n\nThis contains a **breaking** DB change, you will need to remove index DB generated by Docfd version prior to 12.0.0-alpha.13\n\n#### Highlights of changes since 11.0.1\n\n- Moved to using a global word table to reduce index DB size and speed up search (12.0.0-alpha.13)\n\n    - This is the **breaking** DB change stated above\n\n- Added further search speed optimizations (12.0.0)\n\n    - Added an additional document pruning stage\n    - Added a first word candidate pruning stage based on length of the first search word\n        - Searching for short words should now feel much more responsive\n\n- Replaced filter glob with a more powerful filter language, with\n  autocomplete in filter field (12.0.0-alpha.1, 12.0.0-alpha.2,\n  12.0.0-alpha.5, 12.0.0-alpha.6, 12.0.0-alpha.10, 12.0.0-alpha.11)\n\n- Added content view pane scrolling (12.0.0-alpha.5, 12.0.0-alpha.8)\n\n    - Controlled by `-`/`=`\n\n- Added \"save script\" and \"load script\" functionality to make it\n  actually viable to reuse Docfd commands (12.0.0-alpha.8,\n  12.0.0-alpha.9)\n\n- SQL query optimizations for prefix and exact search terms\n  (12.0.0-alpha.3)\n\n- Key binding info grid improvements (12.0.0-alpha.4)\n\n    - Added more key bindings\n\n    - Packed columns more tightly\n\n- Added `--paths-from -` to accept list of paths from stdin\n  (12.0.0-alpha.3)\n\n- Added WSL clipboard integration (12.0.0-alpha.4)\n\n- Added more marking key bindings (12.0.0-alpha.4)\n\n    - `mark listed` (`ml`) marks all currently listed documents\n    - `unmark listed` (`Ml`) unmarks all currently listed documents\n\n- `--open-with` placeholder handling fixes (12.0.0-alpha.4)\n\n    - Using `{page_num}` and `{line_num}` crashes in 11.0.1\n      when there are no search results\n\n- Added sorting to document list (12.0.0-alpha.11, 12.0.0)\n\n    - `s` for sort ascending mode and `Shift+S` for sort descending mode\n    - Under the sort modes, the sort by types are as follows:\n        - `p` sort by path\n        - `d` sort by path date\n        - `s` sort by score\n        - `m` sort by modification time\n        - `f` sort by an interactive fzf search\n            - Selected option will be ranked the highest\n            - Rest of the documents will be ranked using the ranking from fzf\n\n- Adjusted attributes listed in document list entry (12.0.0-alpha.11)\n\n    - Added path date\n    - Replaced last scan time with last modified time\n\n- Reworked the internal architecture of document store snapshots\n  storage and management, which makes the overall interaction\n  between UI and core code much more robust (12.0.0-alpha.11)\n\n#### Changes since 12.0.0-alpha.13\n\n- Added further search speed optimizations:\n\n    - Added an additional document pruning stage\n    - Added a first word candidate pruning stage based on length of the first search word\n        - Searching for short words should now feel much more responsive\n\n- Fixed interaction with fzf (which is used in some selection menus) on macOS\n  due to different behavior of `Unix.waitpid` on macOS compared to Linux\n\n- Document sorting fine tuning\n\n- Fixed document sorting fallback behavior\n\n    - If there is no search expression but sorting method chosen is to\n      sort by score, then sorting method falls back to the option\n      specified by `--sort-no-score`\n\n- Fixed macOS detection\n\n- Updated `--open-with` to accept a list of extensions, e.g.\n  `--open-with ts,js:detached=\"... {path}\"`\n\n- Added sort by fzf functionality\n    - Under sort mode\n        - `f` sort by an interactive fzf search\n            - Selected option will be ranked the highest\n            - Rest of the documents will be ranked using the ranking from fzf\n\n## 12.0.0-alpha.13\n\n- Moved to using a global word table to reduce index DB size and speed up search\n\n    - This is a **breaking** DB change, you will need to remove index DB generated by older versions of Docfd\n\n- Added missing mutexes for caches, should further reduce random crashes\n\n- Added more path date extraction formats\n    - `yyyy-mmm-dd`, `yyyy-mmmm-dd`, `dd-mmm-yyyy`, `dd-mmmm-yyyy`\n    - `-` is an optional separator that is not a digit and not a letter\n\n## 12.0.0-alpha.12\n\n- Made resetting of search result selection and content view offset less aggressive\n\n    - Some changes in 12.0.0-alpha.11 caused some UI counters to reset more frequently than desired\n\n## 12.0.0-alpha.11\n\n- Removed disabling of drop mode key binding `d` when searching or filtering is ongoing\n\n- Fixed content view pane offset not resetting when mouse is used to scroll search result list\n\n- Fixed content view pane staying small while scrolling up when the search result is close to the bottom of the file\n\n- Swapped all mutexes to Eio mutexes to hopefully remove the very random freezes that occur quite rarely\n\n    - They feel like deadlocks due to mixing Eio mutexes\n      (which block fiber) and stdlib mutexes (which block an entire domain)\n\n- Added sorting to document list\n\n    - `s` for sort ascending mode and `Shift+S` for sort descending mode\n    - Under the sort modes, the sort by types are as follows:\n        - `p` sort by path\n        - `d` sort by path date\n        - `s` sort by score\n        - `m` sort by modification time\n\n- Added `--sort` and `--sort-no-score`\n\n    - Latter is mainly useful for when `--files-without-match` is used\n\n- Added `yyyymmdd` path date extraction\n\n- Added `mod-date` to filter language\n\n- Adjusted attributes listed in document list entry\n\n    - Added path date\n    - Replaced last scan time with last modified time\n\n- Reworked `--script` into `--script` and `--start-with-script`\n\n    - `--script` is now only for non-interactive use\n    - `--start-with-script` is only for interactive use\n    - This mirrors the duals `--filter` vs `--start-with-filter` and `--search` vs `--start-with-search`\n\n- Reworked the internal architecture of document store snapshots storage and management\n\n    - Snapshots are now centrally managed by `Document_store_manager`, along with\n      improvements to snapshot handling logic in general\n\n    - This makes the overall interaction between UI and core code\n      much more robust, and eliminates random workarounds used to\n      deal with UI and data synchronization, which have\n      been riddled with random minor bugs\n\n## 12.0.0-alpha.10\n\n- Added basic autocomplete to filter field\n\n- Improved script save autocomplete to insert longest common prefix\n\n- Fixed script save autocomplete so it no longer erases original text when no recommendations are available\n\n## 12.0.0-alpha.9\n\n- Disabled `Tab` handling in edit fields to reduce friction in UX\n\n- Added `nano`-style autocomplete to save commands field with listing of existing scripts\n\n## 12.0.0-alpha.8\n\n- Changed `--commands-from` to `--script`\n\n- Added \"save commands as script\" and \"load script\" functionality to streamline reusing of commands\n\n- Improved content view pane scrolling control\n\n    - The internal counter no longer scrolls past the limit\n\n## 12.0.0-alpha.7\n\n- Fixed interactive use of `--commands-from`\n\n- Added `mark listed` and `unmark listed` to template command history file help info\n\n## 12.0.0-alpha.6\n\n- Fixed `not` operator parsing\n\n    - Previously `not ext:txt and not ext:md` would be parsed as `not (ext:txt and not ext:md)`, which is not what is typically expected\n\n    - `not` now binds tightly, so `not ext:txt and not ext:md` is parsed as `(not ext:txt) and (not ext:md)`\n\n## 12.0.0-alpha.5\n\n- Added content view pane scrolling\n\n    - Controlled by `-`/`=`\n\n- Removed extraneous marking functionality\n\n    - `mark unlisted`\n    - `unmark unlisted`\n\n- Added `\"...\"` as a shorthand to `content:\"...\"` to filter expression\n\n    - For example, `content:keyword AND path-date:>2025-01-01` can be written as `\"keyword\" AND path-date:>2025-01-01`\n\n    - The quotation is necessary to differentiate between typos\n      and actual query, otherwise incorrect input like\n      `pathfuzzy:...` would be parsed as content queries instead\n\n## 12.0.0-alpha.4\n\n- Added additional marking functionality\n\n    - `mark listed` (`ml`) marks all currently listed documents\n    - `mark unlisted` (`mL`) marks all currently unlisted documents\n    - `unmark listed` (`Ml`) unmarks all currently listed documents\n    - `unmark unlisted` (`ML`) unmarks all currently unlisted documents\n\n- `unmark all` is moved to key binding `Ma`\n\n- Reworked key binding info grid to pack columns more tightly\n\n- Added WSL clipboard integration\n\n- Minor fix in command history file template help text\n\n- Added `Tab` key to key binding info grid\n\n- Added key binding info about scrolling through document list and search result list\n\n- Minor fix for `{line_num}` placeholder handling in `--open-with`\n\n    - This should always be usable for text files but previously\n      Docfd crashes when `{line_num}` is specified in `--open-with` \n      and user opens a text file when no search has been made\n\n    - This is fixed by defaulting `{line_num}` to 1 when\n      there are no search results present\n\n- Minor fix for `{page_num}` and `{search_word}` placeholders handling in `--open-with`\n\n    - This should always be usable for PDF files but previously\n      Docfd crashes when `{page_num}` or `{search_word}` is specified in `--open-with`\n      and user opens a PDF file when no search has been made\n\n    - This is fixed by defaulting `{page_num}` to 1\n      and `{search_word}` to empty string when\n      there are no search results present\n\n## 12.0.0-alpha.3\n\n- **Users are advised to recreate the index DB**\n\n- Adjusted SQL indices and swapped to specialized SQL queries\n  for exact and prefix search terms, e.g. `'hello`, `^worl`\n\n    - Handling of these terms is now 10-20% faster depending on the document\n\n- Fixed command history recomputation not using the reloaded version\n  of document store\n\n    - This issue is most noticeable when you've edited a text file after hitting `Enter` in Docfd (after which Docfd reloads the file for you),\n      and you hit `h` to modify the command history\n\n    - The replaying of the command history would use the old copy of the file instead of the new edited version of the text file\n\n- Added missing SQL transaction in code path for reloading a single document\n\n    - Previously, reloading a single document was incredibly slow, which was very noticeable if you edited a text file\n      after hitting `Enter` in Docfd, unless the text file was very small\n\n- Updated `--paths-from` argument handling\n\n    - Added `--paths-from -` for accepting list of paths from stdin\n\n    - Adjusted to accept comma separated list of paths, e.g. `--paths-from path-list0.txt,path-list1.txt`\n\n- Removed builtin piping to fzf triggered by providing `?` as a file path, e.g. `docfd ?`\n\n    - The `--paths-from -` handling makes this obsolete and a lot less flexible by comparison\n\n- Fixed interaction between search and filter\n\n    - Previously, starting a search would incorrectly cancel an ongoing filtering operation.\n      Now only a new filtering operation can cancel an ongoing filtering operation.\n      A new search still cancels an ongoing search.\n\n    - Starting a new filtering operation also still cancels any ongoing search. This is fine since the search results\n      are refreshed after the filtering has been completed.\n\n        - The refreshing of the search results also means that the following sequences of events are still handled correctly,\n          namely they still arrive at the same normal form of the document store:\n\n            - Example 1:\n\n                - (0) Filter `f_exp0` (filtering is canceled by step (2), but the updating of filter expression is never canceled)\n                - (1) Search `s_exp0` (search is canceled by step (2), but the updating of search expression is never canceled)\n                - (2) Filter `f_exp1` (refreshes search results using `s_exp0`)\n\n            - Example 2:\n\n                - (0) Search `s_exp0` (search is canceled by step (1), but updating of search expression is never canceled)\n                - (1) Filter `f_exp0` (this stage is canceled by step (2),\n                    either during the filtering or during the\n                    refreshing of search results, but the updating\n                    of filter expression is never canceled)\n                - (2) Filter `f_exp1` (refreshes search results using `s_exp0`)\n\n- Renaming query expression/language to filter expression/language in help text and documentation\n\n- Added a separate loading indicator for filter field\n\n- Fixed concurrency issue where an update of document store may cause the\n  filter field and search field in UI to be out of sync with the actual\n  filter expression and search expression used by the underlying document store\n\n    - Suppose we have the following sequence of events:\n\n        - (0) Document store `store0` carries filter expression\n            `f_exp0` and search expression `s_exp0`, which we write\n            as pair `(f_exp0, s_exp0)`\n        - (1) User initiates filter/search operation by placing `(f_exp1, s_exp1)` into the input fields.\n\n            We name the document store resulting from this filter/search operation as `store1a`,\n            which carries `(f_exp1, s_exp1)` when finalized.\n        - (2) While filter/search operation is ongoing,\n        user drops a set of documents from the\n            current document store. Since `store1a` is not\n            finalized yet, the current document store is still `store0`, thus the new document store encoding the result of the drop operation, `store1b`, is computed from `store0` instead of `store1a`.\n\n            In other words, both `store1a` and `store1b` share\n            `store0` as their parent.\n            Note that `store1b` carries `(f_exp0, s_exp0)` as\n            inherited from `store0`,\n            since a drop operation does not alter the filter expression or search expression.\n        - (3) As a drop operation immediately updates the document store\n        and cancels ongoing filter/search operation, step (2) canceled the computation of `store1a`, and instead places `store1b` as the current document store.\n\n    - However, this means the input fields are `(f_exp1, s_exp1)`\n      while the current document store `store1b` actually carries\n      `(f_exp0, s_exp0)`.\n\n      The fix in this update is then to add an\n      extra \"sync from input fields\" step whenever a document store\n      is updated. To illustrate, we continue from the above\n      sequence of events, where the updated version of Docfd\n      carries out the following step missing from previous\n      versions.\n\n        - (4) Update input fields to `(f_exp0, s_exp0)`\n\n    - This addresses the mismatch between the underlying document store and the UI input fields.\n\n    - In practice this is very unlikely to occur with human input, as the modes that update document store\n      are disabled if document store manager is carrying out any ongoing filtering or search.\n\n      However, since the UI is async, there will be gaps in timing between UI input/feedback and actual updates of values,\n      opening up to TOCTOU problems.\n      So there is always a chance that a document store update will be requested before the modes are are disabled.\n\n- Made interrupted filter/search operation to not yield a document store at all instead of yielding an empty document store\n  to simplify reasoning about filter/search cancellations and UI fields being in sync\n\n## 12.0.0-alpha.2\n\n- Added `path-date` clause to query expression\n\n    - This allows filtering based on date recognized from document path, for example, `path-date:>=2025-01-01 AND path-date:<2025-02-01`\n      would allow `/home/user/meeting-notes-2025-01-10.md` to pass through\n\n    - This gives a very lightweight method of attaching date information to any document\n\n    - See [relevant Wiki page](https://github.com/darrenldl/docfd/wiki/Document-filtering) for details\n\n## 12.0.0-alpha.1\n\n- Added a more powerful filter mode that replaces the filter glob mode and \"pipe to fzf\" feature\n\n    - Filter query mode uses a proper query language that supports file path globbing and file path fuzzy matching among other features\n\n    - This mode uses key binding `f`\n\n- Removed `q` exit key binding to avoid accidental exiting\n\n## 11.0.1\n\n- Added better search cancellation handling, removing massive lags in some scenarios\n\n## 11.0.0\n\n- Minor fix for search scope narrowing logic:\n\n    - Search scope should also be set to empty if the document is not passing the file filter, not just when the search results are empty\n\n    - The old behavior can be confusing when a document passes an old file filter and thus has search results in memory,\n      but fail to pass a new file filter,\n      yet appears in later searches when file filter is reset\n\n    - It is simpler to make it so if a document is not listed for\n      whatever reason, search scope of that document just becomes\n      empty during narrowing\n\n- Added missing commands in the list of possible commands in the command history file template\n\n    - `clear search`\n\n    - `clear filter`\n\n- Minor breaking change, filter regex mode should have been called filter glob mode\n\n    - The key binding `fr` is changed to `fg`\n\n- Changed UI text \"File path filter\" to \"File path glob\" to be more descriptive\n\n## 10.2.0\n\n- Added `--open-with` to allow customising the command used to open a file based on file extension\n\n    - Example: `--open-with pdf:detached='okular {path}'`\n\n    - Can be specified multiple times\n\n- Added non-interactive use of `--commands-from`\n\n    - Non-interactive use can be triggered by pairing `--commands-from` with `-l`/`--files-with-match`\n\n    - Useful for advanced document management workflow\n\n- Adjustments to search scope narrowing\n\n    - Added `narrow level: 0` for resetting the search scopes of\n      all documents back to full\n\n    - Narrowing now no longer drops unlisted document, so the\n      previous set of documents remain accessible for later\n      searches after resetting the search scopes\n\n- Reworked search into multi-stage pipeline\n\n    - This improves the search speed by around 30%\n\n    - The core search procedure was reworked into an API that\n      generates grouped search jobs which can be easily distributed\n      to threads.\n      This gives a better workload distribution than the current\n      multithreading approach.\n\n## 10.1.3\n\n- Minor fixes\n\n    - \"Reload document\" now removes the document if the document is no longer accessible\n\n    - Docfd now only checks the existence of directly specified files\n      at launch, e.g. `file.txt` in `docfd file.txt`. This means\n      \"reload all documents\" now does not error out due to files becoming\n      no longer accessible.\n\n## 10.1.2\n\n- Minor fix for \"reload all doucments\" when fzf was used to pick documents initially, i.e. `docfd [PATH]... ?`, or any variation where `?` appears anywhere in the path list\n\n    - Under this workflow, later \"reload all\" should use the same selection\n      instead of having the user select again in fzf, which is cumbersome\n\n    - Now Docfd correctly reuses the selection when \"reload all\" is requested,\n      if fzf was used initially to pick documents\n\n    - This does technically mean the functionality is now less flexible,\n      since if `docfd ?` alike is used, \"reload all\" no longer discovers\n      new files\n\n    - But the convenience from reusing the selection outweighs the flexibility\n      in practically all use cases from author's experience\n\n## 10.1.1\n\n- Minor fix for \"filter files via fzf\" functionality\n\n    - Previously, if instead of making a selection,\n      the user quits fzf (e.g. pressing `Ctrl`+`C`, `Ctrl`+`Q`),\n      Docfd also closes with it\n\n    - Now Docfd just discards the interaction and goes back to the main UI\n\n## 10.1.0\n\n- Added back index DB entry pruning\n\n    - Previously missing after swapping to SQLite DB\n\n    - Also renamed `--cache-soft-limit` to `--cache-limit` to\n      reflect the new pruning logic\n\n    - Fixes [issue #12](https://github.com/darrenldl/docfd/issues/12)\n\n- Swapped to a better `doc_id` allocation strategy to minimise\n  `doc_id` size in DB\n\n- Added blinking when drop mode is disabled but `d` is pressed\n\n## 10.0.0\n\n- Reworked document indexing into a multi-stage pipeline\n\n    - This significantly improves the indexing throughput by allowing\n      I/O tasks and computational tasks to run concurrently\n\n    - See [issue #11](https://github.com/darrenldl/docfd/issues/11)\n\n- **Breaking** changes in index DB design - index DBs made by previous version\n  of Docfd are not compatible\n\n    - Optimized DB design, on average the index DB is roughly 60% smaller\n      compared to Docfd 9.0.0 index DB\n\n    - See [issue #11](https://github.com/darrenldl/docfd/issues/11)\n\n- Added functionality to filter files via fzf\n\n    - This is grouped under filter mode. The previous filter mode\n      is renamed to filter regex mode.\n\n    - `f` enters filter mode\n\n        - `f` again activates filter files via fzf functionality\n\n        - `r` activates the filter regex mode, which was previously\n          just called the filter mode\n\n- Fixed incomplete search results when file path filter field is updated while\n  search is ongoing\n\n    - Updating file path filter always cancels the current search (if there is one)\n      and start a new search after the filter is in place\n\n    - Previously, documents with partial search results due to cancellation\n      are kept\n\n    - Docfd now discards said documents, forcing the new search to complete the\n      search results of these documents\n\n- Removed `--no-cache` flag\n\n    - Previously was unused completey\n\n    - It is difficult to share an in-memory SQlite DB\n      between threads, so discarding this flag entirely\n\n    - See [issue #11](https://github.com/darrenldl/docfd/issues/11)\n\n- Swapped to using proper unicode segmentation for tokenisation\n\n    - This should reduce the index size for Western non-English languages\n      significantly\n\n- Added screen split ratios for hiding left or right pane completely\n\n- Minor UI/UX fixes\n\n    - Drop mode is now disabled when search is still ongoing or when either search field or filter field has an error\n\n    - Added missing update of search and filter status when undoing/redoing, or when replaying command history\n\n        - This is most noticeable when the status indicates an error, but undoing does not return it to OK\n\n## 9.0.0\n\n- Swapped over to using SQLite for index\n\n    - Memory usage is much slimmer/stays flat\n\n        - For the sample of 1.4GB worth of PDFs used, after indexing, 9.0.0-rc1 uses\n          1.9GB of memory, while 9.0.0-rc2 uses 39MB\n\n    - Search is a bit slower\n\n    - Added token length limit of 500 bytes to accommodate word table limit in index DB\n\n        - This means during indexing, if Docfd encounters a very long token,\n          e.g. serial number, long hex string, it will be split into chunks of\n          up to 500 bytes\n\n- Added `Ctrl`+`C` exit key binding to key binding info on screen\n\n- Updated exit keys\n\n    - To exit Docfd: `q`, `Ctrl`+`Q` or `Ctrl`+`C`\n\n    - To exit other modes: `Esc`\n\n- Now defaults to not scanning hidden files and directories\n\n    - This behaviour is now enabled via the `--hidden` flag\n\n- Changed to allow `--add-exts` and `--single-line-add-exts` to be specified multiple times\n\n- Changed return code to be 1 when there are no results for `--sample` or `--search`\n\n- Added `--no-pdftotext` and `--no-pandoc` flags\n\n    - Docfd also notes the presence of these flags in error message if there\n      are PDF files but no pdftotext command is available, and same with files\n      relying on pandoc\n\n- Renamed `drop path` command to just `drop`\n\n- Added drop unselected key binding, and the associated command `drop all except`\n\n- Various key binding help info grid adjustments\n\n## 9.0.0-rc1\n\n- Changed default cache size from 100 to 10000\n\n    - Index after compression doesn't take up that much space, and storage is\n      generally cheap enough these days\n\n- Adjusted cache eviction behaviour to be less strict on when eviction happens\n  and thus less expensive\n\n- Renamed `--cache-size` to `--cache-soft-limit`\n\n- Removed periodic GC compact call to avoid freezes when working with many\n  files\n\n- Removed GC compact call during file indexing core loops to reduce overhead\n\n- Added progress bars to initial document processing stage\n\n- Swapped to using C backend for BLAKE2B hashing, this gives >20x speedup depending on CPU\n\n- Swapped from JSON+GZIP to CBOR+GZIP serialization for indices\n\n- Changed help info rotation key from `h` to `?`\n\n- Renamed discard mode to drop mode\n\n- Added command history editing functionality\n\n- Added `--commands-from` command line argument\n\n- Added `--tokens-per-search-scope-level` command line argument\n\n- Concurrency related bug fixes\n\n    - Unlikely to encounter in normal workflows with human input speed\n\n    - https://github.com/darrenldl/docfd/commit/14fcc45b746e6156f29eb989d70700476977a3d7\n\n    - https://github.com/darrenldl/docfd/commit/bfd63d93562f8785ecad8152005aa0f823185699\n\n    - https://github.com/darrenldl/docfd/commit/4e0aa6785ce80630d0cd3cda6e316b7b15a4fb4b\n\n- Replaced print mode with copy mode\n\n- Replaced single file view with key binding to change screen split ratio\n  to remove feature discrepencies\n\n- Added narrow mode for search scope narrowing\n\n- Renamed `--index-chunk-token-count` to `--index-chunk-size`\n\n- Renamed `--sample-count-per-doc` to `--samples-per-doc`\n\n## 8.0.3\n\n- Fixed single file view crash\n\n## 8.0.2\n\n- Reworked asynchronous search/filter UI code to avoid noticeable lag due to\n  waiting for cancellations that take too long\n\n    - Previously there was still a lockstep somewhere that would prevent UI\n      from progressing if previous search was still being canceled\n\n    - The current implementation allows newest requests to override older\n      requests entirely, and not wait for cancellations at all\n\n- Adjusted document counter in multi-file view to be visible even when no files\n  are listed\n\n## 8.0.1\n\n- Fixed missing file path filter field update when undoing or redoing document\n  store updates\n\n- Fixed case insensitive marker handling in glob command line arguments\n\n## 8.0.0\n\n- Removed `--markdown-headings atx` from pandoc commandline\n  arguments\n\n- Removed `Alt`+`U` undo key binding\n\n- Removed `Alt`+`E` redo key binding\n\n- Removed `Ctrl`+`Q` exit key binding\n\n- Added documentation for undo, redo key bindings\n\n- Added clear mode and moved clear search field key binding\n  under this mode for multi-file view\n\n- Added file path filtering functionality to multi-file view\n\n## 7.1.0\n\n- Added initial macOS support\n\n    - Likely to have bugs, but will need macOS users to report back\n\n- Major speedup from letting `pdftotext` output everything in one pass and split\n  on Docfd side instead of asking `pdftotext` to output one page per invocation\n\n    - For very large PDFs the indexing used to take minutes but now only takes\n      seconds\n\n    - Page count may be inaccurate if the PDF page contains form feed character\n      itself (not fully sure if `pdftotext` filters the form feed character from\n      content), but should be rare\n\n- Significant reduction of index file size by adding GZIP\n  compression to the index JSON\n\n## 7.0.0\n\n- Added discard mode to multi-file view\n\n- Changed to using thin bars as pane separators, i.e. tmux style\n\n- Added `g` and `G` key bindings for going to top and bottom of document list respectively\n\n- Added `-l`/`--files-with-match` and `--files-without-match` for printing just paths\n  in non-interactive mode\n\n- Grouped print key bindings under print mode\n\n- Added more print key bindings\n\n- Grouped reload key bindings under reload mode\n\n- Added fixes to ensure Docfd does not exit until all printing is done\n\n- Slimmed down memory usage by switching to OCaml 5.2 which enables use of `Gc.compact`\n\n    - Still no auto-compaction yet, however, will need to wait for a future\n      OCaml release\n\n- Added `h` key binding to rotate key binding info grid\n\n- Added exact, prefix and suffix search syntax from fzf\n\n- Fixed extraneous document path print in non-interactive mode when documents have no search results\n\n- Added \"explicit spaces\" token `~` to match spaces\n\n## 6.0.1\n\n- Fixed random UI freezes when updating search field\n\n    - This is due to a race condition in the search cancellation mechanism that\n      may cause UI fiber to starve and wait forever for a cancellation\n      acknowledgement\n\n    - This mechanism was put in place for asynchronous search since 4.0.0\n\n    - As usual with race conditions, this only manifests under some specific\n      timing by chance\n\n## 6.0.0\n\n- Fixed help message of `--max-linked-token-search-dist`\n\n- Fixed search result printing where output gets chopped off if terminal width is too small\n\n- Added smart additional line grabbing for search result printing\n\n    - `--search-result-print-snippet-min-size N`\n        - If the search result to be printed has fewer than `N` non-space tokens,\n          then Docfd tries to add surrounding lines to the snippet\n          to give better context.\n    - `--search-result-print-snippet-max-add-lines`\n        - Controls maximum number of surrounding lines that can be added in each direction.\n\n- Added search result underlining when output is not a terminal,\n  e.g. redirected to file, piped to another command\n\n- Changed `--search` to show all search results\n\n- Added `--sample` that uses `--search` previous behavior where (by default)\n  only a handful of top search results are picked for each document\n\n- Changed `--search-result-count-per-doc` to `--sample-count-per-doc`\n\n- Added `--color` and `--underline` for controlling behavior of search result\n  printing, they can take one of:\n\n    - `never`\n    - `always`\n    - `auto`\n\n- Removed blinking for `Tab` key presses\n\n## 5.1.0\n\n- Fixed help message of `--max-token-search-dist`\n\n- Adjusted path display in UI to hide current working directory segment when\n  applicable\n\n- Added missing blinking for `Tab` key presses\n\n## 5.0.0\n\n- Added file globbing support in the form of `--glob` argument\n\n- Added single line search mode arguments\n\n    - `--single-line-exts`\n    - `--single-line-add-exts`\n    - `--single-line-glob`\n    - `--single-line`\n\n- Fixed crash on empty file\n\n   - This was due to assertion failure of `max_line_num` in\n     `Content_and_search_result_render.content_snippet`\n\n- Changed search result printing via `Shift+P` and `p` within TUI to not exit\n  after printing, allowing printing of more results\n\n- Added blinking to key binding info grid to give better visual feedback,\n  especially for the new behavior of search result printing\n\n- Changed to allow `--paths-from` to be specified multiple times\n\n- Fixed handling of `.htm` files\n\n    - `htm` is not a valid value for pandoc's `--format` argument\n    - Now it is rewritten to `html` before being passed to pandoc\n\n- Changed `--max-depth`:\n\n    - Changed default from 10 to 100\n    - Changed to accept 0\n\n## 4.0.0\n\n- Made document search asynchronous to search field input, so UI remains\n  smooth even if search is slow\n\n- Added status to search bar:\n\n    - `OK` means Docfd is idling\n    - `...` means Docfd is searching\n    - `ERR` means Docfd failed to parse the search expression\n\n- Added search cancellation. Triggered by editing or clearing search field.\n\n- Added dynamic search distance adjustment based on notion of linked tokens\n\n    - Two tokens are linked if there is no space between them,\n      e.g. `-` and `>` are linked in `->`, but not in `- >`\n\n- Replaced `word` with `token` in the following options for consistency\n\n    - `--max-word-search-dist`\n    - `--index-chunk-word-count`\n\n- Replaced `word` with `token` in user-facing text\n\n## 3.0.0\n\n- Fixed crash from search result snippet being bigger the content view pane\n\n    - Crash was from `Content_and_search_result_render.color_word_image_grid`\n\n- Added key bindings\n\n    - `p`: exit and print search result to stderr\n    - `Shift+P`: exit and print file path to stderr\n\n- Changed `--debug-log -` to use stderr instead of stdout\n\n- Added non-interactive search mode where search results are printed to stdout\n\n    - `--search EXP` invokes non-interactive search mode with search expression `EXP`\n    - `--search-result-count-per-document` sets the number of top search results printed per document\n    - `--search-result-print-text-width`  sets the text width to use when printing\n\n- Added `--start-with-search` to prefill the search field in interactive mode\n\n- Removed content requirement expression from multi-file view\n\n    - Originally designed for file filtering, but I have almost never used\n      it since its addition in 1.0.0\n\n- Added word based line wrapping to following components of document list in multi-file view\n\n    - Document title\n    - Document path\n    - Document content preview\n\n- Added word breaking in word based line wrapping logic so all of the original characters\n  are displayed even when the terminal width is very small or when a word/token is very long\n\n- Added `--paths-from` to specify a file containing list of paths to (also) be scanned\n\n- Fixed search result centering in presence of line wrapping\n\n- Renamed `--max-fuzzy-edit` to `--max-fuzzy-edit-dist` for consistency\n\n- Changed error messages to not be capitalized to follow Rust's and Go's\n  guidelines on error messages\n\n- Added fallback rendering text so Docfd does not crash from trying\n  to render invalid text.\n\n- Added pandoc integration\n\n- Changed the logic of determining when to use stdin as document source\n\n    - Now if any paths are specified, stdin is ignored\n    - This change mostly came from Dune's cram test mechanism\n      not providing a tty to stdin, so previously Docfd would keep\n      trying to source from stdin even when explicit paths are provided\n\n## 2.2.0\n\n- Restored behaviour of skipping file extension checks for top-level\n  user specified files. This behaviour was likely removed during some\n  previous overhaul.\n\n    - This means, for instance, `docfd bin/docfd.ml` will now open the file\n      just fine without `--add-exts ml`\n\n- Bumped default max word search distance from 20 to 50\n\n- Added consideration for balanced opening closing symbols in search result ranking\n\n    - Namely symbol pairs: `()`, `[]`, `{}`\n\n- Fixed crash from reading from stdin\n\n    - This was caused by calling `Notty_unix.Term.release` after closing the underlying\n      file descriptor in stdin input mode\n\n- Added back handling of optional operator `?` in search expression\n\n- Added test corpus to check translation of search expression to search phrases\n\n## 2.1.0\n\n- Added text editor integration for `jed`/`xjed`\n\n    - See [PR #3](https://github.com/darrenldl/docfd/pull/3)\n      by [kseistrup](https://github.com/kseistrup)\n\n## 2.0.0\n\n- Added \"Last scan\" field display to multi-file view and single file view\n\n- Reduced screen flashing by only recreating `Notty_unix.Term.t` when needed\n\n- Added code to recursively mkdir cache directory if needed\n\n- Search procedure parameter tuning\n\n- UI tuning\n\n- Added search expression support\n\n- Adjusted quit key bindings to be: `Esc`, `Ctrl+C`, and `Ctrl+Q`\n\n- Added file selection support via `fzf`\n\n## 1.9.0\n\n- Added PDF viewer integration for:\n\n    - okular\n    - evince\n    - xreader\n    - atril\n    - mupdf\n\n- Fixed change in terminal behavior after invoking text editor\n  by recreating `Notty_unix.Term.t`\n\n- Fixed file auto-reloading to apply to all file types instead of\n  just text files\n\n## 1.8.0\n\n- Swapped to using Nottui at [a337a77](https://github.com/let-def/lwd/commit/a337a778001e6c1dbaed7e758c9e05f300abd388)\n  which fixes event handling, and pasting into edit field works correctly as a result\n\n- Caching is now disabled if number of documents exceeds cache size\n\n- Moved index cache to `XDG_CACHE_HOME/docfd`, which overall\n  defaults to `$HOME/.cache/docfd`\n\n- Added cache related arguments\n\n    - `--cache-dir`\n    - `--cache-size`\n    - `--no-cache`\n\n- Fixed search result centering in content view pane\n\n- Changed `--debug` to `--debug-log` to support outputting debug log to a file\n\n- Fixed file opening failure due to exhausting file descriptors\n\n    - This was caused by not bounding the number of concurrent fibers when loading files\n      via `Document.of_path` in `Eio.Fiber.List.filter_map`\n\n- Added `--index-only` flag\n\n- Fixed document rescanning in multi-file view\n\n## 1.7.3\n\n- Fixed crash from using mouse scrolling in multi-file view\n\n    - The mouse handler did not reset the search result selected\n      when selecting a different document\n    - This leads to out of bound access if the newly selected document\n      does not have enough search results\n\n## 1.7.2\n\n- Fixed content pane sometimes not showing all the lines\n  depending on terminal size and width of lines\n\n- Made chunk size dynamic for parallel search\n\n## 1.7.1\n\n- Parallelization fine-tuning\n\n## 1.7.0\n\n- Added back parallel search\n\n- General optimizations\n\n- Added index file rotation\n\n## 1.6.3\n\n- Further underestimate space available for the purpose of line wrapping\n\n## 1.6.2\n\n- Fixed line wrapping\n\n## 1.6.1\n\n- Fixed line wrapping\n\n## 1.6.0\n\n- Docfd now saves stdin into a tmp file before processing\n  to allow opening in text editor\n\n- Added `--add-exts` argument for additional file extensions\n\n- Added real-time response to terminal size changes\n\n## 1.5.3\n\n- Updated key binding info pane of multi-file view\n\n## 1.5.2\n\n- Added line number into search result ranking consideration\n\n## 1.5.1\n\n- Tuned search procedure and search result ranking\n\n    - Made substring bidirectional matching differently weighted based\n      on direction\n    - Made reverse substring match require at least 3 characters\n    - Case-sensitive bonus only applies if search phrase\n      is not all ascii lowercase\n\n## 1.5.0\n\n- Made substring matching bidirectional\n\n- Tuned search result ranking\n\n## 1.4.0\n\n- Moved reading of environment variables `VISUAL` and `EDITOR` to program start\n\n- Performance tuning\n\n    - Increased cache size for search phrase automata\n\n## 1.3.4\n\n- Added dispatching of search to task pool at file granularity\n\n## 1.3.3\n\n- Performance tuning\n\n    - Switched back to using the old default max word search distance of 20\n    - Reduced default max fuzzy edit distance from 3 to 2 to prevent massive\n      slowdown on long words\n\n## 1.3.2\n\n- Performance tuning\n\n    - Added caching to search phrase automata construction\n    - Removed dispatching of search to task pool\n    - Adjusted search result limits\n\n## 1.3.1\n\n- Added more commandline argument error checking\n\n- Adjusted help messages\n\n- Adjusted max word search range calculation\n\n- Renamed `max-word-search-range` to `max-word-search-dist`\n\n## 1.3.0\n\n- Index data structure optimizations\n\n- Search procedure optimizations\n\n## 1.2.2\n\n- Fixed editor recognition for kakoune\n\n## 1.2.1\n\n- Fixed search results when multiple words are involved\n\n## 1.2.0\n\n- Removed UI components for search cancellation\n\n- Added real time refresh of search\n\n- Added code to open selected text file to selected search result for:\n\n    - nano\n    - neovim/vim/vi\n    - helix\n    - kakoune\n    - emacs\n    - micro\n\n- Added \"rescan for documents\" to multi-file view\n\n## 1.1.1\n\n- Fixed releasing Notty terminal too early\n\n## 1.1.0\n\n- Added index saving and loading\n\n- Added search cancellation\n\n## 1.0.2\n\n- Fixed file tree scan\n\n## 1.0.1\n\n- Minor UI tweaks\n\n## 1.0.0\n\n- Added expression language for file filtering in multi-file view\n\n- Adjusted default file tree depth\n\n- Added `--exts` argument for configuring file extensions recognized\n\n- Fixed parameters passing from binary to library\n\n## 0.9.0\n\n- Added PDF search support via `pdftotext`\n\n- Added UTF-8 support\n\n## 0.8.6\n\n- Minor wording fix\n\n## 0.8.5\n\n- Added check to skip re-searching if search phrase is equivalent to the previous one\n\n## 0.8.4\n\n- Index data structure optimization\n\n- Code cleanup\n\n## 0.8.3\n\n- Optimized multi-file view reload so it does not redo the search over all documents\n\n- Implemented a proper document store\n\n## 0.8.2\n\n- Fixed single file view document reloading not refreshing search results\n\n## 0.8.1\n\n- Replaced shared data structures with multicore safe versions\n\n- Fixed work partitioning for parallel indexing\n\n## 0.8.0\n\n- Added multicore support for indexing and searching\n\n## 0.7.4\n\n- Fixed crashing and incorrect rendering in some cases of files with blank lines\n\n    - This is due to `Index.line_count` being incorrectly calculated\n\n- Added auto refresh on change of file\n\n    - Change detection is based on file modification time\n\n- Added reload file via `r` key\n\n## 0.7.3\n\n- Bumped the default word search range from 15 to 40\n\n    - Since spaces are also counted as words in the index,\n      15 doesn't actually give a lot of range\n\n- Added minor optimization to search\n\n## 0.7.2\n\n- Code refactoring\n\n## 0.7.1\n\n- Delayed `Nottui_unix` term creation so pre TUI\n  printing like `--version` would work\n\n- Added back mouse scrolling support\n\n- Added Page Up and Page Down keys support\n\n## 0.7.0\n\n- Fixed indexing bug\n\n- Added UI mode switch\n\n- Adjusted status bar to show current file name in single file mode\n\n- Adjusted content view to track search result\n\n- Added content view to single file mode\n\n## 0.6.3\n\n- Adjusted status bar to not display index of document selected\n  when in single document mode\n\n- Edited debug message a bit\n\n## 0.6.2\n\n- Fixed typo in error message\n\n## 0.6.1\n\n- Added check of whether provided files exist\n\n## 0.6.0\n\n- Upgraded status bar and help text/key binding info\n\n## 0.5.9\n\n- Changed help text to status bar + help text\n\n## 0.5.8\n\n- Fixed debug print of file paths\n\n- Tuned UI text slightly\n\n## 0.5.7\n\n- Changed word db to do global word recording to further reduce memory footprint\n\n## 0.5.6\n\n- Optimized overall memory footprint\n\n    - Content index memory usage\n\n    - Switched to using content index to render content\n      lines instead of storing file lines again after indexing\n\n## 0.5.5\n\n- Fixed weighing of fuzzy matches\n\n- Fixed bug in scoring of substring matches\n\n## 0.5.4\n\n- Fixed handling of search phrase with uppercase characters\n\n- Prioritized search results that match the case\n\n## 0.5.3\n\n- Cleaned up code\n\n## 0.5.2\n\n- Cleaned up code\n\n## 0.5.1\n\n- Cleaned up code and debug info print a bit\n\n## 0.5.0\n\n- Removed tags handling\n\n- Added stdin piping support\n\n## 0.4.1\n\n- Tuning content search result scoring\n\n## 0.4.0\n\n- Improved content search result scoring\n\n- Added limit on content search results to consider to avoid\n  slowdown\n\n- General optimizations\n\n## 0.3.3\n\n- Fixed crash due to not resetting content search result selection\n  when changing document selection\n\n## 0.3.2\n\n- Fixed internal line numbering, but displayed line numbering\n  still begins at 1\n\n## 0.3.1\n\n- Adjusted line number to begin at 1\n\n## 0.3.0\n\n- Adjusted colouring\n\n## 0.2.9\n\n- Fixed word position tracking in content indexing\n\n## 0.2.8\n\n- Fixed content indexing\n\n## 0.2.7\n\n- Changed to vim style highlighting for content search results\n\n- Color adjustments in general\n\n## 0.2.6\n\n- Added single file UI mode\n\n- Added support for specifying multiple files in command line\n\n## 0.2.5\n\n- Added limit to word search range of each step in content search\n\n    - This speeds up usual search while giving good enough results,\n      and prevents search from becoming very slow in large documents\n\n## 0.2.4\n\n- Adjusted displayed document list size\n\n- Updated style of document list view\n\n## 0.2.3\n\n- Added sanitization to file view text\n\n- Docfd now accepts file being passed as argument\n\n## 0.2.2\n\n- Fixed tokenization of user provided content search input\n\n- Fixed content indexing to not include spaces\n\n## 0.2.1\n\n- Optimized file discovery procedure\n\n- Added `--max-depth` option to limit scanning depth\n\n- Added content search results view\n\n- Adjusted tokenization procedure\n\n## 0.2.0\n\n- Switched to interactive TUI\n\n- Renamed to Docfd\n\n## 0.1.6\n\n- Optimized parsing code slightly\n\n## 0.1.5\n\n- Adjusted parsing code slightly\n\n## 0.1.4\n\n- Adjusted `--tags` and `--ltags` output slightly\n\n## 0.1.3\n\n- Upgraded `--tags` and `--ltags` output to be more human readable\n  when output is terminal\n\n    - Changed behavior to output each tag in individual line when output\n      is not terminal\n\n## 0.1.2\n\n- Fixed output text when output is not terminal\n\n## 0.1.1\n\n- Fixed checking of whether output is terminal\n\n## 0.1.0\n\n- Flipped output positions of file path and tags\n\n## 0.0.9\n\n- Notefd now adds color to title and matching tags if output is terminal\n\n- Improved fuzzy search index building\n\n## 0.0.8\n\n- Code cleanup\n\n## 0.0.7\n\n- Made file recognition more lenient\n\n- Added support for alternative tag section syntax\n\n    - `| ... |`\n    - `@ ... @`\n\n## 0.0.6\n\n- Fixed Notefd to only handle consecutive tag sections\n\n## 0.0.5\n\n- Added `--tags` and `--ltags` flags\n\n- Adjusted parsing to allow multiple tag sections\n\n## 0.0.4\n\n- Fixed tag extraction\n\n## 0.0.3\n\n- Made header extraction more robust to files with very long lines\n\n## 0.0.2\n\n- Added `-s` for case-insensitive substring tag match\n\n- Renamed `-p` to `-e` for exact tag match\n\n## 0.0.1\n\n- Base version\n"
  },
  {
    "path": "LICENSE",
    "content": "MIT License\n\nCopyright (c) 2022 Di Long Li\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
  },
  {
    "path": "Makefile",
    "content": "SRCFILES = lib/*.ml lib/*.mli bin/*.ml bin/*.mli profiling/*.ml tests/*.ml\n\nOCPINDENT = ocp-indent \\\n\t--inplace \\\n\t$(SRCFILES)\n\n.PHONY: all\nall :\n\tpython3 update-version-string.py\n\tdune build @all\n\n.PHONY: podman-build\npodman-build:\n\tpodman build --format docker -t localhost/docfd -f containers/Containerfile.docfd .\n\n.PHONY: podman-build-demo-vhs\npodman-build-demo-vhs:\n\tpodman build --format docker -t localhost/docfd-demo-vhs -f containers/Containerfile.demo-vhs .\n\n.PHONY: lock\nlock:\n\topam lock .\n\n.PHONY: release-build\nrelease-build :\n\tpython3 update-version-string.py\n\tdune build --release bin/docfd.exe\n\tmkdir -p release\n\tcp -f _build/default/bin/docfd.exe release/docfd\n\tchmod 755 release/docfd\n\n.PHONY: release-static-build\nrelease-static-build :\n\tpython3 update-version-string.py\n\tOCAMLPARAM='_,ccopt=-static' dune build --release bin/docfd.exe\n\tmkdir -p release\n\tcp -f _build/default/bin/docfd.exe release/docfd\n\tchmod 755 release/docfd\n\n.PHONY: release-static-build-arm\nrelease-static-build-arm :\n\tpython3 update-version-string.py\n\tOCAMLPARAM='_,ccopt=-static,fPIC' dune build --release bin/docfd.exe\n\tmkdir -p release\n\tcp -f _build/default/bin/docfd.exe release/docfd\n\tchmod 755 release/docfd\n\n.PHONY: tests\ntests :\n\t# Cleaning and rebuilding here to make sure cram tests actually use a recent binary,\n\t# since Dune (as of 3.14.0) doesn't trigger rebuild of binary when\n\t# invoking cram tests, even if the source code has changed.\n\tmake clean\n\tmake\n\tOCAMLRUNPARAM=b dune exec tests/main.exe --no-buffer --force\n\tdune build @file-collection-tests\n\tdune build @line-wrapping-tests\n\tdune build @misc-behavior-tests\n\tdune build @printing-tests\n\tdune build @match-type-tests\n\tdune build @open-with-tests\n\tdune build @non-interactive-mode-return-code-tests\n\tdune build @search-scope-narrowing-tests\n\tdune build @script-tests\n\n.PHONY: demo-vhs\ndemo-vhs :\n\tfor file in demo-vhs-tapes/*; do ./demo-vhs.sh $$file; done\n\trm dummy.gif\n\n.PHONY: profile\nprofile :\n\tOCAMLPARAM='_,ccopt=-static' dune build --release profiling/main.exe\n\n.PHONY: format\nformat :\n\t$(OCPINDENT)\n\n.PHONY : clean\nclean:\n\tdune clean\n"
  },
  {
    "path": "README.md",
    "content": "# Docfd\n\n[Online Demo](https://demo.docfd.sh)\n\nTUI multiline fuzzy document finder\n\nThink interactive grep for text files, PDFs, DOCXs, etc,\nbut word/token based instead of regex and line based,\nso you can search across lines easily.\n\nDocfd aims to provide good UX via integration with common text editors\nand PDF viewers,\nso you can jump directly to a search result with a single key press.\n\n---\n\nInteractive use\n\n![](demo-vhs-gifs/repo.gif)\n\nNon-interactive use\n\n![](demo-vhs-gifs/repo-non-interactive.gif)\n\n## Features\n\n- Multithreaded indexing and searching\n\n- Multiline fuzzy search of multiple files\n\n- Content view pane that shows the snippet surrounding the search result selected\n\n- Text editor and PDF viewer integration\n\n- Editable command history - rewrite/plan your actions in text editor\n\n- Search scope narrowing - limit scope of next search based on current search results\n\n- Clipboard integration\n\n## Why Docfd might be for you\n\n<details>\n<summary>\nYou want a standalone, offline TUI search tool that\nallows you to immediately start searching without any complicated setup.\n</summary>\n\nDocfd only starts processing the current directory or\nspecified directories/files upon start.\nHashing is used to pick out files that have not been indexed yet.\n\nThere is no need to wait for a background indexer to refresh\nbefore you get up-to-date results.\n</details>\n\n<details>\n<summary>\nYou don't want to move everything into a central storage, and want to just keep your current folder structure\n</summary>\n\nThere are no strings attached with using Docfd.\nDocfd does not require you to import your files into\nany special storage system, so you can continue mix and match\ntools to best handle your files.\n</details>\n\n<details>\n<summary>\nYou want to script or record your search\n</summary>\n\nDocfd comes with a simple scripting language,\nwhich is already used to capture your actions in the TUI.\n\nFinally found what you need after many steps?\nSave the session as a script with `Ctrl`+`S`!\nThen open it next time with `Ctrl`+`O`.\n</details>\n\n## Why Docfd might not be for you\n\n<details>\n<summary>\nDocfd is not all-encompassing\n</summary>\n\nDocfd does not try to be a full blown document management system such as Paperless-ngx.\nWhile there may be significant overlaps in terms of the search functionality, Docfd will fall short for almost any other kind of features, such as storage management, tagging, web interface, OCR, email ingestion.\n\n</details>\n\n<details>\n<summary>\nDocfd is not a \"proper\" search engine\n</summary>\n\nDocfd is a search engine in the sense that it uses the same\nfundamental principles, i.e. inverted indices, but it lacks features\nthat you would expect from a \"proper\" search engine solution, e.g.\n[Apache Lucene](https://lucene.apache.org/),\n[Tantivy](https://github.com/quickwit-oss/tantivy),\n[Lnx](https://github.com/lnx-search/lnx).\n\nHere are some of the fundamental features which I think are crucial to a proper search engine, but Docfd lacks:\n- You cannot customize what are indexed by Docfd\n- You cannot add a new type of ranking\n- Docfd lacks support for languages other than English\n- Docfd does not scale very well to very large quantity of documents\n    - Search should still be serviceable when you reach beyond, say, 10k documents, but it will be noticeably more sluggish\n\nSome of these shortcomings are fundamental to the goals of Docfd. For instance,\nDocfd is primarily a standalone desktop TUI tool with quick startup and should not impact other desktop applications.\nAs such, some performance related engineering choices typical for a proper search engine\nare difficult to accommodate as they require longer startup and significantly more memory usage.\n\nOther shortcomings are due to limited time and limited return on efforts - if one is to push Docfd so much to reach the feature parity\nand performance of a proper search engine, then one might as well just use an existing search engine to begin with.\n</details>\n\n<details>\n<summary>\nIf your notes are consistently very short, and you only want to do simple searches, then there are better options\n</summary>\n\nIf you follow note taking methodologies such as Zettelkasten, where each note consists of very few lines, then using a combination of grep and file preview tool can yield a much faster search experience by skipping out on indexing and consideration of word proximity.\n</details>\n\n<details>\n<summary>\nDocfd does not \"stream\" its search results\n</summary>\n\nOne user feedback received was that searching felt slow when Docfd is still conducting the search as UI is not updated result by result. By comparison, fzf felt faster as results start to immediately pop into the screen.\n\nIt is fundamentally more difficult to implement this streaming behavior nicely in Docfd, as Docfd operates with snapshots in mind (e.g. allowing you to undo/redo commands), while fzf does not. More specifically, it is much easier to wait for all search results to be ready, and finalize as a snapshot before presenting onto Docfd UI.\n\nSo while possible to implement in Docfd, it is unclear if the effort is worthwhile with the additional system complexity in mind.\n</details>\n\n## Installation\n\nStatically linked binaries for Linux and macOS are available via\n[GitHub releases](https://github.com/darrenldl/docfd/releases).\n\nDocfd is also packaged on the following platforms for Linux:\n\n- [opam](https://ocaml.org/p/docfd/latest)\n- [AUR](https://aur.archlinux.org/packages/docfd-bin) (as `docfd-bin`)\n    - First packaged by [@kseistrup](https://github.com/kseistrup), now maintained by Dominiquini\n- Nix (as `docfd`)\n    - Packaged by [@chewblacka](https://github.com/chewblacka)\n\nThe only way to use Docfd on Windows right now is via WSL.\n\n**Notes for packagers**: Outside of the OCaml toolchain for building (if you are\npackaging from source), Docfd also requires the following\nexternal tools at run time for full functionality:\n\n- `pdftotext` from `poppler-utils` for PDF support\n- `pandoc` for support of `.epub`, `.odt`, `.docx`, `.fb2`, `.ipynb`, `.html`, and `.htm` files\n- `wl-clibpard` for clipboard support on Wayland\n- `xclip` for clipboard support on X11\n\n## Basic usage\n\nThe typical usage of Docfd is to either `cd` into the directory of interest\nand launch `docfd` directly, or specify the paths as arguments:\n\n```\ndocfd [PATH]...\n```\n\nThe list of paths can contain directories.\nEach directory in the list is scanned recursively for\nfiles with the following extensions by default:\n\n- For multiline search mode:\n    - `.txt`,\n      `.md`,\n      `.pdf`,\n      `.epub`,\n      `.odt`,\n      `.docx`,\n      `.fb2`,\n      `.ipynb`,\n      `.html`,\n      `.htm`\n- For single line search mode:\n    - `.log`,\n      `.csv`,\n      `.tsv`\n\nYou can change the file extensions to use via\n`--exts` and `--single-line-exts`,\nor add onto the list of extensions via\n`--add-exts` and `--single-line-add-exts`.\n\nIf the list `PATH`s is empty,\nthen Docfd defaults to scanning the\ncurrent directory `.`\nunless any of the following is used:\n`--paths-from`, `--glob`, `--single-line-glob`.\n\n## Documentation\n\nSee [GitHub Wiki](https://github.com/darrenldl/docfd/wiki) for\nmore examples/cookbook, and technical details.\n\n## Changelog\n\n[CHANGELOG](CHANGELOG.md)\n\n## Limitations\n\n- Docfd generally expects one intance per index DB\n\n    - You should pick a different cache directory (which houses\n      the index DB) via `--cache-dir`\n      if you need multiple instances\n\n    - There are safe guards to avoid corruptions even if you do run\n      multiple instances of Docfd, but note that the instances of Docfd\n      may exit unexpectedly\n\n    - That being said, running multiple instances of Docfd which are only reading\n      the index DB and not updating it should be fine\n\n- File auto-reloading is not supported for PDF files,\n  as PDF viewers are invoked in the background via shell.\n  It is possible to support this properly\n  in the ways listed below, but requires\n  a lot of engineering for potentially very little gain:\n\n    - Docfd waits for PDF viewer to terminate fully\n      before resuming, but this\n      prohibits viewing multiple search results\n      simultaneously in different PDF viewer instances.\n\n    - Docfd manages the launched PDF viewers completely,\n      but these viewers are closed when Docfd terminates.\n\n    - Docfd invokes the PDF viewers via shell\n      so they stay open when Docfd terminates.\n      Docfd instead periodically checks if they are still running\n      via the PDF viewers' process IDs,\n      but this requires handling forks.\n\n    - Outside of tracking whether the PDF viewer instances\n      interacting with the files are still running,\n      Docfd also needs to set up file update handling\n      either via `inotify` or via checking\n      file modification times periodically.\n\n## Acknowledgement\n\n- Big thanks to [@lunacookies](https://github.com/lunacookies) and\n  [@jthvai](https://github.com/jthvai) for the many UI/UX discussions and\n  suggestions\n- Demo gifs and some screenshots are made using [vhs](https://github.com/charmbracelet/vhs).\n- [ripgrep-all](https://github.com/phiresky/ripgrep-all) was used as reference\n  for text extraction software choices\n- [Marc Coquand](https://mccd.space) (author of\n  [Stitch](https://git.mccd.space/pub/stitch/)) for discussions and inspiration\n  of results narrowing functionality\n- Part of the search syntax was copied from [fzf](https://github.com/junegunn/fzf)\n- Command history editing workflow was inspired by Git interactive rebase workflow, e.g. `git rebase -i`\n- [PDF corpora](https://github.com/pdf-association/pdf-corpora) from PDF association was used to stress test performance\n"
  },
  {
    "path": "bin/BLAKE2B.ml",
    "content": "module B = Digestif.Make_BLAKE2B (struct\n    let digest_size = 64\n  end)\n\nlet hash_of_file ~env ~path =\n  let fs = Eio.Stdenv.fs env in\n  let ctx = ref B.empty in\n  try\n    Eio.Path.(with_open_in (fs / path))\n      (fun flow ->\n         match\n           Eio.Buf_read.parse ~max_size:Params.hash_chunk_size\n             (fun buf ->\n                try\n                  while true do\n                    ctx := B.feed_string !ctx (Eio.Buf_read.take Params.hash_chunk_size buf)\n                  done\n                with\n                | End_of_file ->\n                  ctx := B.feed_string !ctx (Eio.Buf_read.take_all buf)\n             )\n             flow\n         with\n         | Ok () -> Ok (!ctx |> B.get |> B.to_hex)\n         | Error (`Msg msg) -> Error msg\n      )\n  with\n  | _ -> Error (Printf.sprintf \"failed to hash file: %s\" (Filename.quote path))\n"
  },
  {
    "path": "bin/UI.ml",
    "content": "open Docfd_lib\nopen Lwd_infix\n\nmodule Vars = struct\n  let script_name_field = Lwd.var UI_base.empty_text_field\n\n  let script_name_field_focus_handle = Nottui.Focus.make ()\n\n  let path_fuzzy_rank_field = Lwd.var UI_base.empty_text_field\n\n  let path_fuzzy_rank_field_focus_handle = Nottui.Focus.make ()\n\n  let script_files : string Dynarray.t Lwd.var = Lwd.var (Dynarray.create ())\n\n  let usable_script_files : (string Dynarray.t * Int_set.t Dynarray.t option) Lwd.t =\n    let$* arr = Lwd.get script_files in\n    let$* input_mode = Lwd.get UI_base.Vars.input_mode in\n    let$ script_name_specified, _ = Lwd.get script_name_field in\n    match input_mode with\n    | Save_script -> (\n        (Dynarray.filter\n           (CCString.starts_with ~prefix:script_name_specified)\n           arr,\n         None)\n      )\n    | Scripts | Delete_script_confirm (_, _) -> (\n        match Search_exp.parse script_name_specified with\n        | None -> (\n            (Dynarray.filter\n               (CCString.starts_with ~prefix:script_name_specified)\n               arr,\n             None)\n          )\n        | Some exp -> (\n            if Search_exp.is_empty exp then (\n              (arr, None)\n            ) else (\n              let ranking =\n                Misc_utils.fuzzy_rank_assoc\n                  (Stop_signal.make ())\n                  ~get_key:Filename.chop_extension\n                  exp\n                  (Dynarray.to_seq arr)\n              in\n              Dynarray.to_seq ranking\n              |> Seq.map (fun (path, search_result) ->\n                  (path, Misc_utils.highlights_of_search_result search_result)\n                )\n              |> Seq.split\n              |> (fun (s0, s1) ->\n                  (Dynarray.of_seq s0, Some (Dynarray.of_seq s1)))\n            )\n          )\n      )\n    | _ -> (\n        (Dynarray.create (), None)\n      )\nend\n\nlet refresh_script_files () =\n  Lwd.set UI_base.Vars.index_of_script_selected 0;\n  File_utils.list_files_recursive_filter_by_exts\n    ~max_depth:1\n    ~report_progress:(fun () -> ())\n    ~exts:[ Params.docfd_script_ext ]\n    (Seq.return (Params.script_dir ()))\n  |> String_set.to_seq\n  |> Seq.map Filename.basename\n  |> Dynarray.of_seq\n  |> Lwd.set Vars.script_files\n\nlet reload_document (doc : Document.t) =\n  let pool = UI_base.task_pool () in\n  let path = Document.path doc in\n  let doc =\n    match\n      Document.of_path\n        ~env:(UI_base.eio_env ())\n        pool\n        ~already_in_transaction:false\n        (Document.search_mode doc)\n        path\n    with\n    | Ok doc -> Some doc\n    | Error _ -> (\n        None\n      )\n  in\n  let session_state =\n    Session_manager.lock_with_view (fun view ->\n        view.init_state\n      )\n    |> (fun state ->\n        match doc with\n        | Some doc -> (\n            Session.State.add_document pool doc state\n          )\n        | None -> (\n            Session.State.drop (`Path path) state\n          )\n      )\n  in\n  Session_manager.update_starting_state session_state\n\nlet reload_document_selected\n    ~(search_result_groups : Session.search_result_group array)\n  : unit =\n  if Array.length search_result_groups > 0 then (\n    let index = Lwd.peek UI_base.Vars.index_of_document_selected in\n    let doc, _search_results = search_result_groups.(index) in\n    reload_document doc;\n  )\n\nlet toggle_mark ~path =\n  Session_manager.update_from_cur_snapshot\n    (fun cur_snapshot ->\n       let state = Session.Snapshot.state cur_snapshot in\n       let new_command =\n         if\n           String_set.mem\n             path\n             (Session.State.marked_document_paths state)\n         then (\n           `Unmark path\n         ) else (\n           `Mark path\n         )\n       in\n       state\n       |> Session.run_command\n         (UI_base.task_pool ())\n         new_command\n       |> Option.get\n       |> (fun (new_command, state) ->\n           Session.Snapshot.make\n             ~last_command:(Some new_command)\n             state)\n    )\n\nlet drop ~document_count (choice : [`Path of string | `All_except of string | `Marked | `Unmarked | `Listed | `Unlisted]) =\n  let new_command =\n    match choice with\n    | `Path path -> (\n        let n = Lwd.peek UI_base.Vars.index_of_document_selected in\n        UI_base.set_document_selected ~choice_count:(document_count - 1) n;\n        `Drop path\n      )\n    | `All_except path -> (\n        UI_base.set_document_selected ~choice_count:1 0;\n        `Drop_all_except path\n      )\n    | `Marked -> (\n        UI_base.reset_document_selected ();\n        `Drop_marked\n      )\n    | `Unmarked -> (\n        UI_base.reset_document_selected ();\n        `Drop_unmarked\n      )\n    | `Listed -> (\n        UI_base.reset_document_selected ();\n        `Drop_listed\n      )\n    | `Unlisted -> (\n        UI_base.reset_document_selected ();\n        `Drop_unlisted\n      )\n  in\n  Session_manager.update_from_cur_snapshot (fun cur_snapshot ->\n      Session.Snapshot.state cur_snapshot\n      |> Session.run_command\n        (UI_base.task_pool ())\n        new_command\n      |> Option.get\n      |> (fun (new_command, state) ->\n          Session.Snapshot.make\n            ~last_command:(Some new_command)\n            state)\n    )\n\nlet mark (choice : [`Path of string | `Listed]) =\n  let new_command =\n    match choice with\n    | `Path path -> `Mark path\n    | `Listed -> `Mark_listed\n  in\n  Session_manager.update_from_cur_snapshot (fun cur_snapshot ->\n      Session.Snapshot.state cur_snapshot\n      |> Session.run_command\n        (UI_base.task_pool ())\n        new_command\n      |> Option.get\n      |> (fun (new_command, state) ->\n          Session.Snapshot.make\n            ~last_command:(Some new_command)\n            state)\n    )\n\nlet unmark (choice : [`Path of string | `Listed | `All]) =\n  let new_command =\n    match choice with\n    | `Path path -> `Unmark path\n    | `Listed -> `Unmark_listed\n    | `All -> `Unmark_all\n  in\n  Session_manager.update_from_cur_snapshot (fun cur_snapshot ->\n      Session.Snapshot.state cur_snapshot\n      |> Session.run_command\n        (UI_base.task_pool ())\n        new_command\n      |> Option.get\n      |> (fun (new_command, state) ->\n          Session.Snapshot.make\n            ~last_command:(Some new_command)\n            state)\n    )\n\nlet sort (sort_by : Command.Sort_by.t) =\n  UI_base.reset_document_selected ();\n  let new_command = `Sort (sort_by, Command.Sort_by.default_no_score) in\n  Session_manager.update_from_cur_snapshot (fun cur_snapshot ->\n      Session.Snapshot.state cur_snapshot\n      |> Session.run_command\n        (UI_base.task_pool ())\n        new_command\n      |> Option.get\n      |> (fun (new_command, state) ->\n          Session.Snapshot.make\n            ~last_command:(Some new_command)\n            state)\n    )\n\nlet narrow_search_scope_to_level ~level =\n  Session_manager.update_from_cur_snapshot (fun cur_snapshot ->\n      Session.Snapshot.make\n        ~last_command:(Some (`Narrow_level level))\n        (Session.State.narrow_search_scope_to_level\n           ~level\n           (Session.Snapshot.state cur_snapshot))\n    )\n\nlet update_filter ~commit () =\n  let s = fst @@ Lwd.peek UI_base.Vars.filter_field in\n  Session_manager.submit_filter_req ~commit s\n\nlet update_search ~commit () =\n  let s = fst @@ Lwd.peek UI_base.Vars.search_field in\n  Session_manager.submit_search_req ~commit s\n\nlet update_path_fuzzy_rank ~commit () =\n  let s = fst @@ Lwd.peek Vars.path_fuzzy_rank_field in\n  Session_manager.submit_path_fuzzy_rank_req ~commit s\n\nlet compute_save_script_path base_name =\n  let dir = Params.script_dir () in\n  File_utils.mkdir_recursive dir;\n  Filename.concat\n    dir\n    (Fmt.str \"%s%s\" base_name Params.docfd_script_ext)\n\nlet save_script ~path =\n  Session_manager.stop_filter_and_search_and_restore_input_fields ();\n  let lines =\n    Session_manager.lock_with_view (fun view ->\n        view.snapshots\n        |> Dynarray.to_seq\n        |> Seq.filter_map  (fun (snapshot : Session.Snapshot.t) ->\n            Option.map\n              Command.to_string\n              (Session.Snapshot.last_command snapshot)\n          )\n        |> List.of_seq\n      )\n  in\n  try\n    CCIO.with_out path (fun oc ->\n        CCIO.write_lines_l oc lines;\n      )\n  with\n  | Sys_error _ -> (\n      Misc_utils.exit_with_error_msg\n        (Fmt.str \"failed to write script %s\" path)\n    )\n\nmodule Top_pane = struct\n  module Document_list = struct\n    let render_document_entry\n        ~input_mode\n        ~width\n        ~documents_marked\n        ~(search_result_group : Session.search_result_group)\n        ~path_highlights\n        ~selected\n      : Notty.image =\n      let open Notty in\n      let open Notty.Infix in\n      let (doc, search_results) = search_result_group in\n      let search_result_score_image =\n        if Option.is_some !Params.debug_output then (\n          if Array.length search_results = 0 then\n            I.empty\n          else (\n            let x = search_results.(0) in\n            I.strf \"(Best search result score: %f)\" (Search_result.score x)\n          )\n        ) else (\n          I.empty\n        )\n      in\n      let sub_item_base_left_padding = I.string A.empty \"    \" in\n      let sub_item_width = width - I.width sub_item_base_left_padding - 2 in\n      let preview_image =\n        let preview_left_padding_per_line =\n          I.string A.(bg lightgreen) \" \"\n          <|>\n          I.string A.empty \" \"\n        in\n        let preview_line_images =\n          let line_count =\n            min Params.preview_line_count (Index.global_line_count ~doc_id:(Document.doc_id doc))\n          in\n          OSeq.(0 --^ line_count)\n          |> Seq.map (fun global_line_num ->\n              Index.words_of_global_line_num ~doc_id:(Document.doc_id doc) global_line_num\n              |> Dynarray.to_list\n              |> Content_and_search_result_rendering.Text_block_rendering.of_words ~width:sub_item_width\n            )\n          |> Seq.map (fun img ->\n              let left_padding =\n                OSeq.(0 --^ I.height img)\n                |> Seq.map (fun _ -> preview_left_padding_per_line)\n                |> List.of_seq\n                |> I.vcat\n              in\n              left_padding <|> img\n            )\n          |> List.of_seq\n        in\n        I.vcat preview_line_images\n      in\n      let path_highlights =\n        match input_mode with\n        | UI_base.Path_fuzzy_rank -> (\n            String_map.find_opt (Document.path doc) path_highlights\n          )\n        | _ -> (\n            None\n          )\n      in\n      let path_image =\n        Document.path doc\n        |> File_utils.remove_cwd_from_path\n        |> Tokenization.tokenize ~drop_spaces:false\n        |> List.of_seq\n        |> Content_and_search_result_rendering.Text_block_rendering.of_words\n          ~width:sub_item_width\n          ?highlights:path_highlights\n      in\n      let path_image_with_prefix =\n        (I.string A.(fg lightgreen) \"@ \")\n        <|>\n        path_image\n      in\n      let path_date_image =\n        (match Document.path_date doc with\n         | None -> I.void 0 0\n         | Some date -> (\n             I.string A.(fg lightgreen) \"  ⤷ \"\n             <|>\n             I.string A.empty\n               (Timedesc.Date.to_rfc3339 date)\n           )\n        )\n      in\n      let last_modified_image =\n        I.string A.(fg lightgreen) \"Last modified: \"\n        <|>\n        I.string A.empty\n          (Timedesc.to_string ~format:Params.last_modified_format_string (Document.mod_time doc))\n      in\n      let marked = String_set.mem (Document.path doc) documents_marked in\n      let title =\n        let attr =\n          if selected then (\n            A.(fg lightblue ++ st bold)\n          ) else (\n            A.(fg lightblue)\n          )\n        in\n        match Document.title doc with\n        | None ->\n          I.void 0 1\n        | Some title -> (\n            title\n            |> Tokenization.tokenize ~drop_spaces:false\n            |> List.of_seq\n            |> Content_and_search_result_rendering.Text_block_rendering.of_words ~attr ~width\n          )\n      in\n      match input_mode with\n      | UI_base.Path_fuzzy_rank -> (\n          path_image\n        )\n      | _ -> (\n          (\n            (if marked then I.strf \"> \" else I.void 0 1)\n            <|>\n            title\n          )\n          <->\n          (\n            sub_item_base_left_padding\n            <|>\n            I.vcat\n              [ search_result_score_image;\n                path_image_with_prefix;\n                path_date_image;\n                preview_image;\n                last_modified_image;\n              ]\n          )\n        )\n\n    let main\n        ~width\n        ~height\n        ~documents_marked\n        ~(search_result_groups : Session.search_result_group array)\n        ~(document_selected : int)\n      : Nottui.ui Lwd.t =\n      let document_count = Array.length search_result_groups in\n      let$* input_mode = Lwd.get UI_base.Vars.input_mode in\n      let$* (_cur_ver, snapshot) = Session_manager.cur_snapshot in\n      let state = Session.Snapshot.state snapshot in\n      let render_pane () =\n        let rec aux index height_filled acc =\n          if index < document_count\n          && height_filled < height\n          then (\n            let selected = Int.equal document_selected index in\n            let img =\n              render_document_entry\n                ~input_mode\n                ~width\n                ~documents_marked\n                ~search_result_group:search_result_groups.(index)\n                ~path_highlights:(Session.State.path_highlights state)\n                ~selected\n            in\n            aux (index + 1) (height_filled + Notty.I.height img) (img :: acc)\n          ) else (\n            List.rev acc\n            |> List.map Nottui.Ui.atom\n            |> Nottui.Ui.vcat\n          )\n        in\n        if document_count = 0 then (\n          Nottui.Ui.empty\n        ) else (\n          aux document_selected 0 []\n        )\n      in\n      let$ background = UI_base.full_term_sized_background in\n      Nottui.Ui.join_z background (render_pane ())\n      |> Nottui.Ui.mouse_area\n        (UI_base.mouse_handler\n           ~f:(fun direction ->\n               let offset =\n                 match direction with\n                 | `Up -> -1\n                 | `Down -> 1\n               in\n               let document_current_choice =\n                 Lwd.peek UI_base.Vars.index_of_document_selected\n               in\n               UI_base.set_document_selected\n                 ~choice_count:document_count\n                 (document_current_choice + offset);\n             )\n        )\n  end\n\n  module Right_pane = struct\n    module Search_result_list = struct\n      let main\n          ~height\n          ~width\n          ~(search_result_group : Session.search_result_group)\n          ~(index_of_search_result_selected : int Lwd.var)\n        : Nottui.ui Lwd.t =\n        let (document, search_results) = search_result_group in\n        let search_result_selected = Lwd.peek index_of_search_result_selected in\n        let result_count = Array.length search_results in\n        if result_count = 0 then (\n          Lwd.return Nottui.Ui.empty\n        ) else (\n          let images =\n            Misc_utils.array_sub_seq\n              ~start:search_result_selected\n              ~end_exc:(min result_count (search_result_selected + height))\n              search_results\n            |> Seq.map (Content_and_search_result_rendering.search_result\n                          ~doc_id:(Document.doc_id document)\n                          ~render_mode:(UI_base.render_mode_of_document document)\n                          ~width\n                       )\n            |> List.of_seq\n          in\n          let pane =\n            images\n            |> List.map (fun img ->\n                Nottui.Ui.atom Notty.I.(img <-> strf \"\")\n              )\n            |> Nottui.Ui.vcat\n          in\n          let$ background = UI_base.full_term_sized_background in\n          Nottui.Ui.join_z background pane\n          |> Nottui.Ui.mouse_area\n            (UI_base.mouse_handler\n               ~f:(fun direction ->\n                   let n = Lwd.peek index_of_search_result_selected in\n                   let offset =\n                     match direction with\n                     | `Up -> -1\n                     | `Down -> 1\n                   in\n                   UI_base.set_search_result_selected\n                     ~choice_count:result_count\n                     (n + offset)\n                 )\n            )\n        )\n    end\n\n    module Link_list = struct\n      let main\n          ~height\n          ~width\n          ~(search_result_group : Session.search_result_group)\n          ~(index_of_link_selected : int Lwd.var)\n        : Nottui.ui Lwd.t =\n        let (document, _search_results) = search_result_group in\n        let links = Document.links document in\n        let link_selected = Lwd.peek index_of_link_selected in\n        let link_count = Array.length links in\n        if link_count = 0 then (\n          Lwd.return Nottui.Ui.empty\n        ) else (\n          let start = max 0 (link_selected - height / 2) in\n          let end_exc = min link_count (start + height) in\n          let render ~width s =\n            s\n            |> Tokenization.tokenize ~drop_spaces:false\n            |> List.of_seq\n            |> Content_and_search_result_rendering.Text_block_rendering.of_words\n              ~width\n          in\n          let pane =\n            Misc_utils.array_sub_seq ~start ~end_exc\n              links\n            |> Seq.map (fun link -> link.Link.link)\n            |> List.of_seq\n            |> Content_and_search_result_rendering.centered_list\n              ~height\n              ~width\n              ~render\n              (link_selected - start)\n            |> Nottui.Ui.atom\n          in\n          let$ background = UI_base.full_term_sized_background in\n          Nottui.Ui.join_z background pane\n          |> Nottui.Ui.mouse_area\n            (UI_base.mouse_handler\n               ~f:(fun direction ->\n                   let n = Lwd.peek index_of_link_selected in\n                   let offset =\n                     match direction with\n                     | `Up -> -1\n                     | `Down -> 1\n                   in\n                   UI_base.set_link_selected\n                     ~choice_count:link_count\n                     (n + offset)\n                 )\n            )\n        )\n    end\n\n    let main\n        ~width\n        ~height\n        ~(search_result_groups : Session.search_result_group array)\n        ~(document_selected : int)\n        ~show_bottom_right_pane\n      : Nottui.ui Lwd.t =\n      if Array.length search_result_groups = 0 then (\n        let blank ~height =\n          let _ = height in\n          Nottui_widgets.empty_lwd\n        in\n        UI_base.vpane ~width ~height\n          blank blank\n      ) else (\n        let$* input_mode = Lwd.get UI_base.Vars.input_mode in\n        let$* search_result_selected = Lwd.get UI_base.Vars.index_of_search_result_selected in\n        let$* link_selected = Lwd.get UI_base.Vars.index_of_link_selected in\n        let search_result_group = search_result_groups.(document_selected) in\n        if show_bottom_right_pane then (\n          UI_base.vpane ~width ~height\n            (UI_base.Content_view.main\n               ~input_mode\n               ~width\n               ~search_result_group\n               ~search_result_selected\n               ~link_selected)\n            (match input_mode with\n             | Links -> (\n                 Link_list.main\n                   ~width\n                   ~search_result_group\n                   ~index_of_link_selected:UI_base.Vars.index_of_link_selected\n               )\n             | _ -> (\n                 Search_result_list.main\n                   ~width\n                   ~search_result_group\n                   ~index_of_search_result_selected:UI_base.Vars.index_of_search_result_selected\n               ))\n        ) else (\n          UI_base.Content_view.main\n            ~input_mode\n            ~width\n            ~height\n            ~search_result_group\n            ~search_result_selected\n            ~link_selected\n        )\n      )\n  end\n\n  let item_list\n      ~width\n      ~height\n      ~selected\n      ?highlights\n      items\n    : Nottui.ui Lwd.t =\n    let arr = Dynarray.create () in\n    Seq.iter (fun i ->\n        Dynarray.add_last arr (Dynarray.get items i)\n      ) OSeq.(selected --^ Dynarray.length items);\n    Dynarray.to_seq arr\n    |> Seq.mapi (fun i s ->\n        let highlights =\n          Option.map (fun highlights ->\n              Dynarray.get highlights (selected + i)\n            )\n            highlights\n        in\n        s\n        |> Tokenization.tokenize ~drop_spaces:false\n        |> List.of_seq\n        |> Content_and_search_result_rendering.Text_block_rendering.of_words\n          ~width\n          ?highlights\n        |> Nottui.Ui.atom\n      )\n    |> List.of_seq\n    |> Nottui.Ui.vcat\n    |> Nottui.Ui.resize ~w:width ~h:height\n    |> Lwd.return\n\n  let main\n      ~width\n      ~height\n      ~documents_marked\n      ~screen_split\n      ~show_bottom_right_pane\n      ~(search_result_groups : Session.search_result_group array)\n    : Nottui.ui Lwd.t =\n    let$* input_mode = Lwd.get UI_base.Vars.input_mode in\n    let$* script_selected = Lwd.get UI_base.Vars.index_of_script_selected in\n    let$* usable_scripts, usable_script_highlights =\n      Vars.usable_script_files\n    in\n    let file =\n      if script_selected < Dynarray.length usable_scripts then (\n        Some (Dynarray.get usable_scripts script_selected)\n      ) else (\n        None\n      )\n    in\n    let$* selected = Lwd.get UI_base.Vars.index_of_script_selected in\n    match input_mode with\n    | Save_script -> (\n        item_list\n          ~width\n          ~height\n          ~selected\n          usable_scripts\n      )\n    | Scripts | Delete_script_confirm (_, _) -> (\n        let lines =\n          try\n            match file with\n            | None -> []\n            | Some file -> (\n                let dir = Params.script_dir () in\n                CCIO.with_in (Filename.concat dir file) (fun ic ->\n                    CCIO.read_lines_seq ic\n                    |> OSeq.take height\n                    |> List.of_seq\n                  )\n              )\n          with\n          | Sys_error _ -> []\n        in\n        UI_base.hpane ~l_ratio:0.5 ~width ~height\n          (item_list\n             ~height\n             ~selected\n             ?highlights:usable_script_highlights\n             usable_scripts)\n          (fun ~width ->\n             List.map (fun s -> Nottui.Ui.atom (Notty.I.strf \"%s\" s)) lines\n             |> Nottui.Ui.vcat\n             |> Nottui.Ui.resize ~w:width ~h:height\n             |> Lwd.return\n          )\n      )\n    | _ -> (\n        let$* document_selected = Lwd.get UI_base.Vars.index_of_document_selected in\n        let l_ratio =\n          match screen_split with\n          | `Even -> 0.50\n          | `Focus_left -> 1.0\n          | `Wide_left -> 0.618\n          | `Focus_right -> 0.0\n          | `Wide_right -> 1.0 -. 0.618\n        in\n        UI_base.hpane ~l_ratio ~width ~height\n          (Document_list.main\n             ~height\n             ~documents_marked\n             ~search_result_groups\n             ~document_selected)\n          (Right_pane.main\n             ~height\n             ~search_result_groups\n             ~document_selected\n             ~show_bottom_right_pane)\n      )\nend\n\nmodule Bottom_pane = struct\n  let status_bar\n      ~width\n      ~(search_result_groups : Session.search_result_group array)\n      ~(input_mode : UI_base.input_mode)\n    : Nottui.Ui.t Lwd.t =\n    let open Notty.Infix in\n    let input_mode_image =\n      UI_base.Input_mode_map.find input_mode UI_base.Status_bar.input_mode_images\n    in\n    let attr = UI_base.Status_bar.attr in\n    let$* usable_scripts, _usable_script_highlights = Vars.usable_script_files in\n    let$* script_selected = Lwd.get UI_base.Vars.index_of_script_selected in\n    let usable_script_count = Dynarray.length usable_scripts in\n    let selected_script_file_and_path =\n      if usable_script_count > 0 then (\n        let dir = Params.script_dir () in\n        let file = Dynarray.get usable_scripts script_selected in\n        Some (file, Filename.concat dir file)\n      ) else (\n        None\n      )\n    in\n    match input_mode with\n    | Save_script | Scripts -> (\n        let text_field = Vars.script_name_field in\n        let prompt =\n          match input_mode with\n          | Save_script -> \"Save as\"\n          | Scripts -> \"Filter\"\n          | _ -> failwith \"unexpected case\"\n        in\n        let on_tab =\n          match input_mode with\n          | Save_script -> (\n              Some (fun (text, _) ->\n                  let best_fit =\n                    let usable_script_count = Dynarray.length usable_scripts in\n                    if usable_script_count = 0 then (\n                      text\n                    ) else if usable_script_count = 1 then (\n                      Filename.chop_extension (Dynarray.get usable_scripts 0)\n                    ) else (\n                      usable_scripts\n                      |> Dynarray.to_seq\n                      |> String_utils.longest_common_prefix\n                    )\n                  in\n                  Lwd.set text_field (best_fit, String.length best_fit)\n                )\n            )\n          | _ -> None\n        in\n        let on_submit =\n          match input_mode with\n          | Save_script -> (\n              (fun (text, _x) ->\n                 Lwd.set text_field UI_base.empty_text_field;\n                 Nottui.Focus.release Vars.script_name_field_focus_handle;\n                 Lwd.set UI_base.Vars.input_mode\n                   (if String.length text = 0 then\n                      Save_script_no_name\n                    else\n                      Save_script_overwrite text\n                   );\n              )\n            )\n          | Scripts -> (\n              (fun (_text, _x) ->\n                 Option.iter (fun (_file, path) ->\n                     Lwd.set text_field UI_base.empty_text_field;\n                     Nottui.Focus.release Vars.script_name_field_focus_handle;\n                     Lwd.set UI_base.Vars.quit true;\n                     UI_base.Vars.action := Some (Open_script path);\n                     Lwd.set UI_base.Vars.input_mode Navigate;\n                   ) selected_script_file_and_path;\n              )\n            )\n          | _ -> failwith \"unexpected case\"\n        in\n        let on_up_down =\n          match input_mode with\n          | Save_script -> None\n          | Scripts -> (\n              Some (fun up_down _ ->\n                  UI_base.set_script_selected\n                    ~choice_count:usable_script_count\n                    (script_selected +\n                     (match up_down with\n                      | `Up -> (-1)\n                      | `Down -> 1)\n                    ))\n            )\n          | _ -> failwith \"unexpected case\"\n        in\n        let on_ctrl_prefixed =\n          match input_mode with\n          | Scripts -> (\n              Some (fun key (_text, _x) ->\n                  match key with\n                  | (`ASCII 'X', [`Ctrl]) -> (\n                      Option.iter (fun (file, path) ->\n                          Lwd.set UI_base.Vars.input_mode (Delete_script_confirm (file, path))\n                        ) selected_script_file_and_path;\n                      `Handled\n                    )\n                  | _ -> (\n                      `Unhandled\n                    )\n                )\n            )\n          | _ -> None\n        in\n        let$* content =\n          Nottui_widgets.hbox\n            [\n              Lwd.return\n                (Nottui.Ui.atom\n                   (Notty.I.hcat\n                      [\n                        input_mode_image;\n                        UI_base.Status_bar.element_spacer;\n                        Notty.I.strf ~attr \"%s: [ \" prompt;\n                      ]));\n              UI_base.edit_field text_field\n                ~focus:Vars.script_name_field_focus_handle\n                ~on_cancel:(fun (_, _) ->\n                    Lwd.set UI_base.Vars.input_mode Navigate\n                  )\n                ~on_change:(fun (text, x) ->\n                    Lwd.set text_field (text, x);\n                  )\n                ~on_submit\n                ?on_tab\n                ?on_up_down\n                ?on_ctrl_prefixed;\n              Lwd.return (Nottui.Ui.atom (Notty.I.strf ~attr \" ] + %s\" Params.docfd_script_ext));\n            ]\n        in\n        let$ bar = UI_base.Status_bar.background_bar in\n        Nottui.Ui.join_z bar content\n      )\n    | Save_script_overwrite script_name -> (\n        let path = compute_save_script_path script_name in\n        if Sys.file_exists path then (\n          let$* content =\n            Lwd.return\n              (Nottui.Ui.atom\n                 (Notty.I.hcat\n                    [\n                      input_mode_image;\n                      UI_base.Status_bar.element_spacer;\n                      Notty.I.strf ~attr \"%s already exists, overwrite?\"\n                        (Filename.basename path);\n                    ]))\n          in\n          let$ bar = UI_base.Status_bar.background_bar in\n          Nottui.Ui.join_z bar content\n        ) else (\n          save_script ~path;\n          Lwd.set UI_base.Vars.input_mode (Save_script_edit script_name);\n          UI_base.Status_bar.background_bar\n        )\n      )\n    | Save_script_no_name -> (\n        let$* content =\n          Lwd.return\n            (Nottui.Ui.atom\n               (Notty.I.hcat\n                  [\n                    input_mode_image;\n                    UI_base.Status_bar.element_spacer;\n                    Notty.I.strf ~attr \"No name entered, saving skipped\";\n                  ]))\n        in\n        let$ bar = UI_base.Status_bar.background_bar in\n        Nottui.Ui.join_z bar content\n      )\n    | Save_script_edit script_name -> (\n        let path = compute_save_script_path script_name in\n        let$* content =\n          Lwd.return\n            (Nottui.Ui.atom\n               (Notty.I.hcat\n                  [\n                    input_mode_image;\n                    UI_base.Status_bar.element_spacer;\n                    Notty.I.strf ~attr \"Do you want to edit %s to add comments etc?\" (Filename.basename path);\n                  ]))\n        in\n        let$ bar = UI_base.Status_bar.background_bar in\n        Nottui.Ui.join_z bar content\n      )\n    | Delete_script_confirm (script, _) -> (\n        let$* content =\n          Lwd.return\n            (Nottui.Ui.atom\n               (Notty.I.hcat\n                  [\n                    input_mode_image;\n                    UI_base.Status_bar.element_spacer;\n                    Notty.I.strf ~attr \"Confirm deletion of %s?\" script;\n                  ]))\n        in\n        let$ bar = UI_base.Status_bar.background_bar in\n        Nottui.Ui.join_z bar content\n      )\n    | Path_fuzzy_rank -> (\n        let text_field = Vars.path_fuzzy_rank_field in\n        let document_count = Array.length search_result_groups in\n        let$* document_current_choice = Lwd.get UI_base.Vars.index_of_document_selected in\n        let$* content =\n          Nottui_widgets.hbox\n            [\n              Lwd.return\n                (Nottui.Ui.atom\n                   (Notty.I.hcat\n                      [\n                        input_mode_image;\n                        UI_base.Status_bar.element_spacer;\n                      ]));\n              UI_base.edit_field text_field\n                ~focus:Vars.path_fuzzy_rank_field_focus_handle\n                ~on_cancel:(fun (_, _) ->\n                    Lwd.set UI_base.Vars.input_mode Navigate\n                  )\n                ~on_change:(fun (text, x) ->\n                    Lwd.set text_field (text, x);\n                    update_path_fuzzy_rank ~commit:false ();\n                  )\n                ~on_submit:(fun (text, x) ->\n                    Lwd.set text_field (text, x);\n                    update_path_fuzzy_rank ~commit:true ();\n                    Lwd.set UI_base.Vars.input_mode Navigate\n                  )\n                ~on_up_down:(fun ud _ ->\n                    let offset =\n                      match ud with\n                      | `Up -> -1\n                      | `Down -> 1\n                    in\n                    UI_base.set_document_selected\n                      ~choice_count:document_count\n                      (document_current_choice + offset);\n                  )\n              ;\n            ]\n        in\n        let$ bar = UI_base.Status_bar.background_bar in\n        Nottui.Ui.join_z bar content\n      )\n    | _ -> (\n        let$* index_of_document_selected = Lwd.get UI_base.Vars.index_of_document_selected in\n        let document_count = Array.length search_result_groups in\n        let$* (_cur_ver, snapshot) = Session_manager.cur_snapshot in\n        let content =\n          let file_shown_count =\n            Notty.I.strf ~attr\n              \"%5d/%d documents listed\"\n              document_count\n              (Session.Snapshot.state snapshot\n               |> Session.State.size)\n          in\n          let hint =\n            Notty.I.strf ~attr \"< and > to see more key binding info, ? to toggle hide\"\n          in\n          let hint_len = Notty.I.width hint in\n          let hint_overlay =\n            Notty.I.void\n              (width - hint_len) 1\n            <|>\n            hint\n          in\n          let core =\n            if document_count = 0 then (\n              [\n                UI_base.Status_bar.element_spacer;\n                file_shown_count;\n              ]\n            ) else (\n              let index_of_selected =\n                Notty.I.strf ~attr\n                  \"Index of document selected: %d\"\n                  index_of_document_selected\n              in\n              [\n                file_shown_count;\n                UI_base.Status_bar.element_spacer;\n                index_of_selected;\n              ]\n            )\n          in\n          Notty.I.zcat\n            [\n              Notty.I.hcat\n                (input_mode_image\n                 ::\n                 UI_base.Status_bar.element_spacer\n                 ::\n                 core);\n              hint_overlay;\n            ]\n          |> Nottui.Ui.atom\n        in\n        let$ bar = UI_base.Status_bar.background_bar in\n        Nottui.Ui.join_z bar content\n      )\n\n  module Key_binding_info = struct\n    let grid_contents : UI_base.Key_binding_info.grid_contents =\n      let open UI_base.Key_binding_info in\n      let empty_row =\n        [\n          { label = \"\"; msg = \"\" };\n        ]\n      in\n      let navigate_grid =\n        [\n          [\n            { label = \"Enter\"; msg = \"open document\" };\n            { label = \"/\"; msg = \"SEARCH\" };\n            { label = \"↑/↓/j/k\"; msg = \"select document\" };\n            { label = \"s\"; msg = \"SORT-ASC\" };\n            { label = \"Tab\"; msg = \"expand right pane\" };\n            { label = \"y\"; msg = \"COPY\" };\n            { label = \"n\"; msg = \"NARROW\" };\n            { label = \"Space\"; msg = \"toggle mark\" };\n            { label = \"h\"; msg = \"command history\" };\n            { label = \"Ctrl+S\"; msg = \"save session as script\" };\n          ];\n          [\n            { label = \"v\"; msg = \"focus content\" };\n            { label = \"f\"; msg = \"FILTER\" };\n            { label = \"Shift+↑/↓/j/k\"; msg = \"select search result\" };\n            { label = \"Shift+S\"; msg = \"SORT-DESC\" };\n            { label = \"Shift+Tab\"; msg = \"expand left pane\" };\n            { label = \"Shift+Y\"; msg = \"COPY-PATHS\" };\n            { label = \"d\"; msg = \"DROP\" };\n            { label = \"m\"; msg = \"MARK\" };\n            { label = \"\"; msg = \"\" };\n            { label = \"Ctrl+O\"; msg = \"SCRIPTS\" };\n          ];\n          [\n            { label = \"Ctrl+C\"; msg = \"exit\" };\n            { label = \"x\"; msg = \"CLEAR\" };\n            { label = \"-/=\"; msg = \"scroll content view\" };\n            { label = \"l\"; msg = \"LINKS\" };\n            { label = \"\"; msg = \"\" };\n            { label = \"\"; msg = \"\" };\n            { label = \"r\"; msg = \"RELOAD\" };\n            { label = \"Shift+M\"; msg = \"UNMARK\" };\n            { label = \"\"; msg = \"\" };\n            { label = \"\"; msg = \"\" };\n          ];\n        ]\n      in\n      let search_grid =\n        [\n          [\n            { label = \"Enter\"; msg = \"exit SEARCH\" };\n          ];\n        ]\n      in\n      let filter_grid =\n        [\n          [\n            { label = \"Enter\"; msg = \"exit FILTER\" };\n            { label = \"Tab\"; msg = \"autocomplete\" };\n          ];\n        ]\n      in\n      let save_script_grid =\n        [\n          [\n            { label = \"Enter\"; msg = \"confirm answer\" };\n            { label = \"Tab\"; msg = \"autocomplete\" };\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n          empty_row;\n          empty_row;\n        ]\n      in\n      let save_script_confirm_grid =\n        [\n          [\n            { label = \"y\"; msg = \"confirm overwrite\" };\n            { label = \"Esc/n\"; msg = \"cancel\" };\n          ];\n          empty_row;\n          empty_row;\n        ]\n      in\n      let save_script_cancel_grid =\n        [\n          [\n            { label = \"Enter\"; msg = \"confirm\" };\n          ];\n          empty_row;\n          empty_row;\n        ]\n      in\n      let save_script_edit_grid =\n        [\n          [\n            { label = \"y\"; msg = \"open in editor\" };\n            { label = \"Esc/n\"; msg = \"skip\" };\n          ];\n          empty_row;\n          empty_row;\n        ]\n      in\n      let scripts_grid =\n        [\n          [\n            { label = \"Enter\"; msg = \"open\" };\n            { label = \"↑/↓\"; msg = \"select\" };\n            { label = \"Ctrl+X\"; msg = \"delete\" };\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n          empty_row;\n          empty_row;\n        ]\n      in\n      let delete_script_confirm_grid =\n        [\n          [\n            { label = \"y\"; msg = \"confirm deletion\" };\n            { label = \"Esc/n\"; msg = \"cancel\" };\n          ];\n          empty_row;\n          empty_row;\n        ]\n      in\n      let clear_grid =\n        [\n          [\n            { label = \"/\"; msg = \"search field\" };\n            { label = \"f\"; msg = \"filter field\" };\n            { label = \"h\"; msg = \"command history\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n          empty_row;\n        ]\n      in\n      let sort_asc_grid =\n        [\n          [\n            { label = \"s\"; msg = \"score\" };\n            { label = \"p\"; msg = \"path\" };\n            { label = \"d\"; msg = \"path date\" };\n            { label = \"m\"; msg = \"mod time\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n          empty_row;\n        ]\n      in\n      let sort_desc_grid =\n        [\n          [\n            { label = \"s\"; msg = \"score\" };\n            { label = \"p\"; msg = \"path\" };\n            { label = \"d\"; msg = \"path date\" };\n            { label = \"m\"; msg = \"mod time\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n          empty_row;\n        ]\n      in\n      let drop_grid =\n        [\n          [\n            { label = \"d\"; msg = \"selected\" };\n            { label = \"l\"; msg = \"listed\" };\n            { label = \"m\"; msg = \"marked\" };\n          ];\n          [\n            { label = \"Shift+D\"; msg = \"unselected\" };\n            { label = \"Shift+L\"; msg = \"unlisted\" };\n            { label = \"Shift+M\"; msg = \"unmarked\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n        ]\n      in\n      let mark_grid =\n        [\n          [\n            { label = \"l\"; msg = \"listed\" };\n          ];\n          empty_row;\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n        ]\n      in\n      let unmark_grid =\n        [\n          [\n            { label = \"l\"; msg = \"listed\" };\n            { label = \"a\"; msg = \"all\" };\n          ];\n          empty_row;\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n        ]\n      in\n      let copy_grid =\n        [\n          [\n            { label = \"y\"; msg = \"selected search result\" };\n            { label = \"m\"; msg = \"results of marked documents\" };\n            { label = \"l\"; msg = \"results of listed documents\" };\n          ];\n          [\n            { label = \"a\"; msg = \"results of selected document\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n        ]\n      in\n      let copy_paths_grid =\n        [\n          [\n            { label = \"y\"; msg = \"path of selected document\" };\n            { label = \"m\"; msg = \"paths of marked documents\" };\n            { label = \"l\"; msg = \"paths of listed documents\" };\n          ];\n          [\n            { label = \"\"; msg = \"\" };\n            { label = \"Shift+M\"; msg = \"paths of unmarked documents\" };\n            { label = \"Shift+L\"; msg = \"paths of unlisted documents\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n        ]\n      in\n      let narrow_grid =\n        [\n          [\n            { label = \"0-9\"; msg = \"narrow search scope to level N\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n          empty_row;\n        ]\n      in\n      let reload_grid =\n        [\n          [\n            { label = \"r\"; msg = \"selected\" };\n            { label = \"a\"; msg = \"all\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"cancel\" };\n          ];\n          empty_row;\n        ]\n      in\n      let links_grid =\n        [\n          [\n            { label = \"Enter\"; msg = \"open\" };\n            { label = \"o\"; msg = \"open and remain in LINKS\" };\n            { label = \"↑/↓/j/k\"; msg = \"select\" };\n            { label = \"y\"; msg = \"copy\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"exit\" };\n          ];\n          empty_row;\n        ]\n      in\n      let path_fuzzy_rank_grid =\n        [\n          [\n            { label = \"Enter\"; msg = \"pin to top\" };\n            { label = \"↑/↓\"; msg = \"select\" };\n          ];\n          [\n            { label = \"Esc\"; msg = \"exit\" };\n          ];\n          empty_row;\n        ]\n      in\n      [\n        (Navigate, navigate_grid);\n        (Search, search_grid);\n        (Filter, filter_grid);\n        (Clear, clear_grid);\n        (Sort `Asc, sort_asc_grid);\n        (Sort `Desc, sort_desc_grid);\n        (Drop, drop_grid);\n        (Mark, mark_grid);\n        (Unmark, unmark_grid);\n        (Narrow, narrow_grid);\n        (Copy, copy_grid);\n        (Copy_paths, copy_paths_grid);\n        (Reload, reload_grid);\n        (Save_script, save_script_grid);\n        (Save_script_overwrite \"\", save_script_confirm_grid);\n        (Save_script_no_name, save_script_cancel_grid);\n        (Save_script_edit \"\", save_script_edit_grid);\n        (Scripts, scripts_grid);\n        (Delete_script_confirm (\"\", \"\"), delete_script_confirm_grid);\n        (Links, links_grid);\n        (Path_fuzzy_rank, path_fuzzy_rank_grid);\n      ]\n\n    let grid_lookup = UI_base.Key_binding_info.make_grid_lookup grid_contents\n\n    let main ~input_mode ~show_key_binding_info_pane =\n      if show_key_binding_info_pane then (\n        UI_base.Key_binding_info.main ~grid_lookup ~input_mode\n      ) else (\n        Lwd.return (Nottui.Ui.atom (Notty.I.void 0 0))\n      )\n  end\n\n  let autocomplete_grid ~input_mode ~width =\n    match input_mode with\n    | UI_base.Filter -> (\n        let$* l = Lwd.get UI_base.Vars.autocomplete_choices in\n        let max_len =\n          List.fold_left (fun n x ->\n              max n (String.length x)\n            ) 0 l\n        in\n        let cell_len = max_len + 4 in\n        let cells_per_row = width / cell_len in\n        l\n        |> CCList.chunks cells_per_row\n        |> (fun rows ->\n            let row_count = List.length rows in\n            let padding =\n              if row_count < 2 then (\n                CCList.(0 --^ (2 - row_count))\n                |> List.map (fun _ -> [ \"\" ])\n              ) else (\n                []\n              )\n            in\n            rows @ padding\n          )\n        |> List.map (fun row ->\n            List.map (fun s ->\n                let full_background = Notty.I.void cell_len 1 in\n                Notty.I.(strf \"%s\" s </> full_background)\n                |> Nottui.Ui.atom\n                |> Lwd.return\n              ) row\n          )\n        |> Nottui_widgets.grid\n          ~pad:(Nottui.Gravity.make ~h:`Negative ~v:`Negative)\n      )\n    | _ -> Lwd.return (Nottui.Ui.atom (Notty.I.void 0 0))\n\n  let filter_bar =\n    UI_base.Filter_bar.main\n      ~text_field:UI_base.Vars.filter_field\n      ~focus_handle:UI_base.Vars.filter_field_focus_handle\n      ~on_change:(update_filter ~commit:false)\n      ~on_submit:(update_filter ~commit:true)\n\n  let search_bar ~input_mode =\n    UI_base.Search_bar.main ~input_mode\n      ~text_field:UI_base.Vars.search_field\n      ~focus_handle:UI_base.Vars.search_field_focus_handle\n      ~on_change:(update_search ~commit:false)\n      ~on_submit:(update_search ~commit:true)\n\n  let main ~width ~show_key_binding_info_pane ~search_result_groups =\n    let$* input_mode = Lwd.get UI_base.Vars.input_mode in\n    Nottui_widgets.vbox\n      [\n        status_bar ~width ~search_result_groups ~input_mode;\n        Key_binding_info.main ~input_mode ~show_key_binding_info_pane;\n        autocomplete_grid ~input_mode ~width;\n        filter_bar ~input_mode;\n        search_bar ~input_mode;\n      ]\nend\n\nlet keyboard_handler\n    ~(session_state : Session.State.t)\n    ~(search_result_groups : Session.search_result_group array)\n    (key : Nottui.Ui.key)\n  =\n  let document_count =\n    Array.length search_result_groups\n  in\n  let document_current_choice =\n    Lwd.peek UI_base.Vars.index_of_document_selected\n  in\n  let search_result_group =\n    if document_count = 0 then\n      None\n    else\n      Some search_result_groups.(document_current_choice)\n  in\n  let search_result_choice_count =\n    match search_result_group with\n    | None -> 0\n    | Some (_doc, search_results) -> Array.length search_results\n  in\n  let link_choice_count =\n    match search_result_group with\n    | None -> 0\n    | Some (doc, _search_results) -> Array.length (Document.links doc)\n  in\n  let search_result_current_choice =\n    Lwd.peek UI_base.Vars.index_of_search_result_selected\n  in\n  let link_current_choice =\n    Lwd.peek UI_base.Vars.index_of_link_selected\n  in\n  match Lwd.peek UI_base.Vars.input_mode with\n  | Navigate -> (\n      match key with\n      | (`ASCII 'C', [`Ctrl])\n      | (`ASCII 'Q', [`Ctrl]) -> (\n          Lwd.set UI_base.Vars.quit true;\n          UI_base.Vars.action := None;\n          `Handled\n        )\n      | (`ASCII '<', []) -> (\n          UI_base.Key_binding_info.decr_rotation ();\n          `Handled\n        )\n      | (`ASCII '>', []) -> (\n          UI_base.Key_binding_info.incr_rotation ();\n          `Handled\n        )\n      | (`ASCII ' ', []) -> (\n          let index = Lwd.peek UI_base.Vars.index_of_document_selected in\n          if index < Array.length search_result_groups then (\n            let doc, _ = search_result_groups.(index) in\n            toggle_mark ~path:(Document.path doc)\n          );\n          `Handled\n        )\n      | (`ASCII 'm', []) -> (\n          UI_base.set_input_mode Mark;\n          `Handled\n        )\n      | (`ASCII 'M', []) -> (\n          UI_base.set_input_mode Unmark;\n          `Handled\n        )\n      | (`ASCII 'd', []) -> (\n          UI_base.set_input_mode Drop;\n          `Handled\n        )\n      | (`ASCII 'n', []) -> (\n          UI_base.set_input_mode Narrow;\n          `Handled\n        )\n      | (`ASCII 'r', []) -> (\n          UI_base.set_input_mode Reload;\n          `Handled\n        )\n      | (`ASCII 'y', []) -> (\n          UI_base.set_input_mode Copy;\n          `Handled\n        )\n      | (`ASCII 'Y', []) -> (\n          UI_base.set_input_mode Copy_paths;\n          `Handled\n        )\n      | (`Arrow `Left, [])\n      | (`ASCII 'u', [])\n      | (`ASCII 'Z', [`Ctrl]) -> (\n          Session_manager.shift_ver ~offset:(-1);\n          `Handled\n        )\n      | (`Arrow `Right, [])\n      | (`ASCII 'R', [`Ctrl])\n      | (`ASCII 'Y', [`Ctrl]) -> (\n          Session_manager.shift_ver ~offset:1;\n          `Handled\n        )\n      | (`Tab, [])\n      | (`Tab, [`Shift]) -> (\n          let direction =\n            match key with\n            | (_, [`Shift]) -> `Expand_left\n            | (_, _) -> `Expand_right\n          in\n          Session_manager.update_from_cur_snapshot\n            (fun cur_snapshot ->\n               let state = Session.Snapshot.state cur_snapshot in\n               let cur = Session.State.screen_split state in\n               let offset =\n                 match direction with\n                 | `Expand_left -> 1\n                 | `Expand_right -> -1\n               in\n               let next =\n                 Command.screen_split_of_int\n                   (Command.int_of_screen_split cur + offset)\n               in\n               let command = `Split_screen next in\n               state\n               |> Session.run_command\n                 (UI_base.task_pool ())\n                 command\n               |> Option.get\n               |> (fun (command, state) ->\n                   Session.Snapshot.make\n                     ~last_command:(Some command)\n                     state)\n            );\n          `Handled\n        )\n      | (`ASCII '?', [])\n      | (`ASCII 'v', []) -> (\n          let pane =\n            match key with\n            | (`ASCII '?', []) -> `Key_binding_info\n            | (`ASCII 'v', []) -> `Bottom_right\n            | _ -> failwith \"unexpected case\"\n          in\n          Session_manager.update_from_cur_snapshot\n            (fun cur_snapshot ->\n               let state = Session.Snapshot.state cur_snapshot in\n               let cur = Session.State.show_pane state pane in\n               let command =\n                 if cur then (\n                   `Hide_pane pane\n                 ) else (\n                   `Show_pane pane\n                 ) in\n               state\n               |> Session.run_command\n                 (UI_base.task_pool ())\n                 command\n               |> Option.get\n               |> (fun (command, state) ->\n                   Session.Snapshot.make\n                     ~last_command:(Some command)\n                     state)\n            );\n          `Handled\n        )\n      | (`ASCII '=', []) -> (\n          UI_base.incr_content_view_offset ();\n          `Handled\n        )\n      | (`ASCII '-', []) -> (\n          UI_base.decr_content_view_offset ();\n          `Handled\n        )\n      | (`Page `Down, [`Shift])\n      | (`ASCII 'J', [])\n      | (`Arrow `Down, [`Shift]) -> (\n          UI_base.set_search_result_selected\n            ~choice_count:search_result_choice_count\n            (search_result_current_choice+1);\n          `Handled\n        )\n      | (`Page `Up, [`Shift])\n      | (`ASCII 'K', [])\n      | (`Arrow `Up, [`Shift]) -> (\n          UI_base.set_search_result_selected\n            ~choice_count:search_result_choice_count\n            (search_result_current_choice-1);\n          `Handled\n        )\n      | (`Page `Down, [])\n      | (`ASCII 'j', [])\n      | (`Arrow `Down, []) -> (\n          if document_count = 1 then (\n            UI_base.set_search_result_selected\n              ~choice_count:search_result_choice_count\n              (search_result_current_choice+1);\n            `Handled\n          ) else (\n            UI_base.set_document_selected\n              ~choice_count:document_count\n              (document_current_choice+1);\n            `Handled\n          )\n        )\n      | (`Page `Up, [])\n      | (`ASCII 'k', [])\n      | (`Arrow `Up, []) -> (\n          if document_count = 1 then (\n            UI_base.set_search_result_selected\n              ~choice_count:search_result_choice_count\n              (search_result_current_choice-1);\n            `Handled\n          ) else (\n            UI_base.set_document_selected\n              ~choice_count:document_count\n              (document_current_choice-1);\n            `Handled\n          )\n        )\n      | (`ASCII 'g', []) -> (\n          UI_base.set_document_selected\n            ~choice_count:document_count\n            0;\n          `Handled\n        )\n      | (`ASCII 'G', []) -> (\n          UI_base.set_document_selected\n            ~choice_count:document_count\n            (document_count - 1);\n          `Handled\n        )\n      | (`ASCII 'f', []) -> (\n          Nottui.Focus.request UI_base.Vars.filter_field_focus_handle;\n          UI_base.set_input_mode Filter;\n          `Handled\n        )\n      | (`ASCII 'F', []) -> (\n          Nottui.Focus.request Vars.path_fuzzy_rank_field_focus_handle;\n          Lwd.set Vars.path_fuzzy_rank_field UI_base.empty_text_field;\n          UI_base.set_input_mode Path_fuzzy_rank;\n          `Handled\n        )\n      | (`ASCII '/', []) -> (\n          Nottui.Focus.request UI_base.Vars.search_field_focus_handle;\n          UI_base.set_input_mode Search;\n          `Handled\n        )\n      | (`ASCII 'h', []) -> (\n          Lwd.set UI_base.Vars.quit true;\n          UI_base.Vars.action := Some UI_base.Edit_command_history;\n          `Handled\n        )\n      | (`ASCII 'S', [`Ctrl]) -> (\n          UI_base.set_input_mode Save_script;\n          refresh_script_files ();\n          Nottui.Focus.request Vars.script_name_field_focus_handle;\n          `Handled\n        )\n      | (`ASCII 'O', [`Ctrl]) -> (\n          UI_base.set_input_mode Scripts;\n          refresh_script_files ();\n          Nottui.Focus.request Vars.script_name_field_focus_handle;\n          `Handled\n        )\n      | (`ASCII 'x', []) -> (\n          UI_base.set_input_mode Clear;\n          `Handled\n        )\n      | (`ASCII 'l', []) -> (\n          UI_base.set_input_mode Links;\n          if search_result_choice_count > 0  then (\n            let (doc, search_results) = Option.get search_result_group in\n            let search_result = search_results.(search_result_current_choice) in\n            let links = Document.links doc in\n            let avg_pos =\n              List.fold_left (fun min_max_pos search_result ->\n                  let { Search_result.found_word_pos; _ } = search_result in\n                  match min_max_pos with\n                  | None -> Some (found_word_pos, found_word_pos)\n                  | Some (min_pos, max_pos) -> (\n                      Some (min found_word_pos min_pos,\n                            max found_word_pos max_pos)\n                    )\n                )\n                None\n                (Search_result.found_phrase search_result)\n              |> (fun x ->\n                  let (x, y) = Option.get x in\n                  (x + y) / 2)\n            in\n            let before, exact, after = Int_map.split avg_pos (Document.link_index_of_start_pos doc) in\n            let index =\n              match exact with\n              | Some index -> Some index\n              | None -> (\n                  match\n                    Int_map.max_binding_opt before,\n                    Int_map.min_binding_opt after\n                  with\n                  | Some (pos_x, index_x), Some (pos_y, index_y) -> (\n                      let diff_x = Int.to_float (Int.abs (pos_x - avg_pos)) in\n                      let diff_y = Int.to_float (Int.abs (pos_y - avg_pos)) in\n                      (* We prefer picking y (link after search result)\n                         over x (link before search result), as it usually feels more\n                         intuitive to jump forward than backward.\n\n                         But if distance to x is <= 50% the distance\n                         to y, then we resort to x.\n                      *)\n                      if diff_x /. diff_y <= 0.5 then (\n                        Some index_x\n                      ) else (\n                        let link_x = links.(index_x) in\n                        let end_inc_pos_x = link_x.Link.end_inc_pos in\n                        if pos_x <= avg_pos && avg_pos <= end_inc_pos_x then (\n                          Some index_x\n                        ) else (\n                          Some index_y\n                        )\n                      )\n                    )\n                  | Some (_pos, index), None\n                  | None, Some (_pos, index) -> Some index\n                  | None, None -> None\n                )\n            in\n            match index with\n            | None -> ()\n            | Some index -> (\n                UI_base.set_link_selected\n                  ~choice_count:link_choice_count\n                  index\n              )\n          );\n          `Handled\n        )\n      | (`ASCII 's', []) -> (\n          UI_base.set_input_mode (Sort `Asc);\n          `Handled\n        )\n      | (`ASCII 'S', []) -> (\n          UI_base.set_input_mode (Sort `Desc);\n          `Handled\n        )\n      | (`Enter, []) -> (\n          Option.iter (fun (doc, search_results) ->\n              let search_result =\n                if search_result_current_choice < Array.length search_results then\n                  Some search_results.(search_result_current_choice)\n                else\n                  None\n              in\n              Lwd.set UI_base.Vars.quit true;\n              UI_base.Vars.action :=\n                Some (UI_base.Open_file_and_search_result (doc, search_result));\n            )\n            search_result_group;\n          `Handled\n        )\n      | _ -> `Handled\n    )\n  | Sort order -> (\n      let exit =\n        match key with\n        | (`Escape, []) -> true\n        | (`ASCII 's', []) -> (\n            sort (`Score, order);\n            true\n          )\n        | (`ASCII 'p', []) -> (\n            sort (`Path, order);\n            true\n          )\n        | (`ASCII 'd', []) -> (\n            sort (`Path_date, order);\n            true\n          )\n        | (`ASCII 'm', []) -> (\n            sort (`Mod_time, order);\n            true\n          )\n        | _ -> false\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Clear -> (\n      let exit =\n        match key with\n        | (`Escape, []) -> true\n        | (`ASCII '/', []) -> (\n            Lwd.set UI_base.Vars.search_field UI_base.empty_text_field;\n            update_search ~commit:true ();\n            true\n          )\n        | (`ASCII 'f', []) -> (\n            Lwd.set UI_base.Vars.filter_field UI_base.empty_text_field;\n            update_filter ~commit:true ();\n            true\n          )\n        | (`ASCII 'h', []) -> (\n            Lwd.set UI_base.Vars.quit true;\n            UI_base.Vars.action := Some UI_base.Clear_command_history;\n            true\n          )\n        | _ -> false\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Drop -> (\n      let exit =\n        match key with\n        | (`Escape, []) -> true\n        | (`ASCII '>', []) -> (\n            UI_base.Key_binding_info.incr_rotation ();\n            false\n          )\n        | (`ASCII 'd', []) -> (\n            Option.iter (fun (doc, _search_results) ->\n                drop ~document_count (`Path (Document.path doc))\n              ) search_result_group;\n            true\n          )\n        | (`ASCII 'D', []) -> (\n            Option.iter (fun (doc, _search_results) ->\n                drop ~document_count (`All_except (Document.path doc))\n              ) search_result_group;\n            true\n          )\n        | (`ASCII 'l', []) -> (\n            drop ~document_count `Listed;\n            true\n          )\n        | (`ASCII 'L', []) -> (\n            drop ~document_count `Unlisted;\n            true\n          )\n        | (`ASCII 'm', []) -> (\n            drop ~document_count `Marked;\n            true\n          )\n        | (`ASCII 'M', []) -> (\n            drop ~document_count `Unmarked;\n            true\n          )\n        | _ -> false\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Mark -> (\n      let exit =\n        match key with\n        | (`Escape, []) -> true\n        | (`ASCII '>', []) -> (\n            UI_base.Key_binding_info.incr_rotation ();\n            false\n          )\n        | (`ASCII 'l', []) -> (\n            mark `Listed;\n            true\n          )\n        | _ -> false\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Unmark -> (\n      let exit =\n        match key with\n        | (`Escape, []) -> true\n        | (`ASCII '>', []) -> (\n            UI_base.Key_binding_info.incr_rotation ();\n            false\n          )\n        | (`ASCII 'l', []) -> (\n            unmark `Listed;\n            true\n          )\n        | (`ASCII 'a', []) -> (\n            unmark `All;\n            true\n          )\n        | _ -> false\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Narrow -> (\n      let exit =\n        match key with\n        | (`Escape, []) -> true\n        | (`ASCII '>', []) -> (\n            UI_base.Key_binding_info.incr_rotation ();\n            false\n          )\n        | (`ASCII c, []) -> (\n            let code_0 = Char.code '0' in\n            let code_9 = Char.code '9' in\n            let code_c = Char.code c in\n            if code_0 <= code_c && code_c <= code_9 then (\n              let level = code_c - code_0 in\n              narrow_search_scope_to_level ~level;\n              true\n            ) else (\n              false\n            )\n          )\n        | _ -> false\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Copy -> (\n      let copy_search_result_groups (s : Session.search_result_group Seq.t) =\n        Clipboard.pipe_to_clipboard (fun oc ->\n            Printers.search_result_groups\n              ~color:false\n              ~underline:true\n              oc\n              s\n          )\n      in\n      let copy_search_result_group x =\n        copy_search_result_groups (Seq.return x)\n      in\n      let exit =\n        match key with\n        | (`Escape, []) -> true\n        | (`ASCII '>', []) -> (\n            UI_base.Key_binding_info.incr_rotation ();\n            false\n          )\n        | (`ASCII 'y', []) -> (\n            Option.iter (fun (doc, search_results) ->\n                copy_search_result_group\n                  (doc,\n                   (if search_result_current_choice < Array.length search_results then\n                      [|search_results.(search_result_current_choice)|]\n                    else\n                      [||])\n                  )\n              )\n              search_result_group;\n            true\n          )\n        | (`ASCII 'a', []) -> (\n            Option.iter\n              copy_search_result_group\n              search_result_group;\n            true\n          )\n        | (`ASCII 'm', []) -> (\n            let marked =\n              Session.State.marked_document_paths session_state\n            in\n            search_result_groups\n            |> Array.to_seq\n            |> Seq.filter (fun (doc, _) ->\n                String_set.mem (Document.path doc) marked)\n            |> copy_search_result_groups;\n            true\n          )\n        | (`ASCII 'l', []) -> (\n            search_result_groups\n            |> Array.to_seq\n            |> copy_search_result_groups;\n            true\n          )\n        | _ -> false\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Copy_paths -> (\n      let copy_paths s =\n        Clipboard.pipe_to_clipboard (fun oc ->\n            Seq.iter (Printers.path_image ~color:false oc) s\n          )\n      in\n      let exit =\n        match key with\n        | (`Escape, []) -> true\n        | (`ASCII '>', []) -> (\n            UI_base.Key_binding_info.incr_rotation ();\n            false\n          )\n        | (`ASCII 'y', []) -> (\n            Option.iter (fun (doc, _search_results) ->\n                copy_paths (Seq.return (Document.path doc))\n              )\n              search_result_group;\n            true\n          )\n        | (`ASCII 'm', []) -> (\n            String_set.inter\n              (Session.State.usable_document_paths session_state)\n              (Session.State.marked_document_paths session_state)\n            |> String_set.to_seq\n            |> copy_paths;\n            true\n          )\n        | (`ASCII 'M', []) -> (\n            String_set.diff\n              (Session.State.usable_document_paths session_state)\n              (Session.State.marked_document_paths session_state)\n            |> String_set.to_seq\n            |> copy_paths;\n            true\n          )\n        | (`ASCII 'l', []) -> (\n            Session.State.usable_document_paths session_state\n            |> String_set.to_seq\n            |> copy_paths;\n            true\n          )\n        | (`ASCII 'L', []) -> (\n            Session.State.unusable_document_paths session_state\n            |> copy_paths;\n            true\n          )\n        | _ -> false\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Reload -> (\n      let exit =\n        (match key with\n         | (`Escape, []) -> true\n         | (`ASCII '>', []) -> (\n             UI_base.Key_binding_info.incr_rotation ();\n             false\n           )\n         | (`ASCII 'r', []) -> (\n             reload_document_selected ~search_result_groups;\n             true\n           )\n         | (`ASCII 'a', []) -> (\n             UI_base.reset_document_selected ();\n             Lwd.set UI_base.Vars.quit true;\n             UI_base.Vars.action := Some UI_base.Recompute_document_src;\n             true\n           )\n         | _ -> false\n        );\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Links -> (\n      let doc_and_link =\n        if link_choice_count > 0 then (\n          Option.map (fun (doc, _search_results) ->\n              (doc, (Document.links doc).(link_current_choice))\n            ) search_result_group\n        ) else (\n          None\n        )\n      in\n      let set_action_to_open_link () =\n        Option.iter (fun (doc, link) ->\n            Lwd.set UI_base.Vars.quit true;\n            UI_base.Vars.action :=\n              Some (UI_base.Open_link (doc, link))\n          ) doc_and_link\n      in\n      let exit =\n        (match key with\n         | (`Escape, []) -> true\n         | (`Enter, []) -> (\n             set_action_to_open_link ();\n             true\n           )\n         | (`ASCII 'o', []) -> (\n             set_action_to_open_link ();\n             false\n           )\n         | (`Page `Down, [])\n         | (`ASCII 'j', [])\n         | (`Arrow `Down, []) -> (\n             UI_base.set_link_selected\n               ~choice_count:link_choice_count\n               (link_current_choice+1);\n             false\n           )\n         | (`Page `Up, [])\n         | (`ASCII 'k', [])\n         | (`Arrow `Up, []) -> (\n             UI_base.set_link_selected\n               ~choice_count:link_choice_count\n               (link_current_choice-1);\n             false\n           )\n         | (`ASCII 'y', []) -> (\n             Option.iter (fun (_doc, link) ->\n                 Clipboard.pipe_to_clipboard (fun oc ->\n                     output_string oc link.Link.link\n                   );\n               ) doc_and_link;\n             true\n           )\n         | _ -> false\n        );\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Save_script_overwrite script_name -> (\n      (match key with\n       | (`Escape, [])\n       | (`ASCII 'n', []) -> (\n           UI_base.set_input_mode Navigate;\n         )\n       | (`ASCII 'y', []) -> (\n           let path = compute_save_script_path script_name in\n           save_script ~path;\n           UI_base.set_input_mode (Save_script_edit script_name);\n         )\n       | _ -> ()\n      );\n      `Handled\n    )\n  | Save_script_no_name -> (\n      let exit =\n        (match key with\n         | (`Enter, []) -> true\n         | _ -> false\n        );\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Save_script_edit script_name -> (\n      let exit =\n        (match key with\n         | (`Escape, [])\n         | (`ASCII 'n', []) -> true\n         | (`ASCII 'y', []) -> (\n             let path = compute_save_script_path script_name in\n             Lwd.set UI_base.Vars.quit true;\n             UI_base.Vars.action := Some (UI_base.Edit_script path);\n             true\n           )\n         | _ -> false\n        );\n      in\n      if exit then (\n        UI_base.set_input_mode Navigate;\n      );\n      `Handled\n    )\n  | Delete_script_confirm (_script, path) -> (\n      (match key with\n       | (`Escape, [])\n       | (`ASCII 'n', []) -> (\n           UI_base.set_input_mode Scripts;\n         )\n       | (`ASCII 'y', []) -> (\n           Sys.remove path;\n           refresh_script_files ();\n           UI_base.set_input_mode Scripts;\n         )\n       | _ -> ()\n      );\n      `Handled\n    )\n  | _ -> `Unhandled\n\nlet main : Nottui.ui Lwd.t =\n  let$* (_, snapshot) =\n    Session_manager.cur_snapshot\n  in\n  let session_state =\n    Session.Snapshot.state snapshot\n  in\n  let search_result_groups =\n    Session.State.search_result_groups session_state\n  in\n  let document_count = Array.length search_result_groups in\n  UI_base.set_document_selected\n    ~choice_count:document_count\n    (Lwd.peek UI_base.Vars.index_of_document_selected);\n  if document_count > 0 then (\n    UI_base.set_search_result_selected\n      ~choice_count:(Array.length\n                       (snd search_result_groups.(Lwd.peek UI_base.Vars.index_of_document_selected)))\n      (Lwd.peek UI_base.Vars.index_of_search_result_selected)\n  );\n  if document_count > 0 then (\n    UI_base.set_link_selected\n      ~choice_count:(search_result_groups.(Lwd.peek UI_base.Vars.index_of_document_selected)\n                     |> fst\n                     |> Document.links\n                     |> Array.length)\n      (Lwd.peek UI_base.Vars.index_of_link_selected)\n  );\n  let$* (term_width, term_height) = Lwd.get UI_base.Vars.term_width_height in\n  let show_key_binding_info_pane = Session.State.show_pane session_state `Key_binding_info in\n  let$* bottom_pane =\n    Bottom_pane.main\n      ~width:term_width\n      ~search_result_groups\n      ~show_key_binding_info_pane\n  in\n  let bottom_pane_height = Nottui.Ui.layout_height bottom_pane in\n  let top_pane_height = term_height - bottom_pane_height in\n  let screen_split = Session.State.screen_split session_state in\n  let show_bottom_right_pane = Session.State.show_pane session_state `Bottom_right in\n  let$* top_pane =\n    Top_pane.main\n      ~width:term_width\n      ~height:top_pane_height\n      ~documents_marked:(Session.State.marked_document_paths session_state)\n      ~screen_split\n      ~show_bottom_right_pane\n      ~search_result_groups\n  in\n  Nottui_widgets.vbox\n    [\n      Lwd.return (Nottui.Ui.keyboard_area\n                    (keyboard_handler ~session_state ~search_result_groups)\n                    top_pane);\n      Lwd.return bottom_pane;\n    ]\n"
  },
  {
    "path": "bin/UI_base.ml",
    "content": "open Docfd_lib\nopen Lwd_infix\n\ntype input_mode =\n  | Navigate\n  | Search\n  | Filter\n  | Clear\n  | Sort of [ `Asc | `Desc ]\n  | Drop\n  | Mark\n  | Unmark\n  | Narrow\n  | Copy\n  | Copy_paths\n  | Reload\n  | Save_script\n  | Save_script_overwrite of string\n  | Save_script_no_name\n  | Save_script_edit of string\n  | Scripts\n  | Delete_script_confirm of string * string\n  | Links\n  | Path_fuzzy_rank\n[@@deriving ord]\n\nmodule Input_mode_map = Map.Make (struct\n    type t = input_mode\n\n    let compare x y =\n      match x, y with\n      | Save_script_overwrite _, Save_script_overwrite _ -> 0\n      | Save_script_edit _, Save_script_edit _ -> 0\n      | Delete_script_confirm _, Delete_script_confirm _ -> 0\n      | _, _ -> compare_input_mode x y\n  end)\n\ntype top_level_action =\n  | Recompute_document_src\n  | Open_file_and_search_result of Document.t * Search_result.t option\n  | Open_link of (Document.t * Link.t)\n  | Clear_command_history\n  | Edit_command_history\n  | Open_script of string\n  | Edit_script of string\n\ntype search_status = [\n  | `Idle\n  | `Searching\n  | `Parse_error\n]\n\ntype filter_status = [\n  | `Idle\n  | `Filtering\n  | `Parse_error\n]\n\nlet empty_text_field = (\"\", 0)\n\nlet render_mode_of_document (doc : Document.t)\n  : Content_and_search_result_rendering.render_mode =\n  match File_utils.format_of_file (Document.path doc) with\n  | `PDF -> `Page_num_only\n  | `Pandoc_supported_format -> `None\n  | `Text | `Other -> `Line_num_only\n\nmodule Vars = struct\n  let quit = Lwd.var false\n\n  let pool : Task_pool.t option Atomic.t = Atomic.make None\n\n  let action : top_level_action option ref = ref None\n\n  let eio_env : Eio_unix.Stdenv.base option ref = ref None\n\n  let hide_document_list : bool Lwd.var = Lwd.var false\n\n  let input_mode : input_mode Lwd.var = Lwd.var Navigate\n\n  let document_src : Document_src.t ref = ref (Document_src.(Files empty_file_collection))\n\n  let term : Notty_unix.Term.t option ref = ref None\n\n  let term_width_height : (int * int) Lwd.var = Lwd.var (0, 0)\n\n  let content_view_offset = Lwd.var 0\n\n  let autocomplete_choices = Lwd.var []\n\n  let filter_field = Lwd.var empty_text_field\n\n  let filter_field_focus_handle = Nottui.Focus.make ()\n\n  let search_field = Lwd.var empty_text_field\n\n  let search_field_focus_handle = Nottui.Focus.make ()\n\n  let search_ui_status : search_status Lwd.var = Lwd.var `Idle\n\n  let filter_ui_status : filter_status Lwd.var = Lwd.var `Idle\n\n  let index_of_document_selected = Lwd.var 0\n\n  let index_of_search_result_selected = Lwd.var 0\n\n  let index_of_link_selected = Lwd.var 0\n\n  let index_of_script_selected = Lwd.var 0\nend\n\nlet reset_content_view_offset () =\n  Lwd.set Vars.content_view_offset 0\n\nlet decr_content_view_offset () =\n  let x = Lwd.peek Vars.content_view_offset in\n  Lwd.set Vars.content_view_offset (x - 1)\n\nlet incr_content_view_offset () =\n  let x = Lwd.peek Vars.content_view_offset in\n  Lwd.set Vars.content_view_offset (x + 1)\n\nlet reset_document_selected () =\n  reset_content_view_offset ();\n  Lwd.set Vars.index_of_document_selected 0;\n  Lwd.set Vars.index_of_search_result_selected 0;\n  Lwd.set Vars.index_of_link_selected 0\n\nlet set_document_selected ~choice_count n =\n  let n = Misc_utils.bound_selection ~choice_count n in\n  let old = Lwd.peek Vars.index_of_document_selected in\n  if old <> n then (\n    reset_content_view_offset ();\n    Lwd.set Vars.index_of_document_selected n;\n    Lwd.set Vars.index_of_search_result_selected 0;\n    Lwd.set Vars.index_of_link_selected 0;\n  )\n\nlet set_search_result_selected ~choice_count n =\n  let old = Lwd.peek Vars.index_of_search_result_selected in\n  if old <> n then (\n    reset_content_view_offset ();\n    let n = Misc_utils.bound_selection ~choice_count n in\n    Lwd.set Vars.index_of_search_result_selected n\n  )\n\nlet set_link_selected ~choice_count n =\n  let old = Lwd.peek Vars.index_of_link_selected in\n  if old <> n then (\n    reset_content_view_offset ();\n    let n = Misc_utils.bound_selection ~choice_count n in\n    Lwd.set Vars.index_of_link_selected n\n  )\n\nlet set_script_selected ~choice_count n =\n  let old = Lwd.peek Vars.index_of_script_selected in\n  if old <> n then (\n    let n = Misc_utils.bound_selection ~choice_count n in\n    Lwd.set Vars.index_of_script_selected n\n  )\n\nlet task_pool () =\n  Option.get (Atomic.get Vars.pool)\n\nlet eio_env () =\n  Option.get !Vars.eio_env\n\nlet term () =\n  Option.get !Vars.term\n\nlet full_term_sized_background =\n  let$ (term_width, term_height) = Lwd.get Vars.term_width_height in\n  Notty.I.void term_width term_height\n  |> Nottui.Ui.atom\n\nlet vbar ~height =\n  let uc = Uchar.of_int 0x2502 in\n  Notty.I.uchar Notty.A.(fg white) uc 1 height\n  |> Nottui.Ui.atom\n\nlet hbar ~width =\n  let uc = Uchar.of_int 0x2015 in\n  Notty.I.uchar Notty.A.(fg white) uc width 1\n  |> Nottui.Ui.atom\n\nlet hpane\n    ~l_ratio\n    ~width\n    ~height\n    (x : width:int -> Nottui.ui Lwd.t)\n    (y : width:int -> Nottui.ui Lwd.t)\n  : Nottui.ui Lwd.t =\n  let l_width =\n    (* Minus 1 for pane separator bar. *)\n    Int.to_float width *. l_ratio\n    |> Float.floor\n    |> Int.of_float\n    |> (fun x ->\n        if x = 0 || x = width then (\n          x\n        ) else (\n          x - 1\n        ))\n  in\n  let r_width =\n    (* Minus 1 here too just to be conservative. *)\n    width - l_width - 1\n  in\n  let crop w x = Nottui.Ui.resize ~w ~h:height x in\n  let x () =\n    let$ x = x ~width:l_width in\n    crop l_width x\n  in\n  let y () =\n    let$ y = y ~width:r_width in\n    crop r_width y\n  in\n  if l_width = 0 then (\n    y ()\n  ) else if r_width = 0 then (\n    x ()\n  ) else (\n    let$* x = x () in\n    let$ y = y () in\n    Nottui.Ui.hcat [\n      x;\n      vbar ~height;\n      y;\n    ]\n  )\n\nlet vpane\n    ~width\n    ~height\n    (x : height:int -> Nottui.ui Lwd.t)\n    (y : height:int -> Nottui.ui Lwd.t)\n  : Nottui.ui Lwd.t =\n  let t_height =\n    (Misc_utils.div_round_up height 2)\n  in\n  let b_height =\n    (* Minus 1 for pane separator bar. *)\n    (height / 2) - 1\n  in\n  let$* x = x ~height:t_height in\n  let$ y = y ~height:b_height in\n  let crop h x = Nottui.Ui.resize ~w:width ~h x in\n  Nottui.Ui.vcat [\n    crop t_height x;\n    hbar ~width;\n    crop b_height y;\n  ]\n\nlet (mini, maxi, clampi) = Lwd_utils.(mini, maxi, clampi)\n\n(* Modified from upstream Nottui source code. *)\nlet edit_field\n    ~focus\n    ~on_change\n    ~on_submit\n    ~on_cancel\n    ?(on_tab : ((string * int) -> unit) option)\n    ?(on_up_down : ([ `Up | `Down ] -> (string * int) -> unit) option)\n    ?(on_ctrl_prefixed : (Nottui.Ui.key -> (string * int) -> [ `Handled | `Unhandled ]) option)\n    state\n  =\n  let update _focus_h focus (text, pos) =\n    let pos = clampi pos ~min:0 ~max:(String.length text) in\n    let content =\n      Nottui.Ui.atom @@ Notty.I.hcat @@\n      if Nottui.Focus.has_focus focus then (\n        let attr = Notty.A.(bg lightblue) in\n        let len = String.length text in\n        (if pos >= len\n         then [Notty.I.string attr text]\n         else [Notty.I.string attr (String.sub text 0 pos)])\n        @\n        (if pos < String.length text then\n           [Notty.I.string Notty.A.(bg lightred) (String.sub text pos 1);\n            Notty.I.string attr (String.sub text (pos + 1) (len - pos - 1))]\n         else [Notty.I.string Notty.A.(bg lightred) \" \"]);\n      ) else\n        [Notty.I.string Notty.A.(st underline) (if text = \"\" then \" \" else text)]\n    in\n    let handler = function\n      | `Escape, [] -> (\n          on_cancel (text, pos);\n          `Handled\n        )\n      | `ASCII k, [] -> (\n          let text =\n            if pos < String.length text then (\n              String.sub text 0 pos ^ String.make 1 k ^\n              String.sub text pos (String.length text - pos)\n            ) else (\n              text ^ String.make 1 k\n            )\n          in\n          on_change (text, (pos + 1));\n          `Handled\n        )\n      | `Backspace, _ -> (\n          let text =\n            if pos > 0 then (\n              if pos < String.length text then (\n                String.sub text 0 (pos - 1) ^\n                String.sub text pos (String.length text - pos)\n              ) else if String.length text > 0 then (\n                String.sub text 0 (String.length text - 1)\n              ) else text\n            ) else text\n          in\n          let pos = maxi 0 (pos - 1) in\n          on_change (text, pos);\n          `Handled\n        )\n      | `Enter, _ -> (\n          on_submit (text, pos);\n          `Handled\n        )\n      | `Arrow `Left, [] -> (\n          let pos = mini (String.length text) pos in\n          if pos > 0 then (\n            on_change (text, pos - 1);\n            `Handled\n          )\n          else `Unhandled\n        )\n      | `Arrow `Right, [] -> (\n          let pos = pos + 1 in\n          if pos <= String.length text\n          then (on_change (text, pos); `Handled)\n          else `Unhandled\n        )\n      | (`Arrow (`Up as ud), [])\n      | (`Arrow (`Down as ud), []) -> (\n          match on_up_down with\n          | None -> `Unhandled\n          | Some on_up_down -> (\n              on_up_down ud (text, pos);\n              `Handled\n            )\n        )\n      | (`Tab, []) -> (\n          match on_tab with\n          | None -> `Unhandled\n          | Some on_tab -> (\n              on_tab (text, pos);\n              `Handled\n            )\n        )\n      | (_, [`Ctrl]) as x -> (\n          match on_ctrl_prefixed with\n          | None -> `Unhandled\n          | Some f -> (\n              f x (text, pos)\n            )\n        )\n      | _ -> `Unhandled\n    in\n    Nottui.Ui.keyboard_area ~focus handler content\n  in\n  let state = Lwd.get state in\n  let node =\n    Lwd.map2 ~f:(update focus) (Nottui.Focus.status focus) state\n  in\n  let mouse_grab (text, pos) ~x ~y:_ = function\n    | `Left ->\n      if x <> pos then on_change (text, x);\n      Nottui.Focus.request focus;\n      `Handled\n    | _ -> `Unhandled\n  in\n  Lwd.map2 state node ~f:(fun state content ->\n      Nottui.Ui.mouse_area (mouse_grab state) content\n    )\n\nlet mouse_handler\n    ~(f : [ `Up | `Down ] -> unit)\n    ~x ~y\n    (button : Notty.Unescape.button)\n  =\n  let _ = x in\n  let _ = y in\n  match button with\n  | `Scroll `Down -> (\n      f `Down;\n      `Handled\n    )\n  | `Scroll `Up -> (\n      f `Up;\n      `Handled\n    )\n  | _ -> `Unhandled\n\nmodule Content_view = struct\n  let main\n      ~(input_mode : input_mode)\n      ~height\n      ~width\n      ~(search_result_group : Document.t * Search_result.t array)\n      ~(search_result_selected : int)\n      ~(link_selected : int)\n    : Nottui.ui Lwd.t =\n    let (document, search_results) = search_result_group in\n    let links = Document.links document in\n    let data =\n      let search_result_count = Array.length search_results in\n      let link_count = Array.length links in\n      match input_mode with\n      | Links -> (\n          if link_count = 0 then (\n            None\n          ) else (\n            Some (`Link links.(link_selected))\n          )\n        )\n      | _ -> (\n          if search_result_count = 0 then (\n            None\n          ) else (\n            Some (`Search_result search_results.(search_result_selected))\n          )\n        )\n    in\n    let$* _ = Lwd.get Vars.content_view_offset in\n    let content =\n      Content_and_search_result_rendering.content_snippet\n        ~doc_id:(Document.doc_id document)\n        ~view_offset:Vars.content_view_offset\n        ?data\n        ~height\n        ~width\n        ()\n    in\n    let$* background = full_term_sized_background in\n    Nottui.Ui.join_z background (Nottui.Ui.atom content)\n    |> Nottui.Ui.mouse_area\n      (mouse_handler\n         ~f:(fun direction ->\n             match direction with\n             | `Up -> decr_content_view_offset ()\n             | `Down -> incr_content_view_offset ()\n           )\n      )\n    |> Lwd.return\nend\n\nmodule Status_bar = struct\n  let fg_color = Notty.A.black\n\n  let bg_color = Notty.A.white\n\n  let attr = Notty.A.(bg bg_color ++ fg fg_color)\n\n  let background_bar : Nottui.Ui.t Lwd.t =\n    let$ (term_width, _term_height) = Lwd.get Vars.term_width_height in\n    Notty.I.char Notty.A.(bg bg_color) ' ' term_width 1\n    |> Nottui.Ui.atom\n\n  let element_spacing = 4\n\n  let element_spacer =\n    Notty.(I.string\n             A.(bg bg_color ++ fg fg_color))\n      (String.make element_spacing ' ')\n\n  let input_mode_images =\n    let l =\n      [ (Navigate, \"NAVIGATE\")\n      ; (Search, \"SEARCH\")\n      ; (Filter, \"FILTER\")\n      ; (Clear, \"CLEAR\")\n      ; (Sort `Asc, \"SORT-ASC\")\n      ; (Sort `Desc, \"SORT-DESC\")\n      ; (Drop, \"DROP\")\n      ; (Mark, \"MARK\")\n      ; (Unmark, \"UNMARK\")\n      ; (Narrow, \"NARROW\")\n      ; (Copy, \"COPY\")\n      ; (Copy_paths, \"COPY-PATHS\")\n      ; (Reload, \"RELOAD\")\n      ; (Save_script, \"SAVE-SCRIPT\")\n      ; (Save_script_overwrite \"\", \"SAVE-SCRIPT\")\n      ; (Save_script_no_name, \"SAVE-SCRIPT\")\n      ; (Save_script_edit \"\", \"SAVE-SCRIPT\")\n      ; (Scripts, \"SCRIPTS\")\n      ; (Delete_script_confirm (\"\", \"\"), \"DELETE-SCRIPT\")\n      ; (Links, \"LINKS\")\n      ; (Path_fuzzy_rank, \"PATH-FUZZY-RANK\")\n      ]\n    in\n    let max_input_mode_string_len =\n      List.fold_left (fun acc (_, s) ->\n          max acc (String.length s)\n        )\n        0\n        l\n    in\n    let input_mode_string_background =\n      Notty.I.char Notty.A.(bg bg_color) ' ' max_input_mode_string_len 1\n    in\n    List.fold_left (fun m (mode, s) ->\n        let s = Notty.(I.string A.(bg bg_color ++ fg fg_color ++ st bold) s) in\n        Input_mode_map.add mode Notty.I.(s </> input_mode_string_background) m\n      )\n      Input_mode_map.empty\n      l\nend\n\nmodule Key_binding_info = struct\n  let rotation : int Lwd.var = Lwd.var 0\n\n  let incr_rotation () =\n    Lwd.set rotation (Lwd.peek rotation + 1)\n\n  let decr_rotation () =\n    Lwd.set rotation (Lwd.peek rotation - 1)\n\n  let reset_rotation () =\n    Lwd.set rotation 0\n\n  type labelled_msg = {\n    label : string;\n    msg : string;\n  }\n\n  type labelled_msg_line = labelled_msg list\n\n  type grid_contents = (input_mode * (labelled_msg_line list)) list\n\n  type grid_lookup = Nottui.ui Lwd.t Input_mode_map.t\n\n  let grid_lights : (string, Mtime.t ref * bool Lwd.var list) Hashtbl.t = Hashtbl.create 100\n\n  let lock = Eio.Mutex.create ()\n\n  let grid_light_on_req : string Eio.Stream.t = Eio.Stream.create 100\n\n  let grid_light_off_req : (Mtime.t * Mtime.t * string) Eio.Stream.t = Eio.Stream.create 100\n\n  let blink label =\n    Eio.Stream.add grid_light_on_req label\n\n  let grid_light_fiber () =\n    let clock = Eio.Stdenv.mono_clock (eio_env ()) in\n    Eio.Fiber.both\n      (fun () ->\n         while true do\n           let label = Eio.Stream.take grid_light_on_req in\n           let ts_now = Eio.Time.Mono.now clock in\n           Eio.Mutex.use_rw lock ~protect:false (fun () ->\n               match Hashtbl.find_opt grid_lights label with\n               | None -> failwith \"unexpected case\"\n               | Some (ts, l) -> (\n                   ts := ts_now;\n                   List.iter (fun x -> Lwd.set x true) l;\n                   Eio.Stream.add\n                     grid_light_off_req\n                     (ts_now, Option.get (Mtime.(add_span ts_now Params.blink_on_duration)), label);\n                 )\n             )\n         done\n      )\n      (fun () ->\n         while true do\n           let ts_req_time, ts_target_time, label = Eio.Stream.take grid_light_off_req in\n           Eio.Time.Mono.sleep_until clock ts_target_time;\n           Eio.Mutex.use_rw lock ~protect:false (fun () ->\n               match Hashtbl.find_opt grid_lights label with\n               | None -> failwith \"unexpected case\"\n               | Some (ts_last_update, l) -> (\n                   if Mtime.equal !ts_last_update ts_req_time then (\n                     List.iter (fun x -> Lwd.set x false) l;\n                   )\n                 )\n             )\n         done\n      )\n\n  let make_grid_lookup grid_contents : grid_lookup =\n    let max_label_msg_len_lookup : (input_mode * (int * int) Int_map.t) list =\n      grid_contents\n      |> List.map (fun (grid_key, grid) ->\n          let lookup =\n            List.fold_left\n              (fun (acc : (int * int) Int_map.t) (line : labelled_msg_line) ->\n                 line\n                 |> List.to_seq\n                 |> Seq.fold_lefti\n                   (fun (acc : (int * int) Int_map.t) col ({ label; msg } : labelled_msg) ->\n                      let label_len =\n                        Uuseg_string.fold_utf_8 `Grapheme_cluster (fun x _ -> x + 1) 0 label\n                      in\n                      let msg_len =\n                        Uuseg_string.fold_utf_8 `Grapheme_cluster (fun x _ -> x + 1) 0 msg\n                      in\n                      let (max_label_len, max_msg_len) =\n                        match Int_map.find_opt col acc with\n                        | None -> (label_len, msg_len)\n                        | Some (max_label_len, max_msg_len) -> (\n                            (max max_label_len label_len,\n                             max max_msg_len msg_len)\n                          )\n                      in\n                      Int_map.add col (max_label_len, max_msg_len) acc\n                   )\n                   acc\n              )\n              Int_map.empty\n              grid\n          in\n          (grid_key, lookup)\n        )\n    in\n    let label_msg_pair grid_key col { label; msg } : Nottui.ui Lwd.t =\n      let (max_label_len, max_msg_len) =\n        List.assoc grid_key max_label_msg_len_lookup\n        |> Int_map.find col\n      in\n      let light_on_var = Lwd.var false in\n      Eio.Mutex.use_rw lock ~protect:false (fun () ->\n          let x =\n            match Hashtbl.find_opt grid_lights label with\n            | None -> (ref Mtime.min_stamp, [ light_on_var ])\n            | Some (x, l) -> (x, light_on_var :: l)\n          in\n          Hashtbl.replace grid_lights label x\n        );\n      let$ light_on = Lwd.get light_on_var in\n      let label_attr =\n        if light_on then\n          Notty.A.(fg black ++ bg lightyellow ++ st bold)\n        else\n          Notty.A.(fg lightyellow ++ st bold)\n      in\n      let msg_attr = Notty.A.empty in\n      let msg = String.capitalize_ascii msg in\n      let content = Notty.(I.hcat\n                             [ I.(string label_attr label\n                                  </>\n                                  (string label_attr (String.make max_label_len ' ')))\n                             ; I.string A.empty \" \"\n                             ; I.string msg_attr msg\n                             ]\n                          )\n      in\n      let full_background =\n        Notty.I.void (max_label_len + 1 + max_msg_len + 3) 1\n      in\n      Notty.I.(content </> full_background)\n      |> Nottui.Ui.atom\n    in\n    List.fold_left (fun m (grid_key, grid_contents) ->\n        let max_row_size =\n          List.fold_left (fun n l ->\n              max n (List.length l)\n            )\n            0\n            grid_contents\n        in\n        let grid_contents =\n          grid_contents\n          |> List.map (fun l ->\n              let padding =\n                List.init (max_row_size - List.length l)\n                  (fun _ -> { label = \"\"; msg = \"\" })\n              in\n              List.mapi (fun col x ->\n                  label_msg_pair grid_key col x\n                )\n                (l @ padding)\n            )\n        in\n        let grid =\n          let$* rotation = Lwd.get rotation in\n          grid_contents\n          |> List.map (fun l ->\n              Misc_utils.rotate_list\n                (((rotation mod max_row_size) + max_row_size)\n                 mod\n                 max_row_size\n                )\n                l\n            )\n          |> Nottui_widgets.grid\n            ~pad:(Nottui.Gravity.make ~h:`Negative ~v:`Negative)\n        in\n        Input_mode_map.add grid_key grid m\n      )\n      Input_mode_map.empty\n      grid_contents\n\n  let main ~(grid_lookup : grid_lookup) ~(input_mode : input_mode) =\n    Input_mode_map.find input_mode grid_lookup\nend\n\nlet filter_bar_label_string = \"Document filter\"\n\nlet search_bar_label_string = \"Content search\"\n\nlet max_label_length =\n  List.fold_left (fun acc s ->\n      max acc (String.length s)\n    )\n    0\n    [ filter_bar_label_string\n    ; search_bar_label_string\n    ]\n\nlet pad_label_string s =\n  CCString.pad ~side:`Right ~c:' ' max_label_length s\n\nlet autocomplete ~choices (text, pos) : string * int =\n  let left = String.sub text 0 pos in\n  let right = String.sub text pos (String.length text - pos) in\n  let grab_input_word (s : string) =\n    let rec aux acc i s =\n      if i < 0 then (\n        CCString.of_list acc\n      ) else (\n        let c = s.[i] in\n        if Parser_components.is_alphanum c\n        || c = '-'\n        || c = ':'\n        then (\n          aux (c :: acc) (i - 1) s\n        ) else (\n          aux acc (-1) s\n        )\n      )\n    in\n    aux [] (String.length s - 1) s\n  in\n  let current_input_word = grab_input_word left in\n  let usable_choices =\n    List.filter\n      (CCString.prefix ~pre:current_input_word)\n      choices\n  in\n  Lwd.set\n    Vars.autocomplete_choices usable_choices;\n  match usable_choices with\n  | [] -> (text, pos)\n  | _ -> (\n      let best_fit = usable_choices\n        |> List.to_seq\n        |> String_utils.longest_common_prefix\n      in\n      let left =\n        String.sub\n          left\n          0\n          (String.length left - String.length current_input_word)\n      in\n      (String.concat \"\" [ left; best_fit; right ],\n       pos + (String.length best_fit - String.length current_input_word))\n    )\n\nmodule Filter_bar = struct\n  let label_string = pad_label_string filter_bar_label_string\n\n  let label ~(input_mode : input_mode) =\n    let attr =\n      match input_mode with\n      | Filter -> Notty.A.(st bold)\n      | _ -> Notty.A.empty\n    in\n    Notty.I.string attr label_string\n    |> Nottui.Ui.atom\n    |> Lwd.return\n\n  let status =\n    let$* status = Lwd.get Vars.filter_ui_status in\n    (match status with\n     | `Idle -> (\n         Notty.I.string Notty.A.(fg lightgreen)\n           \"  OK\"\n       )\n     | `Filtering -> (\n         Notty.I.string Notty.A.(fg lightyellow)\n           \" ...\"\n       )\n     | `Parse_error -> (\n         Notty.I.string Notty.A.(fg lightred)\n           \" ERR\"\n       )\n    )\n    |> Nottui.Ui.atom\n    |> Lwd.return\n\n  let autocomplete_choices =\n    [ \"path-date:\"\n    ; \"path-fuzzy:\"\n    ; \"path-glob:\"\n    ; \"ext:\"\n    ; \"content:\"\n    ; \"mod-date:\"\n    ]\n\n  let main\n      ~input_mode\n      ~(text_field : (string * int) Lwd.var)\n      ~focus_handle\n      ~on_change\n      ~on_submit\n    : Nottui.ui Lwd.t =\n    Nottui_widgets.hbox\n      [\n        label ~input_mode;\n        status;\n        Lwd.return (Nottui.Ui.atom (Notty.I.strf \": \"));\n        edit_field text_field\n          ~focus:focus_handle\n          ~on_cancel:(fun (_text, _x) -> ())\n          ~on_change:(fun (text, x) ->\n              Lwd.set text_field (text, x);\n              on_change ();\n            )\n          ~on_submit:(fun (text, x) ->\n              Lwd.set text_field (text, x);\n              on_submit ();\n              Lwd.set Vars.autocomplete_choices [];\n              Nottui.Focus.release focus_handle;\n              Lwd.set Vars.input_mode Navigate\n            )\n          ~on_tab:(fun (text, pos) ->\n              let (text, pos) =\n                autocomplete ~choices:autocomplete_choices (text, pos)\n              in\n              Lwd.set text_field (text, pos)\n            );\n      ]\nend\n\nmodule Search_bar = struct\n  let label_string = pad_label_string search_bar_label_string\n\n  let label ~(input_mode : input_mode) =\n    let attr =\n      match input_mode with\n      | Search -> Notty.A.(st bold)\n      | _ -> Notty.A.empty\n    in\n    Notty.I.string attr label_string\n    |> Nottui.Ui.atom\n    |> Lwd.return\n\n  let status =\n    let$* status = Lwd.get Vars.search_ui_status in\n    (match status with\n     | `Idle -> (\n         Notty.I.string Notty.A.(fg lightgreen)\n           \"  OK\"\n       )\n     | `Searching -> (\n         Notty.I.string Notty.A.(fg lightyellow)\n           \" ...\"\n       )\n     | `Parse_error -> (\n         Notty.I.string Notty.A.(fg lightred)\n           \" ERR\"\n       )\n    )\n    |> Nottui.Ui.atom\n    |> Lwd.return\n\n  let main\n      ~input_mode\n      ~(text_field : (string * int) Lwd.var)\n      ~focus_handle\n      ~on_change\n      ~on_submit\n    : Nottui.ui Lwd.t =\n    Nottui_widgets.hbox\n      [\n        label ~input_mode;\n        status;\n        Lwd.return (Nottui.Ui.atom (Notty.I.strf \": \"));\n        edit_field text_field\n          ~focus:focus_handle\n          ~on_cancel:(fun (_text, _x) -> ())\n          ~on_change:(fun (text, x) ->\n              Lwd.set text_field (text, x);\n              on_change ();\n            )\n          ~on_submit:(fun (text, x) ->\n              Lwd.set text_field (text, x);\n              on_submit ();\n              Lwd.set Vars.autocomplete_choices [];\n              Nottui.Focus.release focus_handle;\n              Lwd.set Vars.input_mode Navigate\n            )\n          ~on_tab:(fun (_, _) -> ());\n      ]\nend\n\nlet term' : unit -> Notty_unix.Term.t = term\n\nlet ui_loop ~quit ~term root =\n  let renderer = Nottui.Renderer.make () in\n  let root =\n    let$ root = root in\n    root\n    (* |> Nottui.Ui.event_filter (fun x ->\n        match x with\n        | `Key (`Escape, []) -> (\n            Lwd.set quit true;\n            `Handled\n          )\n        | _ -> `Unhandled\n       ) *)\n  in\n  let rec loop () =\n    if not (Lwd.peek quit) then (\n      let (term_width, term_height) = Notty_unix.Term.size (term' ()) in\n      let (prev_term_width, prev_term_height) = Lwd.peek Vars.term_width_height in\n      if term_width <> prev_term_width || term_height <> prev_term_height then (\n        Lwd.set Vars.term_width_height (term_width, term_height)\n      );\n      Nottui_unix.step\n        ~process_event:true\n        ~timeout:0.05\n        ~renderer\n        term\n        (Lwd.observe @@ root);\n      Eio.Fiber.yield ();\n      loop ()\n    )\n  in\n  loop ()\n\nlet set_input_mode mode =\n  Lwd.set Vars.input_mode mode;\n  Key_binding_info.reset_rotation ()\n"
  },
  {
    "path": "bin/args.ml",
    "content": "open Cmdliner\nopen Docfd_lib\nopen Misc_utils\n\nlet no_pdftotext_arg_name = \"no-pdftotext\"\n\nlet no_pdftotext_arg =\n  let doc =\n    Fmt.str {|Disable use of pdftotext command.\n    Files that require use of pdftotext are excluded.\n    |}\n  in\n  Arg.(value & flag & info [ no_pdftotext_arg_name ] ~doc)\n\nlet no_pandoc_arg_name = \"no-pandoc\"\n\nlet no_pandoc_arg =\n  let doc =\n    Fmt.str {|Disable use of pandoc command.\n    Files that require use of pandoc are excluded.\n    |}\n  in\n  Arg.(value & flag & info [ no_pandoc_arg_name ] ~doc)\n\nlet hidden_arg_name = \"hidden\"\n\nlet hidden_arg =\n  let doc =\n    Fmt.str {|Scan hidden files and directories.\nBy default, hidden files and directories are skipped.\n\nA file or directory is hidden if the base name starts\nwith a dot, e.g. \".gitignore\".\n    |}\n  in\n  Arg.(value & flag & info [ hidden_arg_name ] ~doc)\n\nlet max_depth_arg_name = \"max-depth\"\n\nlet max_depth_arg =\n  let doc =\n    Fmt.str\n      \"Scan up to N levels when exploring file trees.\nThis applies to directory paths provided\nand ** in globs.\nNote that --%s 0 results in no-op when scanning\ndirectories, and --%s 1 means only scanning for\ndirect children.\"\n      max_depth_arg_name\n      max_depth_arg_name\n  in\n  Arg.(\n    value\n    & opt int Params.default_max_file_tree_scan_depth\n    & info [ max_depth_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet exts_arg_name = \"exts\"\n\nlet exts_arg =\n  let doc =\n    \"File extensions to use, comma separated. Leading dots of any extension are removed.\"\n  in\n  Arg.(\n    value\n    & opt string Params.default_recognized_exts\n    & info [ exts_arg_name ] ~doc ~docv:\"EXTS\"\n  )\n\nlet single_file_exts_arg_name = Fmt.str \"single-line-%s\" exts_arg_name\n\nlet single_line_exts_arg =\n  let doc =\n    Fmt.str \"Same as --%s, but use single line search mode instead.\nIf an extension appears in both --%s and --%s,\nthen single line search mode is used for that extension.\"\n      exts_arg_name\n      exts_arg_name\n      single_file_exts_arg_name\n  in\n  Arg.(\n    value\n    & opt string Params.default_recognized_single_line_exts\n    & info [ single_file_exts_arg_name ] ~doc ~docv:\"EXTS\"\n  )\n\nlet add_exts_arg_name = \"add-exts\"\n\nlet add_exts_arg =\n  let doc =\n    \"Additional file extensions to use, comma separated.\nMay be specified multiple times.\"\n  in\n  Arg.(\n    value\n    & opt_all string []\n    & info [ add_exts_arg_name ] ~doc ~docv:\"EXTS\"\n  )\n\nlet single_line_add_exts_arg_name = Fmt.str \"single-line-%s\" add_exts_arg_name\n\nlet single_line_add_exts_arg =\n  let doc =\n    Fmt.str \"Same as --%s, but use single line search mode instead.\" add_exts_arg_name\n  in\n  Arg.(\n    value\n    & opt_all string []\n    & info [ single_line_add_exts_arg_name ] ~doc ~docv:\"EXTS\"\n  )\n\nlet max_fuzzy_edit_dist_arg_name = \"max-fuzzy-edit-dist\"\n\nlet max_fuzzy_edit_dist_arg =\n  let doc =\n    \"Maximum edit distance for fuzzy matches.\"\n  in\n  Arg.(\n    value\n    & opt int Params.default_max_fuzzy_edit_dist\n    & info [ max_fuzzy_edit_dist_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet max_token_search_dist_arg_name = \"max-token-search-dist\"\n\nlet max_token_search_dist_arg =\n  let doc =\n    \"Maximum distance to look for the next matching token in document.\nIf two tokens are adjacent, then they are 1 distance away from each other.\nNote that contiguous spaces count as one token as well.\"\n  in\n  Arg.(\n    value\n    & opt int Params.default_max_token_search_dist\n    & info [ max_token_search_dist_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet max_linked_token_search_dist_arg_name = \"max-linked-token-search-dist\"\n\nlet max_linked_token_search_dist_arg =\n  let doc =\n    Fmt.str\n      {|Similar to %s but for linked tokens.\nTwo tokens are linked if there is no space between them in the search phrase,\ne.g. \"-\" and \">\" are linked in \"->\" but not in \"- >\",\n\"and\" \"/\" \"or\" are linked in \"and/or\" but not in \"and / or\".|}\n      max_token_search_dist_arg_name\n  in\n  Arg.(\n    value\n    & opt int Params.default_max_linked_token_search_dist\n    & info [ max_linked_token_search_dist_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet tokens_per_search_scope_level_arg_name = \"tokens-per-search-scope-level\"\n\nlet tokens_per_search_scope_level_arg =\n  let doc =\n    Fmt.str\n      {|Number of tokens to use around the current search\nresults for each search scope level in narrow mode.|}\n  in\n  Arg.(\n    value\n    & opt int Params.default_tokens_per_search_scope_level\n    & info [ tokens_per_search_scope_level_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet index_chunk_size_arg_name = \"index-chunk-size\"\n\nlet index_chunk_size_arg =\n  let doc =\n    \"Number of tokens to send as a job unit to the thread pool for indexing.\"\n  in\n  Arg.(\n    value\n    & opt int Params.default_index_chunk_size\n    & info [ index_chunk_size_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet cache_dir_arg =\n  let doc =\n    \"Docfd cache directory, mainly for index DB.\"\n  in\n  let cache_home = Xdg_utils.cache_home in\n  Arg.(\n    value\n    & opt string (Filename.concat cache_home \"docfd\")\n    & info [ \"cache-dir\" ] ~doc ~docv:\"DIR\"\n  )\n\nlet cache_limit_arg_name = \"cache-limit\"\n\nlet cache_limit_arg =\n  let doc =\n    \"Maximum number of documents to keep in index.\nDocfd resets the cache to this limit at launch.\"\n  in\n  Arg.(\n    value\n    & opt int Params.default_cache_limit\n    & info [ cache_limit_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet data_dir_arg =\n  let doc =\n    \"Docfd data directory.\"\n  in\n  let data_home = Xdg_utils.data_home in\n  Arg.(\n    value\n    & opt string (Filename.concat data_home \"docfd\")\n    & info [ \"data-dir\" ] ~doc ~docv:\"DIR\"\n  )\n\nlet index_only_arg =\n  let doc =\n    Fmt.str \"Exit after indexing.\"\n  in\n  Arg.(value & flag & info [ \"index-only\" ] ~doc)\n\nlet debug_log_arg =\n  let doc =\n    Fmt.str \"Specify debug log file to use and enable debug mode where\nadditional checks are enabled and additional info is displayed on UI.\nIf FILE is -, then debug log is printed to stderr instead.\nOtherwise FILE is opened in append mode for log writing.\"\n  in\n  Arg.(\n    value\n    & opt (some string) None\n    & info [ \"debug-log\" ] ~doc ~docv:\"FILE\"\n  )\n\nlet start_with_filter_arg_name = \"start-with-filter\"\n\nlet start_with_filter_arg =\n  let doc =\n    Fmt.str \"Start interactive mode with an initial filter using expression EXP.\"\n  in\n  Arg.(\n    value\n    & opt (some string) None\n    & info [ start_with_filter_arg_name ] ~doc ~docv:\"EXP\"\n  )\n\nlet start_with_search_arg_name = \"start-with-search\"\n\nlet start_with_search_arg =\n  let doc =\n    Fmt.str \"Start interactive mode with search expression EXP.\"\n  in\n  Arg.(\n    value\n    & opt (some string) None\n    & info [ start_with_search_arg_name ] ~doc ~docv:\"EXP\"\n  )\n\nlet sample_arg_name = \"sample\"\n\nlet samples_per_doc_arg_name = \"samples-per-doc\"\n\nlet sample_arg =\n  let doc =\n    Fmt.str \"Search with expression EXP in non-interactive mode but only\nshow top N results where N is controlled by --%s.\"\n      samples_per_doc_arg_name\n  in\n  Arg.(\n    value\n    & opt (some string) None\n    & info [ sample_arg_name ] ~doc ~docv:\"EXP\"\n  )\n\nlet samples_per_doc_arg =\n  let doc =\n    Fmt.str\n      \"Number of search results to show per document when --%s is used\nor when samples printing is triggered.\"\n      sample_arg_name\n  in\n  Arg.(\n    value\n    & opt int Params.default_samples_per_document\n    & info [ samples_per_doc_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet search_arg_name = \"search\"\n\nlet search_arg =\n  let doc =\n    \"Search with expression EXP in non-interactive mode and show all results.\"\n  in\n  Arg.(\n    value\n    & opt (some string) None\n    & info [ search_arg_name ] ~doc ~docv:\"EXP\"\n  )\n\nlet filter_arg_name = \"filter\"\n\nlet filter_arg =\n  let doc =\n    Fmt.str\n      \"Filter with expression EXP in non-interactive mode. May be combined with --%s or --%s.\"\n      search_arg_name\n      sample_arg_name\n  in\n  Arg.(\n    value\n    & opt (some string) None\n    & info [ filter_arg_name ] ~doc ~docv:\"EXP\"\n  )\n\nlet sort_arg_name = \"sort\"\n\nlet sort_arg =\n  let doc =\n    Fmt.str\n      \"Sort document by: TYPE,ORDER. TYPE is one of: path, path-date, score, mod-time. ORDER is one of: asc, desc.\"\n  in\n  Arg.(\n    value\n    & opt string Params.default_sort_by_arg\n    & info [ sort_arg_name ] ~doc ~docv:\"TYPE,ORDER\"\n  )\n\nlet style_mode_options = [ (\"never\", `Never); (\"always\", `Always); (\"auto\", `Auto) ]\n\nlet color_arg =\n  let doc =\n    Fmt.str\n      \"Set color mode for search result printing, one of: %s.\"\n      (String.concat \", \" (List.map fst style_mode_options))\n  in\n  Arg.(\n    value\n    & opt (Arg.enum style_mode_options) `Auto\n    & info [ \"color\" ] ~doc ~docv:\"MODE\"\n  )\n\nlet underline_arg =\n  let doc =\n    Fmt.str\n      \"Set underline mode for search result printing, one of: %s.\"\n      (String.concat \", \" (List.map fst style_mode_options))\n  in\n  Arg.(\n    value\n    & opt (Arg.enum style_mode_options) `Auto\n    & info [ \"underline\" ] ~doc ~docv:\"MODE\"\n  )\n\nlet search_result_print_text_width_arg_name = \"search-result-print-text-width\"\n\nlet search_result_print_text_width_arg =\n  let doc =\n    \"Text width to use when printing search results.\"\n  in\n  Arg.(\n    value\n    & opt int Params.default_search_result_print_text_width\n    & info [ search_result_print_text_width_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet search_result_print_snippet_min_size_arg_name = \"search-result-print-snippet-min-size\"\n\nlet search_result_print_snippet_min_size_arg =\n  let doc =\n    \"If the search result to be printed has fewer than N non-space tokens,\nthen Docfd tries to add surrounding lines to the snippet\nto give better context.\"\n  in\n  Arg.(\n    value\n    & opt int Params.default_search_result_print_snippet_min_size\n    & info [ search_result_print_snippet_min_size_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet search_result_print_snippet_max_add_lines_arg_name = \"search-result-print-snippet-max-add-lines\"\n\nlet search_result_print_snippet_max_add_lines_arg =\n  let doc =\n    \"This controls the maximum number of surrounding lines\nDocfd can add in each direction.\"\n  in\n  Arg.(\n    value\n    & opt int Params.default_search_result_print_snippet_max_additional_lines_each_direction\n    & info [ search_result_print_snippet_max_add_lines_arg_name ] ~doc ~docv:\"N\"\n  )\n\nlet script_arg_name = \"script\"\n\nlet script_arg =\n  let doc =\n    Fmt.str \"Read and run Docfd script FILE.\"\n  in\n  Arg.(\n    value\n    & opt (some string) None\n    & info [ script_arg_name ] ~doc ~docv:\"FILE\"\n  )\n\nlet start_with_script_arg_name = \"start-with-script\"\n\nlet start_with_script_arg =\n  let doc =\n    Fmt.str \"Read and run Docfd script FILE, then continue in interactive mode.\"\n  in\n  Arg.(\n    value\n    & opt (some string) None\n    & info [ start_with_script_arg_name ] ~doc ~docv:\"FILE\"\n  )\n\nlet paths_from_arg_name = \"paths-from\"\n\nlet paths_from_arg =\n  let doc =\n    Fmt.str \"Read list of paths from FILES,\nwhich is a comma separated list of files,\nand add to the final list of paths to be scanned.\nFor example, \\\"--%s path-list0.txt,path-list1.txt\\\".\nIf - is in FILES, then stdin is also read for\nlist of paths to be scanned. This is useful\nfor piping, e.g. \\\"find -name '*.txt' | docfd --%s -\\\"\"\n      paths_from_arg_name\n      paths_from_arg_name\n  in\n  Arg.(\n    value\n    & opt_all string []\n    & info [ paths_from_arg_name ] ~doc ~docv:\"FILES\"\n  )\n\nlet glob_arg_name = \"glob\"\n\nlet glob_arg =\n  let doc =\n    \"Add to the final list of paths to be scanned using glob pattern.\nThe pattern should pick up the files directly.\nDirectories picked up by the pattern are not further scanned\nfor files with suitable extensions.\"\n  in\n  Arg.(\n    value\n    & opt_all string []\n    & info [ glob_arg_name ] ~doc ~docv:\"PATTERN\"\n  )\n\nlet single_line_glob_arg_name = Fmt.str \"single-line-%s\" glob_arg_name\n\nlet single_line_glob_arg =\n  let doc =\n    Fmt.str\n      \"Same as --%s, but use single line search mode instead.\nIf the file are picked up by both patterns from --%s and --%s,\nthen single line search mode is used.\"\n      glob_arg_name\n      glob_arg_name\n      single_line_glob_arg_name\n  in\n  Arg.(\n    value\n    & opt_all string []\n    & info [ single_line_glob_arg_name ] ~doc ~docv:\"PATTERN\"\n  )\n\nlet single_line_arg =\n  let doc =\n    \"Use single line search mode by default.\"\n  in\n  Arg.(\n    value\n    & flag\n    & info [ \"single-line\" ] ~doc\n  )\n\nlet open_with_arg_name = \"open-with\"\n\nlet open_with_arg =\n  let doc =\n    Fmt.str \"Specify custom command CMD for\nopening files with file extension EXT.\nMay be specified multiple times.\nLeading dots of EXT are removed.\nLAUNCH_MODE specifies how the command should be\nexecuted:\n`terminal` - for commands which Docfd\nshould run in the terminal and wait for completion,\ne.g. text editors, pagers.\n`detached` - for background\ncommands, such as PDF viewers or\nother GUI tools.\nCMD may contain the following placeholders:\n{path} - file path,\n{page_num} - page number (PDF only),\n{line_num} - line number (not available in PDF),\n{search_word} - most unique word of the page\n(PDF only, useful for passing to PDF viewers as search term).\nExamples: \\\"pdf:detached='okular --page {page_num} --find {search_word} {path}'\\\",\n\\\"txt:terminal='nano +{line_num} {path}'\\\".\n\"\n  in\n  Arg.(\n    value\n    & opt_all string []\n    & info [ open_with_arg_name ] ~doc ~docv:\"EXT:LAUNCH_MODE=CMD\"\n  )\n\nlet files_with_match_arg_name = \"files-with-match\"\n\nlet files_with_match_arg =\n  let doc =\n    Fmt.str \"If paired with\n--%s or --%s,\nthen print the paths of documents with at least one match\ninstead of printing the search results.\nIf paired with --%s, then print paths of documents\nthat would have be listed in the UI\nafter running the commands in interactive mode.\"\n      search_arg_name\n      sample_arg_name\n      script_arg_name\n  in\n  Arg.(\n    value\n    & flag\n    & info [ \"l\"; files_with_match_arg_name ] ~doc\n  )\n\nlet files_without_match_arg_name = \"files-without-match\"\n\nlet files_without_match_arg =\n  let doc =\n    Fmt.str \"If paired with\n--%s or --%s,\nthen print the paths of documents with no matches\ninstead of printing the search results.\nCannot be paired with --%s.\"\n      search_arg_name\n      sample_arg_name\n      script_arg_name\n  in\n  Arg.(\n    value\n    & flag\n    & info [ files_without_match_arg_name ] ~doc\n  )\n\nlet sort_no_score_arg_name = \"sort-no-score\"\n\nlet sort_no_score_arg =\n  let doc =\n    Fmt.str\n      \"Same as --%s but sorting TYPE cannot be score. Used for scenarios when no scores are available, e.g. --%s is used.\"\n      sort_no_score_arg_name\n      files_without_match_arg_name\n  in\n  Arg.(\n    value\n    & opt string Params.default_sort_by_no_score_arg\n    & info [ sort_no_score_arg_name ] ~doc ~docv:\"TYPE,ORDER\"\n  )\n\nlet paths_arg =\n  let doc =\n    Fmt.str\n      \"PATH can be either file or directory.\nDirectories are scanned for files with matching extensions.\nIf no paths are provided,\nthen Docfd defaults to scanning the current working directory\nunless any of the following is used: %a.\nTo use piped stdin as input, the list of paths must be empty.\"\n      Fmt.(list ~sep:comma (fun fmt s -> Fmt.pf fmt \"--%s\" s))\n      [ paths_from_arg_name; glob_arg_name; single_line_glob_arg_name ]\n  in\n  Arg.(value & pos_all string [] & info [] ~doc ~docv:\"PATH\")\n\nlet check\n    ~max_depth\n    ~max_fuzzy_edit_dist\n    ~max_token_search_dist\n    ~max_linked_token_search_dist\n    ~tokens_per_search_scope_level\n    ~index_chunk_size\n    ~cache_limit\n    ~start_with_filter\n    ~start_with_search\n    ~filter_exp\n    ~sample_search_exp\n    ~samples_per_doc\n    ~search_exp\n    ~search_result_print_text_width\n    ~search_result_print_snippet_min_size\n    ~search_result_print_max_add_lines\n    ~start_with_script\n    ~script\n    ~paths_from\n    ~print_files_with_match\n    ~print_files_without_match\n  =\n  if max_depth < 0 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 0\" max_depth_arg_name)\n  );\n  if max_fuzzy_edit_dist < 0 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 0\" max_fuzzy_edit_dist_arg_name)\n  );\n  if max_token_search_dist < 1 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 1\" max_token_search_dist_arg_name)\n  );\n  if max_linked_token_search_dist < 1 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 1\" max_linked_token_search_dist_arg_name)\n  );\n  if tokens_per_search_scope_level < 1 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 1\" tokens_per_search_scope_level_arg_name)\n  );\n  if index_chunk_size < 1 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 1\" index_chunk_size_arg_name)\n  );\n  if cache_limit < 1 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 1\" cache_limit_arg_name)\n  );\n  if samples_per_doc < 1 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 1\" samples_per_doc_arg_name)\n  );\n  if search_result_print_text_width < 1 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 1\" search_result_print_text_width_arg_name)\n  );\n  if search_result_print_snippet_min_size < 0 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 0\" search_result_print_snippet_min_size_arg_name)\n  );\n  if search_result_print_max_add_lines < 0 then (\n    exit_with_error_msg\n      (Fmt.str \"invalid %s: cannot be < 0\" search_result_print_snippet_max_add_lines_arg_name)\n  );\n  if Option.is_some filter_exp then (\n    if not (\n        Option.is_some sample_search_exp\n        ||\n        Option.is_some search_exp\n        ||\n        print_files_with_match\n        ||\n        print_files_without_match\n      )\n    then (\n      exit_with_error_msg\n        (Fmt.str \"--%s must be used with at least one of: --%s, --%s, --%s, --%s\"\n           filter_arg_name\n           search_arg_name\n           sample_arg_name\n           files_with_match_arg_name\n           files_without_match_arg_name\n        )\n    )\n  );\n  let cannot_be_used_together x y =\n    exit_with_error_msg\n      (Fmt.str \"--%s and --%s cannot be used together\" x y)\n  in\n  (match print_files_with_match, print_files_without_match with\n   | true, true -> (\n       cannot_be_used_together\n         files_with_match_arg_name files_without_match_arg_name\n     )\n   | true, false -> (\n       if not (\n           Option.is_some filter_exp\n           ||\n           Option.is_some sample_search_exp\n           ||\n           Option.is_some search_exp\n           ||\n           Option.is_some script\n         )\n       then (\n         exit_with_error_msg\n           (Fmt.str \"--%s cannot be used without one of: --%s, --%s, --%s, --%s\"\n              files_with_match_arg_name\n              filter_arg_name\n              sample_arg_name\n              search_arg_name\n              script_arg_name\n           )\n       )\n     )\n   | false, true -> (\n       if not (\n           Option.is_some filter_exp\n           ||\n           Option.is_some sample_search_exp\n           ||\n           Option.is_some search_exp\n         )\n       then (\n         exit_with_error_msg\n           (Fmt.str \"--%s cannot be used without one of: --%s, --%s, --%s\"\n              files_without_match_arg_name\n              filter_arg_name\n              sample_arg_name\n              search_arg_name\n           )\n       )\n     )\n   | false, false -> ()\n  );\n  (\n    let l = List.filter (fun x -> x = \"-\") paths_from in\n    if List.length l > 1 then (\n      exit_with_error_msg\n        (Fmt.str \"at most one \\\"-\\\" may be supplied to --%s\" paths_from_arg_name)\n    )\n  );\n  (match filter_exp with\n   | None -> ()\n   | Some filter_exp_string -> (\n       match\n         Filter_exp.parse filter_exp_string\n       with\n       | None -> (\n           exit_with_error_msg \"failed to parse filter exp\"\n         )\n       | Some _ -> ()\n     )\n  );\n  (match sample_search_exp, search_exp with\n   | None, None -> ()\n   | Some _, Some _ -> (\n       exit_with_error_msg\n         (Fmt.str \"%s and %s cannot be used together\" sample_arg_name search_arg_name)\n     )\n   | Some search_exp_string, None\n   | None, Some search_exp_string -> (\n       match\n         Search_exp.parse search_exp_string\n       with\n       | None -> (\n           exit_with_error_msg \"failed to parse search exp\"\n         )\n       | Some _ -> ()\n     )\n  );\n  let start_with_arg_check ~arg_name =\n    if Option.is_some filter_exp then (\n      cannot_be_used_together arg_name filter_arg_name\n    );\n    if Option.is_some sample_search_exp then (\n      cannot_be_used_together arg_name sample_arg_name\n    );\n    if Option.is_some search_exp then (\n      cannot_be_used_together arg_name search_arg_name\n    );\n  in\n  if Option.is_some start_with_filter then (\n    start_with_arg_check ~arg_name:start_with_filter_arg_name\n  );\n  if Option.is_some start_with_search then (\n    start_with_arg_check ~arg_name:start_with_search_arg_name\n  );\n  if Option.is_some start_with_script then (\n    start_with_arg_check ~arg_name:start_with_script_arg_name\n  );\n  let script_common_check ~arg_name =\n    if Option.is_some filter_exp then (\n      cannot_be_used_together arg_name filter_arg_name\n    );\n    if Option.is_some sample_search_exp then (\n      cannot_be_used_together arg_name sample_arg_name\n    );\n    if Option.is_some search_exp then (\n      cannot_be_used_together arg_name search_arg_name\n    );\n    if Option.is_some start_with_filter then (\n      cannot_be_used_together arg_name start_with_filter_arg_name\n    );\n    if Option.is_some start_with_search then (\n      cannot_be_used_together arg_name start_with_search_arg_name\n    );\n    if print_files_without_match then (\n      cannot_be_used_together arg_name files_without_match_arg_name\n    );\n  in\n  if Option.is_some script then (\n    script_common_check ~arg_name:script_arg_name;\n    if Option.is_some start_with_script then (\n      cannot_be_used_together script_arg_name start_with_script_arg_name\n    );\n  );\n  if Option.is_some start_with_script then (\n    script_common_check ~arg_name:start_with_script_arg_name;\n  )\n"
  },
  {
    "path": "bin/clipboard.ml",
    "content": "let pipe_to_clipboard (f : out_channel -> unit) : unit =\n  match Params.clipboard_copy_cmd_and_args with\n  | None -> ()\n  | Some (cmd, args) -> (\n      Proc_utils.pipe_to_command f\n        cmd args\n    )\n"
  },
  {
    "path": "bin/command.ml",
    "content": "open Docfd_lib\n\nmodule Sort_by = struct\n  type typ = [\n    | `Path_date\n    | `Path\n    | `Score\n    | `Mod_time\n  ]\n\n  type t = typ * Document.Compare.order\n\n  let default : t = (`Score, `Desc)\n\n  let default_no_score : t = (`Path, `Asc)\n\n  let pp formatter ((typ, order) : t) =\n    Fmt.pf formatter \"%s,%s\"\n      (match typ with\n       | `Path_date -> \"path-date\"\n       | `Path -> \"path\"\n       | `Score -> \"score\"\n       | `Mod_time -> \"mod-time\"\n      )\n      (match order with\n       | `Asc -> \"asc\"\n       | `Desc -> \"desc\"\n      )\n\n  let p ~no_score : t Angstrom.t =\n    let open Angstrom in\n    let open Parser_components in\n    skip_spaces *>\n    (choice (List.filter_map\n               Fun.id ([\n                   Some (string \"path-date\" *> return `Path_date);\n                   Some (string \"path\" *> return `Path);\n                   (if no_score then\n                      None\n                    else\n                      Some (string \"score\" *> return `Score));\n                   Some (string \"mod-time\" *> return `Mod_time);\n                 ]))\n     <|>\n     (take_while (fun c -> is_not_space c && c <> ',') >>=\n      fun s -> fail (Fmt.str \"unrecognized sort by type: %s\" s))\n    )\n    >>= fun typ ->\n    skip_spaces *>\n    char ',' *> skip_spaces *>\n    (choice [\n        string \"asc\" *> return `Asc;\n        string \"desc\" *> return `Desc;\n      ]\n     <|>\n     (take_while is_not_space >>=\n      fun s -> fail (Fmt.str \"unrecognized sort by order: %s\" s))\n    )\n    >>= fun order -> (\n      return (typ, order)\n    )\n\n  let parse ~no_score s =\n    match Angstrom.(parse_string ~consume:Consume.All) (p ~no_score) s with\n    | Ok t -> Ok t\n    | Error msg -> Error msg\nend\n\ntype screen_split = [\n  | `Even\n  | `Focus_left\n  | `Wide_left\n  | `Focus_right\n  | `Wide_right\n]\n\nlet screen_split_of_int (x : int) : screen_split =\n  if x <= 0 then\n    `Focus_right\n  else if x = 1 then\n    `Wide_right\n  else if x = 2 then\n    `Even\n  else if x = 3 then\n    `Wide_left\n  else\n    `Focus_left\n\nlet int_of_screen_split (x : screen_split) =\n  match x with\n  | `Focus_right -> 0\n  | `Wide_right -> 1\n  | `Even -> 2\n  | `Wide_left -> 3\n  | `Focus_left -> 4\n\ntype pane = [\n  | `Bottom_right\n  | `Key_binding_info\n]\n\nlet string_of_pane (x : pane) =\n  match x with\n  | `Bottom_right -> \"bottom-right\"\n  | `Key_binding_info -> \"key-binding-info\"\n\ntype t = [\n  | `Mark of string\n  | `Mark_listed\n  | `Unmark of string\n  | `Unmark_listed\n  | `Unmark_all\n  | `Drop of string\n  | `Drop_all_except of string\n  | `Drop_marked\n  | `Drop_unmarked\n  | `Drop_listed\n  | `Drop_unlisted\n  | `Narrow_level of int\n  | `Sort of Sort_by.t * Sort_by.t\n  | `Path_fuzzy_rank of string * int String_map.t option\n  | `Split_screen of screen_split\n  | `Hide_pane of pane\n  | `Show_pane of pane\n  | `Comment of string\n  | `Focus of string\n  | `Search of string\n  | `Filter of string\n]\n\nlet pp fmt (t : t) =\n  match t with\n  | `Mark s -> Fmt.pf fmt \"mark: %s\" s\n  | `Mark_listed -> Fmt.pf fmt \"mark listed\"\n  | `Unmark s -> Fmt.pf fmt \"unmark: %s\" s\n  | `Unmark_listed -> Fmt.pf fmt \"unmark listed\"\n  | `Unmark_all -> Fmt.pf fmt \"unmark all\"\n  | `Drop s -> Fmt.pf fmt \"drop: %s\" s\n  | `Drop_all_except s -> Fmt.pf fmt \"drop all except: %s\" s\n  | `Drop_marked -> Fmt.pf fmt \"drop marked\"\n  | `Drop_unmarked -> Fmt.pf fmt \"drop unmarked\"\n  | `Drop_listed -> Fmt.pf fmt \"drop listed\"\n  | `Drop_unlisted -> Fmt.pf fmt \"drop unlisted\"\n  | `Narrow_level x -> Fmt.pf fmt \"narrow level: %d\" x\n  | `Sort (x, y) -> (\n      Fmt.pf fmt \"sort by: %a; %a\"\n        Sort_by.pp\n        x\n        Sort_by.pp\n        y\n    )\n  | `Path_fuzzy_rank (s, _ranking) -> (\n      Fmt.pf fmt \"path fuzzy rank: %s\" s\n    )\n  | `Split_screen s -> (\n      Fmt.pf fmt \"split screen: %s\"\n        (match s with\n         | `Even -> \"even\"\n         | `Focus_left -> \"focus-left\"\n         | `Wide_left -> \"wide-left\"\n         | `Focus_right -> \"focus-right\"\n         | `Wide_right -> \"wide-right\"\n        )\n    )\n  | `Hide_pane pane -> (\n      Fmt.pf fmt \"hide-pane: %s\" (string_of_pane pane)\n    )\n  | `Show_pane pane -> (\n      Fmt.pf fmt \"show-pane: %s\" (string_of_pane pane)\n    )\n  | `Comment s -> Fmt.pf fmt \"#%s\" s\n  | `Focus s -> Fmt.pf fmt \"focus: %s\" s\n  | `Search s -> (\n      if String.length s = 0 then (\n        Fmt.pf fmt \"clear search\"\n      ) else (\n        Fmt.pf fmt \"search: %s\" s\n      )\n    )\n  | `Filter s -> (\n      if String.length s = 0 then (\n        Fmt.pf fmt \"clear filter\"\n      ) else (\n        Fmt.pf fmt \"filter: %s\" s\n      )\n    )\n\nlet to_string (t : t) =\n  Fmt.str \"%a\" pp t\n\nmodule Parsers = struct\n  type t' = t\n\n  open Angstrom\n  open Parser_components\n\n  let any_string_trimmed =\n    any_string >>| String.trim\n\n  let p : t' Angstrom.t =\n    skip_spaces *>\n    choice [\n      string \"mark\" *> skip_spaces *> (\n        choice [\n          char ':' *> skip_spaces *>\n          any_string_trimmed >>| (fun s -> (`Mark s));\n          string \"listed\" *> skip_spaces *> return `Mark_listed;\n        ]\n      );\n      string \"unmark\" *> skip_spaces *> (\n        choice [\n          string \"listed\" *> skip_spaces *> return `Unmark_listed;\n          string \"all\" *> skip_spaces *> return `Unmark_all;\n          char ':' *> skip_spaces *>\n          any_string_trimmed >>| (fun s -> (`Unmark s));\n        ]\n      );\n      string \"drop\" *> skip_spaces *> (\n        choice [\n          char ':' *> skip_spaces *>\n          any_string_trimmed >>| (fun s -> (`Drop s));\n          string \"all\" *> skip_spaces *>\n          string \"except\" *> skip_spaces *> char ':' *> skip_spaces *>\n          any_string_trimmed >>| (fun s -> (`Drop_all_except s));\n          string \"listed\" *> skip_spaces *> return `Drop_listed;\n          string \"unlisted\" *> skip_spaces *> return `Drop_unlisted;\n          string \"marked\" *> skip_spaces *> return `Drop_marked;\n          string \"unmarked\" *> skip_spaces *> return `Drop_unmarked;\n        ]\n      );\n      string \"narrow\" *> skip_spaces *> (\n        choice [\n          string \"level\" *> skip_spaces *>\n          char ':' *> skip_spaces *>\n          satisfy (function '0'..'9' -> true | _ -> false) <* skip_spaces >>|\n          (fun c -> `Narrow_level (Char.code c - Char.code '0'));\n        ]\n      );\n      string \"clear\" *> skip_spaces *> (\n        choice [\n          string \"search\" *> skip_spaces *> return (`Search \"\");\n          string \"filter\" *> skip_spaces *> return (`Filter \"\");\n        ]\n      );\n      string \"path\" *> skip_spaces *>\n      string \"fuzzy\" *> skip_spaces *>\n      string \"rank\" *> skip_spaces *>\n      char ':' *> skip_spaces *>\n      any_string_trimmed >>|\n      (fun s -> `Path_fuzzy_rank (s, None));\n      (string \"sort\" *> skip_spaces *>\n       string \"by\" *> skip_spaces *>\n       char ':' *> skip_spaces *>\n       Sort_by.p ~no_score:false >>= fun sort_by ->\n       skip_spaces *>\n       char ';' *>\n       skip_spaces *>\n       Sort_by.p ~no_score:true >>= fun sort_by_no_score ->\n       skip_spaces *>\n       return (`Sort (sort_by, sort_by_no_score)));\n      string \"focus\" *> skip_spaces *>\n      char ':' *> skip_spaces *>\n      any_string_trimmed >>| (fun s -> (`Focus s));\n      string \"split\" *> skip_spaces *>\n      string \"screen\" *> skip_spaces *>\n      char ':' *> skip_spaces *> (\n        choice [\n          string \"even\" *> skip_spaces *> return (`Split_screen `Even);\n          string \"focus-left\" *> skip_spaces *> return (`Split_screen `Focus_left);\n          string \"wide-left\" *> skip_spaces *> return (`Split_screen `Wide_left);\n          string \"focus-right\" *> skip_spaces *> return (`Split_screen `Focus_right);\n          string \"wide-right\" *> skip_spaces *> return (`Split_screen `Wide_right);\n        ]\n      );\n      string \"hide-pane\" *> skip_spaces *>\n      char ':' *> skip_spaces *> (\n        choice [\n          string \"bottom-right\" *> skip_spaces *> return (`Hide_pane `Bottom_right);\n          string \"key-binding-info\" *> skip_spaces *> return (`Hide_pane `Key_binding_info);\n        ]\n      );\n      string \"show-pane\" *> skip_spaces *>\n      char ':' *> skip_spaces *> (\n        choice [\n          string \"bottom-right\" *> skip_spaces *> return (`Show_pane `Bottom_right);\n          string \"key-binding-info\" *> skip_spaces *> return (`Show_pane `Key_binding_info);\n        ]\n      );\n      string \"#\" *> any_string >>| (fun s -> (`Comment s));\n      string \"search\" *> skip_spaces *>\n      char ':' *> skip_spaces *>\n      any_string_trimmed >>| (fun s -> (`Search s));\n      string \"filter\" *> skip_spaces *>\n      char ':' *> skip_spaces *>\n      any_string_trimmed >>| (fun s -> (`Filter s));\n    ]\nend\n\nlet of_string (s : string) : t option =\n  match Angstrom.(parse_string ~consume:Consume.All) Parsers.p s with\n  | Ok t -> Some t\n  | Error _ -> None\n"
  },
  {
    "path": "bin/content_and_search_result_rendering.ml",
    "content": "open Docfd_lib\n\nmodule I = Notty.I\n\nmodule A = Notty.A\n\ntype cell_typ = [\n  | `Plain\n  | `Search_result\n]\n\ntype cell = {\n  word : string;\n  typ : cell_typ;\n}\n\nmodule Text_block_rendering = struct\n  let hchunk_rev ~width (img : Notty.image) : Notty.image list =\n    let open Notty in\n    let rec aux acc img =\n      let img_width = I.width img in\n      if img_width <= width then (\n        img :: acc\n      ) else (\n        let acc = (I.hcrop 0 (img_width - width) img) :: acc in\n        aux acc (I.hcrop width 0 img)\n      )\n    in\n    aux [] img\n\n  let of_cells ?attr ~width ?(underline = false) (cells : cell list) : Notty.image * Int_set.t =\n    let open Notty.Infix in\n    assert (width > 0);\n    let rendered_lines_with_search_result_words = ref Int_set.empty in\n    let grid : Notty.image list list =\n      List.fold_left\n        (fun ((cur_len, acc) : int * Notty.image list list) (cell : cell) ->\n           let attr =\n             match attr with\n             | Some attr -> attr\n             | None -> (match cell.typ with\n                 | `Plain -> A.empty\n                 | `Search_result -> A.(fg black ++ bg lightyellow)\n               )\n           in\n           let word =\n             (match I.string attr cell.word with\n              | s -> s\n              | exception _ -> (\n                  I.string A.(fg lightred) (String.make (String.length cell.word) '?')\n                ))\n           in\n           let word_len = I.width word in\n           let word =\n             match cell.typ with\n             | `Plain -> word\n             | `Search_result -> (\n                 if underline then (\n                   word\n                   <->\n                   (I.string A.empty (String.make word_len '^'))\n                 ) else (\n                   word\n                 )\n               )\n           in\n           let new_len = cur_len + word_len in\n           let cur_len, acc =\n             if new_len <= width then (\n               match acc with\n               | [] -> (new_len, [ [ word ] ])\n               | line :: rest -> (\n                   (new_len, (word :: line) :: rest)\n                 )\n             ) else (\n               if word_len <= width then (\n                 (word_len, [ word ] :: acc)\n               ) else (\n                 let lines =\n                   hchunk_rev ~width word\n                   |> List.map (fun x -> [ x ])\n                 in\n                 (0, [] :: (lines @ acc))\n               )\n             )\n           in\n           (match cell.typ with\n            | `Plain -> ()\n            | `Search_result -> (\n                rendered_lines_with_search_result_words :=\n                  Int_set.add (List.length acc - 1) !rendered_lines_with_search_result_words\n              ));\n           (cur_len, acc)\n        )\n        (0, [])\n        cells\n      |> snd\n      |> List.rev_map List.rev\n    in\n    let img =\n      grid\n      |> List.map I.hcat\n      |> I.vcat\n    in\n    (img, !rendered_lines_with_search_result_words)\n\n  let of_words ?attr ~width ?underline ?(highlights = Int_set.empty) (words : string list) : Notty.image =\n    of_cells\n      ?attr\n      ~width\n      ?underline\n      (List.mapi\n         (fun i word ->\n            if Int_set.mem i highlights then (\n              { word; typ = `Search_result }\n            ) else (\n              { word; typ = `Plain }\n            )\n         )\n         words)\n    |> fst\nend\n\ntype word_grid = {\n  start_global_line_num : int;\n  data : cell array array;\n}\n\nlet start_and_end_inc_global_line_num_of_search_result\n    ~doc_id\n    (search_result : Search_result.t)\n  : (int * int) =\n  match Search_result.found_phrase search_result with\n  | [] -> failwith \"unexpected case\"\n  | l -> (\n      List.fold_left (fun s_e Search_result.{ found_word_pos; _ } ->\n          let loc = Index.loc_of_pos ~doc_id found_word_pos in\n          let line_loc = Index.Loc.line_loc loc in\n          let global_line_num = Index.Line_loc.global_line_num line_loc in\n          match s_e with\n          | None -> (\n              Some (global_line_num, global_line_num)\n            )\n          | Some (s, e) -> (\n              Some (min s global_line_num, max global_line_num e)\n            )\n        )\n        None\n        l\n      |> Option.get\n    )\n\nlet word_grid_of_index\n    ~doc_id\n    ~start_global_line_num\n    ~end_inc_global_line_num\n  : word_grid =\n  let global_line_count = Index.global_line_count ~doc_id in\n  let check x =\n    assert (0 <= x);\n    assert (x <= global_line_count - 1);\n  in\n  check start_global_line_num;\n  check end_inc_global_line_num;\n  if global_line_count = 0 then (\n    { start_global_line_num = 0; data = [||] }\n  ) else (\n    let data =\n      OSeq.(start_global_line_num -- end_inc_global_line_num)\n      |> Seq.map (fun global_line_num ->\n          let data =\n            Index.words_of_global_line_num ~doc_id global_line_num\n            |> Dynarray.to_seq\n            |> Seq.map (fun word -> { word; typ = `Plain })\n            |> Array.of_seq\n          in\n          data\n        )\n      |> Array.of_seq\n    in\n    { start_global_line_num; data }\n  )\n\nlet mark_in_word_grid\n    ~doc_id\n    (grid : word_grid)\n    (positions : int list)\n  : unit =\n  let grid_end_inc_global_line_num = grid.start_global_line_num + Array.length grid.data - 1 in\n  List.iter (fun pos ->\n      let loc = Index.loc_of_pos ~doc_id pos in\n      let line_loc = Index.Loc.line_loc loc in\n      let global_line_num = Index.Line_loc.global_line_num line_loc in\n      if grid.start_global_line_num <= global_line_num\n      && global_line_num <= grid_end_inc_global_line_num\n      then (\n        let pos_in_line = Index.Loc.pos_in_line loc in\n        let row = global_line_num - grid.start_global_line_num in\n        let cell = grid.data.(row).(pos_in_line) in\n        grid.data.(row).(pos_in_line) <- { cell with typ = `Search_result }\n      )\n    )\n    positions\n\nlet mark_search_result_in_word_grid\n    ~doc_id\n    (grid : word_grid)\n    (search_result : Search_result.t)\n  : unit =\n  Search_result.found_phrase search_result\n  |> List.map (fun Search_result.{ found_word_pos; _ } ->\n      found_word_pos\n    )\n  |> mark_in_word_grid ~doc_id grid\n\nlet mark_link_in_word_grid\n    ~doc_id\n    (grid : word_grid)\n    (link : Link.t)\n  : unit =\n  let { Link.start_pos; end_inc_pos; _ } = link in\n  CCList.(start_pos -- end_inc_pos)\n  |> mark_in_word_grid ~doc_id grid\n\ntype render_mode = [\n  | `Page_num_only\n  | `Line_num_only\n  | `Page_and_line_num\n  | `None\n]\n\nlet render_grid\n    ~doc_id\n    ~(view_offset : int Lwd.var option)\n    ~(render_mode : render_mode)\n    ~width\n    ?(height : int option)\n    ?underline\n    (grid : word_grid)\n  : Notty.image =\n  let (_rendered_line_count, rendered_lines_with_search_result_words), images =\n    grid.data\n    |> Array.to_list\n    |> CCList.fold_map_i\n      (fun (rendered_line_count, rendered_lines_with_search_result_words_acc) i cells ->\n         let cells = Array.to_list cells in\n         let global_line_num = grid.start_global_line_num + i in\n         let line_loc = Index.line_loc_of_global_line_num ~doc_id global_line_num in\n         let displayed_line_num = Index.Line_loc.line_num_in_page line_loc + 1 in\n         let displayed_page_num = Index.Line_loc.page_num line_loc + 1 in\n         let left_column_label =\n           match render_mode with\n           | `Page_num_only -> (\n               I.hcat\n                 [ I.strf ~attr:A.(fg lightyellow) \"Page %d\" displayed_page_num\n                 ; I.strf \": \" ]\n             )\n           | `Line_num_only -> (\n               I.hcat\n                 [ I.strf ~attr:A.(fg lightyellow) \"%d\" displayed_line_num\n                 ; I.strf \": \" ]\n             )\n           | `Page_and_line_num -> (\n               I.hcat\n                 [ I.strf ~attr:A.(fg lightyellow) \"Page %d, %d\"\n                     displayed_page_num\n                     displayed_line_num\n                 ; I.strf \": \" ]\n             )\n           | `None -> (\n               I.void 0 1\n             )\n         in\n         let content_width = max 1 (width - I.width left_column_label) in\n         let content, rendered_lines_with_search_result_words =\n           Text_block_rendering.of_cells ?underline ~width:content_width cells\n         in\n         ((rendered_line_count + I.height content,\n           rendered_lines_with_search_result_words\n           |> Int_set.map (fun x -> x + rendered_line_count)\n           |> Int_set.union rendered_lines_with_search_result_words_acc\n          ),\n          I.hcat [ left_column_label; content ])\n      )\n      (0, Int_set.empty)\n  in\n  let img = I.vcat images in\n  match height with\n  | None -> img\n  | Some height -> (\n      let focal_point_offset =\n        match\n          Int_set.min_elt_opt rendered_lines_with_search_result_words,\n          Int_set.max_elt_opt rendered_lines_with_search_result_words\n        with\n        | Some start_rendered_line_num, Some end_inc_rendered_line_num -> (\n            Misc_utils.div_round_to_closest\n              (start_rendered_line_num + end_inc_rendered_line_num)\n              2\n          )\n        | _, _ -> 0\n      in\n      let target_region_start =\n        max 0 (focal_point_offset - (Misc_utils.div_round_to_closest height 2))\n      in\n      let img_height = I.height img in\n      let target_region_end_exc =\n        min\n          img_height\n          (target_region_start + height)\n      in\n      let view_offset_old =\n        match view_offset with\n        | None -> 0\n        | Some x -> Lwd.peek x\n      in\n      let view_offset' =\n        if view_offset_old >= 0 then (\n          min\n            view_offset_old\n            (img_height - target_region_end_exc)\n        ) else (\n          let view_offset_old = Int.abs view_offset_old in\n          - (min\n               view_offset_old\n               (target_region_start - 0))\n        )\n      in\n      Option.iter (fun x ->\n          if view_offset_old <> view_offset' then (\n            Lwd.set x view_offset'\n          )) view_offset;\n      let target_region_start, target_region_end_exc =\n        if view_offset' >= 0 then (\n          (target_region_start + view_offset',\n           target_region_end_exc + view_offset')\n        ) else (\n          (* If the offset is negative (shifting view upwards),\n             then make the bottom border \"sticky\".\n\n             In other words, if the height of view window\n             is smaller than the height of pane,\n             and the view is shifting upwards, don't bother\n             moving the bottom border.\n\n             This prevents the rendered content snippet view staying\n             small and cropping the bottom text when scrolling up\n             if the view started out being small (due to the search\n             result being close to the bottom of the file).\n          *)\n          let view_window_height = target_region_end_exc - target_region_start in\n          (target_region_start + view_offset',\n           if view_window_height < height then (\n             target_region_end_exc\n           ) else (\n             target_region_end_exc + view_offset'\n           )\n          )\n        )\n      in\n      I.vcrop target_region_start (img_height - target_region_end_exc) img\n    )\n\nlet content_snippet\n    ~doc_id\n    ~(view_offset : int Lwd.var)\n    ?(data : [ `Search_result of Search_result.t | `Link of Link.t ] option)\n    ~(width : int)\n    ~(height : int)\n    ?underline\n    ()\n  : Notty.image =\n  let max_end_inc_global_line_num = Index.global_line_count ~doc_id - 1 in\n  assert (height > 0);\n  let compute_final_line_num_range\n      ~(view_offset : int Lwd.var)\n      ~start_global_line_num\n    : int * int =\n    let end_inc_global_line_num =\n      min\n        max_end_inc_global_line_num\n        (start_global_line_num + height - 1)\n    in\n    (* We grow the area in one direction\n       rather than shifting the area, in order\n       to not interfere with the focal point offset computation\n       in render_grid.\n\n       The number of lines to grow is an overapproximation\n       of the actual lines required, as a\n       line may wrap into multiple rendered lines\n       in the rendered view if it is longer than\n       the width of the content pane.\n       But we do not know how many lines (or partial segments\n       of lines) exactly until we\n       actually render the view/word grid.\n    *)\n    let view_offset' = Lwd.peek view_offset in\n    if view_offset' >= 0 then (\n      let end_inc_global_line_num =\n        min\n          max_end_inc_global_line_num\n          (end_inc_global_line_num + view_offset')\n      in\n      (start_global_line_num, end_inc_global_line_num)\n    ) else (\n      let start_global_line_num =\n        max\n          0\n          (start_global_line_num - Int.abs view_offset')\n      in\n      (start_global_line_num, end_inc_global_line_num)\n    )\n  in\n  match data with\n  | None -> (\n      let start_global_line_num, end_inc_global_line_num =\n        compute_final_line_num_range\n          ~view_offset\n          ~start_global_line_num:0\n      in\n      let grid =\n        word_grid_of_index\n          ~doc_id\n          ~start_global_line_num\n          ~end_inc_global_line_num\n      in\n      render_grid\n        ~doc_id\n        ~view_offset:(Some view_offset)\n        ~render_mode:`None\n        ~width\n        ~height\n        ?underline\n        grid\n    )\n  | Some data -> (\n      let focal_line =\n        match data with\n        | `Search_result search_result -> (\n            let (relevant_start_line, relevant_end_inc_line) =\n              start_and_end_inc_global_line_num_of_search_result ~doc_id search_result\n            in\n            let avg = (relevant_start_line + relevant_end_inc_line) / 2 in\n            avg\n          )\n        | `Link link -> (\n            let loc = Index.loc_of_pos ~doc_id link.Link.start_pos in\n            let line_loc = Index.Loc.line_loc loc in\n            Index.Line_loc.global_line_num line_loc\n          )\n      in\n      let start_global_line_num, end_inc_global_line_num =\n        compute_final_line_num_range\n          ~view_offset\n          ~start_global_line_num:(\n            max\n              0\n              (focal_line - (Misc_utils.div_round_to_closest height 2))\n          )\n      in\n      let grid =\n        word_grid_of_index\n          ~doc_id\n          ~start_global_line_num\n          ~end_inc_global_line_num\n      in\n      (match data with\n       | `Search_result search_result -> (\n           mark_search_result_in_word_grid ~doc_id grid search_result\n         )\n       | `Link link -> (\n           mark_link_in_word_grid ~doc_id grid link\n         )\n      );\n      render_grid\n        ~doc_id\n        ~view_offset:(Some view_offset)\n        ~render_mode:`None\n        ~width\n        ~height\n        ?underline\n        grid\n    )\n\nlet word_is_not_space s =\n  String.length s > 0 && not (Parser_components.is_space s.[0])\n\nlet grab_additional_lines\n    ~doc_id\n    ~non_space_word_count\n    start_global_line_num\n    end_inc_global_line_num\n  : int * int =\n  let max_end_inc_global_line_num = Index.global_line_count ~doc_id - 1 in\n  let non_space_word_count_of_line n =\n    Index.words_of_global_line_num ~doc_id n\n    |> Dynarray.to_seq\n    |> Seq.filter word_is_not_space\n    |> Seq.length\n  in\n  let rec aux ~non_space_word_count ~i x y =\n    if i < !Params.search_result_print_snippet_max_additional_lines_each_direction\n    && non_space_word_count < !Params.search_result_print_snippet_min_size\n    then (\n      let x, top_add_count =\n        let n = x - 1 in\n        if n >= 0 then (\n          (n, non_space_word_count_of_line n)\n        ) else (\n          (x, 0)\n        )\n      in\n      let y, bottom_add_count =\n        let n = y + 1 in\n        if n <= max_end_inc_global_line_num then (\n          (n, non_space_word_count_of_line n)\n        ) else (\n          (y, 0)\n        )\n      in\n      let non_space_word_count =\n        non_space_word_count\n        + top_add_count\n        + bottom_add_count\n      in\n      aux ~non_space_word_count ~i:(i + 1) x y\n    ) else (\n      (x, y)\n    )\n  in\n  aux ~non_space_word_count ~i:0 start_global_line_num end_inc_global_line_num\n\nlet search_result\n    ~doc_id\n    ~render_mode\n    ~width\n    ?underline\n    ?(fill_in_context = false)\n    (search_result : Search_result.t)\n  : Notty.image =\n  let open Notty in\n  let open Notty.Infix in\n  let (start_global_line_num, end_inc_global_line_num) =\n    start_and_end_inc_global_line_num_of_search_result ~doc_id search_result\n    |> (fun (x, y) ->\n        if fill_in_context then (\n          let non_space_word_count =\n            Search_result.search_phrase search_result\n            |> Search_phrase.enriched_tokens\n            |> List.filter_map (fun token ->\n                match Search_phrase.Enriched_token.data token with\n                | `Explicit_spaces -> None\n                | `String s -> (\n                    assert (word_is_not_space s);\n                    Some s\n                  )\n              )\n            |> List.length\n          in\n          grab_additional_lines ~doc_id ~non_space_word_count x y\n        ) else (\n          (x, y)\n        )\n      )\n  in\n  let grid =\n    word_grid_of_index\n      ~doc_id\n      ~start_global_line_num\n      ~end_inc_global_line_num\n  in\n  mark_search_result_in_word_grid ~doc_id grid search_result;\n  let img =\n    render_grid\n      ~doc_id\n      ~view_offset:None\n      ~render_mode\n      ~width\n      ?underline\n      grid\n  in\n  if Option.is_some !Params.debug_output then (\n    let score = Search_result.score search_result in\n    I.strf \"(Score: %f)\" score\n    <->\n    img\n  ) else (\n    img\n  )\n\nlet centered_list\n    ~width\n    ~height\n    ~(render : width:int -> 'a -> Notty.image)\n    (selection : int)\n    (l : 'a list)\n  : Notty.image =\n  let height_rendered_before_selection = ref 0 in\n  let img =\n    l\n    |> List.mapi (fun i x ->\n        let img =\n          let x =\n            if i = selection then (\n              let img = render ~width:(width - 2) x in\n              Notty.I.(strf ~attr:A.(fg lightyellow) \"> \"\n                       <|>\n                       img\n                      )\n            ) else (\n              render ~width x\n            )\n          in\n          Notty.I.(x <-> strf \"\")\n        in\n        if i < selection then (\n          height_rendered_before_selection := !height_rendered_before_selection + Notty.I.height img;\n        );\n        img\n      )\n    |> Notty.I.vcat\n  in\n  let focal_point_offset = !height_rendered_before_selection in\n  let img_height = I.height img in\n  let target_region_start =\n    max 0 (focal_point_offset - (Misc_utils.div_round_to_closest height 2))\n  in\n  let target_region_end_exc =\n    min\n      img_height\n      (target_region_start + height)\n  in\n  I.vcrop target_region_start (img_height - target_region_end_exc) img\n"
  },
  {
    "path": "bin/debug_utils.ml",
    "content": "let do_if_debug (f : out_channel -> unit) =\n  match !Params.debug_output with\n  | None -> ()\n  | Some oc -> (\n      f oc\n    )\n"
  },
  {
    "path": "bin/docfd.ml",
    "content": "open Cmdliner\nopen Lwd_infix\nopen Docfd_lib\nopen Debug_utils\nopen Misc_utils\nopen File_utils\n\nlet compute_paths_from_globs ~report_progress globs =\n  Seq.iter (fun s ->\n      match Glob.parse s with\n      | Some _ -> ()\n      | None -> (\n          exit_with_error_msg\n            (Fmt.str \"failed to parse glob pattern: \\\"%s\\\"\" s)\n        )\n    ) globs;\n  list_files_recursive_filter_by_globs ~report_progress globs\n\ntype file_constraints = {\n  no_pdftotext : bool;\n  no_pandoc : bool;\n  paths_were_specified_by_user : bool;\n  exts : string list;\n  single_line_exts : string list;\n  directly_specified_paths : String_set.t;\n  globs : String_set.t;\n  single_line_globs : String_set.t;\n}\n\nlet make_file_constraints\n    ~no_pdftotext\n    ~no_pandoc\n    ~(exts : string list)\n    ~(single_line_exts : string list)\n    ~(paths : string list)\n    ~(paths_from_file_or_stdin : string list option)\n    ~(globs : string list)\n    ~(single_line_globs : string list)\n  : file_constraints =\n  match\n    paths,\n    paths_from_file_or_stdin,\n    globs,\n    single_line_globs\n  with\n  | [], None, [], [] -> (\n      {\n        no_pdftotext;\n        no_pandoc;\n        paths_were_specified_by_user = false;\n        exts;\n        single_line_exts;\n        directly_specified_paths = String_set.of_list [ \".\" ];\n        globs = String_set.empty;\n        single_line_globs = String_set.empty;\n      }\n    )\n  | _, _, _, _ -> (\n      let paths_from_file_or_stdin = Option.value ~default:[] paths_from_file_or_stdin in\n      let directly_specified_paths = String_set.of_list (paths @ paths_from_file_or_stdin) in\n      let globs = String_set.of_list globs in\n      let single_line_globs = String_set.of_list single_line_globs in\n      {\n        no_pdftotext;\n        no_pandoc;\n        paths_were_specified_by_user = true;\n        exts;\n        single_line_exts;\n        directly_specified_paths;\n        globs;\n        single_line_globs;\n      }\n    )\n\nlet files_satisfying_constraints\n    ~interactive\n    (cons : file_constraints)\n  : Document_src.file_collection =\n  let bar =\n    let open Progress.Line in\n    list\n      [ const \"Scanning\"\n      ; spinner ()\n      ]\n  in\n  progress_with_reporter\n    ~interactive\n    bar\n    (fun report_progress : Document_src.file_collection ->\n       let single_line_search_mode_applies file =\n         List.mem (extension_of_file file) cons.single_line_exts\n       in\n       let single_line_search_mode_paths_by_exts, default_search_mode_paths_by_exts =\n         cons.directly_specified_paths\n         |> String_set.to_seq\n         |> list_files_recursive_filter_by_exts\n           ~report_progress\n           ~exts:(cons.exts @ cons.single_line_exts)\n         |> String_set.partition single_line_search_mode_applies\n       in\n       let paths_from_single_line_globs =\n         cons.single_line_globs\n         |> String_set.to_seq\n         |> compute_paths_from_globs ~report_progress\n       in\n       let single_line_search_mode_paths_from_globs, default_search_mode_paths_from_globs =\n         cons.globs\n         |> String_set.to_seq\n         |> compute_paths_from_globs ~report_progress\n         |> String_set.partition single_line_search_mode_applies\n       in\n       let single_line_search_mode_files =\n         single_line_search_mode_paths_by_exts\n         |> String_set.union paths_from_single_line_globs\n         |> String_set.union single_line_search_mode_paths_from_globs\n       in\n       let default_search_mode_files =\n         default_search_mode_paths_by_exts\n         |> String_set.union default_search_mode_paths_from_globs\n         |> (fun s -> String_set.diff s single_line_search_mode_files)\n       in\n       do_if_debug (fun oc ->\n           Printf.fprintf oc \"Checking if single line search mode files and default search mode files are disjoint\\n\";\n           if String_set.is_empty\n               (String_set.inter\n                  single_line_search_mode_files\n                  default_search_mode_files)\n           then (\n             Printf.fprintf oc \"Check successful\\n\"\n           ) else (\n             failwith \"check failed\"\n           );\n           let all_files =\n             single_line_search_mode_paths_by_exts\n             |> String_set.union default_search_mode_paths_by_exts\n             |> String_set.union paths_from_single_line_globs\n             |> String_set.union single_line_search_mode_paths_from_globs\n             |> String_set.union default_search_mode_paths_from_globs\n           in\n           let single_line_search_mode_files', default_search_mode_files' =\n             String_set.partition (fun s ->\n                 single_line_search_mode_applies s\n                 ||\n                 String_set.mem s paths_from_single_line_globs\n               )\n               all_files\n           in\n           Printf.fprintf oc \"Checking if efficiently computed and naively computed results for single line search mode files are consistent\\n\";\n           if String_set.equal\n               single_line_search_mode_files\n               single_line_search_mode_files'\n           then (\n             Printf.fprintf oc \"Check successful\\n\"\n           ) else (\n             failwith \"check failed\"\n           );\n           Printf.fprintf oc \"Checking if efficiently computed and naively computed results for default search mode files are consistent\\n\";\n           if String_set.equal\n               default_search_mode_files\n               default_search_mode_files'\n           then (\n             Printf.fprintf oc \"Check successful\\n\"\n           ) else (\n             failwith \"check failed\"\n           )\n         );\n       let filter_for_no_pdftotext_or_no_pandoc (s : String_set.t) =\n         if cons.no_pdftotext || cons.no_pandoc then (\n           String_set.filter\n             (fun s ->\n                match File_utils.format_of_file s with\n                | `PDF -> not cons.no_pdftotext\n                | `Pandoc_supported_format -> not cons.no_pandoc\n                | `Text | `Other -> true\n             )\n             s\n         ) else (\n           s\n         )\n       in\n       let default_search_mode_files =\n         filter_for_no_pdftotext_or_no_pandoc default_search_mode_files\n       in\n       let single_line_search_mode_files =\n         filter_for_no_pdftotext_or_no_pandoc single_line_search_mode_files\n       in\n       {\n         default_search_mode_files;\n         single_line_search_mode_files;\n       }\n    )\n\nlet init_session_state_of_document_src ~env ~interactive pool (document_src : Document_src.t) =\n  let file_bar ~total_file_count =\n    let open Progress.Line in\n    list\n      [ brackets (elapsed ())\n      ; bar ~width:(`Fixed 20) total_file_count\n      ; percentage_of total_file_count\n      ; const \"ETA: \" ++ eta total_file_count\n      ]\n  in\n  let byte_bar ~total_byte_count =\n    let open Progress.Line in\n    list\n      [ brackets (elapsed ())\n      ; bar ~width:(`Fixed 20) total_byte_count\n      ; percentage_of total_byte_count\n      ; bytes_per_sec\n      ; const \"ETA: \" ++ eta total_byte_count\n      ]\n  in\n  let all_documents : Document.t list list =\n    match document_src with\n    | Document_src.Stdin path -> (\n        match\n          Document.of_path\n            ~env\n            pool\n            ~already_in_transaction:false\n            !Params.default_search_mode\n            path\n        with\n        | Ok x -> [ [ x ] ]\n        | Error msg ->  (\n            exit_with_error_msg msg\n          )\n      )\n    | Files { default_search_mode_files; single_line_search_mode_files } -> (\n        let print_stage_stats ~file_count ~total_byte_count =\n          Printf.printf \"- File count: %6d\\n\" file_count;\n          Printf.printf \"- MiB:        %8.1f\\n\"\n            (Misc_utils.mib_of_bytes total_byte_count);\n        in\n        let total_file_count, files =\n          Seq.append\n            (Seq.map (fun path -> (!Params.default_search_mode, path))\n               (String_set.to_seq default_search_mode_files))\n            (Seq.map (fun path -> (`Single_line, path))\n               (String_set.to_seq single_line_search_mode_files))\n          |> Misc_utils.length_and_list_of_seq\n        in\n        if interactive then (\n          Printf.printf \"Collecting file stats\\n\";\n          flush stdout;\n        );\n        let documents_total_byte_count, document_sizes =\n          progress_with_reporter\n            ~interactive\n            (file_bar ~total_file_count)\n            (fun report_progress ->\n               List.fold_left (fun (total_size, m) (_, path) ->\n                   let res =\n                     match File_utils.file_size path with\n                     | None -> (total_size, m)\n                     | Some x -> (total_size + x, String_map.add path x m)\n                   in\n                   report_progress 1;\n                   res\n                 )\n                 (0, String_map.empty)\n                 files\n            )\n        in\n        if interactive then (\n          print_stage_stats\n            ~file_count:total_file_count\n            ~total_byte_count:documents_total_byte_count\n        );\n        if interactive then (\n          Printf.printf \"Hashing\\n\";\n          flush stdout;\n        );\n        let file_and_hash_list =\n          match files with\n          | [] -> []\n          | _ -> (\n              files\n              |> (fun l ->\n                  progress_with_reporter\n                    ~interactive\n                    (byte_bar ~total_byte_count:documents_total_byte_count)\n                    (fun report_progress ->\n                       Task_pool.filter_map_list pool (fun (search_mode, path) ->\n                           do_if_debug (fun oc ->\n                               Printf.fprintf oc \"Hashing document: %s\\n\" (Filename.quote path);\n                             );\n                           let res =\n                             match BLAKE2B.hash_of_file ~env ~path with\n                             | Ok hash -> Some (search_mode, path, hash)\n                             | Error msg -> (\n                                 do_if_debug (fun oc ->\n                                     Printf.fprintf oc \"Error: %s\\n\" msg\n                                   );\n                                 None\n                               )\n                           in\n                           (match String_map.find_opt path document_sizes with\n                            | None -> ()\n                            | Some x -> report_progress x);\n                           res\n                         )\n                         l\n                    )\n                )\n            )\n        in\n        if interactive then (\n          Printf.printf \"Allocating document IDs\\n\";\n          flush stdout;\n        );\n        progress_with_reporter\n          ~interactive\n          (file_bar ~total_file_count)\n          (fun report_progress ->\n             file_and_hash_list\n             |> List.to_seq\n             |> Seq.map (fun (_, _, doc_hash) ->\n                 report_progress 1;\n                 doc_hash)\n             |> Doc_id_db.allocate_bulk\n          );\n        let indexed_files, unindexed_files =\n          let open Sqlite3_utils in\n          with_stmt\n            Index.is_indexed_sql\n            (fun stmt ->\n               List.partition (fun (_, _, doc_hash) ->\n                   bind_names stmt [ (\"@doc_hash\", TEXT doc_hash) ];\n                   step stmt;\n                   let indexed = data_count stmt > 0 in\n                   reset stmt;\n                   indexed\n                 )\n                 file_and_hash_list\n            )\n        in\n        indexed_files\n        |> List.map (fun (_, _, doc_hash) ->\n            Doc_id_db.doc_id_of_doc_hash doc_hash\n          )\n        |> Index.refresh_last_used_batch;\n        let load_document ~env pool search_mode ~doc_hash path =\n          do_if_debug (fun oc ->\n              Printf.fprintf oc \"Loading document: %s\\n\" (Filename.quote path);\n            );\n          do_if_debug (fun oc ->\n              Printf.fprintf oc \"Using %s search mode for document %s\\n\"\n                (match search_mode with\n                 | `Single_line -> \"single line\"\n                 | `Multiline -> \"multiline\"\n                )\n                (Filename.quote path)\n            );\n          match\n            Document.of_path\n              ~env\n              pool\n              ~already_in_transaction:false\n              search_mode\n              ~doc_hash\n              path\n          with\n          | Ok x -> (\n              do_if_debug (fun oc ->\n                  Printf.fprintf oc \"Document %s loaded successfully\\n\" (Filename.quote path);\n                );\n              Some x\n            )\n          | Error msg -> (\n              do_if_debug (fun oc ->\n                  Printf.fprintf oc \"Error: %s\\n\" msg\n                );\n              None\n            )\n        in\n        if interactive then (\n          Printf.printf \"Processing indexed files\\n\";\n          flush stdout;\n        );\n        let indexed_files =\n          indexed_files\n          |> List.filter_map (fun (search_mode, path, doc_hash) ->\n              load_document ~env pool search_mode ~doc_hash path\n            )\n        in\n        if interactive then (\n          Printf.printf \"Indexing remaining files\\n\";\n          flush stdout;\n        );\n        let unindexed_file_count, unindexed_files_byte_count =\n          List.fold_left (fun (file_count, byte_count) (_, path, _) ->\n              (file_count + 1,\n               byte_count + Option.value ~default:0 (String_map.find_opt path document_sizes))\n            )\n            (0, 0)\n            unindexed_files\n        in\n        if interactive then (\n          print_stage_stats\n            ~file_count:unindexed_file_count\n            ~total_byte_count:unindexed_files_byte_count;\n        );\n        let pipeline = Document_pipeline.make ~env pool in\n        let _, unindexed_files =\n          Eio.Fiber.pair\n            (fun () ->\n               Document_pipeline.run pipeline\n            )\n            (fun () ->\n               (match unindexed_files with\n                | [] -> ()\n                | _ -> (\n                    progress_with_reporter\n                      ~interactive\n                      (byte_bar ~total_byte_count:unindexed_files_byte_count)\n                      (fun report_progress ->\n                         unindexed_files\n                         |> List.iter (fun (search_mode, path, doc_hash) ->\n                             Document_pipeline.feed\n                               pipeline\n                               search_mode\n                               ~doc_hash\n                               path;\n                             (match String_map.find_opt path document_sizes with\n                              | None -> ()\n                              | Some x -> report_progress x\n                             )\n                           )\n                      )\n                  ));\n               if interactive then (\n                 Printf.printf \"Finalizing index\\n\";\n                 flush stdout;\n               );\n               Document_pipeline.finalize pipeline\n            )\n        in\n        [ indexed_files; unindexed_files ]\n      )\n  in\n  let state =\n    all_documents\n    |> List.to_seq\n    |> Seq.flat_map List.to_seq\n    |> Session.State.of_seq pool\n  in\n  Gc.compact ();\n  state\n\nlet parse_sort_by_arg ~no_score (s : string) : Command.Sort_by.t =\n  match Command.Sort_by.parse ~no_score s with\n  | Ok t -> t\n  | Error msg -> (\n      let msg = CCString.chop_prefix ~pre:\": \" msg\n        |> Option.value ~default:msg\n      in\n      exit_with_error_msg\n        (Fmt.str \"failed to parse --%s argument: %s\"\n           (if no_score then (\n               Args.sort_no_score_arg_name\n             ) else (\n              Args.sort_arg_name\n            ))\n           msg\n        )\n    )\n\nlet run\n    ~(eio_env : Eio_unix.Stdenv.base)\n    ~sw\n    (debug_log : string option)\n    (no_pdftotext : bool)\n    (no_pandoc : bool)\n    (scan_hidden : bool)\n    (max_depth : int)\n    (max_fuzzy_edit_dist : int)\n    (max_token_search_dist : int)\n    (max_linked_token_search_dist : int)\n    (tokens_per_search_scope_level : int)\n    (index_chunk_size : int)\n    (exts : string)\n    (single_line_exts : string)\n    (additional_exts : string list)\n    (single_line_additional_exts : string list)\n    (cache_dir : string)\n    (cache_limit : int)\n    (data_dir : string)\n    (index_only : bool)\n    (start_with_filter : string option)\n    (start_with_search : string option)\n    (filter_exp : string option)\n    (sample_search_exp : string option)\n    (samples_per_doc : int)\n    (search_exp : string option)\n    (sort_by : string)\n    (sort_by_no_score : string)\n    (print_color_mode : Params.style_mode)\n    (print_underline_mode : Params.style_mode)\n    (search_result_print_text_width : int)\n    (search_result_print_snippet_min_size : int)\n    (search_result_print_max_add_lines : int)\n    (start_with_script : string option)\n    (script : string option)\n    (paths_from : string list)\n    (globs : string list)\n    (single_line_globs : string list)\n    (single_line_search_mode_by_default : bool)\n    (path_open_specs : string list)\n    (print_files_with_match : bool)\n    (print_files_without_match : bool)\n    (paths : string list)\n  =\n  let env = eio_env in\n  Args.check\n    ~max_depth\n    ~max_fuzzy_edit_dist\n    ~max_token_search_dist\n    ~max_linked_token_search_dist\n    ~tokens_per_search_scope_level\n    ~index_chunk_size\n    ~cache_limit\n    ~start_with_filter\n    ~start_with_search\n    ~filter_exp\n    ~sample_search_exp\n    ~samples_per_doc\n    ~search_exp\n    ~search_result_print_text_width\n    ~search_result_print_snippet_min_size\n    ~search_result_print_max_add_lines\n    ~start_with_script\n    ~script\n    ~paths_from\n    ~print_files_with_match\n    ~print_files_without_match;\n  Params.debug_output := (match debug_log with\n      | None -> None\n      | Some \"-\" -> Some stderr\n      | Some debug_log -> (\n          try\n            Some (\n              open_out_gen\n                [ Open_append; Open_creat; Open_wronly; Open_text ]\n                0o644\n                debug_log\n            )\n          with\n          | Sys_error _ -> (\n              exit_with_error_msg\n                (Fmt.str \"failed to open debug log file %s\" (Filename.quote debug_log))\n            )\n        )\n    );\n  Params.scan_hidden := scan_hidden;\n  Params.max_file_tree_scan_depth := max_depth;\n  Params.max_fuzzy_edit_dist := max_fuzzy_edit_dist;\n  Params.max_token_search_dist := max_token_search_dist;\n  Params.max_linked_token_search_dist := max_linked_token_search_dist;\n  Params.tokens_per_search_scope_level := tokens_per_search_scope_level;\n  Params.index_chunk_size := index_chunk_size;\n  Params.cache_limit := cache_limit;\n  Params.search_result_print_text_width := search_result_print_text_width;\n  Params.search_result_print_snippet_min_size := search_result_print_snippet_min_size;\n  Params.search_result_print_snippet_max_additional_lines_each_direction :=\n    search_result_print_max_add_lines;\n  Params.samples_per_document := samples_per_doc;\n  let sort_by = parse_sort_by_arg ~no_score:false sort_by in\n  let sort_by_no_score = parse_sort_by_arg ~no_score:true sort_by_no_score in\n  Params.cache_dir := (\n    mkdir_recursive cache_dir;\n    Some cache_dir\n  );\n  Params.data_dir := (\n    mkdir_recursive data_dir;\n    Some data_dir\n  );\n  Params.default_search_mode := (\n    if single_line_search_mode_by_default then (\n      `Single_line\n    ) else (\n      `Multiline\n    )\n  );\n  List.iter (fun spec ->\n      match Path_opening.parse_spec spec with\n      | Error msg -> (\n          exit_with_error_msg (Fmt.str \"failed to parse %s, %s\" spec msg)\n        )\n      | Ok (exts, launch_mode, cmd) -> (\n          List.iter (fun ext ->\n              Hashtbl.replace Path_opening.specs ext (launch_mode, cmd)\n            )\n            exts\n        )\n    ) path_open_specs;\n  let db_path = Filename.concat cache_dir Params.db_file_name in\n  (match Docfd_lib.init ~db_path ~document_count_limit:cache_limit with\n   | None -> ()\n   | Some msg -> exit_with_error_msg msg\n  );\n  let interactive =\n    Option.is_none filter_exp\n    &&\n    Option.is_none sample_search_exp\n    &&\n    Option.is_none search_exp\n    &&\n    not print_files_with_match\n    &&\n    not print_files_without_match\n    &&\n    Option.is_none script\n  in\n  if interactive then (\n    Printf.printf \"Initializing in-memory index\\n\";\n    flush stdout;\n  );\n  Word_db.read_from_db ();\n  Index.State.read_from_db ();\n  (match Sys.getenv_opt \"VISUAL\", Sys.getenv_opt \"EDITOR\" with\n   | None, None -> (\n       exit_with_error_msg\n         (Fmt.str \"environment variable VISUAL or EDITOR needs to be set\")\n     )\n   | Some editor, _\n   | None, Some editor -> (\n       Params.text_editor := editor;\n     )\n  );\n  Lwd.unsafe_mutation_logger := (fun () -> ());\n  let recognized_exts =\n    compute_total_recognized_exts ~exts ~additional_exts\n  in\n  let recognized_single_line_exts =\n    compute_total_recognized_exts ~exts:single_line_exts ~additional_exts:single_line_additional_exts\n  in\n  (match recognized_exts, recognized_single_line_exts, globs, single_line_globs with\n   | [], [], [], [] -> (\n       exit_with_error_msg\n         (Fmt.str \"no usable file extensions or glob patterns\")\n     )\n   | _, _, _, _ -> ()\n  );\n  let paths_from_file_or_stdin =\n    match paths_from with\n    | [] -> None\n    | l -> (\n        l\n        |> List.to_seq\n        |> Seq.map (String.split_on_char ',')\n        |> Seq.flat_map List.to_seq\n        |> Seq.flat_map (fun paths_from ->\n            (match paths_from with\n             | \"-\" -> (\n                 CCIO.read_lines_l stdin\n               )\n             | _ -> (\n                 try\n                   CCIO.with_in paths_from CCIO.read_lines_l\n                 with\n                 | Sys_error _ -> (\n                     exit_with_error_msg\n                       (Fmt.str \"failed to read list of paths from %s\" (Filename.quote paths_from))\n                   )\n               )\n            )\n            |> List.to_seq\n          )\n        |> List.of_seq\n        |> Option.some\n      )\n  in\n  let file_constraints =\n    make_file_constraints\n      ~no_pdftotext\n      ~no_pandoc\n      ~exts:recognized_exts\n      ~single_line_exts:recognized_single_line_exts\n      ~paths\n      ~paths_from_file_or_stdin\n      ~globs\n      ~single_line_globs\n  in\n  let pool = Task_pool.make ~sw (Eio.Stdenv.domain_mgr env) in\n  Atomic.set UI_base.Vars.pool (Some pool);\n  String_set.iter (fun path ->\n      if not (Sys.file_exists path) then (\n        exit_with_error_msg\n          (Fmt.str \"path %s does not exist\" (Filename.quote path))\n      )\n    )\n    file_constraints.directly_specified_paths;\n  let compute_if_hide_document_list_initially_and_document_src : unit -> bool * Document_src.t =\n    let stdin_tmp_file = ref None in\n    (fun () ->\n       let file_collection =\n         files_satisfying_constraints ~interactive file_constraints\n       in\n       if file_constraints.paths_were_specified_by_user\n       || stdin_is_atty ()\n       then (\n         let hide_document_list =\n           match Document_src.file_collection_size file_collection with\n           | 0 -> false\n           | 1 -> true\n           | _ -> false\n         in\n         (hide_document_list, Files file_collection)\n       ) else (\n         match !stdin_tmp_file with\n         | None -> (\n             match read_in_channel_to_tmp_file stdin with\n             | Ok tmp_file -> (\n                 stdin_tmp_file := Some tmp_file;\n                 (true, Stdin tmp_file)\n               )\n             | Error msg -> (\n                 exit_with_error_msg msg\n               )\n           )\n         | Some tmp_file -> (\n             (true, Stdin tmp_file)\n           )\n       )\n    )\n  in\n  let compute_document_src () =\n    snd (compute_if_hide_document_list_initially_and_document_src ())\n  in\n  let hide_document_list_initially, init_document_src =\n    compute_if_hide_document_list_initially_and_document_src ()\n  in\n  let clean_up () =\n    match init_document_src with\n    | Stdin tmp_file -> (\n        try\n          Sys.remove tmp_file\n        with\n        | Sys_error _ -> ()\n      )\n    | Files _ -> ()\n  in\n  do_if_debug (fun oc ->\n      Printf.fprintf oc \"Scanning completed\\n\"\n    );\n  do_if_debug (fun oc ->\n      match init_document_src with\n      | Stdin _ -> Printf.fprintf oc \"Document source: stdin\\n\"\n      | Files file_collection -> (\n          Printf.fprintf oc \"Document source: files\\n\";\n          Document_src.seq_of_file_collection file_collection\n          |> Seq.iter (fun file ->\n              Printf.fprintf oc \"File: %s\\n\" (Filename.quote file);\n            )\n        )\n    );\n  (match init_document_src with\n   | Stdin _ -> ()\n   | Files file_collection -> (\n       let pdftotext_exists = Proc_utils.command_exists \"pdftotext\" in\n       let pandoc_exists = Proc_utils.command_exists \"pandoc\" in\n       let formats = Document_src.seq_of_file_collection file_collection\n         |> Seq.map format_of_file\n         |> Seq.fold_left (fun acc x -> File_format_set.add x acc) File_format_set.empty\n       in\n       if not pdftotext_exists && File_format_set.mem `PDF formats then (\n         exit_with_error_msg\n           (Fmt.str \"command pdftotext not found, use --%s to disable use of pdftotext\" Args.no_pdftotext_arg_name)\n       );\n       if File_format_set.mem `Pandoc_supported_format formats then (\n         if not pandoc_exists then (\n           exit_with_error_msg\n             (Fmt.str \"command pandoc not found, use --%s to disable use of pandoc\" Args.no_pandoc_arg_name)\n         );\n       );\n     )\n  );\n  Lwd.set UI_base.Vars.hide_document_list hide_document_list_initially;\n  init_session_state_of_document_src ~env pool ~interactive init_document_src\n  |> (fun state ->\n      Session.run_command pool (`Sort (sort_by, sort_by_no_score)) state\n      |> Option.get\n      |> snd\n    )\n  |> Session_manager.update_starting_state;\n  if index_only then (\n    clean_up ();\n    exit 0\n  );\n  let print_oc = stdout in\n  let print_with_color =\n    match print_color_mode with\n    | `Never -> false\n    | `Always -> true\n    | `Auto -> Out_channel.isatty print_oc\n  in\n  let print_with_underline =\n    match print_underline_mode with\n    | `Never -> false\n    | `Always -> true\n    | `Auto -> not (Out_channel.isatty print_oc)\n  in\n  let snapshots_from_script =\n    match script, start_with_script with\n    | None, None -> None\n    | Some _, Some _ -> failwith \"unexpected case\"\n    | Some script, None\n    | None, Some script -> (\n        let init_state =\n          Session_manager.lock_with_view (fun view ->\n              view.init_state\n            )\n        in\n        match\n          Script.run\n            pool\n            ~init_state\n            ~path:script\n        with\n        | Error msg -> exit_with_error_msg msg\n        | Ok snapshots -> Some snapshots\n      )\n  in\n  (match snapshots_from_script with\n   | None -> ()\n   | Some snapshots -> (\n       Session_manager.load_snapshots snapshots;\n     ));\n  if not interactive then (\n    let filter_exp_and_original_string =\n      match filter_exp with\n      | None -> None\n      | Some s ->\n        Some (Option.get (Filter_exp.parse s), s)\n    in\n    let print_limit_per_doc, search_exp_and_original_string =\n      match sample_search_exp, search_exp with\n      | None, None -> (None, None)\n      | Some _, Some _ -> failwith \"unexpected case\"\n      | Some search_exp_string, None\n      | None, Some search_exp_string -> (\n          let print_limit_per_doc =\n            match sample_search_exp with\n            | Some _ -> Some samples_per_doc\n            | None -> None\n          in\n          let search_exp =\n            Option.get (Search_exp.parse search_exp_string)\n          in\n          do_if_debug (fun oc ->\n              Fmt.pf\n                (Format.formatter_of_out_channel oc)\n                \"Search expression: @[<v>%a@]@.\" Search_exp.pp search_exp\n            );\n          (print_limit_per_doc, Some (search_exp, search_exp_string))\n        )\n    in\n    let session_state =\n      match snapshots_from_script with\n      | None -> (\n          Session_manager.lock_with_view (fun view ->\n              view.init_state\n            )\n          |> (fun state ->\n              match filter_exp_and_original_string with\n              | None -> state\n              | Some (filter_exp, filter_exp_string) -> (\n                  Option.get\n                    (Session.State.update_filter_exp\n                       pool\n                       (Stop_signal.make ())\n                       filter_exp_string\n                       filter_exp\n                       state\n                    )\n                )\n            )\n          |> (fun state ->\n              match search_exp_and_original_string with\n              | None -> state\n              | Some (search_exp, search_exp_string) -> (\n                  Option.get\n                    (Session.State.update_search_exp\n                       pool\n                       (Stop_signal.make ())\n                       search_exp_string\n                       search_exp\n                       state\n                    )\n                )\n            )\n        )\n      | Some _ -> (\n          Session_manager.lock_with_view (fun view ->\n              Dynarray.get_last view.snapshots\n              |> Session.Snapshot.state\n            )\n        )\n    in\n    let oc = stdout in\n    let no_results =\n      if print_files_with_match then (\n        let arr =\n          Session.State.search_result_groups session_state\n        in\n        Array.iter (fun (doc, _search_result) ->\n            Printers.path_image ~color:print_with_color oc (Document.path doc)\n          ) arr;\n        Array.length arr = 0\n      ) else if print_files_without_match then (\n        let arr =\n          Session.State.unusable_documents session_state\n          |> Array.of_seq\n        in\n        let (sort_by_typ, sort_by_order) = sort_by_no_score in\n        let f =\n          match sort_by_typ with\n          | `Path_date -> Document.Compare.path_date sort_by_order\n          | `Mod_time -> Document.Compare.mod_time sort_by_order\n          | `Path -> Document.Compare.path sort_by_order\n          | `Score -> failwith \"unexpected case\"\n        in\n        Array.sort f arr;\n        Array.iter (fun doc ->\n            Printers.path_image ~color:print_with_color oc (Document.path doc)\n          ) arr;\n        Array.length arr = 0\n      ) else (\n        let s =\n          Session.State.search_result_groups session_state\n          |> Array.to_seq\n          |> Seq.map (fun (doc, arr) ->\n              let arr =\n                match print_limit_per_doc with\n                | None -> arr\n                | Some n -> (\n                    Array.sub\n                      arr\n                      0\n                      (min (Array.length arr) n)\n                  )\n              in\n              (doc, arr)\n            )\n        in\n        Printers.search_result_groups\n          ~color:print_with_color\n          ~underline:print_with_underline\n          oc\n          s;\n        Seq.is_empty s\n      )\n    in\n    clean_up ();\n    if no_results then (\n      exit 1\n    ) else (\n      exit 0\n    )\n  );\n  UI_base.Vars.eio_env := Some env;\n  let root : Nottui.ui Lwd.t =\n    let$* (term_width, term_height) = Lwd.get UI_base.Vars.term_width_height in\n    if term_width <= 40 || term_height <= 20 then (\n      let msg = Nottui.Ui.atom (Notty.I.strf \"Terminal size too small\") in\n      let keyboard_handler (key : Nottui.Ui.key) =\n        match key with\n        | (`Escape, [])\n        | (`ASCII 'Q', [`Ctrl])\n        | (`ASCII 'C', [`Ctrl]) -> (\n            Lwd.set UI_base.Vars.quit true;\n            UI_base.Vars.action := None;\n            `Handled\n          )\n        | _ -> `Unhandled\n      in\n      Lwd.return (Nottui.Ui.keyboard_area keyboard_handler msg)\n    ) else (\n      UI.main\n    )\n  in\n  let get_term, close_term =\n    let term_and_tty_fd = ref None in\n    ((fun () ->\n        match !term_and_tty_fd with\n        | None -> (\n            if stdin_is_atty () then (\n              let term = Notty_unix.Term.create () in\n              term_and_tty_fd := Some (term, None);\n              term\n            ) else (\n              let input =\n                Unix.(openfile \"/dev/tty\" [ O_RDONLY ] 0o444)\n              in\n              let term = Notty_unix.Term.create ~input () in\n              term_and_tty_fd := Some (term, Some input);\n              term\n            )\n          )\n        | Some (term, _tty_fd) -> term\n      ),\n     (fun () ->\n        match !term_and_tty_fd with\n        | None -> ()\n        | Some (term, tty_fd) -> (\n            Notty_unix.Term.release term;\n            (match tty_fd with\n             | None -> ()\n             | Some fd -> Unix.close fd);\n            term_and_tty_fd := None\n          )\n     )\n    )\n  in\n  let rec loop () =\n    Sys.command \"clear -x\" |> ignore;\n    let term = get_term () in\n    UI_base.Vars.term := Some term;\n    UI_base.Vars.action := None;\n    Lwd.set UI_base.Vars.quit false;\n    UI_base.ui_loop\n      ~quit:UI_base.Vars.quit\n      ~term\n      root;\n    match !UI_base.Vars.action with\n    | None -> ()\n    | Some action -> (\n        Session_manager.stop_filter_and_search_and_restore_input_fields ();\n        match action with\n        | UI_base.Recompute_document_src -> (\n            close_term ();\n            let new_starting_state =\n              compute_document_src ()\n              |> init_session_state_of_document_src ~env ~interactive pool\n            in\n            Session_manager.update_starting_state new_starting_state;\n            loop ()\n          )\n        | Open_file_and_search_result (doc, search_result) -> (\n            let doc_id = Document.doc_id doc in\n            let path = Document.path doc in\n            let old_stats = Unix.stat path in\n            Path_opening.main\n              ~close_term\n              ~path\n              ~doc_id_and_search_result:(Option.map (fun x -> (doc_id, x)) search_result);\n            let new_stats = Unix.stat path in\n            if\n              Float.abs\n                (new_stats.st_mtime -. old_stats.st_mtime) >= Params.float_compare_margin\n            then (\n              UI.reload_document doc\n            );\n            loop ()\n          )\n        | Open_link (doc, link) -> (\n            Path_opening.open_link\n              ~close_term\n              ~doc\n              link;\n            loop ()\n          )\n        | Edit_command_history -> (\n            let file = Filename.temp_file \"\" Params.docfd_script_ext in\n            let init_snapshots =\n              Session_manager.lock_with_view (fun view ->\n                  view.snapshots\n                )\n            in\n            let init_lines =\n              Seq.append\n                (\n                  init_snapshots\n                  |> Dynarray.to_seq\n                  |> Seq.filter_map  (fun (snapshot : Session.Snapshot.t) ->\n                      Option.map\n                        Command.to_string\n                        (Session.Snapshot.last_command snapshot)\n                    )\n                )\n                (\n                  List.to_seq\n                    [\n                      \"\";\n                      \"; You are viewing/editing Docfd command history.\";\n                      \"; If any change is made to this file, Docfd will replay the commands from the start.\";\n                      \";\";\n                      \"; There are two types of comments:\";\n                      \"; - System comments begin with `;`, and are not preserved after editing of command history.\";\n                      \";   These are for communication of error message to user during command history editing.\";\n                      \"; - User comments begin with `#`, and are preserved after editing of command history.\";\n                      \";\";\n                      \"; If a line is not blank and is not a comment,\";\n                      \"; then the line should contain exactly one command.\";\n                      \"; A command cannot be written across multiple lines.\";\n                      \";\";\n                      \"; Starting point is v0, the state with the full set of documents.\";\n                      \"; Each command adds one to the version number.\";\n                      \"; Command at the top is oldest, command at bottom is the newest.\";\n                      \";\";\n                      \"; Note that for commands that accept text, all text after `:` is trimmed and then used in full.\";\n                      \"; This means \\\" and ' are treated literally and are not used to delimit strings.\";\n                      \";\";\n                      \"; Possible commands:\";\n                      Fmt.str \"; - %a\" Command.pp (`Search \"search phrase\");\n                      Fmt.str \"; - %a\" Command.pp (`Search \"\");\n                      Fmt.str \"; - %a\" Command.pp (`Filter \"path-fuzzy:\\\"file txt\\\"\");\n                      Fmt.str \"; - %a\" Command.pp (`Filter \"\");\n                      Fmt.str \"; - %a\" Command.pp (`Sort (Command.Sort_by.default, Command.Sort_by.default_no_score));\n                      Fmt.str \"; - %a\" Command.pp (`Path_fuzzy_rank (\"readme\", None));\n                      Fmt.str \"; - %a\" Command.pp (`Narrow_level 1);\n                      Fmt.str \"; - %a\" Command.pp (`Mark \"/path/to/document\");\n                      Fmt.str \"; - %a\" Command.pp `Mark_listed;\n                      Fmt.str \"; - %a\" Command.pp (`Unmark \"/path/to/document\");\n                      Fmt.str \"; - %a\" Command.pp `Unmark_listed;\n                      Fmt.str \"; - %a\" Command.pp `Unmark_all;\n                      Fmt.str \"; - %a\" Command.pp (`Drop \"/path/to/document\");\n                      Fmt.str \"; - %a\" Command.pp (`Drop_all_except \"/path/to/document\");\n                      Fmt.str \"; - %a\" Command.pp `Drop_marked;\n                      Fmt.str \"; - %a\" Command.pp `Drop_unmarked;\n                      Fmt.str \"; - %a\" Command.pp `Drop_listed;\n                      Fmt.str \"; - %a\" Command.pp `Drop_unlisted;\n                    ]\n                )\n              |> List.of_seq\n            in\n            let init_state =\n              Session_manager.lock_with_view (fun view ->\n                  view.init_state\n                )\n            in\n            let rec aux rerun snapshots lines : [ `No_changes | `Changes_made of Session.Snapshot.t Dynarray.t ] =\n              CCIO.with_out file (fun oc ->\n                  CCIO.write_lines_l oc lines;\n                );\n              let old_stats = Unix.stat file in\n              close_term ();\n              Path_opening.config_and_cmd_to_open_text_file\n                ~path:file\n                ~line_num:(max 1 (Dynarray.length snapshots - 1))\n                ()\n              |> (fun (config, cmd) ->\n                  Result.get_ok (Path_opening.resolve_cmd config cmd)\n                )\n              |> Sys.command\n              |> ignore;\n              let new_stats = Unix.stat file in\n              if\n                rerun\n                ||\n                Float.abs\n                  (new_stats.st_mtime -. old_stats.st_mtime) >= Params.float_compare_margin\n              then (\n                let state = ref init_state in\n                let snapshots = Dynarray.create () in\n                Dynarray.add_last\n                  snapshots\n                  (Session.Snapshot.make\n                     ~last_command:None\n                     init_state);\n                let rerun = ref false in\n                let lines =\n                  CCIO.with_in file (fun ic ->\n                      CCIO.read_lines_l ic\n                      |> CCList.flat_map (fun line ->\n                          if\n                            String_utils.line_is_blank_or_system_comment line\n                          then (\n                            [ line ]\n                          ) else (\n                            match Command.of_string line with\n                            | None -> (\n                                rerun := true;\n                                [\n                                  line;\n                                  \"# Failed to parse the above command\"\n                                ]\n                              )\n                            | Some command -> (\n                                match Session.run_command pool command !state with\n                                | None -> (\n                                    rerun := true;\n                                    [\n                                      line;\n                                      \"# Failed to run the above command, check if the arguments are correct\"\n                                    ]\n                                  )\n                                | Some (command, x) -> (\n                                    state := x;\n                                    let snapshot =\n                                      Session.Snapshot.make\n                                        ~last_command:(Some command)\n                                        !state\n                                    in\n                                    Dynarray.add_last\n                                      snapshots\n                                      snapshot;\n                                    [ line ]\n                                  )\n                              )\n                          )\n                        )\n                    )\n                in\n                if !rerun then (\n                  aux true snapshots lines\n                ) else (\n                  `Changes_made snapshots\n                )\n              ) else (\n                `No_changes\n              )\n            in\n            (try\n               let res =\n                 aux false init_snapshots init_lines\n               in\n               (try\n                  Sys.remove file;\n                with\n                | Sys_error _ -> ()\n               );\n               (match res with\n                | `No_changes -> ()\n                | `Changes_made snapshots -> (\n                    Session_manager.load_snapshots snapshots\n                  )\n               );\n             with\n             | Sys_error _ -> (\n                 exit_with_error_msg\n                   (Fmt.str \"failed to read or write temporary command history file %s\" (Filename.quote file))\n               ));\n            loop ()\n          )\n        | Clear_command_history -> (\n            let init_state =\n              Session_manager.lock_with_view (fun view ->\n                  view.init_state\n                )\n            in\n            let snapshots = Dynarray.create () in\n            Dynarray.add_last\n              snapshots\n              (Session.Snapshot.make\n                 ~last_command:None\n                 init_state);\n            Session_manager.load_snapshots snapshots;\n            loop ()\n          )\n        | Open_script path -> (\n            let init_state =\n              Session_manager.lock_with_view (fun view ->\n                  view.init_state\n                )\n            in\n            match\n              Script.run\n                pool\n                ~init_state\n                ~path\n            with\n            | Error msg -> exit_with_error_msg msg\n            | Ok snapshots -> (\n                Session_manager.load_snapshots snapshots;\n                loop ()\n              )\n          )\n        | Edit_script path -> (\n            close_term ();\n            Path_opening.config_and_cmd_to_open_text_file\n              ~path\n              ()\n            |> (fun (config, cmd) ->\n                Result.get_ok (Path_opening.resolve_cmd config cmd)\n              )\n            |> Sys.command\n            |> ignore;\n            loop ()\n          )\n      )\n  in\n  Eio.Fiber.any [\n    (fun () ->\n       Eio.Domain_manager.run (Eio.Stdenv.domain_mgr env)\n         (fun () -> Session_manager.worker_fiber pool));\n    Session_manager.manager_fiber;\n    UI_base.Key_binding_info.grid_light_fiber;\n    (fun () ->\n       (match start_with_filter with\n        | None -> ()\n        | Some start_with_filter -> (\n            let start_with_filter_len = String.length start_with_filter in\n            Lwd.set UI_base.Vars.filter_field (start_with_filter, start_with_filter_len);\n            UI.update_filter ~commit:true ();\n          ));\n       (match start_with_search with\n        | None -> ()\n        | Some start_with_search -> (\n            let start_with_search_len = String.length start_with_search in\n            Lwd.set UI_base.Vars.search_field (start_with_search, start_with_search_len);\n            UI.update_search ~commit:true ();\n          ));\n       loop ();\n    );\n  ];\n  close_term ();\n  clean_up ();\n  (match debug_log with\n   | Some \"-\" -> ()\n   | _ -> (\n       match !Params.debug_output with\n       | None -> ()\n       | Some oc -> (\n           close_out oc\n         )\n     )\n  )\n\nlet cmd ~eio_env ~sw =\n  let open Term in\n  let open Args in\n  let doc = \"TUI multiline fuzzy document finder\" in\n  let version = Version_string.s in\n  Cmd.v (Cmd.info \"docfd\" ~version ~doc)\n    (const (run ~eio_env ~sw)\n     $ debug_log_arg\n     $ no_pdftotext_arg\n     $ no_pandoc_arg\n     $ hidden_arg\n     $ max_depth_arg\n     $ max_fuzzy_edit_dist_arg\n     $ max_token_search_dist_arg\n     $ max_linked_token_search_dist_arg\n     $ tokens_per_search_scope_level_arg\n     $ index_chunk_size_arg\n     $ exts_arg\n     $ single_line_exts_arg\n     $ add_exts_arg\n     $ single_line_add_exts_arg\n     $ cache_dir_arg\n     $ cache_limit_arg\n     $ data_dir_arg\n     $ index_only_arg\n     $ start_with_filter_arg\n     $ start_with_search_arg\n     $ filter_arg\n     $ sample_arg\n     $ samples_per_doc_arg\n     $ search_arg\n     $ sort_arg\n     $ sort_no_score_arg\n     $ color_arg\n     $ underline_arg\n     $ search_result_print_text_width_arg\n     $ search_result_print_snippet_min_size_arg\n     $ search_result_print_snippet_max_add_lines_arg\n     $ start_with_script_arg\n     $ script_arg\n     $ paths_from_arg\n     $ glob_arg\n     $ single_line_glob_arg\n     $ single_line_arg\n     $ open_with_arg\n     $ files_with_match_arg\n     $ files_without_match_arg\n     $ paths_arg)\n\nlet () =\n  if Sys.win32 then (\n    exit_with_error_msg \"Windows is not supported\"\n  );\n  Random.self_init ();\n  Eio_posix.run (fun eio_env ->\n      Eio.Switch.run (fun sw ->\n          exit (Cmd.eval (cmd ~eio_env ~sw))\n        ))\n"
  },
  {
    "path": "bin/document.ml",
    "content": "open Result_syntax\nopen Docfd_lib\n\ntype t = {\n  search_mode : Search_mode.t;\n  path : string;\n  path_parts : string list;\n  path_date : Timedesc.Date.t option;\n  mod_time : Timedesc.t;\n  title : string option;\n  doc_id : int64;\n  doc_hash : string;\n  word_ids : Int_set.t;\n  search_scope : Diet.Int.t option;\n  links : Link.t array;\n  link_index_of_start_pos : int Int_map.t;\n  last_scan : Timedesc.t;\n}\n\nlet equal (x : t) (y : t) =\n  x.search_mode = y.search_mode\n  &&\n  String.equal x.path y.path\n  &&\n  String.equal x.doc_hash y.doc_hash\n  &&\n  Option.equal Diet.Int.equal x.search_scope y.search_scope\n\nlet compute_path_parts (path : string) =\n  let path_parts = Tokenization.tokenize ~drop_spaces:false path\n    |> List.of_seq\n  in\n  (path_parts)\n\nlet compute_link_index_of_start_pos links =\n  CCArray.foldi (fun acc i link ->\n      Int_map.add link.Link.start_pos i acc\n    )\n    Int_map.empty\n    links\n\nlet search_mode (t : t) = t.search_mode\n\nlet path (t : t) = t.path\n\nlet path_parts (t : t) = t.path_parts\n\nlet path_date (t : t) = t.path_date\n\nlet mod_time (t : t) = t.mod_time\n\nlet title (t : t) = t.title\n\nlet word_ids (t : t) = t.word_ids\n\nlet doc_hash (t : t) = t.doc_hash\n\nlet doc_id (t : t) = t.doc_id\n\nlet search_scope (t : t) = t.search_scope\n\nlet last_scan (t : t) = t.last_scan\n\nlet links (t : t) = t.links\n\nlet link_index_of_start_pos (t : t) = t.link_index_of_start_pos\n\nlet refresh_modification_time ~path =\n  let time = Unix.time () in\n  Unix.utimes path time time\n\nlet reset_search_scope_to_full (t : t) : t =\n  { t with search_scope = None }\n\nlet inter_search_scope (x : Diet.Int.t) (t : t) : t =\n  let search_scope =\n    match t.search_scope with\n    | None -> x\n    | Some y -> Diet.Int.inter x y\n  in\n  { t with search_scope = Some search_scope }\n\nmodule Compare = struct\n  type order = [\n    | `Asc\n    | `Desc\n  ]\n\n  let mod_time order d0 d1 =\n    match order with\n    | `Asc ->\n      Timedesc.compare_chrono_min (mod_time d0) (mod_time d1)\n    | `Desc ->\n      Timedesc.compare_chrono_min (mod_time d1) (mod_time d0)\n\n  let path order d0 d1 =\n    match order with\n    | `Asc ->\n      String.compare (path d0) (path d1)\n    | `Desc ->\n      String.compare (path d1) (path d0)\n\n  let path_date order d0 d1 =\n    let fallback () = path order d0 d1 in\n    match path_date d0, path_date d1 with\n    | None, None -> fallback ()\n    | None, Some _ -> (\n        (* Always shuffle document with no path date to the back. *)\n        1\n      )\n    | Some _, None -> (\n        (* Always shuffle document with no path date to the back. *)\n        -1\n      )\n    | Some x0, Some x1 -> (\n        match order with\n        | `Asc -> (\n            match Timedesc.Date.compare x0 x1 with\n            | 0 -> fallback ()\n            | n -> n\n          )\n        | `Desc -> (\n            match Timedesc.Date.compare x1 x0 with\n            | 0 -> fallback ()\n            | n -> n\n          )\n      )\nend\n\nmodule Ir0 = struct\n  type t = {\n    search_mode : Search_mode.t;\n    doc_id : int64;\n    doc_hash : string;\n    path : string;\n    last_scan : Timedesc.t;\n  }\n\n  let of_path ~(env : Eio_unix.Stdenv.base) search_mode ?doc_hash path : (t, string) result =\n    let* doc_hash =\n      match doc_hash with\n      | Some x -> Ok x\n      | None -> BLAKE2B.hash_of_file ~env ~path\n    in\n    let doc_id = Doc_id_db.doc_id_of_doc_hash doc_hash in\n    Ok {\n      search_mode;\n      doc_id;\n      doc_hash;\n      path;\n      last_scan = Timedesc.now ~tz_of_date_time:Params.tz ();\n    }\nend\n\nmodule Ir1 = struct\n  type t = {\n    search_mode : Search_mode.t;\n    doc_id : int64;\n    doc_hash : string;\n    path : string;\n    data : [ `Lines of string Dynarray.t | `Pages of string list Dynarray.t ];\n    last_scan : Timedesc.t;\n  }\n\n  let of_path_to_text ~env ~doc_id ~doc_hash search_mode last_scan path : (t, string) result =\n    let fs = Eio.Stdenv.fs env in\n    try\n      let data =\n        Eio.Path.(with_lines (fs / path))\n          (fun lines ->\n             `Lines (Dynarray.of_seq lines)\n          )\n      in\n      Ok {\n        search_mode;\n        doc_id;\n        doc_hash;\n        path;\n        data;\n        last_scan;\n      }\n    with\n    | Failure _\n    | End_of_file\n    | Eio.Buf_read.Buffer_limit_exceeded -> (\n        Error (Printf.sprintf \"failed to read file: %s\" (Filename.quote path))\n      )\n\n  let of_path_to_pdf ~env ~doc_id ~doc_hash search_mode last_scan path : (t, string) result =\n    let proc_mgr = Eio.Stdenv.process_mgr env in\n    let fs = Eio.Stdenv.fs env in\n    try\n      let cmd = [ \"pdftotext\"; path; \"-\" ] in\n      let pages =\n        match Proc_utils.run_return_stdout ~proc_mgr ~fs ~split_mode:`On_form_feed cmd with\n        | None -> Seq.empty\n        | Some pages -> (\n            List.to_seq pages\n            |> Seq.map (fun page -> String.split_on_char '\\n' page)\n          )\n      in\n      let data = `Pages (Dynarray.of_seq pages) in\n      Ok {\n        search_mode;\n        doc_id;\n        doc_hash;\n        path;\n        data;\n        last_scan;\n      }\n    with\n    | Failure _\n    | End_of_file\n    | Eio.Buf_read.Buffer_limit_exceeded -> (\n        Error (Printf.sprintf \"failed to read file: %s\" (Filename.quote path))\n      )\n\n  let of_path_to_pandoc_supported_format ~env ~doc_id ~doc_hash search_mode last_scan path : (t, string) result =\n    let proc_mgr = Eio.Stdenv.process_mgr env in\n    let fs = Eio.Stdenv.fs env in\n    let ext = File_utils.extension_of_file path in\n    let from_format = ext\n      |> String_utils.remove_leading_dots\n      |> (fun s ->\n          match s with\n          | \"htm\" -> \"html\"\n          | _ -> s\n        )\n    in\n    let cmd = [ \"pandoc\"\n              ; \"--from\"\n              ; from_format\n              ; \"--to\"\n              ; \"plain\"\n              ; \"--wrap\"\n              ; \"none\"\n              ; path\n              ]\n    in\n    let error_msg = Fmt.str \"failed to extract text from %s\" (Filename.quote path) in\n    match\n      Proc_utils.run_return_stdout\n        ~proc_mgr\n        ~fs\n        ~split_mode:`On_line_split\n        cmd\n    with\n    | None -> (\n        Error error_msg\n      )\n    | Some lines -> (\n        let data = `Lines (Dynarray.of_list lines) in\n        Ok {\n          search_mode;\n          doc_id;\n          doc_hash;\n          path;\n          data;\n          last_scan;\n        }\n      )\n\n  let of_ir0 ~(env : Eio_unix.Stdenv.base) (ir0 : Ir0.t) : (t, string) result =\n    let { Ir0.search_mode; doc_id; doc_hash; path; last_scan } = ir0 in\n    match File_utils.format_of_file path with\n    | `PDF -> (\n        of_path_to_pdf ~env ~doc_id ~doc_hash search_mode last_scan path\n      )\n    | `Pandoc_supported_format -> (\n        of_path_to_pandoc_supported_format ~env ~doc_id ~doc_hash search_mode last_scan path\n      )\n    | `Text | `Other -> (\n        of_path_to_text ~env ~doc_id ~doc_hash search_mode last_scan path\n      )\nend\n\nmodule Date_extraction = struct\n  let yyyy = \"(\\\\d{4})\"\n\n  let mm = \"([01]\\\\d)\"\n\n  let mmm = \"(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\"\n\n  let mmmm = \"(january|february|march|april|may|june|july|august|september|october|november|december)\"\n\n  let dd = \"([0-3]\\\\d)\"\n\n  let int_of_month_string s =\n    match String.lowercase_ascii (String.sub s 0 3) with\n    | \"jan\" -> 1\n    | \"feb\" -> 2\n    | \"mar\" -> 3\n    | \"apr\" -> 4\n    | \"may\" -> 5\n    | \"jun\" -> 6\n    | \"jul\" -> 7\n    | \"aug\" -> 8\n    | \"sep\" -> 9\n    | \"oct\" -> 10\n    | \"nov\" -> 11\n    | \"dec\" -> 12\n    | _ -> failwith \"unexpected case\"\n\n  let yyyy_mm_dd =\n    let re =\n      Fmt.str\n        \"(?:^|.*[^\\\\d])%s[^\\\\d]%s[^\\\\d]%s(?:$|[^\\\\d])\"\n        yyyy\n        mm\n        dd\n      |> Re.Pcre.re\n      |> Re.compile\n    in\n    fun s ->\n      try\n        let g = Re.exec re s in\n        let start = Re.Group.start g 1 in\n        let y = Re.Group.get g 1 |> int_of_string in\n        let m = Re.Group.get g 2 |> int_of_string in\n        let d = Re.Group.get g 3 |> int_of_string in\n        Some (start, (y, m, d))\n      with\n      | _ -> None\n\n  let yyyy_month_dd ~month ~reverse =\n    let re =\n      let g1, g3 =\n        if not reverse then (\n          (yyyy, dd)\n        ) else (\n          (dd, yyyy)\n        )\n      in\n      Fmt.str\n        \"(?:^|.*[^\\\\d])%s[^A-Za-z\\\\d]?%s[^A-Za-z\\\\d]?%s(?:$|[^\\\\d])\"\n        g1\n        month\n        g3\n      |> Re.Pcre.re\n      |> Re.no_case\n      |> Re.compile\n    in\n    fun s ->\n      try\n        let g = Re.exec re s in\n        let start = Re.Group.start g 1 in\n        let y_group_index, d_group_index =\n          if not reverse then (\n            (1, 3)\n          ) else (\n            (3, 1)\n          )\n        in\n        let y = Re.Group.get g y_group_index |> int_of_string in\n        let m = int_of_month_string (Re.Group.get g 2) in\n        let d = Re.Group.get g d_group_index |> int_of_string in\n        Some (start, (y, m, d))\n      with\n      | _ -> None\n\n  let yyyymmdd =\n    let re =\n      Fmt.str\n        \"(?:^|.*[^\\\\d])%s%s%s\"\n        yyyy\n        mm\n        dd\n      |> Re.Pcre.re\n      |> Re.compile\n    in\n    fun s ->\n      try\n        let g = Re.exec re s in\n        let start = Re.Group.start g 1 in\n        let y = Re.Group.get g 1 |> int_of_string in\n        let m = Re.Group.get g 2 |> int_of_string in\n        let d = Re.Group.get g 3 |> int_of_string in\n        Some (start, (y, m, d))\n      with\n      | _ -> None\n\n  let extract s =\n    let rec aux acc l =\n      match l with\n      | [] -> (\n          match acc with\n          | None -> None\n          | Some (_start_match_pos, (year, month, day)) -> (\n              match Timedesc.Date.Ymd.make ~year ~month ~day with\n              | Ok date -> Some date\n              | Error _ -> None\n            )\n        )\n      | f :: fs -> (\n          let acc =\n            match acc, f s with\n            | None, x -> x\n            | Some x, None -> Some x\n            | Some (start_match_pos, ymd),\n              Some (start_match_pos', ymd') -> (\n                if start_match_pos' > start_match_pos then (\n                  Some (start_match_pos', ymd')\n                ) else (\n                  Some (start_match_pos, ymd)\n                )\n              )\n          in\n          aux acc fs\n        )\n    in\n    aux\n      None\n      [\n        yyyy_mm_dd;\n        yyyy_month_dd ~month:mmm ~reverse:true;\n        yyyy_month_dd ~month:mmm ~reverse:false;\n        yyyy_month_dd ~month:mmmm ~reverse:true;\n        yyyy_month_dd ~month:mmmm ~reverse:false;\n        yyyymmdd;\n      ]\nend\n\nmodule Ir2 = struct\n  type t = {\n    search_mode : Search_mode.t;\n    doc_id : int64;\n    doc_hash : string;\n    path : string;\n    path_parts : string list;\n    path_date : Timedesc.Date.t option;\n    mod_time : Timedesc.t;\n    title : string option;\n    raw : Index.Raw.t;\n    links : Link.t array;\n    last_scan : Timedesc.t;\n  }\n\n  type work_stage =\n    | Title\n    | Content\n\n  let parse_lines pool (s : string Seq.t) : string option * Index.Raw.t =\n    let rec aux (stage : work_stage) title s =\n      match stage with\n      | Content -> (\n          let raw = Index.Raw.of_lines pool s in\n          (title, raw)\n        )\n      | Title -> (\n          match s () with\n          | Seq.Nil -> aux Content title Seq.empty\n          | Seq.Cons (x, xs) -> (\n              aux Content (Some (Misc_utils.sanitize_string x)) (Seq.cons x xs)\n            )\n        )\n    in\n    aux Title None s\n\n  let parse_pages pool (s : string list Seq.t) : string option * Index.Raw.t =\n    let rec aux (stage : work_stage) title s =\n      match stage with\n      | Content -> (\n          let raw = Index.Raw.of_pages pool s in\n          (title, raw)\n        )\n      | Title -> (\n          match s () with\n          | Seq.Nil -> aux Content title Seq.empty\n          | Seq.Cons (x, xs) -> (\n              let title =\n                match x with\n                | [] -> None\n                | x :: _ ->\n                  Some (Misc_utils.sanitize_string x)\n              in\n              aux Content title (Seq.cons x xs)\n            )\n        )\n    in\n    aux Title None s\n\n  let of_ir1 pool (ir : Ir1.t) : t =\n    let { Ir1.search_mode; doc_id; doc_hash; path; data; last_scan } = ir in\n    let path_parts = compute_path_parts path in\n    let path_date = Date_extraction.extract path in\n    let stats = Unix.stat path in\n    let mod_time = Timedesc.of_timestamp_float_s_exn stats.Unix.st_mtime in\n    let title, raw =\n      match data with\n      | `Lines x -> (\n          parse_lines pool (Dynarray.to_seq x)\n        )\n      | `Pages x -> (\n          parse_pages pool (Dynarray.to_seq x)\n        )\n    in\n    let links = Index.Raw.links raw in\n    {\n      search_mode;\n      path;\n      path_parts;\n      path_date;\n      mod_time;\n      doc_id;\n      doc_hash;\n      title;\n      raw;\n      links;\n      last_scan;\n    }\nend\n\nlet of_ir2 db ~already_in_transaction (ir : Ir2.t) : t =\n  let\n    {\n      Ir2.search_mode;\n      path;\n      path_parts;\n      path_date;\n      mod_time;\n      title;\n      doc_id;\n      doc_hash;\n      raw;\n      links;\n      last_scan;\n    } = ir in\n  Word_db.write_to_db db ~already_in_transaction;\n  Index.write_raw_to_db db ~already_in_transaction ~doc_id raw;\n  let link_index_of_start_pos = compute_link_index_of_start_pos links in\n  {\n    search_mode;\n    path;\n    path_parts;\n    path_date;\n    mod_time;\n    title;\n    doc_id;\n    doc_hash;\n    word_ids = Index.Raw.word_ids raw;\n    search_scope = None;\n    links;\n    link_index_of_start_pos;\n    last_scan;\n  }\n\nlet of_path\n    ~(env : Eio_unix.Stdenv.base)\n    pool\n    ~already_in_transaction\n    search_mode\n    ?doc_hash\n    path\n  : (t, string) result =\n  let open Sqlite3_utils in\n  let* doc_hash =\n    match doc_hash with\n    | Some x -> Ok x\n    | None -> BLAKE2B.hash_of_file ~env ~path\n  in\n  if Index.is_indexed ~doc_hash then (\n    let doc_id = Doc_id_db.doc_id_of_doc_hash doc_hash in\n    let title =\n      if Index.global_line_count ~doc_id = 0 then\n        None\n      else\n        Some (Index.line_of_global_line_num ~doc_id 0)\n    in\n    let path_parts = compute_path_parts path in\n    let path_date = Date_extraction.extract path in\n    let stats = Unix.stat path in\n    let mod_time = Timedesc.of_timestamp_float_s_exn stats.Unix.st_mtime in\n    let links = Index.links ~doc_id in\n    Ok\n      {\n        search_mode;\n        path;\n        path_parts;\n        path_date;\n        mod_time;\n        title;\n        doc_id;\n        doc_hash;\n        word_ids = Index.word_ids ~doc_id;\n        search_scope = None;\n        links;\n        link_index_of_start_pos = compute_link_index_of_start_pos links;\n        last_scan = Timedesc.now ~tz_of_date_time:Params.tz ()\n      }\n  ) else (\n    let* ir0 = Ir0.of_path ~env search_mode ~doc_hash path in\n    let* ir1 = Ir1.of_ir0 ~env ir0 in\n    let ir2 = Ir2.of_ir1 pool ir1 in\n    let res =\n      with_db (fun db ->\n          Ok (of_ir2 db ~already_in_transaction ir2)\n        )\n    in\n    res\n  )\n\nlet satisfies_filter_exp pool stop_signal ~global_first_word_candidates_lookup (exp : Filter_exp.t) (t : t) : bool =\n  let open Filter_exp in\n  let date_f (op : Filter_exp.compare_op) =\n    match op with\n    | Eq -> Timedesc.Date.equal\n    | Le -> Timedesc.Date.le\n    | Ge -> Timedesc.Date.ge\n    | Lt -> Timedesc.Date.lt\n    | Gt -> Timedesc.Date.gt\n  in\n  let rec aux exp =\n    match exp with\n    | Empty -> true\n    | Path_date (op, date) -> (\n        match t.path_date with\n        | None -> false\n        | Some path_date -> (\n            date_f op path_date date\n          )\n      )\n    | Mod_date (op, date) -> (\n        date_f op (Timedesc.date t.mod_time) date\n      )\n    | Path_fuzzy exp -> (\n        List.exists (fun phrase ->\n            List.for_all (fun token ->\n                List.exists (fun path_part ->\n                    Search_phrase.Enriched_token.compatible_with_word token path_part\n                  )\n                  t.path_parts\n              )\n              (Search_phrase.enriched_tokens phrase)\n          )\n          (Search_exp.flattened exp)\n      )\n    | Path_glob glob -> (\n        Glob.is_empty glob || Glob.match_ glob t.path\n      )\n    | Ext ext -> (\n        File_utils.extension_of_file t.path = ext\n      )\n    | Content exp -> (\n        try\n          Index.search\n            pool\n            stop_signal\n            ~terminate_on_result_found:true\n            ~doc_id:t.doc_id\n            ~doc_word_ids:(word_ids t)\n            ~global_first_word_candidates_lookup\n            ~within_same_line:false\n            ~search_scope:None\n            exp\n          |> ignore;\n          false\n        with\n        | Index.Search_job.Result_found -> true\n      )\n    | Binary_op (op, e1, e2) -> (\n        match op with\n        | And -> aux e1 && aux e2\n        | Or -> aux e1 || aux e2\n      )\n    | Unary_op (op, e) -> (\n        match op with\n        | Not -> not (aux e)\n      )\n  in\n  aux exp\n"
  },
  {
    "path": "bin/document.mli",
    "content": "open Docfd_lib\n\ntype t\n\nval equal : t -> t -> bool\n\nmodule Compare : sig\n  type order = [\n    | `Asc\n    | `Desc\n  ]\n\n  val mod_time : order -> t -> t -> int\n\n  val path_date : order -> t -> t -> int\n\n  val path : order -> t -> t -> int\nend\n\nval search_mode : t -> Search_mode.t\n\nval path : t -> string\n\nval path_date : t -> Timedesc.Date.t option\n\nval mod_time : t -> Timedesc.t\n\nval title : t -> string option\n\nval word_ids : t -> Int_set.t\n\nval doc_hash : t -> string\n\nval doc_id : t -> int64\n\nval search_scope : t -> Diet.Int.t option\n\nval last_scan : t -> Timedesc.t\n\nval links : t -> Link.t array\n\nval link_index_of_start_pos : t -> int Int_map.t\n\nval satisfies_filter_exp :\n  Task_pool.t ->\n  Stop_signal.t ->\n  global_first_word_candidates_lookup:Int_set.t Search_phrase.Enriched_token.Data_map.t ->\n  Filter_exp.t ->\n  t ->\n  bool\n\nval of_path :\n  env:Eio_unix.Stdenv.base ->\n  Task_pool.t ->\n  already_in_transaction:bool ->\n  Search_mode.t ->\n  ?doc_hash:string ->\n  string ->\n  (t, string) result\n\nval reset_search_scope_to_full : t -> t\n\nval inter_search_scope : Diet.Int.t -> t -> t\n\nmodule Ir0 : sig\n  type t\n\n  val of_path :\n    env:Eio_unix.Stdenv.base ->\n    Search_mode.t ->\n    ?doc_hash:string ->\n    string ->\n    (t, string) result\nend\n\nmodule Ir1 : sig\n  type t\n\n  val of_ir0 :\n    env:Eio_unix.Stdenv.base ->\n    Ir0.t ->\n    (t, string) result\nend\n\nmodule Ir2 : sig\n  type t\n\n  val of_ir1 : Task_pool.t -> Ir1.t -> t\nend\n\nval of_ir2 :\n  Sqlite3.db ->\n  already_in_transaction:bool ->\n  Ir2.t ->\n  t\n"
  },
  {
    "path": "bin/document_pipeline.ml",
    "content": "open Docfd_lib\nopen Debug_utils\n\ntype t = {\n  env : Eio_unix.Stdenv.base;\n  pool : Task_pool.t;\n  ir0_queue : Document.Ir0.t option Eio.Stream.t;\n  ir1_of_ir0_workers_batch_release : Eio.Semaphore.t;\n  ir1_queue : Document.Ir1.t option Eio.Stream.t;\n  ir2_of_ir1_workers_batch_release : Eio.Semaphore.t;\n  ir2_queue : Document.Ir2.t option Eio.Stream.t;\n  documents : Document.t Dynarray.t;\n  result : Document.t Dynarray.t Eio.Stream.t;\n}\n\nlet make ~env pool : t =\n  {\n    env;\n    pool;\n    ir0_queue = Eio.Stream.create 100;\n    ir1_of_ir0_workers_batch_release = Eio.Semaphore.make 0;\n    ir1_queue = Eio.Stream.create 100;\n    ir2_of_ir1_workers_batch_release = Eio.Semaphore.make 0;\n    ir2_queue = Eio.Stream.create 100;\n    documents = Dynarray.create ();\n    result = Eio.Stream.create 1;\n  }\n\nlet ir1_of_ir0_worker (t : t) =\n  let run = ref true in\n  while !run do\n    match Eio.Stream.take t.ir0_queue with\n    | None -> (\n        Eio.Semaphore.release t.ir1_of_ir0_workers_batch_release;\n        run := false\n      )\n    | Some ir0 -> (\n        match Document.Ir1.of_ir0 ~env:t.env ir0 with\n        | Error msg -> (\n            do_if_debug (fun oc ->\n                Printf.fprintf oc \"Error: %s\\n\" msg\n              )\n          )\n        | Ok ir1 -> (\n            Eio.Stream.add t.ir1_queue (Some ir1)\n          )\n      )\n  done\n\nlet ir2_of_ir1_worker (t : t) =\n  let run = ref true in\n  while !run do\n    match Eio.Stream.take t.ir1_queue with\n    | None -> (\n        Eio.Semaphore.release t.ir2_of_ir1_workers_batch_release;\n        run := false\n      )\n    | Some ir -> (\n        Eio.Stream.add t.ir2_queue (Some (Document.Ir2.of_ir1 t.pool ir))\n      )\n  done\n\nlet document_of_ir2_worker (t : t) =\n  let open Sqlite3_utils in\n  let run = ref true in\n  let counter = ref 0 in\n  let outstanding_transaction = ref false in\n  with_db (fun db ->\n      while !run do\n        if !counter = 0 then (\n          step_stmt ~db \"BEGIN IMMEDIATE\" ignore;\n          outstanding_transaction := true;\n        );\n        (match Eio.Stream.take t.ir2_queue with\n         | None -> (\n             run := false\n           )\n         | Some ir -> (\n             let doc = Document.of_ir2 db ~already_in_transaction:true ir in\n             Dynarray.add_last t.documents doc;\n             do_if_debug (fun oc ->\n                 Printf.fprintf oc \"Document %s loaded successfully\\n\" (Filename.quote (Document.path doc));\n               );\n           ));\n        if !counter >= 100 then (\n          step_stmt ~db \"COMMIT\" ignore;\n          outstanding_transaction := false;\n          counter := 0;\n        ) else (\n          incr counter;\n        );\n      done;\n      if !outstanding_transaction then (\n        step_stmt ~db \"COMMIT\" ignore;\n      )\n    )\n\nlet feed (t : t) search_mode ~doc_hash path =\n  do_if_debug (fun oc ->\n      Printf.fprintf oc \"Loading document: %s\\n\" (Filename.quote path);\n    );\n  do_if_debug (fun oc ->\n      Printf.fprintf oc \"Using %s search mode for document %s\\n\"\n        (match search_mode with\n         | `Single_line -> \"single line\"\n         | `Multiline -> \"multiline\"\n        )\n        (Filename.quote path)\n    );\n  match Document.Ir0.of_path ~env:t.env search_mode ~doc_hash path with\n  | Error msg -> (\n      do_if_debug (fun oc ->\n          Printf.fprintf oc \"Error: %s\\n\" msg\n        )\n    )\n  | Ok ir0 -> (\n      Eio.Stream.add t.ir0_queue (Some ir0)\n    )\n\nlet run (t : t) =\n  Eio.Fiber.all\n    (List.concat\n       [ CCList.(0 --^ Task_pool.size)\n         |> List.map (fun _ -> (fun () -> ir1_of_ir0_worker t))\n       ; CCList.(0 --^ Task_pool.size)\n         |> List.map (fun _ -> (fun () -> ir2_of_ir1_worker t))\n       ; [ fun () -> document_of_ir2_worker t ]\n       ]\n    );\n  Eio.Stream.add t.result t.documents\n\nlet finalize (t : t) =\n  for _ = 0 to Task_pool.size - 1 do\n    Eio.Stream.add t.ir0_queue None;\n  done;\n  for _ = 0 to Task_pool.size - 1 do\n    Eio.Semaphore.acquire t.ir1_of_ir0_workers_batch_release;\n  done;\n  for _ = 0 to Task_pool.size - 1 do\n    Eio.Stream.add t.ir1_queue None;\n  done;\n  for _ = 0 to Task_pool.size - 1 do\n    Eio.Semaphore.acquire t.ir2_of_ir1_workers_batch_release;\n  done;\n  Eio.Stream.add t.ir2_queue None;\n  Dynarray.to_list (Eio.Stream.take t.result)\n"
  },
  {
    "path": "bin/document_pipeline.mli",
    "content": "type t\n\nval make : env:Eio_unix.Stdenv.base -> Docfd_lib.Task_pool.t -> t\n\nval feed : t -> Search_mode.t -> doc_hash:string -> string -> unit\n\nval run : t -> unit\n\nval finalize : t -> Document.t list\n"
  },
  {
    "path": "bin/document_src.ml",
    "content": "type file_collection = {\n  default_search_mode_files : String_set.t;\n  single_line_search_mode_files : String_set.t;\n}\n\nlet seq_of_file_collection (x : file_collection) =\n  Seq.append\n    (String_set.to_seq x.default_search_mode_files)\n    (String_set.to_seq x.single_line_search_mode_files)\n\nlet file_collection_size (x : file_collection) =\n  String_set.cardinal x.default_search_mode_files\n  + String_set.cardinal x.single_line_search_mode_files\n\nlet empty_file_collection =\n  {\n    default_search_mode_files = String_set.empty;\n    single_line_search_mode_files = String_set.empty;\n  }\n\ntype t =\n  | Stdin of string\n  | Files of file_collection\n"
  },
  {
    "path": "bin/dune",
    "content": "(rule\n  (targets int_map.ml)\n  (deps ../lib/int_map.ml)\n  (action (copy# %{deps} %{targets}))\n  )\n\n(rule\n  (targets int_set.ml)\n  (deps ../lib/int_set.ml)\n  (action (copy# %{deps} %{targets}))\n  )\n\n(rule\n  (targets string_map.ml)\n  (deps ../lib/string_map.ml)\n  (action (copy# %{deps} %{targets}))\n  )\n\n(rule\n  (targets char_map.ml)\n  (deps ../lib/char_map.ml)\n  (action (copy# %{deps} %{targets}))\n  )\n\n(rule\n  (targets string_set.ml)\n  (deps ../lib/string_set.ml)\n  (action (copy# %{deps} %{targets}))\n  )\n\n(rule\n  (targets parser_components.ml)\n  (deps ../lib/parser_components.ml)\n  (action (copy# %{deps} %{targets}))\n  )\n\n(executable\n (flags     (-w \"+a-4-9-29-37-40-42-44-48-50-32-30-70@8\" -g))\n (name docfd)\n (public_name docfd)\n (preprocess (pps ppx_deriving.show ppx_deriving.ord))\n (libraries docfd_lib\n            containers\n            containers.unix\n            cmdliner\n            fmt\n            notty\n            notty.unix\n            nottui\n            nottui-unix\n            lwd\n            oseq\n            eio\n            eio_main\n            eio_posix\n            digestif.c\n            digestif\n            timedesc\n            timedesc-tzlocal.unix-or-utc\n            re\n            progress\n            diet\n            sqlite3\n )\n)\n"
  },
  {
    "path": "bin/file_utils.ml",
    "content": "open Misc_utils\nopen Debug_utils\n\nlet extension_of_file (s : string) =\n  Filename.extension s\n  |> String.lowercase_ascii\n\ntype file_format = [ `PDF | `Pandoc_supported_format | `Text | `Other ] [@@deriving ord]\n\nmodule File_format_set = CCSet.Make (struct\n    type t = file_format\n\n    let compare = compare_file_format\n  end)\n\nlet format_of_file (s : string) : file_format =\n  let ext = extension_of_file s in\n  if ext = \".pdf\" then (\n    `PDF\n  ) else if List.mem ext Params.pandoc_supported_exts then (\n    `Pandoc_supported_format\n  ) else if String_set.mem ext Params.common_text_file_exts\n         || String_set.mem ext Params.common_code_file_exts\n  then (\n    `Text\n  ) else (\n    `Other\n  )\n\ntype typ = [\n  | `File\n  | `Dir\n]\n\ntype is_link = [\n  | `Is_link\n  | `Not_link\n]\n\nlet typ_of_path (path : string) : (typ * is_link) option =\n  try\n    let stat = Unix.lstat path in\n    match stat.st_kind with\n    | S_REG -> Some (`File, `Not_link)\n    | S_DIR -> Some (`Dir, `Not_link)\n    | S_LNK -> (\n        let stat = Unix.stat path in\n        match stat.st_kind with\n        | S_REG -> Some (`File, `Is_link)\n        | S_DIR -> Some (`Dir, `Is_link)\n        | _ -> None\n      )\n    | _ -> None\n  with\n  | _ -> None\n\nlet cwd_with_trailing_sep () = Sys.getcwd () ^ Filename.dir_sep\n\nlet remove_cwd_from_path (s : string) =\n  let pre = cwd_with_trailing_sep () in\n  match CCString.chop_prefix ~pre s with\n  | None -> s\n  | Some s -> s\n\nlet read_in_channel_to_tmp_file (ic : in_channel) : (string, string) result =\n  let file = Filename.temp_file \"docfd-\" \".txt\" in\n  try\n    CCIO.with_out file (fun oc ->\n        CCIO.copy_into ic oc\n      );\n    Ok file\n  with\n  | _ -> (\n      Error (Fmt.str \"failed to write stdin to %s\" (Filename.quote file))\n    )\n\nlet next_choices path : string Seq.t =\n  try\n    Sys.readdir path\n    |> Array.to_seq\n  with\n  | _ -> Seq.empty\n\nlet list_files_recursive\n    ?(max_depth = !Params.max_file_tree_scan_depth)\n    ~(report_progress : unit -> unit)\n    ~(filter : int -> string -> bool)\n    (path : string)\n  : String_set.t =\n  let acc = ref String_set.empty in\n  let add x =\n    acc := String_set.add x !acc\n  in\n  let rec aux depth path =\n    report_progress ();\n    let hidden =\n      String.starts_with ~prefix:\".\" (Filename.basename path)\n    in\n    if ((hidden && !Params.scan_hidden) || not hidden)\n    && depth <= max_depth then (\n      match typ_of_path path with\n      | Some (`Dir, _) -> (\n          next_choices path\n          |> Seq.iter (fun f ->\n              aux (depth + 1) (Filename.concat path f)\n            )\n        )\n      | Some (`File, _) -> (\n          if filter depth path then (\n            add path\n          )\n        )\n      | _ -> ()\n    )\n  in\n  aux 0 (normalize_path_to_absolute path);\n  !acc\n\nlet list_files_recursive_filter_by_globs\n    ?max_depth\n    ~(report_progress : unit -> unit)\n    (globs : string Seq.t)\n  : String_set.t =\n  let acc = ref String_set.empty in\n  let add x =\n    acc := String_set.add x !acc\n  in\n  let parse_glob ~case_sensitive s =\n    match Glob.parse ~case_sensitive s with\n    | None -> (\n        failwith (Fmt.str \"expected subpath of a valid glob pattern to also be valid: \\\"%s\\\"\" s)\n      )\n    | Some x -> x\n  in\n  let rec aux ~case_sensitive (path_parts : string list) (glob_parts : string list) =\n    report_progress ();\n    let path = path_of_parts path_parts in\n    match typ_of_path path, glob_parts with\n    | Some (`File, _), [] -> add path\n    | Some (`File, _), _ -> ()\n    | Some (`Dir, _), [] -> ()\n    | Some (`Dir, _), x :: xs -> (\n        match x with\n        | \"\" | \".\" -> aux ~case_sensitive path_parts xs\n        | \"..\" -> (\n            let path_parts =\n              match path_parts with\n              | [] -> []\n              | _ :: xs -> xs\n            in\n            aux ~case_sensitive path_parts xs\n          )\n        | \"**\" -> (\n            let glob_string = String.concat Filename.dir_sep (path :: glob_parts) in\n            do_if_debug (fun oc ->\n                Printf.fprintf oc \"Compiling glob using pattern: %s\\n\" glob_string\n              );\n            let glob = parse_glob ~case_sensitive glob_string in\n            path\n            |> list_files_recursive\n              ?max_depth\n              ~report_progress\n              ~filter:(fun _depth path ->\n                  Glob.match_ glob path\n                )\n            |> String_set.iter (fun path ->\n                do_if_debug (fun oc ->\n                    Printf.fprintf oc \"Glob %s matches path %s\\n\" glob_string path\n                  );\n                add path\n              )\n          )\n        | _ -> (\n            let glob = parse_glob ~case_sensitive x in\n            next_choices path\n            |> Seq.iter (fun f ->\n                if Glob.match_ glob f then (\n                  aux ~case_sensitive (f :: path_parts) xs\n                )\n              )\n          )\n      )\n    | None, _ -> ()\n    | exception _ -> ()\n  in\n  Seq.iter (fun glob ->\n      let case_sensitive =\n        Glob.parse glob\n        |> Option.get\n        |> Glob.case_sensitive\n      in\n      let glob_parts = CCString.split ~by:Filename.dir_sep glob in\n      match glob_parts with\n      | \"\" :: l -> (\n          (* Absolute path on Unix-like systems *)\n          aux ~case_sensitive [ \"\" ] l\n        )\n      | _ -> (\n          aux ~case_sensitive (cwd_path_parts ()) glob_parts\n        )\n    ) globs;\n  !acc\n\nlet list_files_recursive_filter_by_exts\n    ?max_depth\n    ~report_progress\n    ~(exts : string list)\n    (paths : string Seq.t)\n  : String_set.t =\n  let filter depth path =\n    let ext = extension_of_file path in\n    depth = 0 || List.mem ext exts\n  in\n  paths\n  |> Seq.map normalize_path_to_absolute\n  |> Seq.map (list_files_recursive ?max_depth ~report_progress ~filter)\n  |> Seq.fold_left String_set.union String_set.empty\n\nlet mkdir_recursive (dir : string) : unit =\n  let rec aux first acc parts =\n    match parts with\n    | [] -> ()\n    | \"\" :: xs -> (\n        if first then\n          aux false Filename.dir_sep xs\n        else\n          aux false \"\" xs\n      )\n    | x :: xs -> (\n        let acc = Filename.concat acc x in\n        match Sys.is_directory acc with\n        | true -> aux false acc xs\n        | false -> (\n            exit_with_error_msg\n              (Fmt.str \"%s is not a directory\" (Filename.quote acc))\n          )\n        | exception (Sys_error _) -> (\n            do_if_debug (fun oc ->\n                Printf.fprintf oc \"Creating directory: %s\\n\" (Filename.quote acc)\n              );\n            (try\n               Sys.mkdir acc 0o755\n             with\n             | _ -> (\n                 exit_with_error_msg\n                   (Fmt.str \"failed to create directory: %s\" (Filename.quote acc))\n               )\n            );\n            aux false acc xs\n          )\n      )\n  in\n  aux true \"\" (CCString.split ~by:Filename.dir_sep dir)\n\nlet file_size (path : string) : int option =\n  try\n    let st = Unix.stat path in\n    Some st.st_size\n  with\n  | _ -> None\n"
  },
  {
    "path": "bin/filter_exp.ml",
    "content": "open Docfd_lib\n\ntype t =\n  | Empty\n  | Path_date of compare_op * Timedesc.Date.t\n  | Path_fuzzy of Search_exp.t\n  | Path_glob of Glob.t\n  | Ext of string\n  | Mod_date of compare_op * Timedesc.Date.t\n  | Content of Search_exp.t\n  | Binary_op of binary_op * t * t\n  | Unary_op of unary_op * t\n\nand binary_op =\n  | And\n  | Or\n\nand unary_op =\n  | Not\n\nand compare_op =\n  | Eq\n  | Le\n  | Ge\n  | Lt\n  | Gt\n\nlet empty = Empty\n\nlet is_empty (e : t) =\n  match e with\n  | Empty -> true\n  | _ -> false\n\nlet equal (e1 : t) (e2 : t) =\n  let rec aux e1 e2 =\n    match e1, e2 with\n    | Empty, Empty -> true\n    | Path_date (op1, x1), Path_date (op2, x2) ->\n      op1 = op2 && Timedesc.Date.equal x1 x2\n    | Path_fuzzy x, Path_fuzzy y -> Search_exp.equal x y\n    | Path_glob x, Path_glob y -> Glob.equal x y\n    | Ext x, Ext y -> String.equal x y\n    | Content x, Content y -> Search_exp.equal x y\n    | Binary_op (op1, x1, y1), Binary_op (op2, x2, y2) ->\n      op1 = op2 && aux x1 x2 && aux y1 y2\n    | Unary_op (op1, x1), Unary_op (op2, x2) ->\n      op1 = op2 && aux x1 x2\n    | _, _ -> false\n  in\n  aux e1 e2\n\nlet all_content_search_exps (t : t) : Search_exp.t list =\n  let rec aux t =\n    match t with\n    | Empty\n    | Path_date _\n    | Path_fuzzy _\n    | Path_glob _\n    | Ext _\n    | Mod_date _ -> []\n    | Content e -> [e]\n    | Binary_op (_op, e1, e2) -> aux e1 @ aux e2\n    | Unary_op (_op, e) -> aux e\n  in\n  aux t\n\nmodule Parsers = struct\n  type exp = t\n\n  open Angstrom\n  open Parser_components\n\n  let alphanum_symbol_string =\n    take_while1 (fun c ->\n        is_letter c\n        ||\n        is_digit c\n        ||\n        (match c with\n         | '&'\n         | '|' -> true\n         | _ -> false\n        )\n      )\n\n  let maybe_quoted_string ?(force_quote = false) () =\n    (\n      (choice\n         [\n           char '\"';\n           char '\\'';\n         ]\n       >>= fun c -> return (Some c)\n      )\n      <|>\n      (if force_quote then\n         fail \"\"\n       else\n         return None)\n    )\n    >>= fun quote_char ->\n    many1 (\n      take_while1 (fun c ->\n          match c with\n          | '\\\\' -> false\n          | c -> (\n              match quote_char with\n              | None -> (\n                  is_not_space c\n                  &&\n                  (match c with\n                   | '('\n                   | ')' -> false\n                   | _ -> true\n                  )\n                )\n              | Some quote_char -> c <> quote_char\n            )\n        )\n      <|>\n      (char '\\\\' *> any_char >>| fun c -> Printf.sprintf \"%c\" c)\n    )\n    >>= fun l ->\n    let s = String.concat \"\" l in\n    (end_of_input *> return s)\n    <|>\n    (match quote_char with\n     | None -> return s\n     | Some quote_char -> char quote_char *> return s)\n\n  let search_exp ?force_quote () =\n    maybe_quoted_string ?force_quote ()\n    >>= fun s ->\n    match Search_exp.parse s with\n    | None -> fail \"\"\n    | Some x -> return x\n\n  let glob =\n    maybe_quoted_string ()\n    >>= fun s ->\n    let s = Misc_utils.normalize_filter_glob_if_not_empty s in\n    match Glob.parse s with\n    | None -> fail \"\"\n    | Some x -> return x\n\n  let ext =\n    maybe_quoted_string ()\n    >>| fun s ->\n    s\n    |> String.lowercase_ascii\n    |> String_utils.remove_leading_dots\n    |> Fmt.str \".%s\"\n\n  let compare_op =\n    choice\n      [\n        char '=' *> skip_spaces *> return Eq;\n        string \"<=\" *> skip_spaces *> return Le;\n        string \">=\" *> skip_spaces *> return Ge;\n        char '<' *> skip_spaces *> return Lt;\n        char '>' *> skip_spaces *> return Gt;\n      ]\n\n  let date =\n    any_string\n    >>= fun s ->\n    match Timedesc.Date.Ymd.of_iso8601 s with\n    | Ok x -> return x\n    | Error _ -> fail \"\"\n\n  let compare_date f =\n    let p =\n      compare_op >>= fun op ->\n      date >>| fun date ->\n      f (op, date)\n    in\n    maybe_quoted_string ()\n    >>= fun s ->\n    match Angstrom.(parse_string ~consume:Consume.All) p s with\n    | Ok x -> return x\n    | Error s -> fail s\n\n  let binary_op op_strings op =\n    alphanum_symbol_string >>= fun s ->\n    skip_spaces *>\n    (\n      if List.mem (String.lowercase_ascii s) op_strings then (\n        return (fun x y -> Binary_op (op, x, y))\n      ) else (\n        fail \"\"\n      )\n    )\n\n  let and_op = binary_op [ \"and\" ] And\n\n  let or_op = binary_op [ \"or\" ] Or\n\n  let unary_op op_strings op =\n    alphanum_symbol_string >>= fun s ->\n    skip_spaces *>\n    (\n      if List.mem (String.lowercase_ascii s) op_strings then (\n        return (fun x -> Unary_op (op, x))\n      ) else (\n        fail \"\"\n      )\n    )\n\n  let not_op = unary_op [ \"not\" ] Not\n\n  let p =\n    skip_spaces *>\n    (\n      (end_of_input *> return empty)\n      <|>\n      fix (fun (exp : exp Angstrom.t) ->\n          let base =\n            choice [\n              (search_exp ~force_quote:true () >>|\n               fun x -> Content x);\n              (string \"content:\" *>\n               search_exp () >>| fun x -> Content x);\n              (string \"path-date:\" *>\n               compare_date (fun (op, date) -> Path_date (op, date)));\n              (string \"path-fuzzy:\" *>\n               search_exp () >>| fun x -> Path_fuzzy x);\n              (string \"path-glob:\" *>\n               glob >>| fun x -> Path_glob x);\n              (string \"mod-date:\" *>\n               compare_date (fun (op, date) -> Mod_date (op, date)));\n              (string \"ext:\" *>\n               ext >>| fun x -> Ext x);\n              (char '(' *> skip_spaces *> exp <* char ')');\n            ]\n            <* skip_spaces\n          in\n          let maybe_neg =\n            (not_op >>= fun f ->\n             skip_spaces *> base >>| f)\n            <|>\n            base\n          in\n          let conj = chainl1 maybe_neg and_op in\n          chainl1 conj or_op\n        )\n    )\n    <* skip_spaces\nend\n\nlet parse s =\n  match Angstrom.(parse_string ~consume:Consume.All) Parsers.p s with\n  | Ok e -> Some e\n  | Error _ -> None\n"
  },
  {
    "path": "bin/glob.ml",
    "content": "type t = {\n  case_sensitive : bool;\n  string : string;\n  re : Re.re;\n}\n\nlet equal x y =\n  x.case_sensitive = y.case_sensitive\n  &&\n  String.equal x.string y.string\n\nlet case_sensitive t = t.case_sensitive\n\nlet string t = t.string\n\nlet is_empty t = String.length t.string = 0\n\nmodule Parsers = struct\n  open Angstrom\n\n  type part = [\n    | `Case_insensitive\n    | `String of string\n  ]\n\n  let parts : part list Angstrom.t =\n    many (\n      (take_while1 (fun c ->\n           match c with\n           | '\\\\' -> false\n           | _ -> true\n         )\n       >>| fun s -> `String s)\n      <|>\n      (char '\\\\' *> any_char >>= fun c ->\n       if c = 'c' then return `Case_insensitive\n       else return (`String (Printf.sprintf \"%c\" c)))\n    )\nend\n\nlet parse ?case_sensitive (s : string) : t option =\n  match Angstrom.(parse_string ~consume:Consume.All) Parsers.parts s with\n  | Error _ -> None\n  | Ok parts -> (\n      let case_insensitive = ref false in\n      let s =\n        parts\n        |> List.filter_map (fun x ->\n            match x with\n            | `String s -> Some s\n            | `Case_insensitive -> (\n                case_insensitive := true;\n                None\n              )\n          )\n        |> String.concat \"\"\n      in\n      let case_sensitive =\n        match case_sensitive with\n        | None -> not !case_insensitive\n        | Some x -> x\n      in\n      try\n        let re =\n          s\n          |> (fun s -> if case_sensitive then s else String.lowercase_ascii s)\n          |> Re.Glob.glob\n            ~anchored:true\n            ~pathname:true\n            ~match_backslashes:false\n            ~period:true\n            ~expand_braces:false\n            ~double_asterisk:true\n          |> Re.compile\n        in\n        Some\n          {\n            case_sensitive;\n            string = s;\n            re;\n          }\n      with\n      | _ -> None\n    )\n\nlet match_ t (s : string) =\n  is_empty t\n  || Re.execp t.re (if t.case_sensitive then s else String.lowercase_ascii s)\n"
  },
  {
    "path": "bin/glob.mli",
    "content": "type t\n\nval parse : ?case_sensitive:bool -> string -> t option\n\nval equal : t -> t -> bool\n\nval is_empty : t -> bool\n\nval case_sensitive : t -> bool\n\nval string : t -> string\n\nval match_ : t -> string -> bool\n"
  },
  {
    "path": "bin/lock_protected_cell.ml",
    "content": "type 'a t = {\n  lock : Eio.Mutex.t;\n  mutable data : 'a option;\n}\n\nlet make () =\n  {\n    lock = Eio.Mutex.create ();\n    data = None;\n  }\n\nlet set (t : 'a t) (x : 'a) =\n  Eio.Mutex.use_rw t.lock ~protect:false (fun () ->\n      t.data <- Some x\n    )\n\nlet unset (t : 'a t) =\n  Eio.Mutex.use_rw t.lock ~protect:false (fun () ->\n      t.data <- None\n    )\n\nlet get (t : 'a t) : 'a option =\n  Eio.Mutex.use_rw t.lock ~protect:false (fun () ->\n      let x = t.data in\n      t.data <- None;\n      x\n    )\n"
  },
  {
    "path": "bin/lock_protected_cell.mli",
    "content": "type 'a t\n\nval make : unit -> 'a t\n\nval set : 'a t -> 'a -> unit\n\nval unset : 'a t -> unit\n\nval get : 'a t -> 'a option\n"
  },
  {
    "path": "bin/misc_utils.ml",
    "content": "open Docfd_lib\ninclude Docfd_lib.Misc_utils'\n\nlet bound_selection ~choice_count (x : int) : int =\n  max 0 (min (choice_count - 1) x)\n\nlet frequencies_of_words_ci (s : string Seq.t) : int String_map.t =\n  Seq.fold_left (fun m word ->\n      let word = String.lowercase_ascii word in\n      let count = Option.value ~default:0\n          (String_map.find_opt word m)\n      in\n      String_map.add word (count + 1) m\n    )\n    String_map.empty\n    s\n\nlet stdin_is_atty () =\n  Unix.isatty Unix.stdin\n\nlet stdout_is_atty () =\n  Unix.isatty Unix.stdout\n\nlet stderr_is_atty () =\n  Unix.isatty Unix.stderr\n\nlet compute_total_recognized_exts ~exts ~additional_exts =\n  let split_on_comma = String.split_on_char ',' in\n  (split_on_comma exts)\n  :: (List.map split_on_comma additional_exts)\n  |> List.to_seq\n  |> Seq.flat_map List.to_seq\n  |> Seq.map (fun s ->\n      s\n      |> CCString.trim\n      |> String_utils.remove_leading_dots\n      |> String.lowercase_ascii\n    )\n  |> Seq.filter (fun s -> s <> \"\")\n  |> Seq.map (fun s -> Printf.sprintf \".%s\" s)\n  |> List.of_seq\n\nlet array_sub_seq : 'a. start:int -> end_exc:int -> 'a array -> 'a Seq.t =\n  fun ~start ~end_exc arr ->\n  let count = Array.length arr in\n  let end_exc = min count end_exc in\n  let rec aux start =\n    if start < end_exc then (\n      Seq.cons arr.(start) (aux (start + 1))\n    ) else (\n      Seq.empty\n    )\n  in\n  aux start\n\nlet rotate_list (x : int) (l : 'a list) : 'a list =\n  let arr = Array.of_list l in\n  let len = Array.length arr in\n  Seq.append\n    (array_sub_seq ~start:x ~end_exc:len arr)\n    (array_sub_seq ~start:0 ~end_exc:x arr)\n  |> List.of_seq\n\nlet drain_eio_stream (x : 'a Eio.Stream.t) =\n  let rec aux () =\n    match Eio.Stream.take_nonblocking x with\n    | None -> ()\n    | Some _ -> aux ()\n  in\n  aux ()\n\nlet mib_of_bytes (x : int) =\n  (Int.to_float x) /. (1024.0 *. 1024.0)\n\nlet progress_with_reporter ~interactive bar f =\n  if interactive then (\n    Progress.with_reporter\n      ~config:(Progress.Config.v ~ppf:Format.std_formatter ())\n      bar\n      (fun report_progress ->\n         let report_progress =\n           let lock = Eio.Mutex.create () in\n           fun x ->\n             Eio.Mutex.use_rw lock ~protect:false (fun () ->\n                 report_progress x\n               )\n         in\n         f report_progress\n      )\n  ) else (\n    f (fun _ -> ())\n  )\n\nlet normalize_filter_glob_if_not_empty (s : string) =\n  if String.length s = 0 then (\n    s\n  ) else (\n    normalize_glob_to_absolute s\n  )\n\nlet trim_angstrom_error_msg (s : string) =\n  CCString.chop_prefix ~pre:\": \" s\n  |> Option.value ~default:s\n\nlet ranking_of_ranked_document_list (l : string list) : int String_map.t =\n  CCList.foldi (fun acc i path ->\n      String_map.add path i acc\n    ) String_map.empty l\n\nlet fuzzy_rank_assoc\n    (stop_signal : Stop_signal.t)\n    ~(get_key : 'a -> string)\n    (exp : Search_exp.t)\n    (items : 'a Seq.t)\n  : ('a * Search_result.t) Dynarray.t =\n  let pick_best_search_result (s : Search_result.t Seq.t) : Search_result.t option =\n    Seq.fold_left (fun best x ->\n        match best with\n        | None -> Some x\n        | Some best -> (\n            if Search_result.score x > Search_result.score best then (\n              Some x\n            ) else (\n              Some best\n            )\n          )\n      )\n      None\n      s\n  in\n  let search_in_line (line : string) (exp : Search_exp.t) : Search_result.t option =\n    let parts = Tokenization.tokenize ~drop_spaces:false line\n      |> List.of_seq\n    in\n    Search_exp.flattened exp\n    |> List.to_seq\n    |> Seq.flat_map (fun phrase ->\n        Search_phrase.enriched_tokens phrase\n        |> List.to_seq\n        |> Seq.map (fun token ->\n            List.to_seq parts\n            |> Seq.mapi (fun i part ->\n                (i, part)\n              )\n            |> Seq.filter_map (fun (i, part) ->\n                if\n                  Search_phrase.Enriched_token.compatible_with_word token part\n                then (\n                  Some (i, part, String.lowercase_ascii part)\n                ) else (\n                  None\n                )\n              )\n          )\n        |> OSeq.cartesian_product\n        |> Seq.map (fun l ->\n            let found_phrase =\n              List.map (fun (i, part, part_ci) ->\n                  Search_result.{\n                    found_word_pos = i;\n                    found_word_ci = part_ci;\n                    found_word = part;\n                  }\n                )\n                l\n            in\n            Search_result.make phrase\n              ~found_phrase\n              ~found_phrase_opening_closing_symbol_match_count:0\n          )\n      )\n    |> pick_best_search_result\n  in\n  Eio.Fiber.first\n    (fun () ->\n       Stop_signal.await stop_signal;\n       Dynarray.create ()\n    )\n    (fun () ->\n       items\n       |> Seq.fold_left (fun acc item ->\n           let line = get_key item in\n           match search_in_line line exp with\n           | None -> acc\n           | Some best_result -> (\n               (item, best_result) :: acc\n             )\n         )\n         []\n       |> List.sort (fun (_x0, r0) (_x1, r1) ->\n           Search_result.compare_relevance r0 r1\n         )\n       |> Dynarray.of_list\n    )\n\nlet highlights_of_search_result (search_result : Search_result.t) =\n  List.fold_left\n    (fun acc (x : Search_result.indexed_found_word) ->\n       Int_set.add x.found_word_pos acc\n    )\n    Int_set.empty\n    (Search_result.found_phrase search_result)\n"
  },
  {
    "path": "bin/params.ml",
    "content": "include Docfd_lib.Params'\n\nlet debug_output : out_channel option ref = ref None\n\nlet scan_hidden = ref false\n\nlet default_max_file_tree_scan_depth = 100\n\nlet max_file_tree_scan_depth = ref default_max_file_tree_scan_depth\n\nlet preview_line_count = 5\n\nlet default_tokens_per_search_scope_level = 100\n\nlet tokens_per_search_scope_level = ref default_tokens_per_search_scope_level\n\nlet docfd_script_ext = \".docfd-script\"\n\nlet pandoc_supported_exts =\n  [ \".epub\"\n  ; \".odt\"\n  ; \".docx\"\n  ; \".fb2\"\n  ; \".ipynb\"\n  ; \".html\"\n  ; \".htm\"\n  ]\n\nlet common_text_file_exts =\n  String_set.of_list\n    [ \".txt\"\n    ; \".md\"\n    ; \".markdown\"\n    ; \".rst\"\n    ; \".adoc\"\n    ; \".org\"\n    ; \".json\"\n    ; \".yaml\"\n    ; \".yml\"\n    ; \".toml\"\n    ; \".xml\"\n    ; \".csv\"\n    ; \".tsv\"\n    ; \".conf\"\n    ; \".cfg\"\n    ; \".ini\"\n    ; \".env\"\n    ; \".log\"\n    ]\n\nlet common_code_file_exts =\n  String_set.of_list\n    [ \".sh\"\n    ; \".bash\"\n    ; \".zsh\"\n    ; \".fish\"\n    ; \".ksh\"\n    ; \".csh\"\n    ; \".tcsh\"\n    ; \".ps1\"\n    ; \".psm1\"\n    ; \".psd1\"\n    ; \".cmd\"\n    ; \".bat\"\n    ; \".py\"\n    ; \".pyi\"\n    ; \".pyw\"\n    ; \".ipynb\"\n    ; \".pxd\"\n    ; \".pxi\"\n    ; \".js\"\n    ; \".mjs\"\n    ; \".cjs\"\n    ; \".jsx\"\n    ; \".ts\"\n    ; \".tsx\"\n    ; \".vite\"\n    ; \".webpack\"\n    ; \".html\"\n    ; \".htm\"\n    ; \".xhtml\"\n    ; \".css\"\n    ; \".scss\"\n    ; \".sass\"\n    ; \".less\"\n    ; \".vue\"\n    ; \".svelte\"\n    ; \".svg\"\n    ; \".c\"\n    ; \".h\"\n    ; \".i\"\n    ; \".cpp\"\n    ; \".cc\"\n    ; \".cxx\"\n    ; \".hpp\"\n    ; \".hh\"\n    ; \".hxx\"\n    ; \".m\"\n    ; \".mm\"\n    ; \".java\"\n    ; \".kt\"\n    ; \".kts\"\n    ; \".scala\"\n    ; \".groovy\"\n    ; \".clj\"\n    ; \".cljs\"\n    ; \".cljc\"\n    ; \".edn\"\n    ; \".go\"\n    ; \".rs\"\n    ; \".zig\"\n    ; \".nim\"\n    ; \".nims\"\n    ; \".hs\"\n    ; \".lhs\"\n    ; \".ml\"\n    ; \".mli\"\n    ; \".re\"\n    ; \".rei\"\n    ; \".elm\"\n    ; \".idr\"\n    ; \".agda\"\n    ; \".lua\"\n    ; \".pl\"\n    ; \".pm\"\n    ; \".rb\"\n    ; \".tcl\"\n    ; \".raku\"\n    ; \".jl\"\n    ; \".r\"\n    ; \".cs\"\n    ; \".csx\"\n    ; \".fs\"\n    ; \".fsi\"\n    ; \".fsx\"\n    ; \".vb\"\n    ; \".asm\"\n    ; \".s\"\n    ; \".S\"\n    ; \".ll\"\n    ; \".bc\"\n    ; \".wat\"\n    ; \".wasm\"\n    ; \".swift\"\n    ; \".storyboard\"\n    ; \".xib\"\n    ; \".json\"\n    ; \".jsonc\"\n    ; \".json5\"\n    ; \".yaml\"\n    ; \".yml\"\n    ; \".toml\"\n    ; \".xml\"\n    ; \".ini\"\n    ; \".cfg\"\n    ; \".conf\"\n    ; \".env\"\n    ; \".properties\"\n    ; \".sql\"\n    ; \".psql\"\n    ; \".cql\"\n    ; \".gql\"\n    ; \".graphql\"\n    ; \".mk\"\n    ; \".make\"\n    ; \".cmake\"\n    ; \".gradle\"\n    ; \".gradle.kts\"\n    ; \".bazel\"\n    ; \".bzl\"\n    ; \".ninja\"\n    ; \".build\"\n    ; \".Dockerfile\"\n    ; \".dockerignore\"\n    ; \".compose\"\n    ; \".compose.yml\"\n    ; \".tf\"\n    ; \".tfvars\"\n    ; \".hcl\"\n    ; \".ansible\"\n    ; \".playbook\"\n    ; \".md\"\n    ; \".markdown\"\n    ; \".rst\"\n    ; \".adoc\"\n    ; \".org\"\n    ; \".tex\"\n    ; \".vim\"\n    ; \".vimrc\"\n    ; \".emacs\"\n    ; \".editorconfig\"\n    ; \".awk\"\n    ; \".sed\"\n    ; \".pug\"\n    ; \".jade\"\n    ; \".haml\"\n    ; \".qs\"\n    ; \".kql\"\n    ]\n\nlet default_recognized_exts =\n  ([ \"txt\"; \"md\"; \"pdf\" ]\n   @\n   pandoc_supported_exts\n  )\n  |> List.map String_utils.remove_leading_dots\n  |> String.concat \",\"\n\nlet default_recognized_single_line_exts =\n  [ \"log\"; \"csv\"; \"tsv\" ]\n  |> List.map String_utils.remove_leading_dots\n  |> String.concat \",\"\n\nlet default_search_mode : Search_mode.t ref = ref `Multiline\n\nlet default_sort_by_arg = \"score,desc\"\n\nlet default_sort_by_no_score_arg = \"path,asc\"\n\nlet index_file_ext = \".index\"\n\nlet db_file_name = \"index.db\"\n\nlet hash_chunk_size = 4096\n\nlet text_editor = ref \"\"\n\nlet default_samples_per_document = 5\n\nlet samples_per_document = ref default_samples_per_document\n\ntype style_mode = [ `Never | `Always | `Auto ]\n\nlet default_search_result_print_text_width = 80\n\nlet search_result_print_text_width = ref default_search_result_print_text_width\n\nlet default_search_result_print_snippet_min_size = 10\n\nlet search_result_print_snippet_min_size = ref default_search_result_print_snippet_min_size\n\nlet default_search_result_print_snippet_max_additional_lines_each_direction = 2\n\nlet search_result_print_snippet_max_additional_lines_each_direction =\n  ref default_search_result_print_snippet_max_additional_lines_each_direction\n\nlet default_cache_limit = 10_000\n\nlet cache_limit = ref default_cache_limit\n\nlet cache_dir : string option ref = ref None\n\nlet data_dir : string option ref = ref None\n\nlet script_dir () =\n  Filename.concat\n    (Option.get !data_dir)\n    \"scripts\"\n\nlet tz : Timedesc.Time_zone.t =\n  Option.value ~default:Timedesc.Time_zone.utc\n    (Timedesc.Time_zone.local ())\n\nlet last_scan_format_string =\n  \"{year}-{mon:0X}-{day:0X} {hour:0X}:{min:0X}:{sec:0X}\"\n  ^\n  (match Timedesc.Time_zone.local () with\n   | None -> \"Z\"\n   | Some _ -> \"\")\n\nlet last_modified_format_string =\n  \"{year}-{mon:0X}-{day:0X} {hour:0X}:{min:0X}\"\n  ^\n  (match Timedesc.Time_zone.local () with\n   | None -> \"Z\"\n   | Some _ -> \"\")\n\nlet blink_on_duration : Mtime.span = Mtime.Span.(140 * ms)\n\nlet session_manager_request_debounce_interval = Mtime.Span.(200 * ms)\n\nlet session_manager_request_debounce_wait_buffer = Mtime.Span.(5 * ms)\n\nlet os_typ : [ `Darwin | `Linux ] =\n  let s = CCUnix.call_stdout \"uname\"\n    |> String.trim\n    |> String.lowercase_ascii\n  in\n  match s with\n  | \"darwin\" -> `Darwin\n  | _ -> `Linux\n\nlet clipboard_copy_cmd_and_args =\n  match os_typ with\n  | `Darwin -> Some (\"pbcopy\", [||])\n  | `Linux -> (\n      if Proc_utils.command_exists \"clip.exe\" then (\n        Some (\"clip.exe\", [||])\n      ) else (\n        match Sys.getenv_opt \"XDG_SESSION_TYPE\" with\n        | None -> None\n        | Some s -> (\n            match String.lowercase_ascii s with\n            | \"x11\" -> Some (\"xclip\", [| \"-sel\"; \"clip\" |])\n            | \"wayland\" -> Some (\"wl-copy\", [|\"-n\"|])\n            | _ -> None\n          )\n      )\n    )\n"
  },
  {
    "path": "bin/path_opening.ml",
    "content": "open Docfd_lib\nopen Debug_utils\n\ntype launch_mode = [ `Terminal | `Detached ]\n\ntype spec = string list * launch_mode * string\n\nlet specs : (string, launch_mode * string) Hashtbl.t = Hashtbl.create 128\n\nmodule Parsers = struct\n  open Angstrom\n  open Parser_components\n\n  let expected_char c =\n    fail (Fmt.str \"expected char %c\" c)\n\n  let inner ~path ~page_num ~line_num ~search_word : string t =\n    choice [\n      string \"path\" *> commit *> return path;\n      string \"page_num\" *> commit\n      >>= (fun _ ->\n          match page_num with\n          | None -> fail \"page_num not available\"\n          | Some n -> return (Fmt.str \"%d\" n)\n        );\n      string \"line_num\" *> commit\n      >>= (fun _ ->\n          match line_num with\n          | None -> fail \"line_num not available\"\n          | Some n -> return (Fmt.str \"%d\" n)\n        );\n      string \"search_word\" *> commit\n      >>= (fun _ ->\n          match search_word with\n          | None -> fail \"search_word not available\"\n          | Some s -> return (Fmt.str \"'%s'\" s));\n    ]\n    <|>\n    fail \"invalid placeholder\"\n\n  let cmd ~path ~page_num ~line_num ~search_word : string t =\n    let single =\n      choice [\n        (string \"{{\" >>| fun _ -> Fmt.str \"{\");\n        (char '{' *> commit *>\n         inner ~path ~page_num ~line_num ~search_word <*\n         (char '}' <|> expected_char '}'));\n        (take_while1 (function '{' -> false | _ -> true));\n      ]\n    in\n    many single\n    >>| fun l -> String.concat \"\" l\n\n  let spec : spec t =\n    sep_by (char ',')\n      (take_while1 (function ':' | ',' -> false | _ -> true))\n    >>= fun exts ->\n    (char ':' <|> expected_char ':')*>\n    (choice [\n        string \"terminal\" *> return `Terminal;\n        string \"detached\" *> return `Detached;\n      ]\n     <|>\n     fail \"invalid launch mode\")\n    >>= fun launch_mode ->\n    (char '=' <|> expected_char '=') *> any_string\n    >>= fun cmd ->\n    return (exts, launch_mode, cmd)\nend\n\nmodule Config = struct\n  type t = {\n    quote_path : bool;\n    path : string;\n    page_num : int option;\n    line_num : int option;\n    search_word : string option;\n    launch_mode : launch_mode;\n  }\n\n  let make ?(quote_path = true) ~path ?page_num ?line_num ?search_word ~launch_mode () : t =\n    {\n      quote_path;\n      path;\n      page_num;\n      line_num;\n      search_word;\n      launch_mode;\n    }\nend\n\nlet resolve_cmd (config : Config.t) (s : string) : (string, string) result =\n  let open Angstrom in\n  let { Config.quote_path; path; page_num; line_num; search_word } = config in\n  let path =\n    if quote_path then\n      Filename.quote path\n    else\n      path\n  in\n  match\n    parse_string ~consume:All (Parsers.cmd ~path ~page_num ~line_num ~search_word) s\n  with\n  | Error msg -> Error (Misc_utils.trim_angstrom_error_msg msg)\n  | Ok s -> Ok s\n\nlet parse_spec (s : string) : (spec, string) result =\n  let open Angstrom in\n  match\n    parse_string ~consume:All Parsers.spec s\n  with\n  | Error msg -> Error (Misc_utils.trim_angstrom_error_msg msg)\n  | Ok (exts', launch_mode, cmd) -> (\n      let rec aux acc exts =\n        match exts with\n        | [] -> Ok (List.rev acc, launch_mode, cmd)\n        | ext :: rest -> (\n            let ext = ext\n              |> String.lowercase_ascii\n              |> String_utils.remove_leading_dots\n              |> Fmt.str \".%s\"\n            in\n            let config =\n              if ext = \".pdf\" then (\n                Config.make\n                  ~path:\"path\"\n                  ~page_num:1\n                  ~search_word:\"word\"\n                  ~launch_mode:`Detached\n                  ()\n              ) else (\n                Config.make\n                  ~path:\"path\"\n                  ~line_num:1\n                  ~launch_mode:`Terminal\n                  ()\n              )\n            in\n            match\n              resolve_cmd config cmd\n            with\n            | Error msg -> Error msg\n            | Ok _ -> aux (ext :: acc) rest\n          )\n      in\n      aux [] exts'\n    )\n\nlet xdg_open_cmd =\n  \"xdg-open {path}\"\n\nlet pandoc_supported_format_config_and_cmd ~path =\n  (Config.make ~path ~launch_mode:`Detached (),\n   xdg_open_cmd)\n\nlet fallback_cmd : string =\n  match Params.os_typ with\n  | `Linux -> xdg_open_cmd\n  | `Darwin -> \"open {path}\"\n\nlet compute_most_unique_word_and_residing_page_num ~doc_id found_phrase =\n  let page_nums = found_phrase\n    |> List.map (fun word ->\n        word.Search_result.found_word_pos\n        |> (fun pos -> Index.loc_of_pos ~doc_id pos)\n        |> Index.Loc.line_loc\n        |> Index.Line_loc.page_num\n      )\n    |> List.sort_uniq Int.compare\n  in\n  let frequency_of_word_of_page_ci : int String_map.t Int_map.t =\n    List.fold_left (fun acc page_num ->\n        let m = Misc_utils.frequencies_of_words_ci\n            (Index.words_of_page_num ~doc_id page_num\n             |> Dynarray.to_seq)\n        in\n        Int_map.add page_num m acc\n      )\n      Int_map.empty\n      page_nums\n  in\n  found_phrase\n  |> List.map (fun word ->\n      let page_num =\n        Index.loc_of_pos ~doc_id word.Search_result.found_word_pos\n        |> Index.Loc.line_loc\n        |> Index.Line_loc.page_num\n      in\n      let m = Int_map.find page_num frequency_of_word_of_page_ci in\n      let freq =\n        String_map.fold (fun word_on_page_ci freq acc_freq ->\n            if\n              CCString.find ~sub:word.Search_result.found_word_ci word_on_page_ci >= 0\n            then (\n              acc_freq + freq\n            ) else (\n              acc_freq\n            )\n          )\n          m\n          0\n      in\n      (word, page_num, freq)\n    )\n  |> List.fold_left (fun best x ->\n      let (_x_word, _x_page_num, x_freq) = x in\n      match best with\n      | None -> Some x\n      | Some (_best_word, _best_page_num, best_freq) -> (\n          if x_freq < best_freq then\n            Some x\n          else\n            best\n        )\n    )\n    None\n  |> Option.get\n  |> (fun (word, page_num, _freq) ->\n      (word.found_word, page_num))\n\nlet pdf_config_and_cmd ~path ~doc_id_and_search_result : Config.t * string =\n  let config =\n    let page_num, search_word =\n      match doc_id_and_search_result with\n      | None -> (\n          (1, \"\")\n        )\n      | Some (doc_id, search_result) -> (\n          let found_phrase = Search_result.found_phrase search_result in\n          let (most_unique_word, most_unique_word_page_num) =\n            compute_most_unique_word_and_residing_page_num ~doc_id found_phrase\n          in\n          let page_num = most_unique_word_page_num + 1 in\n          (page_num, most_unique_word)\n        )\n    in\n    Config.make ~path ~page_num ~search_word ~launch_mode:`Detached ()\n  in\n  let cmd =\n    match Params.os_typ with\n    | `Linux -> (\n        match Xdg_utils.default_desktop_file_path `PDF with\n        | None -> fallback_cmd\n        | Some viewer_desktop_file_path -> (\n            let flatpak_package_name =\n              let s = Filename.basename viewer_desktop_file_path in\n              Option.value ~default:s\n                (CCString.chop_suffix ~suf:\".desktop\" s)\n            in\n            let viewer_desktop_file_path_lowercase_ascii =\n              String.lowercase_ascii viewer_desktop_file_path\n            in\n            let contains sub =\n              CCString.find ~sub viewer_desktop_file_path_lowercase_ascii >= 0\n            in\n            let make_command name args =\n              if contains \"flatpak\" then\n                Fmt.str \"flatpak run %s %s\" flatpak_package_name args\n              else\n                Fmt.str \"%s %s\" name args\n            in\n            match doc_id_and_search_result with\n            | None -> fallback_cmd\n            | Some _ -> (\n                if contains \"okular\" then\n                  make_command \"okular\"\n                    \"--page {page_num} --find {search_word} {path}\"\n                else if contains \"evince\" then\n                  make_command \"evince\"\n                    \"--page-index {page_num} --find {search_word} {path}\"\n                else if contains \"xreader\" then\n                  make_command \"xreader\"\n                    \"--page-index {page_num} --find {search_word} {path}\"\n                else if contains \"atril\" then\n                  make_command \"atril\"\n                    \"--page-index {page_num} --find {search_word} {path}\"\n                else if contains \"zathura\" then\n                  (* Check zathura before mupdf as desktop file for\n                     zathura might be `org.pwmt.zathura-pdf-mupdf.desktop`\n                  *)\n                  make_command \"zathura\"\n                    \"--page {page_num} --find {search_word} {path}\"\n                else if contains \"mupdf\" then\n                  make_command \"mupdf\" \"{path} {page_num}\"\n                else\n                  fallback_cmd\n              )\n          )\n      )\n    | `Darwin -> fallback_cmd\n  in\n  (config, cmd)\n\nlet config_and_cmd_to_open_text_file ~path ?(line_num = 1) () : Config.t * string =\n  let editor = !Params.text_editor in\n  let config =\n    Config.make ~path ~line_num ~launch_mode:`Terminal ()\n  in\n  let cmd =\n    match Filename.basename editor with\n    | \"nano\" ->\n      Fmt.str \"%s +{line_num} {path}\" editor\n    | \"nvim\" | \"vim\" | \"vi\" ->\n      Fmt.str \"%s +{line_num} {path}\" editor\n    | \"kak\" ->\n      Fmt.str \"%s +{line_num} {path}\" editor\n    | \"hx\" ->\n      Fmt.str \"%s {path}:{line_num}\" editor\n    | \"emacs\" ->\n      Fmt.str \"%s +{line_num} {path}\" editor\n    | \"micro\" ->\n      Fmt.str \"%s {path}:{line_num}\" editor\n    | \"jed\" | \"xjed\" ->\n      Fmt.str \"%s {path} -g {line_num}\" editor\n    | _ ->\n      Fmt.str \"%s {path}\" editor\n  in\n  (config, cmd)\n\nlet text_config_and_cmd ~path ~doc_id_and_search_result : Config.t * string =\n  let line_num =\n    match doc_id_and_search_result with\n    | None -> None\n    | Some (doc_id, search_result) -> (\n        let first_word = List.hd @@ Search_result.found_phrase search_result in\n        let first_word_loc =\n          Index.loc_of_pos ~doc_id first_word.Search_result.found_word_pos\n        in\n        first_word_loc\n        |> Index.Loc.line_loc\n        |> Index.Line_loc.line_num_in_page\n        |> (fun x -> x + 1)\n        |> Option.some\n      )\n  in\n  config_and_cmd_to_open_text_file\n    ~path\n    ?line_num\n    ()\n\nlet main ~close_term ~path ~doc_id_and_search_result =\n  let ext = File_utils.extension_of_file path in\n  let config, cmd =\n    (match File_utils.format_of_file path with\n     | `PDF -> (\n         pdf_config_and_cmd ~path ~doc_id_and_search_result\n       )\n     | `Pandoc_supported_format -> (\n         pandoc_supported_format_config_and_cmd ~path\n       )\n     | `Text -> (\n         text_config_and_cmd ~path ~doc_id_and_search_result\n       )\n     | `Other -> (\n         (Config.make ~path ~launch_mode:`Detached (), fallback_cmd)\n       )\n    )\n    |> (fun (config, cmd) ->\n        match Hashtbl.find_opt specs ext with\n        | None -> (\n            (config, cmd)\n          )\n        | Some (launch_mode, cmd) -> (\n            ({ config with launch_mode }, cmd)\n          )\n      )\n    |> (fun (config, cmd) ->\n        (config, Result.get_ok (resolve_cmd config cmd)))\n  in\n  match config.launch_mode with\n  | `Terminal -> (\n      let cmd =\n        if Misc_utils.stdin_is_atty () then (\n          cmd\n        ) else (\n          Fmt.str \"</dev/tty %s\" cmd\n        )\n      in\n      close_term ();\n      do_if_debug (fun oc ->\n          Printf.fprintf oc \"System command: %s\\n\" cmd\n        );\n      Sys.command cmd |> ignore\n    )\n  | `Detached -> (\n      do_if_debug (fun oc ->\n          Printf.fprintf oc \"System command: %s\\n\" cmd\n        );\n      Proc_utils.run_in_background cmd |> ignore\n    )\n\nlet find_project_root path =\n  let rec aux arr =\n    let cur = CCString.concat_seq ~sep:Filename.dir_sep (Dynarray.to_seq arr) in\n    if Dynarray.length arr = 0 then (\n      None\n    ) else if Dynarray.length arr = 3\n           && Dynarray.get arr 0 = \"home\"\n    then (\n      Some cur\n    ) else (\n      let candidates =\n        try\n          Some (Sys.readdir cur)\n        with\n        | _ -> None\n      in\n      match candidates with\n      | None -> (\n          None\n        )\n      | Some candidates -> (\n          let root_indicator_exists =\n            Array.exists (fun name ->\n                List.mem name\n                  [ \".git\"\n                  ; \".hg\"\n                  ; \".svn\"\n                  ; \".obsidian\"\n                  ; \".logseq\"\n                  ; \".tangent\"\n                  ]\n              )\n              candidates\n          in\n          if root_indicator_exists then (\n            Some cur\n          ) else (\n            Dynarray.pop_last arr |> ignore;\n            aux arr\n          )\n        )\n    )\n  in\n  let arr = Dynarray.of_list (CCString.split ~by:Filename.dir_sep path) in\n  aux arr\n\nlet open_link ~close_term ~doc link =\n  let { Link.typ; link; _ } = link in\n  let doc_path = Document.path doc in\n  let doc_dir = Filename.dirname doc_path in\n  let doc_ext = Filename.extension doc_path in\n  let resolve_wiki_link link =\n    let link =\n      Option.value ~default:link\n        (CCString.chop_prefix ~pre:\"/\" link)\n    in\n    let link_with_ext = link ^ doc_ext in\n    if link.[0] = '.' then (\n      Filename.concat doc_dir link_with_ext\n    ) else (\n      let wiki_root =\n        Option.value ~default:doc_dir (find_project_root doc_dir)\n      in\n      let candidates = File_utils.list_files_recursive\n          ~report_progress:(fun () -> ())\n          ~filter:(fun _depth path ->\n              let path_no_ext =\n                try\n                  Filename.chop_extension path\n                with\n                | _ -> path\n              in\n              CCString.suffix ~suf:link path_no_ext\n            )\n          wiki_root\n      in\n      match\n        String_set.find_first_opt (fun path ->\n            CCString.suffix ~suf:link_with_ext path\n          ) candidates\n      with\n      | Some x -> x\n      | None -> (\n          match String_set.min_elt_opt candidates with\n          | Some x -> x\n          | None -> Filename.concat wiki_root link_with_ext\n        )\n    )\n  in\n  if String.length link > 0 then (\n    match typ with\n    | `Markdown -> (\n        let path =\n          if Filename.is_relative link then (\n            Filename.concat doc_dir link\n          ) else (\n            let project_root =\n              Option.value ~default:doc_dir (find_project_root doc_dir)\n            in\n            Filename.concat project_root link\n          )\n        in\n        main ~close_term ~path ~doc_id_and_search_result:None\n      )\n    | `Wiki -> (\n        let path = resolve_wiki_link link in\n        main ~close_term ~path ~doc_id_and_search_result:None\n      )\n    | `URL -> (\n        let config = Config.make ~path:link ~launch_mode:`Detached () in\n        resolve_cmd config fallback_cmd\n        |> Result.get_ok\n        |> Proc_utils.run_in_background\n        |> ignore\n      )\n  )\n"
  },
  {
    "path": "bin/ping.ml",
    "content": "type t = {\n  queue : unit Eio.Stream.t;\n}\n\nlet make () =\n  {\n    queue = Eio.Stream.create Int.max_int;\n  }\n\nlet ping (t : t) =\n  Eio.Stream.add t.queue ()\n\nlet wait (t : t) =\n  Eio.Stream.take t.queue;\n  Misc_utils.drain_eio_stream t.queue\n\nlet clear (t : t) =\n  Misc_utils.drain_eio_stream t.queue\n"
  },
  {
    "path": "bin/ping.mli",
    "content": "type t\n\nval make : unit -> t\n\nval ping : t -> unit\n\nval wait : t -> unit\n\nval clear : t -> unit\n"
  },
  {
    "path": "bin/printers.ml",
    "content": "let output_image ~color (oc : out_channel) (img : Notty.image) : unit =\n  let open Notty in\n  let buf = Buffer.create 1024 in\n  let cap =\n    if color then\n      Cap.ansi\n    else\n      Cap.dumb\n  in\n  Render.to_buffer buf cap (0, 0) (I.width img, I.height img) img;\n  Buffer.output_buffer oc buf\n\nlet newline_image oc =\n  Notty_unix.eol (Notty.I.void 0 1)\n  |> output_image ~color:false oc\n\nlet path_image ~color oc path =\n  Notty.I.string Notty.A.(fg magenta) path\n  |> Notty_unix.eol\n  |> output_image ~color oc\n\nlet search_result_group ~color ~underline (oc : out_channel) ((document, results) : Session.search_result_group) =\n  let path = Document.path document in\n  path_image ~color oc path;\n  Array.iteri (fun i search_result ->\n      if i > 0 then (\n        newline_image oc\n      );\n      let img =\n        Content_and_search_result_rendering.search_result\n          ~doc_id:(Document.doc_id document)\n          ~render_mode:(UI_base.render_mode_of_document document)\n          ~width:!Params.search_result_print_text_width\n          ~underline\n          ~fill_in_context:true\n          search_result\n      in\n      Notty_unix.eol img\n      |> output_image ~color oc;\n    ) results\n\nlet search_result_groups\n    ~color\n    ~underline\n    (oc : out_channel)\n    (s : Session.search_result_group Seq.t)\n  =\n  Seq.iteri (fun i x ->\n      if i > 0 then (\n        newline_image oc;\n      );\n      search_result_group ~color ~underline oc x\n    ) s\n"
  },
  {
    "path": "bin/proc_utils.ml",
    "content": "open Misc_utils\n\nlet command_exists (cmd : string) : bool =\n  Sys.command (Fmt.str \"command -v %s 2>/dev/null 1>/dev/null\" (Filename.quote cmd)) = 0\n\nlet run_in_background (cmd : string) =\n  Sys.command (Fmt.str \"%s 2>/dev/null 1>/dev/null &\" cmd)\n\nlet run_return_stdout\n    ~proc_mgr\n    ~fs\n    ~(split_mode : [ `On_line_split | `On_form_feed ])\n    (cmd : string list)\n  : string list option =\n  let form_feed = Char.chr 0x0C in\n  Eio.Path.(with_open_out\n              ~create:`Never\n              (fs / \"/dev/null\"))\n    (fun stderr ->\n       let output =\n         try\n           let lines =\n             Eio.Process.parse_out proc_mgr\n               (match split_mode with\n                | `On_line_split -> Eio.Buf_read.(map List.of_seq lines)\n                | `On_form_feed -> (\n                    let p =\n                      let open Eio.Buf_read in\n                      let open Syntax in\n                      let* c = peek_char in\n                      (match c with\n                       | None -> return ()\n                       | Some c -> (\n                           if c = form_feed then (\n                             skip 1\n                           ) else (\n                             return ()\n                           )\n                         ))\n                      *>\n                      (take_while (fun c -> c <> form_feed))\n                    in\n                    Eio.Buf_read.(map List.of_seq (seq p))\n                  )\n               )\n               ~stderr cmd\n           in\n           Some lines\n         with\n         | _ -> None\n       in\n       output\n    )\n\nlet pipe_to_command (f : out_channel -> unit) command args =\n  if not (command_exists command) then (\n    exit_with_error_msg\n      (Fmt.str \"command %s not found\" command)\n  );\n  let oc =\n    Unix.open_process_args_out\n      command (Array.append [|command|] args)\n  in\n  f oc;\n  Out_channel.flush oc;\n  Out_channel.close oc\n"
  },
  {
    "path": "bin/result_syntax.ml",
    "content": "let ( let* ) = Result.bind\n\nlet ( let+ ) x y = Result.map y x\n"
  },
  {
    "path": "bin/script.ml",
    "content": "let run pool ~init_state ~path\n  : (Session.Snapshot.t Dynarray.t, string) result =\n  let exception Error_with_msg of string in\n  let snapshots = Dynarray.create () in\n  try\n    let lines =\n      try\n        CCIO.with_in path CCIO.read_lines_l\n      with\n      | Sys_error _ -> (\n          raise (Error_with_msg (Fmt.str \"failed to read script %s\" (Filename.quote path)))\n        )\n    in\n    Dynarray.add_last\n      snapshots\n      (Session.Snapshot.make\n         ~last_command:None\n         init_state);\n    lines\n    |> CCList.foldi (fun state i line ->\n        let line_num_in_error_msg = i + 1 in\n        if String_utils.line_is_blank_or_system_comment line then (\n          state\n        ) else (\n          match Command.of_string line with\n          | None -> (\n              raise (Error_with_msg\n                       (Fmt.str \"failed to parse command on line %d: %s\"\n                          line_num_in_error_msg line))\n            )\n          | Some command -> (\n              match Session.run_command pool command state with\n              | None -> (\n                  raise (Error_with_msg\n                           (Fmt.str \"failed to run command on line %d: %s\"\n                              line_num_in_error_msg line))\n                )\n              | Some (command, state) -> (\n                  let snapshot =\n                    Session.Snapshot.make\n                      ~last_command:(Some command)\n                      state\n                  in\n                  Dynarray.add_last snapshots snapshot;\n                  state\n                )\n            )\n        )\n      ) init_state\n    |> ignore;\n    Ok snapshots\n  with\n  | Error_with_msg msg -> Error msg\n"
  },
  {
    "path": "bin/search_mode.ml",
    "content": "type t = [\n  | `Single_line\n  | `Multiline\n]\n"
  },
  {
    "path": "bin/session.ml",
    "content": "open Docfd_lib\n\ntype search_result_group = Document.t * Search_result.t array\n\nmodule State = struct\n  module Sort_by = struct\n    type typ = [\n      | `Path_date\n      | `Path\n      | `Score\n      | `Mod_time\n      | `Ranking of int String_map.t\n    ]\n\n    type t = typ * Document.Compare.order\n  end\n\n  type t = {\n    all_documents : Document.t String_map.t;\n    filter_exp : Filter_exp.t;\n    filter_exp_string : string;\n    documents_passing_filter : String_set.t;\n    documents_marked : String_set.t;\n    search_exp : Search_exp.t;\n    search_exp_string : string;\n    search_results : Search_result.t array String_map.t;\n    sort_by : Sort_by.t;\n    sort_by_no_score : Sort_by.t;\n    screen_split : Command.screen_split;\n    show_bottom_right_pane : bool;\n    show_key_binding_info_pane : bool;\n    focus_list : string list;\n    path_highlights : Int_set.t String_map.t;\n  }\n\n  let equal (x : t) (y : t) =\n    String_map.equal Document.equal x.all_documents y.all_documents\n    &&\n    String.equal x.filter_exp_string y.filter_exp_string\n    &&\n    String_set.equal x.documents_passing_filter y.documents_passing_filter\n    &&\n    String_set.equal x.documents_marked y.documents_marked\n    &&\n    String.equal x.search_exp_string y.search_exp_string\n    &&\n    String_map.equal\n      (Array.for_all2 Search_result.equal)\n      x.search_results\n      y.search_results\n\n  let size (t : t) =\n    String_map.cardinal t.all_documents\n\n  let empty : t =\n    {\n      all_documents = String_map.empty;\n      filter_exp = Filter_exp.empty;\n      filter_exp_string = \"\";\n      documents_passing_filter = String_set.empty;\n      documents_marked = String_set.empty;\n      search_exp = Search_exp.empty;\n      search_exp_string = \"\";\n      search_results = String_map.empty;\n      sort_by = Command.Sort_by.default\n        |> (fun (typ, order) -> ((typ :> Sort_by.typ), order));\n      sort_by_no_score = Command.Sort_by.default_no_score\n        |> (fun (typ, order) -> ((typ :> Sort_by.typ), order));\n      screen_split = `Even;\n      show_bottom_right_pane = true;\n      show_key_binding_info_pane = true;\n      focus_list = [];\n      path_highlights = String_map.empty;\n    }\n\n  let filter_exp (t : t) = t.filter_exp\n\n  let filter_exp_string (t : t) = t.filter_exp_string\n\n  let search_exp (t : t) = t.search_exp\n\n  let search_exp_string (t : t) = t.search_exp_string\n\n  let path_highlights (t : t) = t.path_highlights\n\n  let clear_path_highlights (t : t) =\n    { t with path_highlights = String_map.empty }\n\n  let screen_split (t : t) = t.screen_split\n\n  let show_pane (t : t) (pane : Command.pane) =\n    match pane with\n    | `Bottom_right -> t.show_bottom_right_pane\n    | `Key_binding_info -> t.show_key_binding_info_pane\n\n  let refresh_search_results pool stop_signal (t : t) : t option =\n    let cancellation_notifier = Atomic.make false in\n    let updates =\n      Eio.Fiber.first\n        (fun () ->\n           Stop_signal.await stop_signal;\n           Atomic.set cancellation_notifier true;\n           String_map.empty)\n        (fun () ->\n           let global_first_word_candidates_lookup =\n             Index.generate_global_first_word_candidates_lookup\n               pool\n               t.search_exp\n           in\n           let usable_doc_ids =\n             let bv = CCBV.empty () in\n             Search_phrase.Enriched_token.Data_map.iter\n               (fun _word word_ids ->\n                  Int_set.iter (fun word_id ->\n                      Index.State.union_doc_ids_of_word_id_into_bv ~word_id ~into:bv\n                    )\n                    word_ids\n               )\n               global_first_word_candidates_lookup;\n             bv\n           in\n           let documents_to_search_through =\n             t.documents_passing_filter\n             |> String_set.to_seq\n             |> Seq.map (fun path -> (path, String_map.find path t.all_documents))\n             |> Seq.filter (fun (path, doc) ->\n                 Option.is_none (String_map.find_opt path t.search_results)\n                 && CCBV.get usable_doc_ids (Int64.to_int @@ Document.doc_id doc)\n               )\n             |> List.of_seq\n           in\n           documents_to_search_through\n           |> Task_pool.map_list pool (fun (path, doc) ->\n               let within_same_line =\n                 match Document.search_mode doc with\n                 | `Single_line -> true\n                 | `Multiline -> false\n               in\n               Index.make_search_job_groups\n                 stop_signal\n                 ~cancellation_notifier\n                 ~doc_id:(Document.doc_id doc)\n                 ~doc_word_ids:(Document.word_ids doc)\n                 ~global_first_word_candidates_lookup\n                 ~within_same_line\n                 ~search_scope:(Document.search_scope doc)\n                 t.search_exp\n               |> Seq.map (fun x -> (path, x))\n               |> List.of_seq\n             )\n           |> List.concat\n           |> Task_pool.map_list pool (fun (path, search_job_group) ->\n               let heap = Index.Search_job_group.run search_job_group in\n               (path, heap)\n             )\n           |> List.fold_left (fun acc (path, heap) ->\n               Eio.Fiber.yield ();\n               let heap =\n                 String_map.find_opt path acc\n                 |> Option.value ~default:Search_result_heap.empty\n                 |> Search_result_heap.merge heap\n               in\n               String_map.add path heap acc\n             )\n             String_map.empty\n           |> String_map.map (fun v ->\n               Eio.Fiber.yield ();\n               let arr =\n                 Search_result_heap.to_seq v\n                 |> Array.of_seq\n               in\n               Array.sort Search_result.compare_relevance arr;\n               arr\n             )\n        )\n    in\n    if Atomic.get cancellation_notifier then (\n      None\n    ) else (\n      let search_results =\n        String_map.union (fun _k v1 _v2 -> Some v1)\n          updates\n          t.search_results\n      in\n      Some { t with search_results }\n    )\n\n  let update_filter_exp\n      pool\n      stop_signal\n      filter_exp_string\n      filter_exp\n      (t : t)\n    : t option =\n    if Filter_exp.equal filter_exp t.filter_exp then (\n      Some { t with filter_exp_string }\n    ) else (\n      let cancellation_notifier = Atomic.make false in\n      let documents_passing_filter =\n        Eio.Fiber.first\n          (fun () ->\n             Stop_signal.await stop_signal;\n             Atomic.set cancellation_notifier true;\n             String_set.empty\n          )\n          (fun () ->\n             let global_first_word_candidates_lookup =\n               Filter_exp.all_content_search_exps filter_exp\n               |> List.fold_left (fun acc search_exp ->\n                   Index.generate_global_first_word_candidates_lookup\n                     pool\n                     ~acc\n                     search_exp\n                 )\n                 Search_phrase.Enriched_token.Data_map.empty\n             in\n             t.all_documents\n             |> String_map.to_seq\n             |> Seq.map snd\n             |> (fun s ->\n                 if Filter_exp.is_empty filter_exp then (\n                   s\n                 ) else (\n                   Seq.filter (fun s ->\n                       Eio.Fiber.yield ();\n                       Document.satisfies_filter_exp\n                         pool\n                         stop_signal\n                         ~global_first_word_candidates_lookup\n                         filter_exp\n                         s\n                     ) s\n                 )\n               )\n             |> Seq.map Document.path\n             |> String_set.of_seq\n          )\n      in\n      if Atomic.get cancellation_notifier then (\n        None\n      ) else (\n        { t with\n          filter_exp_string;\n          filter_exp;\n          documents_passing_filter;\n        }\n        |> refresh_search_results pool stop_signal\n      )\n    )\n\n  let update_search_exp pool stop_signal search_exp_string search_exp (t : t) : t option =\n    if Search_exp.equal search_exp t.search_exp then (\n      Some { t with search_exp_string }\n    ) else (\n      { t with\n        search_exp;\n        search_exp_string;\n        search_results = String_map.empty;\n      }\n      |> refresh_search_results pool stop_signal\n    )\n\n  let add_document pool (doc : Document.t) (t : t) : t =\n    let within_same_line =\n      match Document.search_mode doc with\n      | `Single_line -> true\n      | `Multiline -> false\n    in\n    let path = Document.path doc in\n    let documents_passing_filter =\n      let global_first_word_candidates_lookup =\n        Filter_exp.all_content_search_exps t.filter_exp\n        |> List.fold_left (fun acc search_exp ->\n            Index.generate_global_first_word_candidates_lookup\n              pool\n              ~acc\n              search_exp\n          )\n          Search_phrase.Enriched_token.Data_map.empty\n      in\n      if Document.satisfies_filter_exp pool (Stop_signal.make ()) ~global_first_word_candidates_lookup t.filter_exp doc\n      then\n        String_set.add path t.documents_passing_filter\n      else\n        t.documents_passing_filter\n    in\n    let search_results =\n      let global_first_word_candidates_lookup =\n        Index.generate_global_first_word_candidates_lookup\n          pool\n          t.search_exp\n      in\n      String_map.add\n        path\n        (Index.search\n           pool\n           (Stop_signal.make ())\n           ~doc_id:(Document.doc_id doc)\n           ~doc_word_ids:(Document.word_ids doc)\n           ~global_first_word_candidates_lookup\n           ~within_same_line\n           ~search_scope:(Document.search_scope doc)\n           t.search_exp\n         |> Option.get\n        )\n        t.search_results\n    in\n    { t with\n      all_documents =\n        String_map.add\n          path\n          doc\n          t.all_documents;\n      documents_passing_filter;\n      search_results;\n    }\n\n  let of_seq pool (s : Document.t Seq.t) =\n    Seq.fold_left (fun t doc ->\n        add_document pool doc t\n      )\n      empty\n      s\n\n  module Compare_search_result_group = struct\n    let mod_time order (d0, _s0) (d1, _s1) =\n      Document.Compare.mod_time order d0 d1\n\n    let path_date order (d0, _s0) (d1, _s1) =\n      Document.Compare.path_date order d0 d1\n\n    let path order (d0, _s0) (d1, _s1) =\n      Document.Compare.path order d0 d1\n\n    let ranking ranking order (d0, _s0) (d1, _s1) =\n      match\n        String_map.find_opt (Document.path d0) ranking,\n        String_map.find_opt (Document.path d1) ranking\n      with\n      | None, None -> Document.Compare.path order d0 d1\n      | None, Some _ -> (\n          (* Always shuffle document with no ranking to the back. *)\n          1\n        )\n      | Some _, None -> (\n          (* Always shuffle document with no ranking to the back. *)\n          -1\n        )\n      | Some x0, Some x1 -> (\n          match order with\n          | `Asc -> Int.compare x0 x1\n          | `Desc -> Int.compare x1 x0\n        )\n\n    let score order (_d0, s0) (_d1, s1) =\n      assert (Array.length s0 > 0);\n      assert (Array.length s1 > 0);\n      (* Search_result.compare_relevance puts the more relevant\n         result to the front, so we flip the comparison here to\n         obtain an ordering of \"lowest score\" first to match the\n         usual definition of \"sort by score in ascending order\".\n      *)\n      match order with\n      | `Asc -> Search_result.compare_relevance s1.(0) s0.(0)\n      | `Desc -> Search_result.compare_relevance s0.(0) s1.(0)\n  end\n\n  let search_result_groups\n      (t : t)\n    : (Document.t * Search_result.t array) array =\n    let no_search_exp = Search_exp.is_empty t.search_exp in\n    let arr =\n      t.documents_passing_filter\n      |> String_set.to_seq\n      |> Seq.map (fun path ->\n          (path, String_map.find path t.all_documents)\n        )\n      |> (fun s ->\n          if no_search_exp then (\n            Seq.map (fun (_path, doc) -> (doc, [||])) s\n          ) else (\n            Seq.filter_map (fun (path, doc) ->\n                match String_map.find_opt path t.search_results with\n                | None -> None\n                | Some search_results -> (\n                    if Array.length search_results = 0 then\n                      None\n                    else\n                      Some (doc, search_results)\n                  )\n              ) s\n          )\n        )\n      |> Array.of_seq\n    in\n    let rec f (sort_by_typ, sort_by_order) =\n      match sort_by_typ with\n      | `Path_date -> Compare_search_result_group.path_date sort_by_order\n      | `Mod_time -> Compare_search_result_group.mod_time sort_by_order\n      | `Path -> Compare_search_result_group.path sort_by_order\n      | `Score -> (\n          if no_search_exp then (\n            f t.sort_by_no_score\n          ) else (\n            Compare_search_result_group.score sort_by_order\n          )\n        )\n      | `Ranking ranking -> (\n          Compare_search_result_group.ranking ranking sort_by_order\n        )\n    in\n    Array.sort (f t.sort_by) arr;\n    let focus_ranking =\n      List.rev t.focus_list\n      |> CCList.foldi (fun ranking i x ->\n          String_map.add x i ranking) String_map.empty\n    in\n    Array.stable_sort (fun (d0, _) (d1, _) ->\n        match\n          String_map.find_opt (Document.path d0) focus_ranking,\n          String_map.find_opt (Document.path d1) focus_ranking\n        with\n        | Some x0, Some x1 -> Int.compare x0 x1\n        | Some _, None -> -1\n        | None, Some _ -> 1\n        | None, None -> 0\n      ) arr;\n    arr\n\n  let usable_document_paths (t : t) : String_set.t =\n    search_result_groups t\n    |> Array.to_seq\n    |> Seq.map (fun (doc, _) -> Document.path doc)\n    |> String_set.of_seq\n\n  let marked_document_paths (t : t) =\n    t.documents_marked\n\n  let all_document_paths (t : t) : string Seq.t =\n    t.all_documents\n    |> String_map.to_seq\n    |> Seq.map fst\n\n  let unusable_documents (t : t) =\n    let s = usable_document_paths t in\n    t.all_documents\n    |> String_map.to_seq\n    |> Seq.filter (fun (path, _doc) ->\n        not (String_set.mem path s))\n    |> Seq.map snd\n\n  let unusable_document_paths (t : t) =\n    unusable_documents t\n    |> Seq.map Document.path\n\n  let mark\n      (choice :\n         [ `Path of string\n         | `Usable\n         | `Unusable ])\n      t\n    : t =\n    match choice with\n    | `Path path -> (\n        match String_map.find_opt path t.all_documents with\n        | None -> t\n        | Some _ -> (\n            let documents_marked =\n              String_set.add path t.documents_marked\n            in\n            { t with documents_marked }\n          )\n      )\n    | `Usable -> (\n        let documents_marked =\n          String_set.union\n            t.documents_marked\n            (usable_document_paths t)\n        in\n        { t with documents_marked }\n      )\n    | `Unusable -> (\n        let documents_marked =\n          Seq.fold_left\n            (fun acc x -> String_set.add x acc)\n            t.documents_marked\n            (unusable_document_paths t)\n        in\n        { t with documents_marked }\n      )\n\n  let unmark\n      (choice :\n         [ `Path of string\n         | `Usable\n         | `Unusable\n         | `All ])\n      t\n    : t =\n    match choice with\n    | `Path path -> (\n        let documents_marked =\n          String_set.remove path t.documents_marked\n        in\n        { t with documents_marked }\n      )\n    | `Usable -> (\n        let documents_marked =\n          String_set.diff\n            t.documents_marked\n            (usable_document_paths t)\n        in\n        { t with documents_marked }\n      )\n    | `Unusable -> (\n        let documents_marked =\n          Seq.fold_left\n            (fun acc x -> String_set.remove x acc)\n            t.documents_marked\n            (unusable_document_paths t)\n        in\n        { t with documents_marked }\n      )\n    | `All -> (\n        { t with documents_marked = String_set.empty }\n      )\n\n  let drop\n      (choice :\n         [ `Path of string\n         | `All_except of string\n         | `Marked\n         | `Unmarked\n         | `Usable\n         | `Unusable ])\n      (t : t)\n    : t =\n    let aux ~(keep : string -> bool) =\n      let keep' : 'a. string -> 'a -> bool =\n        fun path _ ->\n        keep path\n      in\n      { all_documents = String_map.filter keep' t.all_documents;\n        filter_exp = t.filter_exp;\n        filter_exp_string = t.filter_exp_string;\n        documents_passing_filter = String_set.filter keep t.documents_passing_filter;\n        documents_marked = String_set.filter keep t.documents_marked;\n        search_exp = t.search_exp;\n        search_exp_string = t.search_exp_string;\n        search_results = String_map.filter keep' t.search_results;\n        sort_by = t.sort_by;\n        sort_by_no_score = t.sort_by_no_score;\n        screen_split = t.screen_split;\n        show_bottom_right_pane = t.show_bottom_right_pane;\n        show_key_binding_info_pane = t.show_key_binding_info_pane;\n        focus_list = t.focus_list;\n        path_highlights = t.path_highlights;\n      }\n    in\n    match choice with\n    | `Path path -> (\n        { all_documents = String_map.remove path t.all_documents;\n          filter_exp = t.filter_exp;\n          filter_exp_string = t.filter_exp_string;\n          documents_passing_filter = String_set.remove path t.documents_passing_filter;\n          documents_marked = String_set.remove path t.documents_marked;\n          search_exp = t.search_exp;\n          search_exp_string = t.search_exp_string;\n          search_results = String_map.remove path t.search_results;\n          sort_by = t.sort_by;\n          sort_by_no_score = t.sort_by_no_score;\n          screen_split = t.screen_split;\n          show_bottom_right_pane = t.show_bottom_right_pane;\n          show_key_binding_info_pane = t.show_key_binding_info_pane;\n          focus_list = t.focus_list;\n          path_highlights = t.path_highlights;\n        }\n      )\n    | `All_except path -> (\n        let keep path' =\n          String.equal path' path\n        in\n        aux ~keep\n      )\n    | `Marked -> (\n        let keep path =\n          not (String_set.mem path t.documents_marked)\n        in\n        aux ~keep\n      )\n    | `Unmarked -> (\n        let keep path =\n          String_set.mem path t.documents_marked\n        in\n        aux ~keep\n      )\n    | `Usable | `Unusable -> (\n        let usable_document_paths =\n          usable_document_paths t\n        in\n        let document_is_usable path =\n          String_set.mem path usable_document_paths\n        in\n        let keep path =\n          match choice with\n          | `Usable -> not (document_is_usable path)\n          | `Unusable -> document_is_usable path\n          | _ -> failwith \"unexpected case\"\n        in\n        aux ~keep\n      )\n\n  let update_path_fuzzy_ranking stop_signal exp (t : t) : t option =\n    let cancellation_notifier = Atomic.make false in\n    let ranking, path_highlights =\n      Eio.Fiber.first\n        (fun () ->\n           Stop_signal.await stop_signal;\n           Atomic.set cancellation_notifier true;\n           (String_map.empty, String_map.empty)\n        )\n        (fun () ->\n           let l =\n             usable_document_paths t\n             |> String_set.to_seq\n             |> Seq.map File_utils.remove_cwd_from_path\n             |> Misc_utils.fuzzy_rank_assoc\n               (Stop_signal.make ())\n               ~get_key:Fun.id\n               exp\n             |> Dynarray.to_list\n             |> List.map (fun (path, x) ->\n                 (Misc_utils.normalize_path_to_absolute path, x))\n           in\n           let ranking =\n             List.map fst l\n             |> Misc_utils.ranking_of_ranked_document_list\n           in\n           let highlights =\n             List.fold_left (fun acc (path, search_result) ->\n                 String_map.add\n                   path\n                   (Misc_utils.highlights_of_search_result search_result)\n                   acc\n               )\n               String_map.empty\n               l\n           in\n           (ranking, highlights)\n        )\n    in\n    if Atomic.get cancellation_notifier then (\n      None\n    ) else (\n      let sort_by = (`Ranking ranking, `Asc) in\n      Some {\n        t with sort_by;\n               sort_by_no_score = sort_by;\n               path_highlights;\n      }\n    )\n\n  let narrow_search_scope_to_level ~level (t : t) : t =\n    let all_documents =\n      if level = 0 then (\n        String_map.mapi (fun _path doc ->\n            Document.reset_search_scope_to_full doc\n          )\n          t.all_documents\n      ) else (\n        String_map.mapi (fun path doc ->\n            let doc_id = Document.doc_id doc in\n            let search_scope =\n              match String_map.find_opt path t.search_results with\n              | None -> Diet.Int.empty\n              | Some search_results -> (\n                  if String_set.mem path t.documents_passing_filter then (\n                    Array.fold_left (fun scope search_result ->\n                        let s, e =\n                          List.fold_left (fun s_e Search_result.{ found_word_pos; _ } ->\n                              match s_e with\n                              | None -> Some (found_word_pos, found_word_pos)\n                              | Some (s, e) -> (\n                                  Some (min s found_word_pos, max found_word_pos e)\n                                )\n                            )\n                            None\n                            (Search_result.found_phrase search_result)\n                          |> Option.get\n                        in\n                        let offset = level * !Params.tokens_per_search_scope_level in\n                        let s, e =\n                          (max 0 (s - offset), min (Index.max_pos ~doc_id) (e + offset))\n                        in\n                        Diet.Int.add\n                          (Diet.Int.Interval.make s e)\n                          scope\n                      )\n                      Diet.Int.empty\n                      search_results\n                  ) else (\n                    Diet.Int.empty\n                  )\n                )\n            in\n            Document.inter_search_scope\n              search_scope\n              doc\n          )\n          t.all_documents\n      )\n    in\n    { t with all_documents }\nend\n\nlet run_command pool (command : Command.t) (st : State.t) : (Command.t * State.t) option =\n  let open State in\n  let reset_focus_list st = { st with focus_list = [] } in\n  match command with\n  | `Mark path -> (\n      Some (command, mark (`Path path) st)\n    )\n  | `Mark_listed -> (\n      Some (command, mark `Usable st)\n    )\n  | `Unmark path -> (\n      Some (command, unmark (`Path path) st)\n    )\n  | `Unmark_listed -> (\n      Some (command, unmark `Usable st)\n    )\n  | `Unmark_all -> (\n      Some (command, unmark `All st)\n    )\n  | `Drop s -> (\n      Some (command, drop (`Path s) st)\n    )\n  | `Drop_all_except s -> (\n      Some (command, drop (`All_except s) st)\n    )\n  | `Drop_marked -> (\n      Some (command, drop `Marked st)\n    )\n  | `Drop_unmarked -> (\n      Some (command, drop `Unmarked st)\n    )\n  | `Drop_listed -> (\n      Some (command, drop `Usable st)\n    )\n  | `Drop_unlisted -> (\n      Some (command, drop `Unusable st)\n    )\n  | `Narrow_level level -> (\n      Some (command, narrow_search_scope_to_level ~level st)\n    )\n  | `Focus path -> (\n      let focus_list = path :: st.focus_list in\n      Some (command, { st with focus_list })\n    )\n  | `Sort (sort_by, sort_by_no_score) -> (\n      let st = reset_focus_list st in\n      let sort_by =\n        sort_by\n        |> (fun (typ, order) -> ((typ :> Sort_by.typ), order))\n      in\n      let sort_by_no_score =\n        sort_by_no_score\n        |> (fun (typ, order) -> ((typ :> Sort_by.typ), order))\n      in\n      Some (command, { st with sort_by; sort_by_no_score })\n    )\n  | `Path_fuzzy_rank (s, ranking) -> (\n      let st = reset_focus_list st in\n      match Search_exp.parse s with\n      | None -> None\n      | Some exp -> (\n          update_path_fuzzy_ranking\n            (Stop_signal.make ())\n            exp\n            st\n          |> Option.map (fun state ->\n              let command = `Path_fuzzy_rank (s, ranking) in\n              (command, state)\n            )\n        )\n    )\n  | `Split_screen screen_split -> (\n      Some (command, { st with screen_split })\n    )\n  | `Hide_pane pane -> (\n      let st =\n        match pane with\n        | `Bottom_right -> { st with show_bottom_right_pane = false }\n        | `Key_binding_info -> { st with show_key_binding_info_pane = false }\n      in\n      Some (command, st)\n    )\n  | `Show_pane pane -> (\n      let st =\n        match pane with\n        | `Bottom_right -> { st with show_bottom_right_pane = true }\n        | `Key_binding_info -> { st with show_key_binding_info_pane = true }\n      in\n      Some (command, st)\n    )\n  | `Comment _ -> (\n      Some (command, st)\n    )\n  | `Search s -> (\n      match Search_exp.parse s with\n      | None -> None\n      | Some search_exp -> (\n          update_search_exp\n            pool\n            (Stop_signal.make ())\n            s\n            search_exp\n            st\n          |> Option.map (fun state -> (command, state))\n        )\n    )\n  | `Filter s -> (\n      match Filter_exp.parse s with\n      | None -> None\n      | Some exp -> (\n          update_filter_exp\n            pool\n            (Stop_signal.make ())\n            s\n            exp\n            st\n          |> Option.map (fun state -> (command, state))\n        )\n    )\n\nmodule Snapshot = struct\n  let counter = ref 0\n\n  type t = {\n    last_command : Command.t option;\n    state : State.t;\n    committed : bool;\n    id : int;\n  }\n\n  let committed t = t.committed\n\n  let last_command t = t.last_command\n\n  let state t = t.state\n\n  let id t = t.id\n\n  let equal_id x y =\n    id x = id y\n\n  let make ?(committed = true) ~last_command state : t =\n    let id = !counter in\n    counter := id + 1;\n    { last_command; state; id; committed }\n\n  let make_empty ?committed () =\n    make ?committed ~last_command:None State.empty\n\n  let update_state state t =\n    { t with state }\nend\n"
  },
  {
    "path": "bin/session.mli",
    "content": "open Docfd_lib\n\ntype search_result_group = Document.t * Search_result.t array\n\nmodule State : sig\n  type t\n\n  val equal : t -> t -> bool\n\n  val size : t -> int\n\n  val empty : t\n\n  val update_filter_exp :\n    Task_pool.t ->\n    Stop_signal.t ->\n    string ->\n    Filter_exp.t ->\n    t ->\n    t option\n\n  val update_search_exp :\n    Task_pool.t ->\n    Stop_signal.t ->\n    string ->\n    Search_exp.t ->\n    t ->\n    t option\n\n  val filter_exp : t -> Filter_exp.t\n\n  val filter_exp_string : t -> string\n\n  val search_exp : t -> Search_exp.t\n\n  val search_exp_string : t -> string\n\n  val path_highlights : t -> Int_set.t String_map.t\n\n  val clear_path_highlights : t -> t\n\n  val add_document : Task_pool.t -> Document.t -> t -> t\n\n  val of_seq : Task_pool.t -> Document.t Seq.t -> t\n\n  val search_result_groups : t -> search_result_group array\n\n  val usable_document_paths : t -> String_set.t\n\n  val unusable_documents : t -> Document.t Seq.t\n\n  val unusable_document_paths : t -> string Seq.t\n\n  val all_document_paths : t -> string Seq.t\n\n  val marked_document_paths : t -> String_set.t\n\n  val mark : [ `Path of string | `Usable | `Unusable ] -> t -> t\n\n  val unmark : [ `Path of string | `Usable | `Unusable | `All ] -> t -> t\n\n  val drop : [ `Path of string | `All_except of string | `Marked | `Unmarked | `Usable | `Unusable ] -> t -> t\n\n  val update_path_fuzzy_ranking : Stop_signal.t -> Search_exp.t -> t -> t option\n\n  val narrow_search_scope_to_level : level:int -> t -> t\n\n  val screen_split : t -> Command.screen_split\n\n  val show_pane : t -> Command.pane -> bool\nend\n\nval run_command : Task_pool.t -> Command.t -> State.t -> (Command.t * State.t) option\n\nmodule Snapshot : sig\n  type t\n\n  val committed : t -> bool\n\n  val last_command : t -> Command.t option\n\n  val state : t -> State.t\n\n  val id : t -> int\n\n  val equal_id : t -> t -> bool\n\n  val make : ?committed:bool -> last_command:Command.t option -> State.t -> t\n\n  val make_empty : ?committed:bool -> unit -> t\n\n  val update_state : State.t -> t -> t\nend\n"
  },
  {
    "path": "bin/session_manager.ml",
    "content": "open Docfd_lib\n\nlet last_request_timestamp : Mtime.t Atomic.t =\n  Atomic.make (Mtime_clock.now ())\n\nlet search_request : (bool * string) Lock_protected_cell.t =\n  Lock_protected_cell.make ()\n\nlet filter_request : (bool * string) Lock_protected_cell.t =\n  Lock_protected_cell.make ()\n\nlet version_shift_request : int Lock_protected_cell.t =\n  Lock_protected_cell.make ()\n\nlet path_fuzzy_rank_request : (bool * string) Lock_protected_cell.t =\n  Lock_protected_cell.make ()\n\nlet worker_ping : Ping.t = Ping.make ()\n\nlet _requester_lock = Eio.Mutex.create ()\n\nlet lock_as_requester : type a. (unit -> a) -> a =\n  fun f ->\n  Eio.Mutex.use_rw ~protect:false _requester_lock f\n\n(* let requester_ping : Ping.t = Ping.make () *)\n\ntype egress_payload =\n  | Search_exp_parse_error\n  | Searching\n  | Search_cancelled\n  | Search_done of int * Session.Snapshot.t\n  | Filter_parse_error\n  | Filtering\n  | Filtering_cancelled\n  | Filtering_done of int * Session.Snapshot.t\n  | Path_fuzzy_rank_done of int * Session.Snapshot.t * bool\n\nlet egress : egress_payload Eio.Stream.t =\n  Eio.Stream.create 0\n\nlet egress_ack : unit Eio.Stream.t =\n  Eio.Stream.create 0\n\nlet stop_filter_signal = Atomic.make (Stop_signal.make ())\n\nlet stop_search_signal = Atomic.make (Stop_signal.make ())\n\nlet stop_path_fuzzy_rank_signal = Atomic.make (Stop_signal.make ())\n\nlet stop_filter () =\n  let x = Atomic.exchange stop_filter_signal (Stop_signal.make ()) in\n  Stop_signal.broadcast x\n\nlet stop_search () =\n  let x = Atomic.exchange stop_search_signal (Stop_signal.make ()) in\n  Stop_signal.broadcast x\n\nlet stop_path_fuzzy_rank () =\n  let x = Atomic.exchange stop_path_fuzzy_rank_signal (Stop_signal.make ()) in\n  Stop_signal.broadcast x\n\nlet _worker_state_lock = Eio.Mutex.create ()\n\nlet lock_worker_state : type a. (unit -> a) -> a =\n  fun f ->\n  Eio.Mutex.use_rw ~protect:false _worker_state_lock f\n\nlet init_state : Session.State.t ref = ref Session.State.empty\n\nlet snapshots =\n  let arr = Dynarray.create () in\n  Dynarray.add_last arr (Session.Snapshot.make_empty ());\n  arr\n\nlet cur_ver = ref 0\n\nlet cur_snapshot_var = Lwd.var (0, Session.Snapshot.make_empty ())\n\nlet cur_snapshot = Lwd.get cur_snapshot_var\n\ntype view = {\n  init_state : Session.State.t;\n  snapshots : Session.Snapshot.t Dynarray.t;\n  cur_ver : int;\n}\n\nlet sync_input_fields_from_snapshot\n    (x : Session.Snapshot.t)\n  =\n  let state = Session.Snapshot.state x in\n  Session.State.filter_exp_string state\n  |> (fun s ->\n      Lwd.set UI_base.Vars.filter_field (s, String.length s));\n  Session.State.search_exp_string state\n  |> (fun s ->\n      Lwd.set UI_base.Vars.search_field (s, String.length s))\n\nlet lock_for_external_editing ~clean_up f =\n  (* This blocks further requests from being made. *)\n  lock_as_requester (fun () ->\n      (* We try to get worker to finish ASAP. *)\n      stop_filter ();\n      stop_search ();\n      (* Locking worker also locks the manager, as egress_ack forces\n         lock-step progression of the system.\n      *)\n      lock_worker_state (fun () ->\n          (* Clear any outstanding requests. *)\n          Lock_protected_cell.unset filter_request;\n          Lock_protected_cell.unset search_request;\n          let result = f () in\n          if clean_up then (\n            Lwd.set UI_base.Vars.search_ui_status `Idle;\n            Lwd.set UI_base.Vars.filter_ui_status `Idle;\n            let snapshot = Dynarray.get snapshots !cur_ver in\n            Lwd.set cur_snapshot_var (!cur_ver, snapshot);\n            sync_input_fields_from_snapshot snapshot;\n          );\n          result\n        )\n    )\n\nlet lock_with_view : type a. (view -> a) -> a =\n  fun f ->\n  lock_for_external_editing ~clean_up:false (fun () ->\n      f\n        {\n          init_state = !init_state;\n          snapshots = Dynarray.copy snapshots;\n          cur_ver = !cur_ver;\n        }\n    )\n\nlet update_starting_state (starting_state : Session.State.t) =\n  lock_for_external_editing ~clean_up:true (fun () ->\n      let pool = UI_base.task_pool () in\n      init_state := starting_state;\n      let starting_snapshot =\n        Session.Snapshot.make\n          ~last_command:None\n          starting_state\n      in\n      Dynarray.set snapshots 0 starting_snapshot;\n      for i=1 to Dynarray.length snapshots - 1 do\n        let prev = Dynarray.get snapshots (i - 1) in\n        let prev_state = Session.Snapshot.state prev in\n        let cur = Dynarray.get snapshots i in\n        let state =\n          match Session.Snapshot.last_command cur with\n          | None -> prev_state\n          | Some command ->\n            Session.run_command pool command prev_state\n            |> Option.map snd\n            |> Option.value ~default:prev_state\n        in\n        Dynarray.set\n          snapshots\n          i\n          (Session.Snapshot.update_state state cur)\n      done;\n      cur_ver := (Dynarray.length snapshots - 1);\n    )\n\nlet load_snapshots snapshots' =\n  assert (Dynarray.length snapshots' > 0);\n  lock_for_external_editing ~clean_up:true (fun () ->\n      assert\n        (Session.State.equal\n           (Session.Snapshot.state @@ Dynarray.get snapshots' 0)\n           !init_state);\n      Dynarray.clear snapshots;\n      Dynarray.append snapshots snapshots';\n      cur_ver := (Dynarray.length snapshots - 1);\n    )\n\nlet stop_filter_and_search_and_restore_input_fields () =\n  lock_for_external_editing ~clean_up:true (fun () ->\n      ()\n    )\n\nlet shift_ver ~offset =\n  lock_for_external_editing ~clean_up:true (fun () ->\n      let new_ver = !cur_ver + offset in\n      if 0 <= new_ver && new_ver < Dynarray.length snapshots then (\n        cur_ver := new_ver;\n      )\n    )\n\nlet update_from_cur_snapshot f =\n  lock_for_external_editing ~clean_up:true (fun () ->\n      Dynarray.truncate snapshots (!cur_ver + 1);\n      let next_snapshot = f (Dynarray.get_last snapshots) in\n      Dynarray.add_last snapshots next_snapshot;\n      cur_ver := Dynarray.length snapshots - 1;\n    )\n\nlet manager_fiber () =\n  (* This fiber handles updates of Lwd.var which are not thread-safe,\n     and thus cannot be done by worker_fiber directly\n  *)\n  let update_snapshot ?(reset_document_selected = true) ver snapshot =\n    if reset_document_selected then (\n      UI_base.reset_document_selected ();\n    );\n    Lwd.set cur_snapshot_var (ver, snapshot);\n  in\n  while true do\n    let payload = Eio.Stream.take egress in\n    (match payload with\n     | Search_exp_parse_error -> (\n         Lwd.set UI_base.Vars.search_ui_status `Parse_error\n       )\n     | Searching -> (\n         Lwd.set UI_base.Vars.search_ui_status `Searching\n       )\n     | Filtering -> (\n         Lwd.set UI_base.Vars.filter_ui_status `Filtering\n       )\n     | Search_cancelled -> (\n       )\n     | Search_done (ver, snapshot) -> (\n         update_snapshot ver snapshot;\n         Lwd.set UI_base.Vars.search_ui_status `Idle\n       )\n     | Filter_parse_error -> (\n         Lwd.set UI_base.Vars.filter_ui_status `Parse_error\n       )\n     | Filtering_cancelled -> (\n       )\n     | Filtering_done (ver, snapshot) -> (\n         update_snapshot ver snapshot;\n         Lwd.set UI_base.Vars.filter_ui_status `Idle\n       )\n     | Path_fuzzy_rank_done (ver, snapshot, commit) -> (\n         let snapshot =\n           if commit then (\n             let state =\n               Session.Snapshot.state snapshot\n               |> Session.State.clear_path_highlights\n             in\n             Session.Snapshot.update_state state snapshot\n           ) else (\n             snapshot\n           )\n         in\n         update_snapshot ~reset_document_selected:false ver snapshot;\n       )\n    );\n    Eio.Stream.add egress_ack ();\n  done\n\nlet worker_fiber pool =\n  (* This fiber runs in a background domain to allow the UI code in the main\n     domain to immediately continue running after key presses that trigger\n     searches or search cancellations.\n\n     This removes the need to make the code of session module always yield\n     frequently.\n  *)\n  let get_cur_snapshot () =\n    Dynarray.get snapshots !cur_ver\n  in\n  let add_snapshot\n      ?(overwrite_if_last_snapshot_satisfies = fun _ -> false)\n      snapshot\n    =\n    Dynarray.truncate snapshots (!cur_ver + 1);\n    let last_snapshot = Dynarray.get_last snapshots in\n    if !cur_ver > 0 && overwrite_if_last_snapshot_satisfies last_snapshot then (\n      Dynarray.set snapshots !cur_ver snapshot;\n    ) else (\n      Dynarray.add_last snapshots snapshot;\n      incr cur_ver;\n    );\n  in\n  let send_to_manager x =\n    Eio.Stream.add egress x;\n    Eio.Stream.take egress_ack;\n  in\n  let cancelled_search_request : (bool * string) option ref = ref None in\n  let process_search_req stop_signal ~commit (s : string) =\n    cancelled_search_request := None;\n    match Search_exp.parse s with\n    | None -> (\n        send_to_manager Search_exp_parse_error\n      )\n    | Some search_exp -> (\n        send_to_manager Searching;\n        let state =\n          get_cur_snapshot ()\n          |> Session.Snapshot.state\n          |> Session.State.update_search_exp\n            pool\n            stop_signal\n            s\n            search_exp\n        in\n        match state with\n        | None -> (\n            send_to_manager Search_cancelled;\n            cancelled_search_request := Some (commit, s);\n          )\n        | Some state -> (\n            let command = Some (`Search s) in\n            let snapshot =\n              Session.Snapshot.make\n                ~committed:commit\n                ~last_command:command\n                state\n            in\n            add_snapshot\n              ~overwrite_if_last_snapshot_satisfies:(fun snapshot ->\n                  match Session.Snapshot.last_command snapshot with\n                  | Some (`Search s') -> (\n                      not (Session.Snapshot.committed snapshot)\n                      ||\n                      s' = s\n                    )\n                  | _ -> false\n                )\n              snapshot;\n            send_to_manager (Search_done (!cur_ver, snapshot))\n          )\n      )\n  in\n  let process_filter_req stop_signal ~commit (s : string) =\n    match Filter_exp.parse s with\n    | Some filter_exp -> (\n        send_to_manager Filtering;\n        let state =\n          get_cur_snapshot ()\n          |> Session.Snapshot.state\n          |> Session.State.update_filter_exp\n            pool\n            stop_signal\n            s\n            filter_exp\n        in\n        match state with\n        | None -> (\n            send_to_manager Filtering_cancelled\n          )\n        | Some state -> (\n            let command = Some (`Filter s) in\n            let snapshot =\n              Session.Snapshot.make\n                ~committed:commit\n                ~last_command:command\n                state\n            in\n            add_snapshot\n              ~overwrite_if_last_snapshot_satisfies:(fun snapshot ->\n                  match Session.Snapshot.last_command snapshot with\n                  | Some (`Filter s') -> (\n                      not (Session.Snapshot.committed snapshot)\n                      ||\n                      s' = s\n                    )\n                  | _ -> false\n                )\n              snapshot;\n            send_to_manager (Filtering_done (!cur_ver, snapshot))\n          )\n      )\n    | None -> (\n        send_to_manager Filter_parse_error\n      )\n  in\n  let process_path_fuzzy_rank_req stop_signal ~commit (s : string) =\n    match Search_exp.parse s with\n    | None -> ()\n    | Some exp -> (\n        let state =\n          get_cur_snapshot ()\n          |> Session.Snapshot.state\n          |> Session.State.update_path_fuzzy_ranking\n            stop_signal\n            exp\n        in\n        match state with\n        | None -> (\n          )\n        | Some state -> (\n            let command = Some (`Path_fuzzy_rank (s, None)) in\n            let snapshot =\n              Session.Snapshot.make\n                ~committed:commit\n                ~last_command:command\n                state\n            in\n            add_snapshot\n              ~overwrite_if_last_snapshot_satisfies:(fun snapshot ->\n                  match Session.Snapshot.last_command snapshot with\n                  | Some (`Path_fuzzy_rank (s', _)) -> (\n                      not (Session.Snapshot.committed snapshot)\n                      ||\n                      s' = s\n                    )\n                  | _ -> false\n                )\n              snapshot;\n            send_to_manager (Path_fuzzy_rank_done (!cur_ver, snapshot, commit))\n          )\n      )\n  in\n  while true do\n    Ping.wait worker_ping;\n    let time_since_last_request =\n      Mtime.span\n        (Atomic.get last_request_timestamp)\n        (Mtime_clock.now ())\n    in\n    if\n      Mtime.Span.is_shorter\n        time_since_last_request\n        ~than:Params.session_manager_request_debounce_interval\n    then (\n      let clock = Eio.Stdenv.mono_clock (UI_base.eio_env ()) in\n      let sleep_duration_s =\n        Mtime.Span.abs_diff\n          time_since_last_request\n          Params.session_manager_request_debounce_interval\n        |> Mtime.Span.add Params.session_manager_request_debounce_wait_buffer\n        |> Mtime.Span.to_float_ns\n        |> (fun x -> x /. 1_000_000_000.0)\n      in\n      Eio.Time.Mono.sleep clock sleep_duration_s;\n      Ping.ping worker_ping\n    ) else (\n      lock_worker_state (fun () ->\n          (match Lock_protected_cell.get filter_request with\n           | None -> ()\n           | Some (commit, s) -> (\n               process_filter_req (Atomic.get stop_filter_signal) ~commit s\n             )\n          );\n          (match Lock_protected_cell.get search_request with\n           | None -> !cancelled_search_request\n           | Some (commit, s) -> Some (commit, s)\n          )\n          |> Option.iter (fun (commit, s) ->\n              process_search_req (Atomic.get stop_search_signal) ~commit s\n            );\n          Lock_protected_cell.get path_fuzzy_rank_request\n          |> Option.iter (fun (commit, s) ->\n              process_path_fuzzy_rank_req (Atomic.get stop_path_fuzzy_rank_signal) ~commit s\n            );\n        )\n    )\n  done\n\nlet submit_filter_req ~commit (s : string) =\n  lock_as_requester (fun () ->\n      stop_filter ();\n      stop_search ();\n      Lock_protected_cell.set filter_request (commit, s);\n      Ping.ping worker_ping\n    )\n\nlet submit_search_req ~commit (s : string) =\n  lock_as_requester (fun () ->\n      stop_search ();\n      Lock_protected_cell.set search_request (commit, s);\n      Ping.ping worker_ping\n    )\n\nlet submit_path_fuzzy_rank_req ~commit (s : string) =\n  lock_as_requester (fun () ->\n      stop_path_fuzzy_rank ();\n      Lock_protected_cell.set path_fuzzy_rank_request (commit, s);\n      Ping.ping worker_ping\n    )\n"
  },
  {
    "path": "bin/session_manager.mli",
    "content": "open Docfd_lib\n\nval manager_fiber : unit -> unit\n\nval worker_fiber : Task_pool.t -> unit\n\nval cur_snapshot : (int * Session.Snapshot.t) Lwd.t\n\ntype view = {\n  init_state : Session.State.t;\n  snapshots : Session.Snapshot.t Dynarray.t;\n  cur_ver : int;\n}\n\nval lock_with_view : (view -> 'a) -> 'a\n\nval update_starting_state : Session.State.t -> unit\n\nval load_snapshots : Session.Snapshot.t Dynarray.t -> unit\n\nval shift_ver : offset:int -> unit\n\nval update_from_cur_snapshot : (Session.Snapshot.t -> Session.Snapshot.t) -> unit\n\nval submit_filter_req : commit:bool -> string -> unit\n\nval submit_search_req : commit:bool -> string -> unit\n\nval submit_path_fuzzy_rank_req : commit:bool -> string -> unit\n\nval stop_filter_and_search_and_restore_input_fields : unit -> unit\n"
  },
  {
    "path": "bin/string_utils.ml",
    "content": "let remove_leading_dots (s : string) =\n  let str_len = String.length s in\n  if str_len = 0 then (\n    \"\"\n  ) else (\n    let rec aux pos =\n      if pos < str_len then (\n        if String.get s pos = '.' then\n          aux (pos + 1)\n        else (\n          String.sub s pos (str_len - pos)\n        )\n      ) else (\n        \"\"\n      )\n    in\n    aux 0\n  )\n\nlet line_is_system_comment line =\n  CCString.starts_with ~prefix:\";\" line\n\nlet line_is_blank_or_system_comment line =\n  line_is_system_comment line\n  ||\n  String.length (String.trim line) = 0\n\nlet longest_common_prefix (seq : string Seq.t) : string =\n  let prefix = ref \"\" in\n  Seq.iteri (fun i s ->\n      if i = 0 then (\n        prefix := s\n      ) else (\n        let match_len = ref 0 in\n        let prefix_len = String.length !prefix in\n        String.iteri (fun i c ->\n            if !match_len = i\n            && i < prefix_len\n            && !prefix.[i] = c then (\n              incr match_len\n            )\n          ) s;\n        prefix :=\n          String.sub !prefix 0 (min !match_len prefix_len)\n      )\n    ) seq;\n  !prefix\n"
  },
  {
    "path": "bin/version_string.ml",
    "content": "let s = \"13.0.0\"\n"
  },
  {
    "path": "bin/xdg_utils.ml",
    "content": "let all_desktop_files () : string Seq.t =\n  match Sys.getenv_opt \"XDG_DATA_DIRS\" with\n  | None -> Seq.empty\n  | Some s -> (\n      String.split_on_char ':' s\n      |> List.to_seq\n      |> Seq.flat_map (fun dir ->\n          let dir = Filename.concat dir \"applications\" in\n          try\n            Sys.readdir dir\n            |> Array.to_seq\n            |> Seq.map (Filename.concat dir)\n          with\n          | _ -> Seq.empty\n        )\n    )\n\nlet path_of_desktop_file file =\n  let rec aux paths =\n    match paths () with\n    | Seq.Nil -> None\n    | Seq.Cons (path, rest) -> (\n        if String.equal file (Filename.basename path) then (\n          Some path\n        ) else (\n          aux rest\n        )\n      )\n  in\n  aux (all_desktop_files ())\n\nlet default_desktop_file_path (typ : [ `PDF ]) =\n  let mime_typ =\n    match typ with\n    | `PDF -> \"application/pdf\"\n    (* | `ODT -> \"application/vnd.oasis.opendocument.text\"\n       | `DOCX -> \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\" *)\n  in\n  let (stdout, _, ret) = CCUnix.call \"xdg-mime query default %s\" mime_typ in\n  if ret = 0 then (\n    path_of_desktop_file (CCString.trim stdout)\n  ) else (\n    None\n  )\n\nlet data_home =\n  let home_dir =\n    match Sys.getenv_opt \"HOME\" with\n    | None -> (\n        Misc_utils.exit_with_error_msg \"environment variable HOME is not set\";\n      )\n    | Some home -> home\n  in\n  match Params.os_typ with\n  | `Linux -> (\n      match Sys.getenv_opt \"XDG_DATA_HOME\" with\n      | None -> Filename.concat home_dir \".local/share\"\n      | Some x -> x\n    )\n  | `Darwin -> (\n      Filename.concat home_dir\n        (Filename.concat \"Library\" \"Application Support\")\n    )\n\nlet cache_home =\n  let home_dir =\n    match Sys.getenv_opt \"HOME\" with\n    | None -> (\n        Misc_utils.exit_with_error_msg \"environment variable HOME is not set\";\n      )\n    | Some home -> home\n  in\n  match Params.os_typ with\n  | `Linux -> (\n      match Sys.getenv_opt \"XDG_CACHE_HOME\" with\n      | None -> Filename.concat home_dir \".cache\"\n      | Some x -> x\n    )\n  | `Darwin -> (\n      Filename.concat home_dir\n        (Filename.concat \"Library\" \"Caches\")\n    )\n"
  },
  {
    "path": "containers/Containerfile.demo-vhs",
    "content": "FROM ghcr.io/charmbracelet/vhs\n\nRUN apt-get update\n\nRUN apt-get install -y poppler-utils\n\nRUN apt-get install -y neovim\n"
  },
  {
    "path": "containers/Containerfile.docfd",
    "content": "FROM docker.io/alpine:3.22\n\nUSER root\nRUN apk add linux-headers\nRUN apk add poppler-utils\nRUN apk add sqlite sqlite-libs sqlite-dev sqlite-static\nRUN apk add sqlite-analyzer\nRUN apk add make\nRUN apk add clang\nRUN apk add opam\nRUN apk add git\nRUN apk add python3\n\n# RUN ln -s $(which opam-2.2) /usr/local/bin/opam\nRUN opam --version\nRUN apk add bash\nRUN opam init --disable-sandboxing\nRUN opam install --yes dune\nRUN opam install --yes utop ocp-indent\n\nCOPY . /root/docfd\n# RUN chown -R root:root /home/opam/docfd\n\nWORKDIR /root/docfd\nRUN eval $(opam env) && dune build docfd.opam\nRUN opam install --yes . --deps-only --with-test\nRUN echo 'eval $(opam env)' >> /root/.bash_profile\n"
  },
  {
    "path": "demo-vhs-tapes/repo-non-interactive.tape",
    "content": "Output demo-vhs-gifs/repo-non-interactive.gif\n\nSet Padding 0\nSet Framerate 10\n\nSet Width  1366\nSet Height  768\nSet FontSize 15\n\nSet TypingSpeed 100ms\n\nType@50ms 'docfd README.md --sample \"interactive grep across lines\"'\nEnter\n\nSleep 4s\n"
  },
  {
    "path": "demo-vhs-tapes/repo.tape",
    "content": "Output demo-vhs-gifs/repo.gif\n\nSet Padding 0\nSet Framerate 10\n\nSet Width  1366\nSet Height  768\nSet FontSize 15\n\nSet TypingSpeed 100ms\n\nType@50ms \"docfd *.md\"\nSleep 1s\nEnter\n\nSleep 1s\n\nType \"/\"\nType \"fuzz search\"\nEnter\nSleep 1s\n\nType \"f\"\nType \"path-fuzzy:readme\"\nEnter\nSleep 1s\n\nEnter\nSleep 1s\nType \"zz\"\nSleep 1s\nType \"O\"\nType@50ms \"Docfd opens the editor to where the search result is when we hit Enter.\"\nEscape\nSleep 2s\nType@200ms \":q!\"\nEnter\n\nSleep 4s\n"
  },
  {
    "path": "demo-vhs-tapes/ui-screenshot.tape",
    "content": "Output dummy.gif\n\nSet Padding 0\nSet Framerate 10\n\nSet Width  1366\nSet Height  768\nSet FontSize 15\n\nSet TypingSpeed 100ms\n\nType@10ms \"docfd *.md\"\nEnter\nSleep 1s\n\nScreenshot screenshots/ui0.png\n\nSleep 1s\n"
  },
  {
    "path": "demo-vhs.sh",
    "content": "#!/usr/bin/env bash\n\npodman run --rm -v $PWD:/vhs \\\n  --env 'VISUAL=nvim' \\\n  -v $PWD/release/docfd:/usr/bin/docfd \\\n  localhost/docfd-demo-vhs \\\n  \"$@\"\n"
  },
  {
    "path": "docfd.opam",
    "content": "# This file is generated by dune, edit dune-project instead\nopam-version: \"2.0\"\nsynopsis: \"TUI multiline fuzzy document finder\"\ndescription: \"\"\"\n\nThink interactive grep for text files, PDFs, DOCXs, etc,\nbut word/token based instead of regex and line based,\nso you can search across lines easily.\n\nDocfd aims to provide good UX via integration with common text editors\nand PDF viewers,\nso you can jump directly to a search result with a single key press.\n\nFeatures:\n\n- Multithreaded indexing and searching\n\n- Multiline fuzzy search of multiple files or a single file\n\n- Swap between multi-file view and single file view on the fly\n\n- Content view pane that shows the snippet surrounding the search result selected\n\n- Text editor and PDF viewer integration\n              \"\"\"\nmaintainer: [\"Darren Li\"]\nauthors: [\"Darren Li\"]\nlicense: \"MIT\"\ntags: [\"fuzzy\" \"document\" \"finder\"]\nhomepage: \"https://github.com/darrenldl/docfd\"\ndoc: \"https://github.com/darrenldl/docfd\"\nbug-reports: \"https://github.com/darrenldl/docfd/issues\"\ndepends: [\n  \"ocaml\" {>= \"5.2\"}\n  \"dune\" {>= \"3.4\"}\n  \"fmt\" {>= \"0.9.0\"}\n  \"angstrom\" {>= \"0.15.0\"}\n  \"containers\" {>= \"3.12\"}\n  \"oseq\"\n  \"spelll\"\n  \"notty-community\"\n  \"nottui\" {>= \"0.4\"}\n  \"nottui-unix\" {>= \"0.4\"}\n  \"lwd\"\n  \"cmdliner\" {>= \"2.0.0\"}\n  \"eio\" {>= \"0.14\"}\n  \"digestif\"\n  \"eio_main\" {>= \"1.3\"}\n  \"containers-data\"\n  \"timedesc\" {>= \"3.1.0\"}\n  \"re\" {>= \"1.11.0\"}\n  \"ppx_deriving\" {>= \"5.0\"}\n  \"decompress\"\n  \"progress\" {>= \"0.5.0\"}\n  \"diet\"\n  \"sqlite3\"\n  \"uuseg\"\n  \"uucp\"\n  \"alcotest\" {with-test}\n  \"qcheck-alcotest\" {with-test}\n  \"qcheck\" {with-test}\n  \"odoc\" {with-doc}\n]\ndev-repo: \"git+https://github.com/darrenldl/docfd.git\"\nbuild: [\n  [\"dune\" \"subst\"] {dev}\n  [\n    \"dune\"\n    \"build\"\n    \"-p\"\n    name\n    \"-j\"\n    jobs\n    \"@install\"\n    \"@doc\" {with-doc}\n  ]\n]\n"
  },
  {
    "path": "docfd.opam.locked",
    "content": "opam-version: \"2.0\"\nname: \"docfd\"\nversion: \"3.0.0\"\nsynopsis: \"TUI multiline fuzzy document finder\"\nmaintainer: \"Darren Li\"\nauthors: \"Darren Li\"\nlicense: \"MIT\"\ntags: [\"fuzzy\" \"document\" \"finder\"]\nhomepage: \"https://github.com/darrenldl/docfd\"\ndoc: \"https://github.com/darrenldl/docfd\"\nbug-reports: \"https://github.com/darrenldl/docfd/issues\"\ndepends: [\n  \"alcotest\" {= \"1.8.0\" & with-test}\n  \"angstrom\" {= \"0.16.1\"}\n  \"astring\" {= \"0.8.5\" & with-test}\n  \"base-bigarray\" {= \"base\"}\n  \"base-bytes\" {= \"base\"}\n  \"base-domains\" {= \"base\"}\n  \"base-nnp\" {= \"base\"}\n  \"base-threads\" {= \"base\"}\n  \"base-unix\" {= \"base\"}\n  \"bigstringaf\" {= \"0.10.0\"}\n  \"checkseum\" {= \"0.5.2\"}\n  \"cmdliner\" {= \"1.3.0\"}\n  \"conf-pkg-config\" {= \"3\"}\n  \"conf-sqlite3\" {= \"1\"}\n  \"containers\" {= \"3.15\"}\n  \"containers-data\" {= \"3.15\"}\n  \"cppo\" {= \"1.8.0\"}\n  \"csexp\" {= \"1.5.2\"}\n  \"cstruct\" {= \"6.2.0\"}\n  \"decompress\" {= \"1.5.3\"}\n  \"diet\" {= \"0.4\"}\n  \"digestif\" {= \"1.2.0\"}\n  \"domain-local-await\" {= \"1.0.1\"}\n  \"dune\" {= \"3.17.1\"}\n  \"dune-configurator\" {= \"3.17.1\"}\n  \"eio\" {= \"1.2\"}\n  \"eio_linux\" {= \"1.2\"}\n  \"eio_main\" {= \"1.2\"}\n  \"eio_posix\" {= \"1.2\"}\n  \"either\" {= \"1.0.0\"}\n  \"eqaf\" {= \"0.10\"}\n  \"fmt\" {= \"0.9.0\"}\n  \"hmap\" {= \"0.8.1\"}\n  \"host-arch-x86_64\" {= \"1\"}\n  \"host-system-other\" {= \"1\"}\n  \"iomux\" {= \"0.3\"}\n  \"logs\" {= \"0.7.0\"}\n  \"lwd\" {= \"0.3\"}\n  \"lwt\" {= \"5.9.0\"}\n  \"lwt-dllist\" {= \"1.0.1\"}\n  \"mtime\" {= \"2.1.0\"}\n  \"nottui\" {= \"0.3\"}\n  \"notty\" {= \"0.2.3\"}\n  \"ocaml\" {= \"5.2.1\"}\n  \"ocaml-base-compiler\" {= \"5.2.1\"}\n  \"ocaml-compiler-libs\" {= \"v0.17.0\"}\n  \"ocaml-config\" {= \"3\"}\n  \"ocaml-options-vanilla\" {= \"1\"}\n  \"ocaml-syntax-shims\" {= \"1.0.0\"}\n  \"ocamlbuild\" {= \"0.15.0\"}\n  \"ocamlfind\" {= \"1.9.6\"}\n  \"ocplib-endian\" {= \"1.2\"}\n  \"optint\" {= \"0.3.0\"}\n  \"oseq\" {= \"0.5.1\"}\n  \"ounit2\" {= \"2.2.7\" & with-test}\n  \"ppx_derivers\" {= \"1.2.1\"}\n  \"ppx_deriving\" {= \"6.0.3\"}\n  \"ppxlib\" {= \"0.33.0\"}\n  \"progress\" {= \"0.4.0\"}\n  \"psq\" {= \"0.2.1\"}\n  \"ptime\" {= \"1.2.0\"}\n  \"qcheck\" {= \"0.23\" & with-test}\n  \"qcheck-alcotest\" {= \"0.23\" & with-test}\n  \"qcheck-core\" {= \"0.23\" & with-test}\n  \"qcheck-ounit\" {= \"0.23\" & with-test}\n  \"re\" {= \"1.12.0\"}\n  \"result\" {= \"1.5\"}\n  \"seq\" {= \"base\"}\n  \"sexplib0\" {= \"v0.17.0\"}\n  \"spelll\" {= \"0.4\"}\n  \"sqlite3\" {= \"5.2.0\"}\n  \"stdlib-shims\" {= \"0.3.0\"}\n  \"terminal\" {= \"0.4.0\"}\n  \"thread-table\" {= \"1.0.0\"}\n  \"timedesc\" {= \"3.1.0\"}\n  \"timedesc-tzdb\" {= \"3.1.0\"}\n  \"timedesc-tzlocal\" {= \"3.1.0\"}\n  \"topkg\" {= \"1.0.7\"}\n  \"uring\" {= \"0.9\"}\n  \"uucp\" {= \"16.0.0\"}\n  \"uuseg\" {= \"16.0.0\"}\n  \"uutf\" {= \"1.0.3\"}\n  \"vector\" {= \"1.0.0\"}\n]\nbuild: [\n  [\"dune\" \"subst\"] {dev}\n  [\"dune\" \"build\" \"-p\" name \"-j\" jobs \"@install\" \"@doc\" {with-doc}]\n]\ndev-repo: \"git+https://github.com/darrenldl/docfd.git\"\npin-depends: [\n  [\n    \"nottui.0.3\"\n    \"git+https://github.com/let-def/lwd.git#a337a778001e6c1dbaed7e758c9e05f300abd388\"\n  ]\n  [\n  \"notty.0.2.3\"\n  \"git+https://github.com/ocaml-dune/notty.git#b6e1036c61521be3b1f4d585895ac598bdf4ab8d\"\n]\n  [\n  \"ocaml-base-compiler.5.2.1\"\n  \"https://github.com/ocaml/ocaml/releases/download/5.2.1/ocaml-5.2.1.tar.gz\"\n]\n]\ndescription: \"\"\"\\\nThink interactive grep for text files, PDFs, DOCXs, etc,\nbut word/token based instead of regex and line based,\nso you can search across lines easily.\n\nDocfd aims to provide good UX via integration with common text editors\nand PDF viewers,\nso you can jump directly to a search result with a single key press.\n\nFeatures:\n\n- Multithreaded indexing and searching\n\n- Multiline fuzzy search of multiple files or a single file\n\n- Swap between multi-file view and single file view on the fly\n\n- Content view pane that shows the snippet surrounding the search result selected\n\n- Text editor and PDF viewer integration\"\"\"\n"
  },
  {
    "path": "docfd.opam.template",
    "content": "build: [\n  [\"dune\" \"subst\"] {dev}\n  [\n    \"dune\"\n    \"build\"\n    \"-p\"\n    name\n    \"-j\"\n    jobs\n    \"@install\"\n    \"@doc\" {with-doc}\n  ]\n]\n"
  },
  {
    "path": "dune-project",
    "content": "(lang dune 3.4)\n\n(name docfd)\n\n(generate_opam_files true)\n\n(source\n (github darrenldl/docfd))\n\n(authors \"Darren Li\")\n\n(maintainers \"Darren Li\")\n\n(license MIT)\n\n(package\n (name docfd)\n (synopsis \"TUI multiline fuzzy document finder\")\n (description \"\nThink interactive grep for text files, PDFs, DOCXs, etc,\nbut word/token based instead of regex and line based,\nso you can search across lines easily.\n\nDocfd aims to provide good UX via integration with common text editors\nand PDF viewers,\nso you can jump directly to a search result with a single key press.\n\nFeatures:\n\n- Multithreaded indexing and searching\n\n- Multiline fuzzy search of multiple files or a single file\n\n- Swap between multi-file view and single file view on the fly\n\n- Content view pane that shows the snippet surrounding the search result selected\n\n- Text editor and PDF viewer integration\n              \")\n (documentation https://github.com/darrenldl/docfd)\n (depends\n   (ocaml (>= \"5.2\"))\n   dune\n   (fmt (>= \"0.9.0\"))\n   (angstrom (>= \"0.15.0\"))\n   (containers (>= \"3.12\"))\n   oseq\n   spelll\n   notty-community\n   (nottui (>= \"0.4\"))\n   (nottui-unix (>= \"0.4\"))\n   lwd\n   (cmdliner (>= \"2.0.0\"))\n   (eio (>= \"0.14\"))\n   digestif\n   (eio_main (>= \"1.3\"))\n   containers-data\n   (timedesc (>= \"3.1.0\"))\n   (re (>= \"1.11.0\"))\n   (ppx_deriving (>= \"5.0\"))\n   decompress\n   (progress (>= \"0.5.0\"))\n   diet\n   sqlite3\n   uuseg\n   uucp\n   (alcotest :with-test)\n   (qcheck-alcotest :with-test)\n   (qcheck :with-test)\n   )\n (tags\n  (\"fuzzy\" \"document\" \"finder\"\n   ))\n )\n"
  },
  {
    "path": "file-collection-tests.t/dune",
    "content": "(cram\n  (deps ../bin/docfd.exe))\n"
  },
  {
    "path": "file-collection-tests.t/run.t",
    "content": "Setup:\n  $ touch no-ext\n  $ touch empty-paths.txt\n  $ echo \"test.txt\" >> paths\n  $ echo \"test-symlink.txt\" >> paths\n  $ echo \"test0\" >> paths\n  $ echo \"test1/ijkl\" >> paths\n  $ echo \"test2/\" >> paths\n  $ echo \"test3/\" >> paths\n  $ echo \"test.log\" >> single-path0.txt\n  $ echo \"test.txt\" >> single-path1.txt\n  $ touch test.ext0\n  $ touch test.log\n  $ touch test.md\n  $ touch test.txt\n  $ mkdir test0\n  $ touch test0/1234.md\n  $ touch test0/abcd.txt\n  $ mkdir test0/abcd\n  $ touch test0/abcd/efgh.md\n  $ touch test0/abcd/efgh.txt\n  $ mkdir test1\n  $ touch test1/5678.md\n  $ touch test1/ijkl.txt\n  $ mkdir test1/ijkl\n  $ touch test1/ijkl/mnop.md\n  $ touch test1/ijkl/mnop.txt\n  $ mkdir test2\n  $ touch test2/1234.md\n  $ mkdir test2/abcd\n  $ touch test2/abcd/efgh.md\n  $ ln -s $(pwd)/test0/abcd/efgh.txt test2/abcd/efgh.txt\n  $ ln -s ../test1/5678.md test2/56.md\n  $ ln -s ../test1/ijkl test2/ijkl\n  $ ln -s test2 test3\n  $ ln -s test.txt test-symlink.txt\n  $ mkdir miXeD-CaSe\n  $ touch miXeD-CaSe/AbCd.md\n  $ tree\n  .\n  |-- dune -> ../../../../default/file-collection-tests.t/dune\n  |-- empty-paths.txt\n  |-- miXeD-CaSe\n  |   `-- AbCd.md\n  |-- no-ext\n  |-- paths\n  |-- single-path0.txt\n  |-- single-path1.txt\n  |-- test-symlink.txt -> test.txt\n  |-- test.ext0\n  |-- test.log\n  |-- test.md\n  |-- test.txt\n  |-- test0\n  |   |-- 1234.md\n  |   |-- abcd\n  |   |   |-- efgh.md\n  |   |   `-- efgh.txt\n  |   `-- abcd.txt\n  |-- test1\n  |   |-- 5678.md\n  |   |-- ijkl\n  |   |   |-- mnop.md\n  |   |   `-- mnop.txt\n  |   `-- ijkl.txt\n  |-- test2\n  |   |-- 1234.md\n  |   |-- 56.md -> ../test1/5678.md\n  |   |-- abcd\n  |   |   |-- efgh.md\n  |   |   `-- efgh.txt -> $TESTCASE_ROOT/test0/abcd/efgh.txt\n  |   `-- ijkl -> ../test1/ijkl\n  `-- test3 -> test2\n  \n  7 directories, 26 files\n\nBasic invocation for reference:\n  $ docfd --debug-log - --cache-dir .cache --index-only . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n\n--max-depth 0:\n  $ docfd --debug-log - --cache-dir .cache --index-only --max-depth 0 . 2>&1 | grep '^Using .* search mode' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --max-depth 0 test.txt 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n\n--max-depth 1:\n  $ docfd --debug-log - --cache-dir .cache --index-only --max-depth 1 . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  $ docfd --debug-log - --cache-dir .cache --index-only --max-depth 1 --glob '**/*.md' 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n\n--max-depth 2:\n  $ docfd --debug-log - --cache-dir .cache --index-only --max-depth 2 . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  $ docfd --debug-log - --cache-dir .cache --index-only --max-depth 2 --glob '**/*.md' 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n\nDefault path is not picked if --paths-from is used:\n  $ docfd --debug-log - --cache-dir .cache --index-only --paths-from empty-paths.txt 2>&1 | grep '^Using .* search mode' | sort\n\nDefault path is not picked if --glob is used:\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n\nDefault path is not picked if --single-line-glob is used:\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n\nMultiple --paths-from:\n  $ docfd --debug-log - --cache-dir .cache --index-only --paths-from single-path0.txt --paths-from single-path1.txt 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n\nEmpty --exts:\n  $ docfd --debug-log - --cache-dir .cache --index-only --exts \"\" . 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n\nEmpty --single-line-exts:\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-exts \"\" . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n\nEmpty --exts and --single-line-exts:\n  $ docfd --debug-log - --cache-dir .cache --index-only --exts \"\" --single-line-exts \"\" .\n  Initializing in-memory index\n  error: no usable file extensions or glob patterns\n  [1]\n\n--add-exts:\n  $ docfd --debug-log - --cache-dir .cache --index-only --add-exts ext0 . 2>&1 | grep '^Using .* search mode' | sort | grep \"ext0\"\n  Using multiline search mode for document '$TESTCASE_ROOT/test.ext0'\n\n--single-line-add-exts:\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-add-exts ext0 . 2>&1 | grep '^Using .* search mode' | sort | grep \"ext0\"\n  Using single line search mode for document '$TESTCASE_ROOT/test.ext0'\n\nPicking via multiple --glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '*.txt' --glob '*.md' --glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n\nPicking via multiple --single-line-glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-glob '*.txt' --single-line-glob '*.md' --single-line-glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n\nPicking via multiple --glob and --single-line-glob:\n  $ # --single-line-glob for .txt files\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-glob '*.txt' --glob '*.md' --glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using single line search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n  $ # --single-line-glob for .md files\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '*.txt' --single-line-glob '*.md' --glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n  $ # --single-line-glob for .log files\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '*.txt' --glob '*.md' --single-line-glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  $ # --glob for .txt files\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '*.txt' --single-line-glob '*.md' --single-line-glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n  $ # --glob for .md files\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-glob '*.txt' --glob '*.md' --single-line-glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using single line search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n  $ # --glob for .log files\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-glob '*.txt' --single-line-glob '*.md' --glob '*.log' 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n\n--single-line-exts and --exts:\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-exts md --exts md . 2>&1 | grep -e '^Using .* search mode' -e '^Checking.*search mode' | sort\n  Checking if efficiently computed and naively computed results for default search mode files are consistent\n  Checking if efficiently computed and naively computed results for single line search mode files are consistent\n  Checking if single line search mode files and default search mode files are disjoint\n  Using single line search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n\n--single-line-exts and --glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --exts \"\" --single-line-exts md --glob '**/*.md' 2>&1 | grep -e '^Using .* search mode' -e '^Checking.*search mode' | sort\n  Checking if efficiently computed and naively computed results for default search mode files are consistent\n  Checking if efficiently computed and naively computed results for single line search mode files are consistent\n  Checking if single line search mode files and default search mode files are disjoint\n  Using single line search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n\n--single-line-glob and --exts:\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line-glob '*.md' --exts md . 2>&1 | grep -e '^Using .* search mode' -e '^Checking.*search mode'| sort\n  Checking if efficiently computed and naively computed results for default search mode files are consistent\n  Checking if efficiently computed and naively computed results for single line search mode files are consistent\n  Checking if single line search mode files and default search mode files are disjoint\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n\n--single-line-glob and --glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --exts \"\" --single-line-glob '*.md' --glob '**/*.md' 2>&1 | grep -e '^Using .* search mode' -e '^Checking.*search mode' | sort\n  Checking if efficiently computed and naively computed results for default search mode files are consistent\n  Checking if efficiently computed and naively computed results for single line search mode files are consistent\n  Checking if single line search mode files and default search mode files are disjoint\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n\n--exts applies to directories in FILE in --paths-from FILE:\n  $ docfd --debug-log - --cache-dir .cache --index-only --paths-from paths --exts md 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n\n--single-line-exts apply to directories in FILE in --paths-from FILE:\n  $ docfd --debug-log - --cache-dir .cache --index-only --paths-from paths --single-line-exts md 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n\nTop-level symlinks:\n  $ docfd --debug-log - --cache-dir .cache --index-only test-symlink.txt 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only test3 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n\nTop-level files and --single-line-exts:\n  $ docfd --debug-log - --cache-dir .cache --index-only test.txt --single-line-exts txt 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n\nTop-level files and --single-line-glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only test.txt --single-line-glob '*.txt' 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n\n--glob and unrecognized extensions:\n  $ docfd --debug-log - --cache-dir .cache --index-only --exts md --glob \"*.txt\" . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n\n--single-line-glob and unrecognized extensions:\n  $ docfd --debug-log - --cache-dir .cache --index-only --exts md --single-line-glob \"*.txt\" . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n\n--single-line:\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line . 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.log'\n  Using single line search mode for document '$TESTCASE_ROOT/test.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/5678.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/1234.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/56.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n\n--glob and --single-line:\n  $ docfd --debug-log - --cache-dir .cache --index-only --single-line --glob '**/*.txt' 2>&1 | grep '^Using .* search mode' | sort\n  Using single line search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using single line search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n\nTop-level files with unrecognized extensions are still picked:\n  $ docfd --debug-log - --cache-dir .cache --index-only --exts md test.txt . 2>&1 | grep '^Using .* search mode' | sort | grep 'test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n\nTop-level files without extensions are still picked:\n  $ docfd --debug-log - --cache-dir .cache --index-only --exts md no-ext . 2>&1 | grep '^Using .* search mode' | sort | grep 'no-ext'\n  Using multiline search mode for document '$TESTCASE_ROOT/no-ext'\n\nCurrent working directory is symlink:\n  $ cd test3\n  $ docfd --debug-log - --cache-dir .cache --index-only . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '*.txt' . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '**/*.txt' . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob \"$(pwd)/**/*.txt\" . 2>&1 | grep '^Using .* search mode' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/1234.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/56.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n  $ cd ..\n\n'./' in glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob './*.txt' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n\n'..' in glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'test1/../*.txt' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n\nDirectories in glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '.' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '..' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'test0/' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'test3' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n\nCrossing symlinks explicitly in glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'test3/../*.txt' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'test3/abcd/*.txt' --glob 'test3/abcd/*.md' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'test3/ijkl/*.txt' --glob 'test3/ijkl/*.md' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.md'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n\n'**' in glob:\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '**/*.txt' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/empty-paths.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/single-path0.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/single-path1.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test-symlink.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test0/abcd.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test0/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test1/ijkl.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test1/ijkl/mnop.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test2/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test2/ijkl/mnop.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test3/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test3/ijkl/mnop.txt\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '**/**/*.txt' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/empty-paths.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/single-path0.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/single-path1.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test-symlink.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test0/abcd.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test0/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test1/ijkl.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test1/ijkl/mnop.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test2/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test2/ijkl/mnop.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test3/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test3/ijkl/mnop.txt\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob \"$(pwd)/**/*.txt\" 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/empty-paths.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/single-path0.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/single-path1.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test-symlink.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test0/abcd.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test0/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test1/ijkl.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test1/ijkl/mnop.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test2/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test2/ijkl/mnop.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test3/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/*.txt matches path $TESTCASE_ROOT/test3/ijkl/mnop.txt\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob \"$(pwd)/**/**/*.txt\" 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/empty-paths.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/single-path0.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/single-path1.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test-symlink.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test0/abcd.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test0/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test1/ijkl.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test1/ijkl/mnop.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test2/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test2/ijkl/mnop.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test3/abcd/efgh.txt\n  Glob $TESTCASE_ROOT/**/**/*.txt matches path $TESTCASE_ROOT/test3/ijkl/mnop.txt\n  Using multiline search mode for document '$TESTCASE_ROOT/empty-paths.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path0.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/single-path1.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test-symlink.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test2/ijkl/mnop.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/abcd/efgh.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test3/ijkl/mnop.txt'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob \"**/test[01]/*.txt\" 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Glob $TESTCASE_ROOT/**/test[01]/*.txt matches path $TESTCASE_ROOT/test0/abcd.txt\n  Glob $TESTCASE_ROOT/**/test[01]/*.txt matches path $TESTCASE_ROOT/test1/ijkl.txt\n  Using multiline search mode for document '$TESTCASE_ROOT/test0/abcd.txt'\n  Using multiline search mode for document '$TESTCASE_ROOT/test1/ijkl.txt'\n\nCase insensitive marker:\n  $ # Exact match without marker\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'miXeD-CaSe/AbCd.md' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ # All lowercase glob\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'mixed-case/abcd.md' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '\\cmixed-case/abcd.md' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'mixed-\\ccase/abcd.md' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'mixed-case/\\cabcd.md' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'mixed-case/abcd.md\\c' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ # All uppercase glob\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MIXED-CASE/ABCD.MD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '\\cMIXED-CASE/ABCD.MD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MIX\\cED-CASE/ABCD.MD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MIXED-CASE/\\cABCD.MD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MIXED-CASE/ABCD.MD\\c' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ # Mixed case glob\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MixeD-CaSE/aBcD.mD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '\\cMixeD-CaSE/aBcD.mD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MixeD\\c-CaSE/aBcD.mD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MixeD-CaSE/\\caBcD.mD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MixeD-CaSE/aBcD.mD\\c' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  Using multiline search mode for document '$TESTCASE_ROOT/miXeD-CaSe/AbCd.md'\n\nDouble escape characters:\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob '\\\\cMixeD-CaSE/AbCd.mD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MixeD\\\\c-CaSE/AbCd.mD' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n  $ docfd --debug-log - --cache-dir .cache --index-only --glob 'MixeD-CaSE/AbCd.mD\\\\c' 2>&1 | grep -e '^Using .* search mode' -e '^Glob' | sort\n"
  },
  {
    "path": "lib/GZIP.ml",
    "content": "(* Basically fully copied from examples in Decompress manual *)\n\nlet time () =\n  Int32.of_float (Unix.gettimeofday ())\n\nlet compress (s : string) : string =\n  let i = De.bigstring_create De.io_buffer_size in\n  let o = De.bigstring_create De.io_buffer_size in\n  let config = Gz.Higher.configuration Gz.Unix time in\n  let w = De.Lz77.make_window ~bits:15 in\n  let q = De.Queue.create 1024 in\n  let res = Buffer.create 4096 in\n  let cur = ref 0 in\n  let refill buf =\n    let len = min (String.length s - !cur) De.io_buffer_size in\n    Bigstringaf.blit_from_string s ~src_off:!cur buf ~dst_off:0 ~len;\n    cur := !cur + len;\n    len\n  in\n  let flush buf len =\n    let str = Bigstringaf.substring buf ~off:0 ~len in\n    Buffer.add_string res str\n  in\n  Gz.Higher.compress ~w ~q ~level:4 ~refill ~flush () config i o;\n  Buffer.contents res\n\nlet decompress (s : string) : string option =\n  let i = De.bigstring_create De.io_buffer_size in\n  let o = De.bigstring_create De.io_buffer_size in\n  let r = Buffer.create 0x1000 in\n  let cur = ref 0 in\n  let refill buf =\n    let len = min (String.length s - !cur) De.io_buffer_size in\n    Bigstringaf.blit_from_string s ~src_off:!cur buf ~dst_off:0 ~len;\n    cur := !cur + len;\n    len\n  in\n  let flush buf len =\n    let str = Bigstringaf.substring buf ~off:0 ~len in\n    Buffer.add_string r str\n  in\n  match Gz.Higher.uncompress ~refill ~flush i o with\n  | Ok _ -> Some (Buffer.contents r)\n  | Error _ -> None\n"
  },
  {
    "path": "lib/char_map.ml",
    "content": "include CCMap.Make (Char)\n"
  },
  {
    "path": "lib/doc_id_db.ml",
    "content": "type t = {\n  lock : Eio.Mutex.t;\n  doc_id_of_doc_hash : (string, int64) Hashtbl.t;\n}\n\nlet t : t =\n  {\n    lock = Eio.Mutex.create ();\n    doc_id_of_doc_hash = Hashtbl.create 10_000;\n  }\n\nlet lock : type a. (unit -> a) -> a =\n  fun f ->\n  Eio.Mutex.use_rw ~protect:true t.lock f\n\nlet allocate_bulk (doc_hashes : string Seq.t) : unit =\n  let open Sqlite3_utils in\n  lock (fun () ->\n      with_db (fun db ->\n          with_stmt ~db\n            {|\n  INSERT INTO doc_info\n  (id, hash, status)\n  VALUES\n  (\n    (SELECT\n      IFNULL(\n        (\n          SELECT a.id - 1 AS id\n          FROM doc_info a\n          LEFT JOIN doc_info b ON a.id - 1 = b.id\n          WHERE b.id IS NULL AND a.id - 1 >= 0\n\n          UNION\n\n          SELECT a.id + 1 AS id\n          FROM doc_info a\n          LEFT JOIN doc_info b ON a.id + 1 = b.id\n          WHERE b.id IS NULL\n\n          ORDER BY id\n          LIMIT 1\n        ),\n        0\n      )\n    ),\n    @doc_hash,\n    'ONGOING'\n  )\n  ON CONFLICT(hash) DO NOTHING\n  |}\n            (fun stmt ->\n               Seq.iter (fun doc_hash ->\n                   bind_names stmt [ (\"@doc_hash\", TEXT doc_hash) ];\n                   step stmt;\n                   reset stmt;\n                 )\n                 doc_hashes\n            );\n          with_stmt ~db\n            {|\n    SELECT id\n    FROM doc_info\n    WHERE hash = @doc_hash\n    |}\n            (fun stmt ->\n               Seq.iter (fun doc_hash ->\n                   bind_names stmt [ (\"@doc_hash\", TEXT doc_hash) ];\n                   step stmt;\n                   Hashtbl.add t.doc_id_of_doc_hash doc_hash (column_int64 stmt 0);\n                   reset stmt;\n                 )\n                 doc_hashes\n            )\n        )\n    )\n\nlet doc_id_of_doc_hash (doc_hash : string) : int64 =\n  let test =\n    lock (fun () ->\n        Hashtbl.find_opt t.doc_id_of_doc_hash doc_hash\n      )\n  in\n  match test with\n  | Some id -> id\n  | None -> (\n      allocate_bulk (Seq.return doc_hash);\n      lock (fun () ->\n          Hashtbl.find t.doc_id_of_doc_hash doc_hash\n        )\n    )\n"
  },
  {
    "path": "lib/doc_id_db.mli",
    "content": "val allocate_bulk : string Seq.t -> unit\n\nval doc_id_of_doc_hash : string -> int64\n"
  },
  {
    "path": "lib/docfd_lib.ml",
    "content": "module Index = Index\n\nmodule Doc_id_db = Doc_id_db\n\nmodule Link = Link\n\nmodule Search_result = Search_result\n\nmodule Search_phrase = Search_phrase\n\nmodule Search_exp = Search_exp\n\nmodule Search_result_heap = Search_result_heap\n\nmodule Word_db = Word_db\n\nmodule Tokenization = Tokenization\n\nmodule Params' = Params\n\nmodule Task_pool = Task_pool\n\nmodule Stop_signal = Stop_signal\n\nmodule Parser_components = Parser_components\n\nmodule Misc_utils' = Misc_utils\n\nmodule Sqlite3_utils = Sqlite3_utils\n\nlet init ~db_path ~document_count_limit =\n  let open Sqlite3_utils in\n  let db = db_open db_path in\n  let db_res =\n    Sqlite3.exec db Params.db_schema\n  in\n  let res =\n    if not (Rc.is_success db_res) then (\n      Some (Fmt.str\n              \"failed to initialize index DB: %s\" (Rc.to_string db_res))\n    ) else (\n      Params.db_path := Some db_path;\n      if Index.document_count () >= document_count_limit then (\n        Index.prune_old_documents ~keep_n_latest:document_count_limit\n      );\n      None\n    )\n  in\n  while not (db_close db) do Unix.sleepf 0.1 done;\n  res\n"
  },
  {
    "path": "lib/dune",
    "content": "(library\n (flags     (-w \"+a-4-9-29-37-40-42-44-48-50-32-30-70@8\"))\n (name docfd_lib)\n (preprocess (pps\n               ppx_deriving.show\n               ppx_deriving.ord\n               ppx_deriving.eq\n               ))\n (libraries containers\n            containers-data\n            angstrom\n            fmt\n            spelll\n            oseq\n            eio\n            decompress.gz\n            bigstringaf\n            unix\n            diet\n            sqlite3\n            uuseg\n            uucp\n            timedesc\n            timedesc-tzlocal.unix-or-utc\n )\n)\n"
  },
  {
    "path": "lib/index.ml",
    "content": "module Line_loc = struct\n  type t = {\n    page_num : int;\n    line_num_in_page : int;\n    global_line_num : int;\n  }\n  [@@deriving eq]\n\n  let page_num t = t.page_num\n\n  let line_num_in_page t = t.line_num_in_page\n\n  let global_line_num t = t.global_line_num\n\n  let compare (x : t) (y : t) =\n    Int.compare x.global_line_num y.global_line_num\nend\n\nmodule Loc = struct\n  type t = {\n    line_loc : Line_loc.t;\n    pos_in_line : int;\n  }\n  [@@deriving eq]\n\n  let line_loc t = t.line_loc\n\n  let pos_in_line t =  t.pos_in_line\nend\n\nmodule Raw = struct\n  type t = {\n    pos_s_of_word : Int_set.t Int_map.t;\n    loc_of_pos : Loc.t Int_map.t;\n    line_loc_of_global_line_num : Line_loc.t Int_map.t;\n    start_end_inc_pos_of_global_line_num : (int * int) Int_map.t;\n    start_end_inc_pos_of_page_num : (int * int) Int_map.t;\n    word_of_pos : int Int_map.t;\n    line_count_of_page_num : int Int_map.t;\n    page_count : int;\n    global_line_count : int;\n    links : Link.t array;\n  }\n\n  type multi_indexed_word = {\n    pos : int;\n    loc : Loc.t;\n    word : string;\n  }\n\n  type chunk = multi_indexed_word array\n\n  let make () : t = {\n    pos_s_of_word = Int_map.empty;\n    loc_of_pos = Int_map.empty;\n    line_loc_of_global_line_num = Int_map.empty;\n    start_end_inc_pos_of_global_line_num = Int_map.empty;\n    start_end_inc_pos_of_page_num = Int_map.empty;\n    word_of_pos = Int_map.empty;\n    line_count_of_page_num = Int_map.empty;\n    page_count = 0;\n    global_line_count = 0;\n    links = [||];\n  }\n\n  let word_ids (t : t) : Int_set.t =\n    Int_map.fold (fun word_id _pos_s acc ->\n        Int_set.add word_id acc\n      )\n      t.pos_s_of_word\n      Int_set.empty\n\n  let links (t : t) = t.links\n\n  let union (x : t) (y : t) =\n    {\n      pos_s_of_word =\n        Int_map.union (fun _k s0 s1 -> Some (Int_set.union s0 s1))\n          x.pos_s_of_word\n          y.pos_s_of_word;\n      loc_of_pos =\n        Int_map.union (fun _k x _ -> Some x)\n          x.loc_of_pos\n          y.loc_of_pos;\n      line_loc_of_global_line_num =\n        Int_map.union (fun _k x _ -> Some x)\n          x.line_loc_of_global_line_num\n          y.line_loc_of_global_line_num;\n      start_end_inc_pos_of_global_line_num =\n        Int_map.union (fun _k (start_x, end_inc_x) (start_y, end_inc_y) ->\n            Some (min start_x start_y, max end_inc_x end_inc_y))\n          x.start_end_inc_pos_of_global_line_num\n          y.start_end_inc_pos_of_global_line_num;\n      start_end_inc_pos_of_page_num =\n        Int_map.union (fun _k (start_x, end_inc_x) (start_y, end_inc_y) ->\n            Some (min start_x start_y, max end_inc_x end_inc_y))\n          x.start_end_inc_pos_of_page_num\n          y.start_end_inc_pos_of_page_num;\n      word_of_pos =\n        Int_map.union (fun _k x _ -> Some x)\n          x.word_of_pos\n          y.word_of_pos;\n      line_count_of_page_num =\n        Int_map.union (fun _k x y -> Some (max x y))\n          x.line_count_of_page_num\n          y.line_count_of_page_num;\n      page_count = max x.page_count y.page_count;\n      global_line_count = max x.global_line_count y.global_line_count;\n      links = [||];\n    }\n\n  let words_of_lines\n      (s : (Line_loc.t * string) Seq.t)\n    : multi_indexed_word Seq.t =\n    s\n    |> Seq.flat_map (fun (line_loc, s) ->\n        let seq = Tokenization.tokenize_with_pos ~drop_spaces:false s in\n        if Seq.is_empty seq then (\n          let empty_word = ({ Loc.line_loc; pos_in_line = 0 }, \"\") in\n          Seq.return empty_word\n        ) else (\n          Seq.map (fun (pos_in_line, word) ->\n              ({ Loc.line_loc; pos_in_line }, word))\n            seq\n        )\n      )\n    |> Seq.mapi (fun pos (loc, word) ->\n        { pos; loc; word })\n\n  let of_chunk (arr : chunk) : t =\n    Array.fold_left\n      (fun\n        { pos_s_of_word;\n          loc_of_pos;\n          line_loc_of_global_line_num;\n          start_end_inc_pos_of_global_line_num;\n          start_end_inc_pos_of_page_num;\n          word_of_pos;\n          line_count_of_page_num;\n          page_count;\n          global_line_count;\n        }\n        { pos; loc; word } ->\n\n        let word_id =\n          Word_db.add word\n        in\n\n        let line_loc = loc.Loc.line_loc in\n        let global_line_num = line_loc.global_line_num in\n        let page_num = line_loc.page_num in\n        let pos_s =\n          Int_map.find_opt word_id pos_s_of_word\n          |> Option.value ~default:Int_set.empty\n          |> Int_set.add pos\n        in\n        let cur_page_line_count =\n          Option.value ~default:0\n            (Int_map.find_opt page_num line_count_of_page_num)\n        in\n        { pos_s_of_word = Int_map.add word_id pos_s pos_s_of_word;\n          loc_of_pos = Int_map.add pos loc loc_of_pos;\n          line_loc_of_global_line_num =\n            Int_map.add global_line_num line_loc line_loc_of_global_line_num;\n          start_end_inc_pos_of_global_line_num =\n            Int_map.add\n              global_line_num\n              (match Int_map.find_opt global_line_num start_end_inc_pos_of_global_line_num with\n               | None -> (pos, pos)\n               | Some (x, y) -> (min x pos, max y pos))\n              start_end_inc_pos_of_global_line_num;\n          start_end_inc_pos_of_page_num =\n            Int_map.add\n              page_num\n              (match Int_map.find_opt page_num start_end_inc_pos_of_page_num with\n               | None -> (pos, pos)\n               | Some (x, y) -> (min x pos, max y pos))\n              start_end_inc_pos_of_page_num;\n          word_of_pos = Int_map.add pos word_id word_of_pos;\n          line_count_of_page_num =\n            Int_map.add line_loc.page_num (max cur_page_line_count (line_loc.line_num_in_page + 1)) line_count_of_page_num;\n          page_count = max page_count (line_loc.page_num + 1);\n          global_line_count = max global_line_count (global_line_num + 1);\n          links = [||];\n        }\n      )\n      (make ())\n      arr\n\n  let chunks_of_words (s : multi_indexed_word Seq.t) : chunk Seq.t =\n    let empty_word =\n      let line_loc =\n        { Line_loc.page_num = 0; line_num_in_page = 0; global_line_num = 0 }\n      in\n      let loc = { Loc.line_loc; pos_in_line = 0 } in\n      { pos = 0; loc; word = \"\" }\n    in\n    (if Seq.is_empty s then (\n        Seq.return empty_word\n      ) else (\n       s\n     ))\n    |> OSeq.chunks !Params.index_chunk_size\n\n  let extract_links (t : t) : Link.t array =\n    let flush_buf link_typ ~acc ~buf =\n      match buf with\n      | [] -> (acc, buf)\n      | _ -> (\n          let start_pos, end_inc_pos, strings =\n            List.fold_left (fun (start, end_inc, strings) (pos, word) ->\n                let start =\n                  match start with\n                  | None -> Some pos\n                  | Some start -> Some (min pos start)\n                in\n                let end_inc =\n                  match end_inc with\n                  | None -> Some pos\n                  | Some end_inc -> Some (max pos end_inc)\n                in\n                (start, end_inc, word :: strings)\n              )\n              (None, None, [])\n              buf\n          in\n          let start_pos = Option.get start_pos in\n          let end_inc_pos = Option.get end_inc_pos in\n          let link = Link.{\n              start_pos;\n              end_inc_pos;\n              typ = link_typ;\n              link = String.concat \"\" strings;\n            }\n          in\n          (link :: acc, [])\n        )\n    in\n    let process_line (line : (int * string) Dynarray.t) : Link.t list =\n      assert (Dynarray.length line > 0);\n      let word_at pos = Dynarray.get line pos in\n      let word_string_at pos = snd @@ word_at pos in\n      let word_ci_string_at pos = String.lowercase_ascii @@ word_string_at pos in\n      let rec aux\n          (state : [ `Scanning | `In_link of Link.typ ])\n          ~(acc : Link.t list)\n          ~(buf : (int * string) list)\n          (cur : int)\n        : Link.t list\n        =\n        if cur >= Dynarray.length line then (\n          match state with\n          | `Scanning -> (\n              acc\n            )\n          | `In_link link_typ -> (\n              let acc, _buf = flush_buf link_typ ~acc ~buf in\n              acc\n            )\n        ) else (\n          let words_left = Dynarray.length line - cur - 1 in\n          let pos, word = word_at cur in\n          let word_ci = String.lowercase_ascii word in\n          match state with\n          | `Scanning -> (\n              let link_typ =\n                if List.mem word_ci [ \"https\"; \"http\"; \"file\" ]\n                && words_left >= 3\n                && word_ci_string_at (cur + 1) = \":\"\n                && word_ci_string_at (cur + 2) = \"/\"\n                && word_ci_string_at (cur + 3) = \"/\"\n                then (\n                  Some `URL\n                ) else if cur >= 2\n                       && word_ci_string_at (cur - 2) = \"]\"\n                       && word_ci_string_at (cur - 1) = \"(\"\n                then (\n                  Some `Markdown\n                ) else if cur >= 2\n                       && word_ci_string_at (cur - 2) = \"[\"\n                       && word_ci_string_at (cur - 1) = \"[\"\n                then (\n                  Some `Wiki\n                ) else (\n                  None\n                )\n              in\n              match link_typ with\n              | Some link_typ -> (\n                  aux (`In_link link_typ) ~acc ~buf:((pos, word) :: buf) (cur + 1)\n                )\n              | None -> (\n                  aux `Scanning ~acc ~buf (cur + 1)\n                )\n            )\n          | `In_link link_typ -> (\n              let link_ended =\n                String.length word = 0\n                || Parser_components.is_space word.[0]\n                || List.mem word\n                  [ \"]\"\n                  ; \")\"\n                  ; \"|\"\n                  ; \"\\\"\"\n                  ; \"<\"\n                  ; \">\"\n                  ; \"{\"\n                  ; \"}\"\n                  ; \"^\"\n                  ; \"\\\\\"\n                  ]\n              in\n              if link_ended then (\n                let acc, buf = flush_buf link_typ ~acc ~buf in\n                aux `Scanning ~acc ~buf (cur + 1)\n              ) else (\n                aux state ~acc ~buf:((pos, word) :: buf) (cur + 1)\n              )\n            )\n        )\n      in\n      aux `Scanning ~acc:[] ~buf:[] 0\n    in\n    let lines_with (mode : [ `Any_of | `All_of ]) l =\n      let line_nums_by_word =\n        l\n        |> List.to_seq\n        |> Seq.filter_map Word_db.id_of_word\n        |> Seq.map (fun word_id ->\n            Int_map.find_opt word_id t.pos_s_of_word\n            |> Option.value ~default:Int_set.empty\n            |> Int_set.to_seq\n            |> Seq.fold_left (fun acc pos ->\n                let loc = Int_map.find pos t.loc_of_pos in\n                let line_loc = Loc.line_loc loc in\n                Int_set.add line_loc.global_line_num acc\n              )\n              Int_set.empty\n          )\n      in\n      line_nums_by_word\n      |> Seq.fold_left\n        (match mode with\n         | `Any_of -> (fun acc s ->\n             let acc = Option.value ~default:Int_set.empty acc in\n             Some (Int_set.union acc s)\n           )\n         | `All_of -> (fun acc s ->\n             match acc with\n             | None -> Some s\n             | Some acc -> Some (Int_set.inter acc s)\n           )\n        )\n        None\n      |> Option.value ~default:Int_set.empty\n    in\n    let url_line_candidates =\n      Int_set.inter\n        (lines_with `Any_of [ \"http\"; \"https\"; \"file\" ])\n        (lines_with `All_of [ \":\"; \"/\" ])\n    in\n    let markdown_and_wiki_line_candidates =\n      lines_with `All_of [ \"[\"; \"]\" ]\n    in\n    let line_candidates =\n      Int_set.union\n        url_line_candidates\n        markdown_and_wiki_line_candidates\n      |> Int_set.to_seq\n    in\n    let rec aux acc line_nums =\n      match line_nums () with\n      | Seq.Nil -> List.rev acc\n      | Seq.Cons (cur, rest) -> (\n          let start, end_inc = Int_map.find cur t.start_end_inc_pos_of_global_line_num in\n          let links =\n            OSeq.(start -- end_inc)\n            |> Seq.map (fun pos -> (pos, Int_map.find pos t.word_of_pos))\n            |> Seq.map (fun (pos, word_id) -> (pos, Word_db.word_of_id word_id))\n            |> Dynarray.of_seq\n            |> process_line\n          in\n          aux (links @ acc) rest\n        )\n    in\n    aux [] line_candidates\n    |> Array.of_list\n\n  let of_seq pool (s : (Line_loc.t * string) Seq.t) : t =\n    let indices =\n      s\n      |> Seq.map (fun (line_loc, s) -> (line_loc, Misc_utils.sanitize_string s))\n      |> words_of_lines\n      |> chunks_of_words\n      |> List.of_seq\n      |> Task_pool.map_list pool of_chunk\n    in\n    let res =\n      List.fold_left (fun acc index ->\n          union acc index\n        )\n        (make ())\n        indices\n    in\n    let links = extract_links res in\n    { res with links }\n\n  let of_lines pool (s : string Seq.t) : t =\n    s\n    |> Seq.mapi (fun global_line_num line ->\n        ({ Line_loc.page_num = 0; line_num_in_page = global_line_num; global_line_num }, line)\n      )\n    |> of_seq pool\n\n  let of_pages pool (s : string list Seq.t) : t =\n    s\n    |> Seq.mapi (fun page_num page ->\n        (page_num, page)\n      )\n    |> Seq.flat_map (fun (page_num, page) ->\n        match page with\n        | [] -> (\n            let empty_line = ({ Line_loc.page_num; line_num_in_page = 0; global_line_num = 0 }, \"\") in\n            Seq.return empty_line\n          )\n        | _ -> (\n            List.to_seq page\n            |> Seq.mapi (fun line_num_in_page line ->\n                ({ Line_loc.page_num; line_num_in_page; global_line_num = 0 }, line)\n              )\n          )\n      )\n    |> Seq.mapi (fun global_line_num ((line_loc : Line_loc.t), line) ->\n        ({ line_loc with global_line_num }, line)\n      )\n    |> of_seq pool\n\nend\n\nmodule State : sig\n  val add_word_id_doc_id_link : word_id:int -> doc_id:int64 -> unit\n\n  val read_from_db : unit -> unit\n\n  val union_doc_ids_of_word_id_into_bv : word_id:int -> into:CCBV.t -> unit\nend = struct\n  type t = {\n    lock : Eio.Mutex.t;\n    doc_ids_of_word_id : (int, CCBV.t) Hashtbl.t;\n  }\n\n  let t : t =\n    {\n      lock = Eio.Mutex.create ();\n      doc_ids_of_word_id = Hashtbl.create 100_000;\n    }\n\n  let lock : type a. (unit -> a) -> a =\n    fun f ->\n    Eio.Mutex.use_rw ~protect:true t.lock f\n\n  let find_doc_ids_bv ~word_id =\n    match Hashtbl.find_opt t.doc_ids_of_word_id word_id with\n    | Some doc_ids -> doc_ids\n    | None -> (\n        let bv = CCBV.empty () in\n        Hashtbl.replace t.doc_ids_of_word_id word_id bv;\n        bv\n      )\n\n  let union_doc_ids_of_word_id_into_bv ~word_id ~into =\n    lock (fun () ->\n        let bv = find_doc_ids_bv ~word_id in\n        CCBV.union_into ~into bv\n      )\n\n  let add_word_id_doc_id_link ~word_id ~doc_id =\n    lock (fun () ->\n        let doc_ids = find_doc_ids_bv ~word_id in\n        CCBV.set doc_ids (Int64.to_int doc_id)\n      )\n\n  let read_from_db () : unit =\n    let open Sqlite3_utils in\n    lock (fun () ->\n        with_db (fun db ->\n            iter_stmt ~db\n              {|\n  SELECT word_id, doc_id\n  FROM word_id_doc_id_link\n  |}\n              ~names:[]\n              (fun data ->\n                 let word_id = Data.to_int_exn data.(0) in\n                 let doc_id = Data.to_int_exn data.(1) in\n                 let doc_ids = find_doc_ids_bv ~word_id in\n                 CCBV.set doc_ids doc_id\n              )\n          )\n      )\nend\n\nlet now_int64 () =\n  Timedesc.Timestamp.now ()\n  |> Timedesc.Timestamp.get_s\n\nlet refresh_last_used_batch (doc_ids : int64 list) : unit =\n  let open Sqlite3_utils in\n  let now = now_int64 () in\n  with_db (fun db ->\n      step_stmt ~db \"BEGIN IMMEDIATE\" ignore;\n      List.iter (fun doc_id ->\n          step_stmt ~db\n            {|\n  UPDATE doc_info\n  SET last_used = @now\n  WHERE id = @doc_id\n  |}\n            ~names:[ (\"@doc_id\", INT doc_id)\n                   ; (\"@now\", INT now)\n                   ]\n            ignore;\n        )\n        doc_ids;\n      step_stmt ~db \"COMMIT\" ignore;\n    )\n\nlet document_count () : int =\n  let open Sqlite3_utils in\n  with_db (fun db ->\n      step_stmt ~db \"SELECT COUNT(1) FROM doc_info\"\n        (fun stmt ->\n           Int64.to_int (column_int64 stmt 0)\n        )\n    )\n\nlet prune_old_documents ~keep_n_latest : unit =\n  let open Sqlite3_utils in\n  with_db (fun db ->\n      step_stmt ~db \"BEGIN IMMEDIATE\" ignore;\n      step_stmt ~db \"DROP TABLE IF EXISTS temp.docs_to_drop\" ignore;\n      step_stmt ~db \"CREATE TEMP TABLE docs_to_drop (hash TEXT, id INTEGER)\" ignore;\n      step_stmt ~db\n        {|\n    INSERT INTO temp.docs_to_drop\n    SELECT hash, id\n    FROM doc_info\n    ORDER BY last_used DESC\n    LIMIT -1\n    OFFSET @offset\n    |}\n        ~names:[(\"@offset\", INT (Int64.of_int keep_n_latest))]\n        ignore;\n      let drop_based_on_doc_id ?(id_column = \"doc_id\") table =\n        step_stmt ~db\n          (Fmt.str\n             {|\n      DELETE FROM %s\n      WHERE EXISTS (\n        SELECT 1 FROM temp.docs_to_drop WHERE %s.%s = temp.docs_to_drop.id\n      )\n      |}\n             table\n             table\n             id_column\n          )\n          ignore\n      in\n      drop_based_on_doc_id ~id_column:\"id\" \"doc_info\";\n      drop_based_on_doc_id \"line_info\";\n      drop_based_on_doc_id \"page_info\";\n      drop_based_on_doc_id \"position\";\n      drop_based_on_doc_id \"word_id_doc_id_link\";\n      step_stmt ~db \"DROP TABLE temp.docs_to_drop\" ignore;\n      step_stmt ~db \"COMMIT\" ignore;\n    )\n\nlet write_raw_to_db db ~already_in_transaction ~doc_id (x : Raw.t) : unit =\n  let open Sqlite3_utils in\n  let now = now_int64 () in\n  with_db ~db (fun db ->\n      step_stmt ~db\n        {|\n  UPDATE doc_info\n  SET page_count = @page_count,\n      global_line_count = @global_line_count,\n      max_pos = @max_pos,\n      last_used = @now,\n      status = 'ONGOING'\n  WHERE\n      id = @doc_id\n  |}\n        ~names:[ (\"@doc_id\", INT doc_id)\n               ; (\"@page_count\", INT (Int64.of_int x.page_count))\n               ; (\"@global_line_count\", INT (Int64.of_int x.global_line_count))\n               ; (\"@max_pos\", INT (Int64.of_int (Int_map.max_binding x.word_of_pos |> fst)))\n               ; (\"@now\", INT now)\n               ]\n        ignore;\n      if not already_in_transaction then (\n        step_stmt ~db \"BEGIN IMMEDIATE\" ignore;\n      );\n      with_stmt ~db\n        {|\n  INSERT INTO page_info\n  (doc_id, page_num, line_count, start_pos, end_inc_pos)\n  VALUES\n  (@doc_id, @page_num, @line_count, @start_pos, @end_inc_pos)\n  ON CONFLICT(doc_id, page_num) DO NOTHING\n  |}\n        (fun stmt ->\n           Int_map.iter (fun page_num line_count ->\n               let (start_pos, end_inc_pos) =\n                 Int_map.find page_num x.start_end_inc_pos_of_page_num\n               in\n               bind_names stmt [ (\"@doc_id\", INT doc_id)\n                               ; (\"@page_num\", INT (Int64.of_int page_num))\n                               ; (\"@line_count\", INT (Int64.of_int line_count))\n                               ; (\"@start_pos\", INT (Int64.of_int start_pos))\n                               ; (\"@end_inc_pos\", INT (Int64.of_int end_inc_pos))\n                               ];\n               step stmt;\n               reset stmt;\n             )\n             x.line_count_of_page_num\n        );\n      with_stmt ~db\n        {|\n  INSERT INTO line_info\n  (doc_id, global_line_num, start_pos, end_inc_pos, page_num, line_num_in_page)\n  VALUES\n  (@doc_id, @global_line_num, @start_pos, @end_inc_pos, @page_num, @line_num_in_page)\n  ON CONFLICT(doc_id, global_line_num) DO NOTHING\n  |}\n        (fun stmt ->\n           Int_map.iter (fun line_num line_loc ->\n               let (start_pos, end_inc_pos) =\n                 Int_map.find line_num x.start_end_inc_pos_of_global_line_num\n               in\n               let page_num = line_loc.Line_loc.page_num in\n               let line_num_in_page = line_loc.Line_loc.line_num_in_page in\n               bind_names stmt [ (\"@doc_id\", INT doc_id)\n                               ; (\"@global_line_num\", INT (Int64.of_int line_num))\n                               ; (\"@start_pos\", INT (Int64.of_int start_pos))\n                               ; (\"@end_inc_pos\", INT (Int64.of_int end_inc_pos))\n                               ; (\"@page_num\", INT (Int64.of_int page_num))\n                               ; (\"@line_num_in_page\", INT (Int64.of_int line_num_in_page))\n                               ];\n               step stmt;\n               reset stmt;\n             )\n             x.line_loc_of_global_line_num;\n        );\n      with_stmt ~db\n        {|\n  INSERT INTO position\n  (doc_id, pos, word_id)\n  VALUES\n  (@doc_id, @pos, @word_id)\n  ON CONFLICT(doc_id, pos) DO NOTHING\n    |}\n        (fun stmt ->\n           Int_map.iter (fun word_id pos_s ->\n               Int_set.iter (fun pos ->\n                   bind_names stmt\n                     [ (\"@doc_id\", INT doc_id)\n                     ; (\"@pos\", INT (Int64.of_int pos))\n                     ; (\"@word_id\", INT (Int64.of_int word_id))\n                     ];\n                   step stmt;\n                   reset stmt;\n                 )\n                 pos_s\n             )\n             x.pos_s_of_word\n        );\n      with_stmt ~db\n        {|\n  INSERT INTO link\n  (doc_id, start_pos, end_inc_pos, typ, link)\n  VALUES\n  (@doc_id, @start_pos, @end_inc_pos, @typ, @link)\n  ON CONFLICT(doc_id, start_pos, end_inc_pos) DO NOTHING\n    |}\n        (fun stmt ->\n           Array.iter (fun link ->\n               let { Link.start_pos; end_inc_pos; typ; link } = link in\n               let typ = Link.string_of_typ typ in\n               bind_names stmt\n                 [ (\"@doc_id\", INT doc_id)\n                 ; (\"@start_pos\", INT (Int64.of_int start_pos))\n                 ; (\"@end_inc_pos\", INT (Int64.of_int end_inc_pos))\n                 ; (\"@typ\", TEXT typ)\n                 ; (\"@link\", TEXT link)\n                 ];\n               step stmt;\n               reset stmt;\n             )\n             x.links\n        );\n      with_stmt ~db\n        {|\n  INSERT INTO word_id_doc_id_link\n  (word_id, doc_id)\n  VALUES\n  (@word_id, @doc_id)\n  ON CONFLICT(word_id, doc_id) DO NOTHING\n    |}\n        (fun stmt ->\n           Int_map.iter (fun word_id _pos_s ->\n               State.add_word_id_doc_id_link ~word_id ~doc_id;\n               bind_names stmt\n                 [ (\"@word_id\", INT (Int64.of_int word_id))\n                 ; (\"@doc_id\", INT doc_id)\n                 ];\n               step stmt;\n               reset stmt;\n             )\n             x.pos_s_of_word\n        );\n      step_stmt ~db\n        {|\n      UPDATE doc_info\n      SET status = 'COMPLETED'\n      WHERE id = @doc_id\n    |}\n        ~names:[ (\"@doc_id\", INT doc_id) ]\n        ignore;\n      if not already_in_transaction then (\n        step_stmt ~db \"COMMIT\" ignore;\n      );\n    )\n\nlet global_line_count =\n  let open Sqlite3_utils in\n  fun ~doc_id ->\n    step_stmt\n      {|\n    SELECT global_line_count FROM doc_info\n    WHERE id = @doc_id\n    |}\n      ~names:[ (\"@doc_id\", INT doc_id) ]\n      (fun stmt ->\n         column_int stmt 0\n      )\n\nlet page_count ~doc_id =\n  let open Sqlite3_utils in\n  step_stmt\n    {|\n    SELECT page_count FROM doc_info\n    WHERE id = @doc_id\n    |}\n    ~names:[(\"@doc_id\", INT doc_id)]\n    (fun stmt ->\n       column_int stmt 0\n    )\n\nlet ccvector_of_int_map\n  : 'a . 'a Int_map.t -> 'a CCVector.ro_vector =\n  fun m ->\n  Int_map.to_seq m\n  |> Seq.map snd\n  |> CCVector.of_seq\n  |> CCVector.freeze\n\nlet is_indexed_sql =\n  {|\n    SELECT 1\n    FROM doc_info\n    WHERE hash = @doc_hash\n    AND status = 'COMPLETED'\n    |}\n\nlet is_indexed ~doc_hash =\n  let open Sqlite3_utils in\n  step_stmt\n    is_indexed_sql\n    ~names:[ (\"@doc_hash\", TEXT doc_hash) ]\n    (fun stmt ->\n       data_count stmt > 0\n    )\n\nlet word_of_pos ~doc_id pos : string =\n  let open Sqlite3_utils in\n  step_stmt\n    {|\n    SELECT word.word\n    FROM position p\n    JOIN word\n        ON word.id = p.word_id\n    WHERE p.doc_id = @doc_id\n    AND p.pos = @pos\n    |}\n    ~names:[ (\"@doc_id\", INT doc_id)\n           ; (\"@pos\", INT (Int64.of_int pos)) ]\n    (fun stmt ->\n       column_text stmt 0\n    )\n\nlet word_ci_of_pos ~doc_id pos : string =\n  word_of_pos ~doc_id pos\n  |> String.lowercase_ascii\n\nlet words_between_start_and_end_inc : doc_id:int64 -> int * int -> string Dynarray.t =\n  let lock = Eio.Mutex.create () in\n  let cache =\n    CCCache.lru ~eq:(fun (x0, y0, z0) (x1, y1, z1) ->\n        Int64.equal x0 x1\n        && Int.equal y0 y1\n        && Int.equal z0 z1\n      )\n      10240\n  in\n  fun ~doc_id (start, end_inc) ->\n    Eio.Mutex.use_rw ~protect:false lock (fun () ->\n        CCCache.with_cache cache (fun (doc_id, start, end_inc) ->\n            let open Sqlite3_utils in\n            let acc = Dynarray.create () in\n            iter_stmt\n              {|\n    SELECT word.word\n    FROM position p\n    JOIN word\n      ON word.id = p.word_id\n    WHERE p.doc_id = @doc_id\n    AND p.pos BETWEEN @start AND @end_inc\n    ORDER BY p.pos\n    |}\n              ~names:[ (\"@doc_id\", INT doc_id)\n                     ; (\"@start\", INT (Int64.of_int start))\n                     ; (\"@end_inc\", INT (Int64.of_int end_inc))\n                     ]\n              (fun data ->\n                 Dynarray.add_last acc (Data.to_string_exn data.(0))\n              );\n            acc\n          )\n          (doc_id, start, end_inc)\n      )\n\nlet words_of_global_line_num : doc_id:int64 -> int -> string Dynarray.t =\n  let lock = Eio.Mutex.create () in\n  let cache =\n    CCCache.lru ~eq:(fun (x0, y0) (x1, y1) ->\n        Int64.equal x0 x1 && Int.equal y0 y1)\n      10240\n  in\n  fun ~doc_id x ->\n    Eio.Mutex.use_rw ~protect:false lock (fun () ->\n        CCCache.with_cache cache (fun (doc_id, x) ->\n            let open Sqlite3_utils in\n            if x >= global_line_count ~doc_id then (\n              invalid_arg \"Index.words_of_global_line_num: global_line_num out of range\"\n            ) else (\n              let start, end_inc =\n                step_stmt\n                  {|\n        SELECT start_pos, end_inc_pos\n        FROM line_info\n        WHERE doc_id = @doc_id\n        AND global_line_num = @x\n        |}\n                  ~names:[ (\"@doc_id\", INT doc_id)\n                         ; (\"@x\", INT (Int64.of_int x))\n                         ]\n                  (fun stmt ->\n                     (column_int stmt 0, column_int stmt 1)\n                  )\n              in\n              words_between_start_and_end_inc ~doc_id (start, end_inc)\n            )\n          )\n          (doc_id, x)\n      )\n\nlet words_of_page_num ~doc_id x : string Dynarray.t =\n  let open Sqlite3_utils in\n  if x >= page_count ~doc_id then (\n    invalid_arg \"Index.words_of_page_num: page_num out of range\"\n  ) else (\n    let start, end_inc =\n      step_stmt\n        {|\n        SELECT start_pos, end_inc_pos\n        FROM page_info\n        WHERE doc_id = @doc_id\n        AND page_num = @x\n        |}\n        ~names:[ (\"@doc_id\", INT doc_id)\n               ; (\"@x\", INT (Int64.of_int x))\n               ]\n        (fun stmt ->\n           (column_int stmt 0, column_int stmt 1)\n        )\n    in\n    words_between_start_and_end_inc ~doc_id (start, end_inc)\n  )\n\nlet line_of_global_line_num ~doc_id x =\n  if x >= global_line_count ~doc_id then (\n    invalid_arg \"Index.line_of_global_line_num: global_line_num out of range\"\n  ) else (\n    words_of_global_line_num ~doc_id x\n    |> Dynarray.to_list\n    |> String.concat \"\"\n  )\n\nlet line_loc_of_global_line_num ~doc_id global_line_num : Line_loc.t =\n  let open Sqlite3_utils in\n  if global_line_num >= global_line_count ~doc_id then (\n    invalid_arg \"Index.line_loc_of_global_line_num: global_line_num out of range\"\n  ) else (\n    let page_num, line_num_in_page =\n      step_stmt\n        {|\n        SELECT page_num, line_num_in_page\n        FROM line_info\n        WHERE doc_id = @doc_id\n        AND global_line_num = @global_line_num\n        |}\n        ~names:[ (\"@doc_id\", INT doc_id)\n               ; (\"@global_line_num\", INT (Int64.of_int global_line_num)) ]\n        (fun stmt ->\n           (column_int stmt 0, column_int stmt 1)\n        )\n    in\n    { Line_loc.page_num; line_num_in_page; global_line_num }\n  )\n\nlet loc_of_pos ~doc_id pos : Loc.t =\n  let open Sqlite3_utils in\n  let pos_in_line, global_line_num =\n    step_stmt\n      {|\n      SELECT @pos - start_pos, global_line_num\n      FROM line_info\n      WHERE doc_id = @doc_id\n      AND @pos BETWEEN start_pos AND end_inc_pos\n      |}\n      ~names:[ (\"@doc_id\", INT doc_id)\n             ; (\"@pos\", INT (Int64.of_int pos)) ]\n      (fun stmt ->\n         (column_int stmt 0, column_int stmt 1)\n      )\n  in\n  let line_loc = line_loc_of_global_line_num ~doc_id global_line_num in\n  { line_loc; pos_in_line }\n\nlet max_pos ~doc_id =\n  let open Sqlite3_utils in\n  step_stmt\n    {|\n    SELECT max_pos\n    FROM doc_info\n    WHERE id = @doc_id\n    |}\n    ~names:[ (\"@doc_id\", INT doc_id) ]\n    (fun stmt ->\n       column_int stmt 0\n    )\n\nlet line_count_of_page_num ~doc_id page : int =\n  let open Sqlite3_utils in\n  step_stmt\n    {|\n    SELECT line_count\n    FROM page_info\n    WHERE doc_id = @doc_id\n    AND page = @page\n    |}\n    ~names:[ (\"@doc_id\", INT doc_id)\n           ; (\"@page\", INT (Int64.of_int page)) ]\n    (fun stmt ->\n       column_int stmt 0\n    )\n\nlet start_end_inc_pos_of_global_line_num ~doc_id global_line_num =\n  let open Sqlite3_utils in\n  if global_line_num >= global_line_count ~doc_id then (\n    invalid_arg \"Index.start_end_inc_pos_of_global_line_num: global_line_num out of range\"\n  ) else (\n    step_stmt\n      {|\n      SELECT start_pos, end_inc_pos\n      FROM line_info\n      WHERE doc_id = @doc_id\n      AND global_line_num = @global_line_num\n      |}\n      ~names:[ (\"@doc_id\", INT doc_id)\n             ; (\"@global_line_num\", INT (Int64.of_int global_line_num)) ]\n      (fun stmt ->\n         (column_int stmt 0, column_int stmt 1)\n      )\n  )\n\nmodule Search = struct\n  module ET = Search_phrase.Enriched_token\n\n  let positions_of_words\n      ~doc_id\n      (words : int Seq.t)\n    : int Dynarray.t =\n    let open Sqlite3_utils in\n    let acc = Dynarray.create () in\n    let f data =\n      Dynarray.add_last acc (Data.to_int_exn data.(0))\n    in\n    with_stmt\n      {|\n    SELECT\n      p.pos\n    FROM position p\n    WHERE doc_id = @doc_id\n    AND word_id = @word_id\n    ORDER BY p.pos\n    |}\n      (fun stmt ->\n         Seq.iter (fun word_id ->\n             bind_names stmt [ (\"@doc_id\", INT doc_id)\n                             ; (\"@word_id\", INT (Int64.of_int word_id))\n                             ];\n             Rc.check (iter stmt ~f);\n             reset stmt;\n           )\n           words\n      );\n    acc\n\n  let usable_positions\n      ~doc_id\n      ?within\n      ~around_pos\n      (token : Search_phrase.Enriched_token.t)\n    : int Seq.t =\n    let open Sqlite3_utils in\n    Eio.Fiber.yield ();\n    let match_typ = ET.match_typ token in\n    let start, end_inc =\n      let start, end_inc =\n        if ET.is_linked_to_prev token then (\n          match match_typ with\n          | `Fuzzy ->\n            (around_pos - !Params.max_linked_token_search_dist,\n             around_pos + !Params.max_linked_token_search_dist)\n          | `Exact | `Prefix | `Suffix ->\n            (around_pos + 1,\n             around_pos + 1)\n        ) else (\n          (around_pos - !Params.max_token_search_dist,\n           around_pos + !Params.max_token_search_dist)\n        )\n      in\n      match within with\n      | None -> (start, end_inc)\n      | Some (within_start_pos, within_end_inc_pos) -> (\n          (max within_start_pos start, min within_end_inc_pos end_inc)\n        )\n    in\n    let positions : int Dynarray.t =\n      let acc : int Dynarray.t =\n        Dynarray.create ()\n      in\n      let cache : (string, bool) Hashtbl.t = Hashtbl.create 100 in\n      let f data =\n        Eio.Fiber.yield ();\n        let indexed_word = Data.to_string_exn data.(0) in\n        let pos = Data.to_int_exn data.(1) in\n        let compatible =\n          match Hashtbl.find_opt cache indexed_word with\n          | None -> (\n              let compatible = ET.compatible_with_word token indexed_word in\n              Hashtbl.replace cache indexed_word compatible;\n              compatible\n            )\n          | Some compatible -> compatible\n        in\n        if compatible then (\n          Dynarray.add_last acc pos\n        )\n      in\n      (\n        let extra_sql =\n          match ET.data token with\n          | `Explicit_spaces -> (\n              {|AND (\n                  word LIKE ' %'\n                  OR\n                  word LIKE char(9) || '%'\n                  OR\n                  word LIKE char(10) || '%'\n                  OR\n                  word LIKE char(13) || '%'\n                )\n            |}\n            )\n          | `String search_word -> (\n              let search_word = search_word\n                |> CCString.replace ~sub:\"'\" ~by:\"''\"\n                |> CCString.replace ~sub:\"\\\\\" ~by:\"\\\\\\\\\"\n                |> CCString.replace ~sub:\"%\" ~by:\"\\\\%\"\n              in\n              match match_typ with\n              | `Fuzzy | `Suffix -> \"\"\n              | `Exact -> (\n                  Fmt.str \"AND word LIKE '%s' ESCAPE '\\\\'\" search_word\n                )\n              | `Prefix -> (\n                  Fmt.str \"AND word LIKE '%s%%' ESCAPE '\\\\'\" search_word\n                )\n            )\n        in\n        iter_stmt\n          (Fmt.str\n             {|\n              SELECT\n                word.word AS word,\n                p.pos as pos\n              FROM position p\n              JOIN word\n                  ON p.word_id = word.id\n              WHERE p.doc_id = @doc_id\n              AND p.pos BETWEEN @start AND @end_inc\n              %s\n              |}\n             extra_sql)\n          ~names:[ (\"@doc_id\", INT doc_id)\n                 ; (\"@start\", INT (Int64.of_int start))\n                 ; (\"@end_inc\", INT (Int64.of_int end_inc))\n                 ]\n          f\n      );\n      acc\n    in\n    Dynarray.to_seq positions\n\n  let search_around_pos\n      ~doc_id\n      ~(within : (int * int) option)\n      (around_pos : int)\n      (l : Search_phrase.Enriched_token.t list)\n    : int list Seq.t =\n    let rec aux around_pos l =\n      Eio.Fiber.yield ();\n      match l with\n      | [] -> Seq.return []\n      | token :: rest -> (\n          usable_positions\n            ~doc_id\n            ?within\n            ~around_pos\n            token\n          |> Seq.flat_map (fun pos ->\n              aux pos rest\n              |> Seq.map (fun l -> pos :: l)\n            )\n        )\n    in\n    aux around_pos l\n\n  let search_result_heap_merge_with_yield x y =\n    Eio.Fiber.yield ();\n    Search_result_heap.merge x y\n\n  module Search_job = struct\n    exception Result_found\n\n    type t = {\n      stop_signal : Stop_signal.t;\n      terminate_on_result_found : bool;\n      cancellation_notifier : bool Atomic.t;\n      doc_id : int64;\n      within_same_line : bool;\n      phrase : Search_phrase.t;\n      start_pos : int;\n      search_limit_per_start : int;\n    }\n\n    let make\n        stop_signal\n        ~terminate_on_result_found\n        ~cancellation_notifier\n        ~doc_id\n        ~within_same_line\n        ~phrase\n        ~start_pos\n        ~search_limit_per_start\n      =\n      {\n        stop_signal;\n        terminate_on_result_found;\n        cancellation_notifier;\n        doc_id;\n        within_same_line;\n        phrase;\n        start_pos;\n        search_limit_per_start;\n      }\n\n    let run (t : t) : Search_result_heap.t =\n      match Search_phrase.enriched_tokens t.phrase with\n      | [] -> Search_result_heap.empty\n      | _ :: rest -> (\n          let doc_id = t.doc_id in\n          let within =\n            if t.within_same_line then (\n              let loc = loc_of_pos ~doc_id t.start_pos in\n              Some (start_end_inc_pos_of_global_line_num ~doc_id loc.line_loc.global_line_num)\n            ) else (\n              None\n            )\n          in\n          Eio.Fiber.first\n            (fun () ->\n               Stop_signal.await t.stop_signal;\n               Atomic.set t.cancellation_notifier true;\n               Search_result_heap.empty)\n            (fun () ->\n               search_around_pos\n                 ~doc_id\n                 ~within\n                 t.start_pos\n                 rest\n               |> Seq.map (fun l -> t.start_pos :: l)\n               |> Seq.map (fun (l : int list) ->\n                   if t.terminate_on_result_found then (\n                     raise Result_found\n                   );\n                   Eio.Fiber.yield ();\n                   let opening_closing_symbol_pairs =\n                     List.map (fun pos -> word_of_pos ~doc_id pos) l\n                     |>  Misc_utils.opening_closing_symbol_pairs\n                   in\n                   let found_phrase_opening_closing_symbol_match_count =\n                     let pos_arr : int array = Array.of_list l in\n                     List.fold_left (fun total (x, y) ->\n                         let pos_x = pos_arr.(x) in\n                         let pos_y = pos_arr.(y) in\n                         let c_x = String.get (word_of_pos ~doc_id pos_x) 0 in\n                         let c_y = String.get (word_of_pos ~doc_id pos_y) 0 in\n                         assert (List.exists (fun (x, y) -> c_x = x && c_y = y)\n                                   Params.opening_closing_symbols);\n                         if pos_x < pos_y then (\n                           let outstanding_opening_symbol_count =\n                             OSeq.(pos_x + 1 --^ pos_y)\n                             |> Seq.fold_left (fun count pos ->\n                                 match count with\n                                 | Some count -> (\n                                     let word = word_of_pos ~doc_id pos in\n                                     if String.length word = 1 then (\n                                       if String.get word 0 = c_x then (\n                                         Some (count + 1)\n                                       ) else if String.get word 0 = c_y then (\n                                         if count = 0 then (\n                                           None\n                                         ) else (\n                                           Some (count - 1)\n                                         )\n                                       ) else (\n                                         Some count\n                                       )\n                                     ) else (\n                                       Some count\n                                     )\n                                   )\n                                 | None -> None\n                               )\n                               (Some 0)\n                           in\n                           match outstanding_opening_symbol_count with\n                           | Some 0 -> total + 1\n                           | _ -> total\n                         ) else (\n                           total\n                         )\n                       )\n                       0\n                       opening_closing_symbol_pairs\n                   in\n                   Search_result.make\n                     t.phrase\n                     ~found_phrase:(List.map\n                                      (fun pos ->\n                                         Search_result.{\n                                           found_word_pos = pos;\n                                           found_word_ci = word_ci_of_pos ~doc_id pos;\n                                           found_word = word_of_pos ~doc_id pos;\n                                         }) l)\n                     ~found_phrase_opening_closing_symbol_match_count\n                 )\n               |> Seq.fold_left (fun best_results r ->\n                   Eio.Fiber.yield ();\n                   let best_results = Search_result_heap.add best_results r in\n                   if Search_result_heap.size best_results <= t.search_limit_per_start then (\n                     best_results\n                   ) else (\n                     let best_results, _ = Search_result_heap.take_exn best_results in\n                     best_results\n                   )\n                 )\n                 Search_result_heap.empty\n            )\n        )\n  end\n\n  module Search_job_group = struct\n    type t = {\n      terminate_on_result_found : bool;\n      stop_signal : Stop_signal.t;\n      cancellation_notifier : bool Atomic.t;\n      doc_id : int64;\n      within_same_line : bool;\n      phrase : Search_phrase.t;\n      possible_start_pos_list : int list;\n      search_limit_per_start : int;\n    }\n\n    let unpack (group : t) : Search_job.t Seq.t =\n      let\n        {\n          stop_signal;\n          terminate_on_result_found;\n          cancellation_notifier;\n          doc_id;\n          within_same_line;\n          phrase;\n          possible_start_pos_list;\n          search_limit_per_start;\n        } = group in\n      List.to_seq possible_start_pos_list\n      |> Seq.map (fun start_pos ->\n          Search_job.make\n            stop_signal\n            ~terminate_on_result_found\n            ~cancellation_notifier\n            ~doc_id\n            ~within_same_line\n            ~phrase\n            ~start_pos\n            ~search_limit_per_start\n        )\n\n    let run (t : t) =\n      unpack t\n      |> Seq.map Search_job.run\n      |> Seq.fold_left Search_result_heap.merge Search_result_heap.empty\n  end\n\n  let make_search_job_groups\n      stop_signal\n      ?(terminate_on_result_found = false)\n      ~(cancellation_notifier : bool Atomic.t)\n      ~doc_id\n      ~doc_word_ids\n      ~global_first_word_candidates_lookup\n      ~within_same_line\n      ~(search_scope : Diet.Int.t option)\n      (exp : Search_exp.t)\n    : Search_job_group.t Seq.t =\n    if Search_exp.is_empty exp then (\n      Seq.empty\n    ) else (\n      Search_exp.flattened exp\n      |> List.to_seq\n      |> Seq.flat_map (fun phrase ->\n          let first_word_candidates =\n            match Search_phrase.enriched_tokens phrase with\n            | [] -> failwith \"unexpected case\"\n            | first_word :: _ -> (\n                Search_phrase.Enriched_token.Data_map.find\n                  (Search_phrase.Enriched_token.data first_word)\n                  global_first_word_candidates_lookup\n                |> Int_set.inter doc_word_ids\n              )\n          in\n          let possible_starts =\n            first_word_candidates\n            |> Int_set.to_seq\n            |> positions_of_words ~doc_id\n            |> (fun arr ->\n                match search_scope with\n                | None -> arr\n                | Some search_scope -> (\n                    Dynarray.filter (fun x ->\n                        Diet.Int.mem x search_scope\n                      ) arr\n                  )\n              )\n          in\n          let possible_start_count = Dynarray.length possible_starts in\n          if possible_start_count = 0 then (\n            Seq.empty\n          ) else (\n            let search_limit_per_start =\n              max\n                Params.search_result_min_per_start\n                (\n                  (Params.default_search_result_total_per_document + possible_start_count - 1) / possible_start_count\n                )\n            in\n            let search_chunk_size =\n              max 10 (possible_start_count / Task_pool.size)\n            in\n            OSeq.(0 --^ possible_start_count)\n            |> OSeq.chunks search_chunk_size\n            |> Seq.map (fun index_arr ->\n                Array.map (fun i ->\n                    Dynarray.get possible_starts i\n                  ) index_arr\n                |> Array.to_list\n              )\n            |> Seq.map (fun possible_start_pos_list ->\n                {\n                  Search_job_group.stop_signal;\n                  terminate_on_result_found;\n                  cancellation_notifier;\n                  doc_id;\n                  within_same_line;\n                  phrase;\n                  possible_start_pos_list;\n                  search_limit_per_start;\n                }\n              )\n          )\n        )\n    )\n\n  let search\n      pool\n      stop_signal\n      ?terminate_on_result_found\n      ~cancellation_notifier\n      ~doc_id\n      ~doc_word_ids\n      ~global_first_word_candidates_lookup\n      ~within_same_line\n      ~search_scope\n      (exp : Search_exp.t)\n    : Search_result_heap.t =\n    make_search_job_groups\n      stop_signal\n      ?terminate_on_result_found\n      ~cancellation_notifier\n      ~doc_id\n      ~doc_word_ids\n      ~global_first_word_candidates_lookup\n      ~within_same_line\n      ~search_scope\n      exp\n    |> List.of_seq\n    |> Task_pool.map_list pool Search_job_group.run\n    |> List.fold_left search_result_heap_merge_with_yield Search_result_heap.empty\nend\n\nlet search\n    pool\n    stop_signal\n    ?terminate_on_result_found\n    ~doc_id\n    ~doc_word_ids\n    ~global_first_word_candidates_lookup\n    ~within_same_line\n    ~search_scope\n    (exp : Search_exp.t)\n  : Search_result.t array option =\n  let cancellation_notifier = Atomic.make false in\n  let arr =\n    Search.search\n      pool\n      stop_signal\n      ?terminate_on_result_found\n      ~cancellation_notifier\n      ~doc_id\n      ~doc_word_ids\n      ~global_first_word_candidates_lookup\n      ~within_same_line\n      ~search_scope\n      exp\n    |> Search_result_heap.to_seq\n    |> Array.of_seq\n  in\n  if Atomic.get cancellation_notifier then (\n    None\n  ) else (\n    Array.sort Search_result.compare_relevance arr;\n    Some arr\n  )\n\nmodule Search_job = Search.Search_job\n\nmodule Search_job_group = Search.Search_job_group\n\nlet make_search_job_groups = Search.make_search_job_groups\n\nmodule Word_candidate_heap = CCHeap.Make_from_compare (struct\n    type t = int * float\n\n    let compare (_x0, y0) (_x1, y1) = Float.compare y0 y1\n  end)\n\nlet generate_global_first_word_candidates_lookup\n    pool\n    ?(acc = Search_phrase.Enriched_token.Data_map.empty)\n    (exp : Search_exp.t)\n  : Int_set.t Search_phrase.Enriched_token.Data_map.t =\n  Search_exp.flattened exp\n  |> List.fold_left (fun acc phrase ->\n      match Search_phrase.enriched_tokens phrase with\n      | [] -> failwith \"unexpected case\"\n      | first_word :: _rest -> (\n          let data = Search_phrase.Enriched_token.data first_word in\n          match Search_phrase.Enriched_token.Data_map.find_opt data acc with\n          | None -> (\n              let score ~search_word ~found_word =\n                let search_word_ci = String.lowercase_ascii search_word in\n                let search_word_len = Int.to_float (String.length search_word) in\n                let found_word_ci = String.lowercase_ascii found_word in\n                let found_word_len = Int.to_float (String.length found_word) in\n                if String.equal search_word found_word then (\n                  1.0\n                ) else if String.equal search_word_ci found_word_ci then (\n                  0.9\n                ) else if CCString.find ~sub:search_word found_word >= 0 then (\n                  search_word_len\n                  /.\n                  found_word_len\n                ) else if CCString.find ~sub:search_word_ci found_word_ci >= 0 then (\n                  0.9\n                  *.\n                  (search_word_len\n                   /.\n                   found_word_len)\n                ) else (\n                  1.0\n                  -.\n                  (Int.to_float (Spelll.edit_distance search_word_ci found_word_ci)\n                   /.\n                   search_word_len\n                  )\n                )\n              in\n              let candidates =\n                Word_db.filter\n                  pool\n                  (Search_phrase.Enriched_token.compatible_with_word first_word)\n              in\n              let candidates =\n                match data with\n                | `Explicit_spaces -> (\n                    Dynarray.fold_left (fun acc (id, _word) ->\n                        Int_set.add id acc\n                      )\n                      Int_set.empty\n                      candidates\n                  )\n                | `String s -> (\n                    let s_len = Int.to_float (String.length s) in\n                    let max_candidate_count = min 500 (Int.of_float @@ Float.pow 2.5 s_len) in\n                    Dynarray.fold_left (fun acc (id, word) ->\n                        let acc = Word_candidate_heap.add acc (id, score ~search_word:s ~found_word:word) in\n                        if Word_candidate_heap.size acc <= max_candidate_count then (\n                          acc\n                        ) else (\n                          let acc, _ = Word_candidate_heap.take_exn acc in\n                          acc\n                        )\n                      )\n                      Word_candidate_heap.empty\n                      candidates\n                    |> Word_candidate_heap.to_seq\n                    |> Seq.map (fun (id, _score) ->\n                        id\n                      )\n                    |> Int_set.of_seq\n                  )\n              in\n              Search_phrase.Enriched_token.Data_map.add\n                data\n                candidates\n                acc\n            )\n          | Some _ -> acc\n        )\n    )\n    acc\n\nlet word_ids ~doc_id =\n  let open Sqlite3_utils in\n  with_db (fun db ->\n      fold_stmt ~db\n        {|\n    SELECT word_id_doc_id_link.word_id\n    FROM word_id_doc_id_link\n    WHERE word_id_doc_id_link.doc_id = @doc_id\n    |}\n        ~names:[ (\"@doc_id\", INT doc_id) ]\n        (fun acc data ->\n           Int_set.add (Data.to_int_exn data.(0)) acc\n        )\n        Int_set.empty\n    )\n\nlet links ~doc_id : Link.t array =\n  let open Sqlite3_utils in\n  let acc = Dynarray.create () in\n  with_db (fun db ->\n      iter_stmt ~db\n        {|\n    SELECT start_pos, end_inc_pos, typ, link\n    FROM link\n    WHERE link.doc_id = @doc_id\n    |}\n        ~names:[ (\"@doc_id\", INT doc_id) ]\n        (fun data ->\n           let start_pos = Data.to_int_exn data.(0) in\n           let end_inc_pos = Data.to_int_exn data.(1) in\n           let typ =\n             Data.to_string_exn data.(2)\n             |> Link.typ_of_string\n             |> Option.get\n           in\n           let link = Data.to_string_exn data.(3) in\n           Dynarray.add_last acc { Link.start_pos; end_inc_pos; typ; link }\n        );\n    );\n  Dynarray.to_array acc\n"
  },
  {
    "path": "lib/index.mli",
    "content": "module Line_loc : sig\n  type t\n\n  val page_num : t -> int\n\n  val line_num_in_page : t -> int\n\n  val global_line_num : t -> int\n\n  val compare : t -> t -> int\nend\n\nmodule Loc : sig\n  type t\n\n  val line_loc : t -> Line_loc.t\n\n  val pos_in_line : t -> int\nend\n\nval word_ci_of_pos : doc_id:int64 -> int -> string\n\nval word_of_pos : doc_id:int64 -> int -> string\n\nval words_of_global_line_num : doc_id:int64 -> int -> string Dynarray.t\n\nval line_of_global_line_num : doc_id:int64 -> int -> string\n\nval line_loc_of_global_line_num : doc_id:int64 -> int -> Line_loc.t\n\nval loc_of_pos : doc_id:int64 -> int -> Loc.t\n\nval max_pos : doc_id:int64 -> int\n\nval words_of_page_num : doc_id:int64 -> int -> string Dynarray.t\n\nval line_count_of_page_num : doc_id:int64 -> int -> int\n\nval generate_global_first_word_candidates_lookup :\n  Task_pool.t ->\n  ?acc:Int_set.t Search_phrase.Enriched_token.Data_map.t ->\n  Search_exp.t ->\n  Int_set.t Search_phrase.Enriched_token.Data_map.t\n\nval search :\n  Task_pool.t ->\n  Stop_signal.t ->\n  ?terminate_on_result_found : bool ->\n  doc_id:int64 ->\n  doc_word_ids:Int_set.t ->\n  global_first_word_candidates_lookup:Int_set.t Search_phrase.Enriched_token.Data_map.t ->\n  within_same_line:bool ->\n  search_scope:Diet.Int.t option ->\n  Search_exp.t ->\n  Search_result.t array option\n\nmodule Search_job : sig\n  exception Result_found\n\n  type t\n\n  val run : t -> Search_result_heap.t\nend\n\nmodule Search_job_group : sig\n  type t\n\n  val unpack : t -> Search_job.t Seq.t\n\n  val run : t -> Search_result_heap.t\nend\n\nval make_search_job_groups :\n  Stop_signal.t ->\n  ?terminate_on_result_found : bool ->\n  cancellation_notifier:bool Atomic.t ->\n  doc_id:int64 ->\n  doc_word_ids:Int_set.t ->\n  global_first_word_candidates_lookup:Int_set.t Search_phrase.Enriched_token.Data_map.t ->\n  within_same_line:bool ->\n  search_scope:Diet.Int.t option ->\n  Search_exp.t ->\n  Search_job_group.t Seq.t\n\nval global_line_count : doc_id:int64 -> int\n\nval page_count : doc_id:int64 -> int\n\nval is_indexed_sql : string\n\nval is_indexed : doc_hash:string -> bool\n\nval refresh_last_used_batch : int64 list -> unit\n\nval document_count : unit -> int\n\nval prune_old_documents : keep_n_latest:int -> unit\n\nmodule Raw : sig\n  type t\n\n  val word_ids : t -> Int_set.t\n\n  val of_lines : Task_pool.t -> string Seq.t -> t\n\n  val of_pages : Task_pool.t -> string list Seq.t -> t\n\n  val links : t -> Link.t array\nend\n\nval word_ids : doc_id:int64 -> Int_set.t\n\nval write_raw_to_db :\n  Sqlite3.db ->\n  already_in_transaction:bool ->\n  doc_id:int64 ->\n  Raw.t ->\n  unit\n\nmodule State : sig\n  val union_doc_ids_of_word_id_into_bv : word_id:int -> into:CCBV.t -> unit\n\n  val read_from_db : unit -> unit\nend\n\nval links : doc_id:int64 -> Link.t array\n"
  },
  {
    "path": "lib/int_map.ml",
    "content": "include CCMap.Make (Int)\n"
  },
  {
    "path": "lib/int_set.ml",
    "content": "include CCSet.Make (Int)\n"
  },
  {
    "path": "lib/link.ml",
    "content": "type typ = [\n  | `Markdown\n  | `Wiki\n  | `URL\n]\n\nlet string_of_typ (typ : typ) =\n  match typ with\n  | `Markdown -> \"markdown\"\n  | `Wiki -> \"wiki\"\n  | `URL -> \"url\"\n\nlet typ_of_string (s : string) : typ option =\n  match String.lowercase_ascii s with\n  | \"markdown\" -> Some `Markdown\n  | \"wiki\" -> Some `Wiki\n  | \"url\" -> Some `URL\n  | _ -> None\n\ntype t = {\n  start_pos : int;\n  end_inc_pos : int;\n  typ : typ;\n  link : string;\n}\n"
  },
  {
    "path": "lib/misc_utils.ml",
    "content": "let exit_with_error_msg (msg : string) =\n  Printf.printf \"error: %s\\n\" msg;\n  exit 1\n\nlet ci_string_set_of_list (l : string list) =\n  l\n  |> List.map String.lowercase_ascii\n  |> String_set.of_list\n\nlet first_n_chars_of_string_contains ~n s c =\n  let s_len = String.length s in\n  let s =\n    if s_len <= n then\n      s\n    else\n      String.sub s 0 n\n  in\n  String.contains s c\n\nlet char_is_usable c =\n  let code = Char.code c in\n  (0x20 <= code && code <= 0x7E)\n\nlet sanitize_string s =\n  let s_len = String.length s in\n  let bytes = Bytes.make s_len ' ' in\n  let rec aux pos =\n    if pos >= s_len then\n      String.of_bytes bytes\n    else (\n      let decode = String.get_utf_8_uchar s pos in\n      if Uchar.utf_decode_is_valid decode then (\n        let c = String.get_uint8 s pos in\n        if c land 0b1000_0000 = 0b0000_0000 then (\n          if 0x20 <= c && c <= 0x7E then (\n            Bytes.blit_string s pos bytes pos 1\n          );\n          aux (pos+1)\n        ) else (\n          let len = Uchar.utf_decode_length decode in\n          Bytes.blit_string s pos bytes pos len;\n          aux (pos+len)\n        )\n      ) else (\n        aux (pos+1)\n      )\n    )\n  in\n  aux 0\n\nlet length_and_list_of_seq (s : 'a Seq.t) : int * 'a list =\n  let len, acc =\n    Seq.fold_left (fun (len, acc) x ->\n        (len + 1, x :: acc)\n      )\n      (0, [])\n      s\n  in\n  (len, List.rev acc)\n\nlet div_round_to_closest x y =\n  (x + (y / 2)) / y\n\nlet div_round_up x y =\n  (x + (y - 1)) / y\n\nlet opening_closing_symbol_pairs (l : string list) : (int * int) list =\n  let _, pairs =\n    CCList.foldi\n      (fun ((m, pairs) : (int list Char_map.t) * ((int * int) list)) i s ->\n         if String.length s = 1 then (\n           let c = String.get s 0 in\n           match List.assoc_opt c Params.opening_closing_symbols with\n           | Some _ -> (\n               let stack =\n                 match Char_map.find_opt c m with\n                 | None -> []\n                 | Some l -> l\n               in\n               (Char_map.add c (i :: stack) m, pairs)\n             )\n           | None -> (\n               match List.assoc_opt c Params.opening_closing_symbols_flipped with\n               | Some corresponding_open_symbol -> (\n                   let stack =\n                     match Char_map.find_opt corresponding_open_symbol m with\n                     | None -> []\n                     | Some l -> l\n                   in\n                   match stack with\n                   | [] -> (m, pairs)\n                   | x :: xs -> (\n                       (Char_map.add corresponding_open_symbol xs m, (x, i) :: pairs)\n                     )\n                 )\n               | None -> (m, pairs)\n             )\n         ) else (\n           (m, pairs)\n         )\n      )\n      (Char_map.empty, [])\n      l\n  in\n  pairs\n\nlet cwd_path_parts () =\n  Sys.getcwd ()\n  |> CCString.split ~by:Filename.dir_sep\n  |> List.rev\n\nlet path_of_parts parts =\n  match List.rev parts with\n  | [] | [ \"\" ] -> Filename.dir_sep\n  | [ x ] -> x\n  | l -> String.concat Filename.dir_sep l\n\nlet normalize_glob_to_absolute glob =\n  let rec aux acc parts =\n    match parts with\n    | [] -> path_of_parts acc\n    | x :: xs -> (\n        match x with\n        | \"\" | \".\" -> aux acc xs\n        | \"..\" -> (\n            let acc =\n              match acc with\n              | [] -> []\n              | _ :: xs -> xs\n            in\n            aux acc xs\n          )\n        | \"**\" -> (\n            aux (List.rev parts @ acc) []\n          )\n        | _ -> (\n            aux (x :: acc) xs\n          )\n      )\n  in\n  let glob_parts = CCString.split ~by:Filename.dir_sep glob in\n  match glob_parts with\n  | \"\" :: l -> (\n      (* Absolute path on Unix-like systems *)\n      aux [ \"\" ] l\n    )\n  | _ -> (\n      aux (cwd_path_parts ()) glob_parts\n    )\n\nlet normalize_path_to_absolute path =\n  let rec aux acc path_parts =\n    match path_parts with\n    | [] -> path_of_parts acc\n    | x :: xs -> (\n        match x with\n        | \"\" | \".\" -> aux acc xs\n        | \"..\" -> (\n            let acc =\n              match acc with\n              | [] -> []\n              | _ :: xs -> xs\n            in\n            aux acc xs\n          )\n        | _ -> (\n            aux (x :: acc) xs\n          )\n      )\n  in\n  let path_parts = CCString.split ~by:Filename.dir_sep path in\n  match path_parts with\n  | \"\" :: l -> (\n      (* Absolute path on Unix-like systems *)\n      aux [ \"\" ] l\n    )\n  | _ -> (\n      aux (cwd_path_parts ()) path_parts\n    )\n"
  },
  {
    "path": "lib/option_syntax.ml",
    "content": "let ( let* ) = Option.bind\n\nlet ( let+ ) x y = Option.map y x\n"
  },
  {
    "path": "lib/params.ml",
    "content": "let default_search_result_total_per_document = 50\n\nlet search_result_min_per_start = 5\n\nlet max_token_size = 500\n\nlet default_max_token_search_dist = 50\n\nlet max_token_search_dist = ref default_max_token_search_dist\n\nlet default_max_linked_token_search_dist = 2\n\nlet max_linked_token_search_dist = ref default_max_linked_token_search_dist\n\nlet default_index_chunk_size = 5000\n\nlet index_chunk_size = ref default_index_chunk_size\n\nlet search_word_automaton_cache_size = 200\n\nlet float_compare_margin = 0.000_001\n\nlet opening_closing_symbols = [ ('(', ')')\n                              ; ('[', ']')\n                              ; ('{', '}')\n                              ]\n\nlet opening_closing_symbols_flipped = List.map (fun (x, y) -> (y, x)) opening_closing_symbols\n\nlet default_max_fuzzy_edit_dist = 2\n\nlet max_fuzzy_edit_dist = ref default_max_fuzzy_edit_dist\n\nlet db_schema =\n  {|\nCREATE TABLE IF NOT EXISTS line_info (\n  doc_id INTEGER,\n  global_line_num INTEGER,\n  start_pos INTEGER,\n  end_inc_pos INTEGER,\n  page_num INTEGER,\n  line_num_in_page INTEGER,\n  PRIMARY KEY (doc_id, global_line_num)\n) WITHOUT ROWID;\n\nCREATE INDEX IF NOT EXISTS line_info_index_1 ON line_info (start_pos);\n\nCREATE INDEX IF NOT EXISTS line_info_index_2 ON line_info (end_inc_pos);\n\nCREATE TABLE IF NOT EXISTS position (\n  doc_id INTEGER,\n  pos INTEGER,\n  word_id INTEGER,\n  PRIMARY KEY (doc_id, pos)\n) WITHOUT ROWID;\n\nCREATE INDEX IF NOT EXISTS position_index_1 ON position (doc_id, word_id, pos);\n\nCREATE TABLE IF NOT EXISTS page_info (\n  doc_id INTEGER,\n  page_num INTEGER,\n  line_count INTEGER,\n  start_pos INTEGER,\n  end_inc_pos INTEGER,\n  PRIMARY KEY (doc_id, page_num)\n) WITHOUT ROWID;\n\nCREATE TABLE IF NOT EXISTS doc_info (\n  hash TEXT PRIMARY KEY,\n  id INTEGER,\n  page_count INTEGER,\n  global_line_count INTEGER,\n  max_pos INTEGER,\n  last_used INTEGER,\n  status TEXT\n) WITHOUT ROWID;\n\nCREATE INDEX IF NOT EXISTS doc_info_index_1 ON doc_info (id);\n\nCREATE INDEX IF NOT EXISTS doc_info_index_2 ON doc_info (last_used);\n\nCREATE TABLE IF NOT EXISTS link (\n  doc_id INTEGER,\n  start_pos INTEGER,\n  end_inc_pos INTEGER,\n  typ TEXT,\n  link TEXT,\n  PRIMARY KEY (doc_id, start_pos, end_inc_pos)\n) WITHOUT ROWID;\n\nCREATE TABLE IF NOT EXISTS word (\n  id INTEGER,\n  word TEXT,\n  PRIMARY KEY (id)\n) WITHOUT ROWID;\n\nCREATE INDEX IF NOT EXISTS word_index_1 ON word (word);\n\nCREATE INDEX IF NOT EXISTS word_index_2 ON word (word COLLATE NOCASE);\n\nCREATE TABLE IF NOT EXISTS word_id_doc_id_link (\n  word_id INTEGER,\n  doc_id INTEGER,\n  PRIMARY KEY (word_id, doc_id)\n) WITHOUT ROWID;\n\nCREATE INDEX IF NOT EXISTS word_id_doc_id_link_index_1 ON word_id_doc_id_link (doc_id);\n  |}\n\nlet db_path : string option ref = ref None\n"
  },
  {
    "path": "lib/parser_components.ml",
    "content": "open Angstrom\n\nlet is_space c =\n  match c with\n  | ' '\n  | '\\t'\n  | '\\n'\n  | '\\r' -> true\n  | _ -> false\n\nlet skip_spaces = skip_while is_space\n\nlet is_not_space c = not (is_space c)\n\nlet any_string : string t = take_while (fun _ -> true)\n\nlet is_letter c =\n  match c with\n  | 'A'..'Z'\n  | 'a'..'z' -> true\n  | _ -> false\n\nlet is_digit c =\n  match c with\n  | '0'..'9' -> true\n  | _ -> false\n\nlet is_alphanum c =\n  is_letter c || is_digit c\n\nlet is_possibly_utf_8 c =\n  let c = Char.code c in\n  c land 0b1000_0000 <> 0b0000_0000\n\nlet utf_8_char =\n  peek_char >>= fun c ->\n  match c with\n  | None -> fail \"Eof\"\n  | Some c -> (\n      let c = Char.code c in\n      if c land 0b1000_0000 = 0b0000_0000 then (\n        take 1\n      ) else if c land 0b1110_0000 = 0b1100_0000 then (\n        take 2\n      ) else if c land 0b1111_0000 = 0b1110_0000 then (\n        take 3\n      ) else if c land 0b1111_1000 = 0b1111_0000 then (\n        take 4\n      ) else (\n        fail \"Invalid UTF-8\"\n      )\n    )\n\n(* Copied from Angstrom README. *)\nlet chainl1 e op =\n  let rec go acc =\n    (lift2 (fun f x -> f acc x) op e >>= go) <|> return acc in\n  e >>= fun init -> go init\n"
  },
  {
    "path": "lib/search_exp.ml",
    "content": "type match_typ_marker = [ `Exact | `Prefix | `Suffix ]\n[@@deriving show]\n\ntype exp = [\n  | `Word of string\n  | `Match_typ_marker of match_typ_marker\n  | `Explicit_spaces\n  | `List of exp list\n  | `Paren of exp\n  | `Binary_op of binary_op * exp * exp\n  | `Optional of exp\n]\n[@@deriving show]\n\nand binary_op =\n  | Or\n[@@deriving show]\n\ntype t = {\n  exp : exp;\n  flattened : Search_phrase.t list;\n}\n[@@deriving show]\n\nlet flattened (t : t) = t.flattened\n\nlet empty : t = {\n  exp = `List [];\n  flattened = [];\n}\n\nlet is_empty (t : t) =\n  t.flattened = []\n  ||\n  List.for_all Search_phrase.is_empty t.flattened\n\nlet equal (t1 : t) (t2 : t) =\n  let rec aux (e1 : exp) (e2 : exp) =\n    match e1, e2 with\n    | `Word x1, `Word x2 -> String.equal x1 x2\n    | `List l1, `List l2 -> List.equal aux l1 l2\n    | `Paren e1, `Paren e2 -> aux e1 e2\n    | `Binary_op (Or, e1x, e1y), `Binary_op (Or, e2x, e2y) ->\n      aux e1x e2x && aux e1y e2y\n    | `Optional e1, `Optional e2 -> aux e1 e2\n    | _, _ -> false\n  in\n  aux t1.exp t2.exp\n\nlet as_paren x : exp = `Paren x\n\nlet as_list l : exp = `List l\n\nlet as_word s : exp = `Word s\n\nlet as_word_list (l : string list) : exp = `List (List.map as_word  l)\n\nmodule Parsers = struct\n  open Angstrom\n  open Parser_components\n\n  let phrase : string list Angstrom.t =\n    many1 (\n      take_while1 (fun c ->\n          match c with\n          | '?'\n          | '|'\n          | '\\\\'\n          | '('\n          | ')'\n          | '\\''\n          | '^'\n          | '$'\n          | '~' -> false\n          | _ -> true\n        )\n      <|>\n      (char '\\\\' *> any_char >>| fun c -> Printf.sprintf \"%c\" c)\n    )\n    >>| fun l ->\n    String.concat \"\" l\n    |> Tokenization.tokenize ~drop_spaces:false\n    |> List.of_seq\n\n  let or_op =\n    char '|' *> skip_spaces *> return (fun x y -> `Binary_op (Or, x, y))\n\n  let p : exp Angstrom.t =\n    fix (fun (exp : exp Angstrom.t) : exp Angstrom.t ->\n        let base =\n          choice [\n            (phrase >>| as_word_list);\n            (char '\\'' *> return (`Match_typ_marker `Exact));\n            (char '^' *> return (`Match_typ_marker `Prefix));\n            (char '$' *> return (`Match_typ_marker `Suffix));\n            (char '~' *> return (`Explicit_spaces));\n            (string \"()\" *> return (as_word_list []));\n            (char '(' *> exp <* char ')' >>| as_paren);\n          ]\n        in\n        let opt_base =\n          choice [\n            (char '?' *> skip_spaces *> phrase\n             >>| fun l ->\n             match l with\n             | [] -> failwith \"unexpected case\"\n             | x :: xs -> (\n                 as_list [ `Optional (as_word x); as_word_list xs ]\n               )\n            );\n            (char '?' *> skip_spaces *> base >>| fun p -> `Optional p);\n            base;\n          ]\n        in\n        let opt_bases =\n          many1 opt_base\n          >>| fun l -> `List l\n        in\n        chainl1 opt_bases or_op\n      )\n    <* skip_spaces\nend\n\nlet flatten_nested_lists (exp : exp) : exp =\n  let rec aux (exp : exp) =\n    match exp with\n    | `Word _\n    | `Match_typ_marker _\n    | `Explicit_spaces -> exp\n    | `List l -> (\n        `List\n          (CCList.flat_map (fun e ->\n               match aux e with\n               | `List l -> l\n               | x -> [ x ]\n             ) l)\n      )\n    | `Paren e -> `Paren (aux e)\n    | `Binary_op (op, x, y) -> `Binary_op (op, aux x, aux y)\n    | `Optional e -> `Optional (aux e)\n  in\n  aux exp\n\nlet flatten (exp : exp) : Search_phrase.t list =\n  let get_group_id =\n    let counter = ref 0 in\n    fun () ->\n      let x = !counter in\n      counter := x + 1;\n      x\n  in\n  let rec aux group_id (exp : exp) : Search_phrase.annotated_token list Seq.t =\n    match exp with\n    | `Match_typ_marker x -> (\n        Seq.return [\n          Search_phrase.{ data = `Match_typ_marker x; group_id }\n        ]\n      )\n    | `Word s ->\n      Seq.return [\n        Search_phrase.{ data = `String s; group_id }\n      ]\n    | `Explicit_spaces ->\n      Seq.return [\n        Search_phrase.{ data = `Explicit_spaces; group_id }\n      ]\n    | `List l -> (\n        l\n        |> List.to_seq\n        |> Seq.map (aux group_id)\n        |> OSeq.cartesian_product\n        |> Seq.map List.concat\n      )\n    | `Paren e -> (\n        aux (get_group_id ()) e\n      )\n    | `Binary_op (Or, x, y) -> (\n        Seq.append\n          (aux group_id x)\n          (aux group_id y)\n      )\n    | `Optional x -> (\n        Seq.cons [] (aux (get_group_id ()) x)\n      )\n  in\n  aux (get_group_id ()) exp\n  |> Seq.map (fun l ->\n      List.to_seq l\n      |> Search_phrase.of_annotated_tokens)\n  |> List.of_seq\n  |> List.sort_uniq Search_phrase.compare\n\nlet parse s =\n  if String.length s = 0 || String.for_all Parser_components.is_space s then (\n    Some empty\n  ) else (\n    match Angstrom.(parse_string ~consume:Consume.All) Parsers.p s with\n    | Ok exp -> (\n        let exp = flatten_nested_lists exp in\n        Some\n          { exp;\n            flattened = flatten exp;\n          }\n      )\n    | Error _ -> None\n  )\n"
  },
  {
    "path": "lib/search_exp.mli",
    "content": "type t\n\nval pp : Format.formatter -> t -> unit\n\nval empty : t\n\nval is_empty : t -> bool\n\nval flattened : t -> Search_phrase.t list\n\nval parse : string -> t option\n\nval equal : t -> t -> bool\n"
  },
  {
    "path": "lib/search_phrase.ml",
    "content": "type match_typ = [\n  | `Fuzzy\n  | `Exact\n  | `Suffix\n  | `Prefix\n]\n[@@deriving show, ord]\n\ntype match_typ_marker = [ `Exact | `Prefix | `Suffix ]\n[@@deriving show, ord]\n\nlet char_of_match_typ_marker (x : match_typ_marker) =\n  match x with\n  | `Exact -> '\\''\n  | `Prefix -> '^'\n  | `Suffix -> '$'\n\nlet string_of_match_typ_marker (x : match_typ_marker) =\n  match x with\n  | `Exact -> \"\\'\"\n  | `Prefix -> \"^\"\n  | `Suffix -> \"$\"\n\ntype annotated_token = {\n  data : [\n    | `String of string\n    | `Match_typ_marker of match_typ_marker\n    | `Explicit_spaces\n  ];\n  group_id : int;\n}\n[@@deriving show]\n\ntype ir0 = {\n  data : [\n    | `String of string\n    | `Match_typ_marker of match_typ_marker\n    | `Explicit_spaces\n  ];\n  is_linked_to_prev : bool;\n  is_linked_to_next : bool;\n  match_typ : match_typ option;\n}\n\nmodule Enriched_token = struct\n  type data = [ `String of string | `Explicit_spaces ]\n  [@@deriving ord]\n\n  module Data_map = Map.Make (struct\n      type t = data\n\n      let compare = compare_data\n    end)\n\n  let pp_data formatter data =\n    Fmt.pf formatter \"%s\"\n      (match data with\n       | `String s -> s\n       | `Explicit_spaces -> \" \")\n\n  type t = {\n    data : data;\n    is_linked_to_prev : bool;\n    is_linked_to_next : bool;\n    automaton : Spelll.automaton;\n    match_typ : match_typ;\n  }\n\n  let make data ~is_linked_to_prev ~is_linked_to_next automaton match_typ =\n    { data; is_linked_to_prev; is_linked_to_next; automaton; match_typ }\n\n  let pp fmt (x : t) =\n    Fmt.pf fmt \"%a:%b:%b:%a\"\n      pp_data\n      x.data\n      x.is_linked_to_prev\n      x.is_linked_to_next\n      pp_match_typ\n      x.match_typ\n\n  let data (t : t) =\n    t.data\n\n  let match_typ (t : t) =\n    t.match_typ\n\n  let automaton (t : t) =\n    t.automaton\n\n  let is_linked_to_prev (t : t) =\n    t.is_linked_to_prev\n\n  let is_linked_to_next (t : t) =\n    t.is_linked_to_next\n\n  let compare (x : t) (y : t) =\n    match compare_data x.data y.data with\n    | 0 -> (\n        match Bool.compare x.is_linked_to_prev y.is_linked_to_prev with\n        | 0 -> (\n            match Bool.compare x.is_linked_to_next y.is_linked_to_next with\n            | 0 -> (\n                compare_match_typ x.match_typ y.match_typ\n              )\n            | n -> n\n          )\n        | n -> n\n      )\n    | n -> n\n\n  let equal (x : t) (y : t) =\n    compare x y = 0\n\n  let compatible_with_word (token : t) indexed_word =\n    String.length indexed_word > 0\n    &&\n    (match data token with\n     | `Explicit_spaces -> (\n         Parser_components.is_space indexed_word.[0]\n       )\n     | `String search_word -> (\n         let search_word_ci = String.lowercase_ascii search_word in\n         let indexed_word_ci = String.lowercase_ascii indexed_word in\n         let use_ci_match = String.equal search_word search_word_ci in\n         let search_word_len = String.length search_word in\n         let indexed_word_len = String.length indexed_word in\n         let edit_dist_based_match_min_len = !Params.max_fuzzy_edit_dist + 1 in\n         if Parser_components.is_possibly_utf_8 indexed_word.[0] then (\n           String.equal search_word indexed_word\n         ) else (\n           match match_typ token with\n           | `Fuzzy -> (\n               String.equal search_word_ci indexed_word_ci\n               || CCString.find ~sub:search_word_ci indexed_word_ci >= 0\n               || (indexed_word_len >= 2\n                   && CCString.find ~sub:indexed_word_ci search_word_ci >= 0)\n               || (search_word_len >= edit_dist_based_match_min_len\n                   && indexed_word_len >= edit_dist_based_match_min_len\n                   && Misc_utils.first_n_chars_of_string_contains ~n:3 indexed_word_ci search_word_ci.[0]\n                   && Spelll.match_with (automaton token) indexed_word_ci)\n             )\n           | `Exact -> (\n               if use_ci_match then (\n                 String.equal search_word_ci indexed_word_ci\n               ) else (\n                 String.equal search_word indexed_word\n               )\n             )\n           | `Prefix -> (\n               if use_ci_match then (\n                 CCString.prefix ~pre:search_word_ci indexed_word_ci\n               ) else (\n                 CCString.prefix ~pre:search_word indexed_word\n               )\n             )\n           | `Suffix -> (\n               if use_ci_match then (\n                 CCString.suffix ~suf:search_word_ci indexed_word_ci\n               ) else (\n                 CCString.suffix ~suf:search_word indexed_word\n               )\n             )\n         )\n       )\n    )\nend\n\ntype t = {\n  annotated_tokens : annotated_token list;\n  enriched_tokens : Enriched_token.t list;\n}\n\nlet is_empty (t : t) =\n  List.is_empty t.enriched_tokens\n\nlet pp fmt (t : t) =\n  Fmt.pf fmt \"%a\"\n    Fmt.(list ~sep:sp Enriched_token.pp)\n    t.enriched_tokens\n\ntype cache = {\n  mutex : Eio.Mutex.t;\n  cache : (string, Spelll.automaton) CCCache.t;\n}\n\nlet cache = {\n  mutex = Eio.Mutex.create ();\n  cache = CCCache.lru ~eq:String.equal Params.search_word_automaton_cache_size;\n}\n\nlet compare (t1 : t) (t2 : t) =\n  List.compare Enriched_token.compare t1.enriched_tokens t2.enriched_tokens\n\nlet equal (t1 : t) (t2 : t) =\n  compare t1 t2 = 0\n\nlet empty : t =\n  {\n    annotated_tokens = [];\n    enriched_tokens = [];\n  }\n\nlet ir0_s_of_annotated_tokens (tokens : annotated_token Seq.t) : ir0 list =\n  let token_is_space (token : annotated_token) =\n    match token.data with\n    | `String s -> Parser_components.is_space (String.get s 0)\n    | _ -> false\n  in\n  let rec aux acc (prev_token : annotated_token option) (tokens : annotated_token Seq.t) =\n    match tokens () with\n    | Seq.Nil -> List.rev acc\n    | Seq.Cons (token, rest) -> (\n        let is_linked_to_prev =\n          match prev_token with\n          | None -> false\n          | Some prev_token -> (\n              (prev_token.group_id = token.group_id)\n              &&\n              (not (token_is_space prev_token))\n            )\n        in\n        if token_is_space token then (\n          aux acc None rest\n        ) else (\n          let ir0 : ir0 = \n            { data = token.data;\n              is_linked_to_prev;\n              is_linked_to_next = false;\n              match_typ = None;\n            }\n          in\n          aux (ir0 :: acc) (Some token) rest\n        )\n      )\n  in\n  aux [] None tokens\n\nlet ir0_s_link_forward (ir0_s : ir0 list) : ir0 list =\n  List.rev ir0_s\n  |> List.fold_left (fun (acc, next) x ->\n      match next with\n      | None -> (x :: acc, Some x)\n      | Some next -> (\n          let x = { x with is_linked_to_next = next.is_linked_to_prev } in\n          (x :: acc, Some x)\n        )\n    )\n    ([], None)\n  |> fst\n\nlet ir0_process_exact_prefix_match_typ_markers (ir0_s : ir0 list) : ir0 list =\n  let rec aux\n      (acc : ir0 list)\n      token_removed\n      (marker : ([ `Exact ] * [ `Exact | `Prefix ]) option)\n      (ir0_s : ir0 list)\n    =\n    match ir0_s with\n    | [] -> List.rev acc\n    | x :: xs -> (\n        match marker with\n        | None -> (\n            let default () =\n              aux (x :: acc) false None xs\n            in\n            match x.data with\n            | `String _ | `Explicit_spaces ->\n              default ()\n            | `Match_typ_marker m -> (\n                match x.match_typ with\n                | None -> (\n                    if x.is_linked_to_next then (\n                      match m with\n                      | `Exact ->\n                        aux acc true (Some (`Exact, `Exact)) xs\n                      | `Prefix ->\n                        aux acc true (Some (`Exact, `Prefix)) xs\n                      | _ ->\n                        default ()\n                    ) else (\n                      default ()\n                    )\n                  )\n                | Some _ ->\n                  default ()\n              )\n          )\n        | Some (m, m_last) -> (\n            let x =\n              if x.is_linked_to_prev then (\n                { x with\n                  is_linked_to_prev = not token_removed;\n                  match_typ = Some (\n                      if x.is_linked_to_next then\n                        (m :> match_typ)\n                      else\n                        (m_last :> match_typ)\n                    );\n                }\n              ) else (\n                x\n              )\n            in\n            let marker =\n              if x.is_linked_to_next then\n                marker\n              else\n                None\n            in\n            aux (x :: acc) false marker xs\n          )\n      )\n  in\n  aux [] false None ir0_s\n\nlet ir0_process_suffix_match_typ_markers (ir0_s : ir0 list) : ir0 list =\n  let rec aux\n      (acc : ir0 list)\n      token_removed\n      (marker : ([ `Suffix ] * [ `Exact ]) option)\n      (ir0_s : ir0 list)\n    =\n    match ir0_s with\n    | [] -> acc\n    | x :: xs -> (\n        match marker with\n        | None -> (\n            let default () =\n              aux (x :: acc) false None xs\n            in\n            match x.data with\n            | `String _ | `Explicit_spaces ->\n              default ()\n            | `Match_typ_marker m -> (\n                match x.match_typ with\n                | None -> (\n                    if x.is_linked_to_prev then (\n                      match m with\n                      | `Suffix ->\n                        aux acc true (Some (`Suffix, `Exact)) xs\n                      | _ ->\n                        default ()\n                    ) else (\n                      default ()\n                    )\n                  )\n                | Some _ ->\n                  default ()\n              )\n          )\n        | Some (m_first, m) -> (\n            let x =\n              if x.is_linked_to_next then (\n                { x with\n                  is_linked_to_next = not token_removed;\n                  match_typ = Some (\n                      if x.is_linked_to_prev then\n                        (m :> match_typ)\n                      else\n                        (m_first :> match_typ)\n                    );\n                }\n              ) else (\n                x\n              )\n            in\n            let marker =\n              if x.is_linked_to_prev then\n                marker\n              else\n                None\n            in\n            aux (x :: acc) false marker xs\n          )\n      )\n  in\n  aux [] false None (List.rev ir0_s)\n\nlet enriched_tokens_of_ir0 (ir0_s : ir0 list) : Enriched_token.t list =\n  List.map (fun (ir0 : ir0) ->\n      let data =\n        match ir0.data with\n        | `String s -> `String s\n        | `Match_typ_marker m ->\n          `String (string_of_match_typ_marker m)\n        | `Explicit_spaces -> `Explicit_spaces\n      in\n      let is_linked_to_prev = ir0.is_linked_to_prev in\n      let is_linked_to_next = ir0.is_linked_to_next in\n      let automaton =\n        match data with\n        | `String string -> (\n            Eio.Mutex.use_rw cache.mutex ~protect:false (fun () ->\n                let automaton =\n                  CCCache.with_cache cache.cache\n                    (Spelll.of_string ~limit:!Params.max_fuzzy_edit_dist)\n                    string\n                in\n                automaton\n              )\n          )\n        | `Explicit_spaces ->\n          Spelll.of_string ~limit:0 \"\"\n      in\n      Enriched_token.make\n        data\n        ~is_linked_to_prev\n        ~is_linked_to_next\n        automaton\n        (Option.value ~default:`Fuzzy ir0.match_typ)\n    ) ir0_s\n\nlet of_annotated_tokens\n    (annotated_tokens : annotated_token Seq.t)\n  =\n  let enriched_tokens =\n    annotated_tokens\n    |> ir0_s_of_annotated_tokens\n    |> ir0_s_link_forward\n    |> ir0_process_exact_prefix_match_typ_markers\n    |> ir0_process_suffix_match_typ_markers\n    |> enriched_tokens_of_ir0\n  in\n  {\n    annotated_tokens = List.of_seq annotated_tokens;\n    enriched_tokens;\n  }\n\nlet of_tokens\n    (tokens : string Seq.t)\n  =\n  tokens\n  |> Seq.map (fun s -> { data = `String s; group_id = 0 })\n  |> of_annotated_tokens\n\nlet parse phrase =\n  phrase\n  |> Tokenization.tokenize ~drop_spaces:false\n  |> of_tokens\n\nlet annotated_tokens t =\n  t.annotated_tokens\n\nlet enriched_tokens t =\n  t.enriched_tokens\n"
  },
  {
    "path": "lib/search_phrase.mli",
    "content": "type match_typ = [\n  | `Fuzzy\n  | `Exact\n  | `Suffix\n  | `Prefix\n]\n[@@deriving show, ord]\n\ntype match_typ_marker = [ `Exact | `Prefix | `Suffix ]\n[@@deriving show]\n\ntype annotated_token = {\n  data : [\n    | `String of string\n    | `Match_typ_marker of match_typ_marker\n    | `Explicit_spaces\n  ];\n  group_id : int;\n}\n[@@deriving show]\n\nmodule Enriched_token : sig\n  type data = [ `String of string | `Explicit_spaces ]\n  [@@deriving ord]\n\n  module Data_map : Map.S with type key = data\n\n  type t\n\n  val make :\n    data ->\n    is_linked_to_prev:bool ->\n    is_linked_to_next:bool ->\n    Spelll.automaton ->\n    match_typ ->\n    t\n\n  val data : t -> data\n\n  val equal : t -> t -> bool\n\n  val pp : Format.formatter -> t -> unit\n\n  val match_typ : t -> match_typ\n\n  val is_linked_to_prev : t -> bool\n\n  val is_linked_to_next : t -> bool\n\n  val automaton : t -> Spelll.automaton\n\n  val compatible_with_word : t -> string -> bool\nend\n\ntype t\n\nval empty : t\n\nval compare : t -> t -> int\n\nval equal : t -> t -> bool\n\nval pp : Format.formatter -> t -> unit\n\nval of_annotated_tokens : annotated_token Seq.t -> t\n\nval of_tokens : string Seq.t -> t\n\nval parse : string -> t\n\nval is_empty : t -> bool\n\nval annotated_tokens : t -> annotated_token list\n\nval enriched_tokens : t -> Enriched_token.t list\n"
  },
  {
    "path": "lib/search_result.ml",
    "content": "type indexed_found_word = {\n  found_word_pos : int;\n  found_word_ci : string;\n  found_word : string;\n}\n\ntype t = {\n  score : float;\n  search_phrase : Search_phrase.t;\n  found_phrase : indexed_found_word list;\n}\n\nlet equal t1 t2 =\n  Search_phrase.equal t1.search_phrase t2.search_phrase\n  && List.length t1.found_phrase = List.length t2.found_phrase\n  && List.for_all2 (fun x1 x2 -> x1.found_word_pos = x2.found_word_pos)\n    t1.found_phrase t2.found_phrase\n\nmodule Score = struct\n  module ET = Search_phrase.Enriched_token\n\n  type stats = {\n    total_search_char_count : float;\n    total_found_char_count : float;\n    exact_match_found_char_count : float;\n    ci_exact_match_found_char_count : float;\n    sub_match_search_word_in_found_word_char_count : float;\n    sub_match_found_word_in_search_word_char_count : float;\n    sub_match_search_char_count : float;\n    sub_match_found_char_count : float;\n    ci_sub_match_search_word_in_found_word_char_count : float;\n    ci_sub_match_found_word_in_search_word_char_count : float;\n    ci_sub_match_search_char_count : float;\n    ci_sub_match_found_char_count : float;\n    fuzzy_match_edit_distance : float;\n    fuzzy_match_search_char_count : float;\n    fuzzy_match_found_char_count : float;\n  }\n\n  let empty_stats = {\n    total_search_char_count = 0.0;\n    total_found_char_count = 0.0;\n    exact_match_found_char_count = 0.0;\n    ci_exact_match_found_char_count = 0.0;\n    sub_match_search_word_in_found_word_char_count = 0.0;\n    sub_match_found_word_in_search_word_char_count = 0.0;\n    sub_match_search_char_count = 0.0;\n    sub_match_found_char_count = 0.0;\n    ci_sub_match_search_word_in_found_word_char_count = 0.0;\n    ci_sub_match_found_word_in_search_word_char_count = 0.0;\n    ci_sub_match_search_char_count = 0.0;\n    ci_sub_match_found_char_count = 0.0;\n    fuzzy_match_edit_distance = 0.0;\n    fuzzy_match_search_char_count = 0.0;\n    fuzzy_match_found_char_count = 0.0;\n  }\n\n  let score\n      (search_phrase : Search_phrase.t)\n      ~(found_phrase : indexed_found_word list)\n      ~(found_phrase_opening_closing_symbol_match_count : int)\n    : float =\n    assert (not (Search_phrase.is_empty search_phrase));\n    assert (List.length (Search_phrase.enriched_tokens search_phrase) = List.length found_phrase);\n    let found_phrase_opening_closing_symbol_match_count =\n      Int.to_float found_phrase_opening_closing_symbol_match_count\n    in\n    let quite_close_to_zero x =\n      Float.abs x < Params.float_compare_margin\n    in\n    let add_sub_match_search_and_found_char_count ~search_word_len ~found_word_len stats =\n      { stats with\n        sub_match_search_char_count =\n          stats.sub_match_search_char_count +. search_word_len;\n        sub_match_found_char_count =\n          stats.sub_match_found_char_count +. found_word_len;\n      }\n    in\n    let add_ci_sub_match_search_and_found_char_count ~search_word_len ~found_word_len stats =\n      { stats with\n        ci_sub_match_search_char_count =\n          stats.ci_sub_match_search_char_count +. search_word_len;\n        ci_sub_match_found_char_count =\n          stats.ci_sub_match_found_char_count +. found_word_len;\n      }\n    in\n    let stats =\n      List.fold_left2 (fun (stats : stats) (token : ET.t) { found_word_ci; found_word; _ } ->\n          let found_word_len = Int.to_float (String.length found_word) in\n          match ET.data token with\n          | `Explicit_spaces -> (\n              { stats with\n                total_search_char_count =\n                  stats.total_search_char_count +. 1.0;\n                total_found_char_count =\n                  stats.total_found_char_count +. 1.0;\n                exact_match_found_char_count =\n                  stats.exact_match_found_char_count +. 1.0;\n              }\n            )\n          | `String search_word -> (\n              let search_word_len = Int.to_float (String.length search_word) in\n              let search_word_ci = String.lowercase_ascii search_word in\n              let stats =\n                { stats with\n                  total_search_char_count =\n                    stats.total_search_char_count +. search_word_len;\n                  total_found_char_count =\n                    stats.total_found_char_count +. found_word_len;\n                }\n              in\n              if String.equal search_word found_word then (\n                { stats with\n                  exact_match_found_char_count =\n                    stats.exact_match_found_char_count +. found_word_len;\n                }\n              ) else if String.equal search_word_ci found_word_ci then (\n                { stats with\n                  ci_exact_match_found_char_count =\n                    stats.ci_exact_match_found_char_count +. found_word_len;\n                }\n              ) else if CCString.find ~sub:search_word found_word >= 0 then (\n                { stats with\n                  sub_match_search_word_in_found_word_char_count =\n                    stats.sub_match_search_word_in_found_word_char_count +. search_word_len;\n                }\n                |> add_sub_match_search_and_found_char_count\n                  ~search_word_len\n                  ~found_word_len\n              ) else if CCString.find ~sub:found_word search_word >= 0 then (\n                { stats with\n                  sub_match_found_word_in_search_word_char_count =\n                    stats.sub_match_found_word_in_search_word_char_count +. found_word_len;\n                }\n                |> add_sub_match_search_and_found_char_count\n                  ~search_word_len\n                  ~found_word_len\n              ) else if CCString.find ~sub:search_word_ci found_word_ci >= 0 then (\n                { stats with\n                  ci_sub_match_search_word_in_found_word_char_count =\n                    stats.ci_sub_match_search_word_in_found_word_char_count +. search_word_len;\n                }\n                |> add_ci_sub_match_search_and_found_char_count\n                  ~search_word_len\n                  ~found_word_len\n              ) else if CCString.find ~sub:found_word_ci search_word_ci >= 0 then (\n                { stats with\n                  ci_sub_match_found_word_in_search_word_char_count =\n                    stats.ci_sub_match_found_word_in_search_word_char_count +. found_word_len;\n                }\n                |> add_ci_sub_match_search_and_found_char_count\n                  ~search_word_len\n                  ~found_word_len\n              ) else (\n                { stats with\n                  fuzzy_match_edit_distance =\n                    stats.fuzzy_match_edit_distance\n                    +. Int.to_float (Spelll.edit_distance search_word_ci found_word_ci);\n                  fuzzy_match_search_char_count =\n                    stats.fuzzy_match_search_char_count +. search_word_len;\n                  fuzzy_match_found_char_count =\n                    stats.fuzzy_match_found_char_count +. found_word_len;\n                }\n              )\n            )\n        )\n        empty_stats\n        (Search_phrase.enriched_tokens search_phrase)\n        found_phrase\n    in\n    let search_phrase_length =\n      Search_phrase.enriched_tokens search_phrase\n      |> List.length\n      |> Int.to_float\n    in\n    let unique_match_count =\n      found_phrase\n      |> List.map (fun x -> x.found_word_pos)\n      |> List.sort_uniq Int.compare\n      |> List.length\n      |> Int.to_float\n    in\n    let (total_distance, out_of_order_match_count, _) =\n      List.fold_left\n        (fun (total_dist, out_of_order_match_count, last_pos) { found_word_pos = pos; _ } ->\n           match last_pos with\n           | None -> (total_dist, out_of_order_match_count, Some pos)\n           | Some last_pos ->\n             let total_dist = total_dist +. Int.to_float (abs (pos - last_pos)) in\n             let out_of_order_match_count =\n               if last_pos <= pos then\n                 out_of_order_match_count\n               else\n                 out_of_order_match_count +. 1.0\n             in\n             (total_dist, out_of_order_match_count, Some pos)\n        )\n        (0.0, 0.0, None)\n        found_phrase\n    in\n    let average_distance =\n      let gaps = unique_match_count -. 1.0 in\n      assert (gaps >= 0.0);\n      if quite_close_to_zero gaps then (\n        0.0\n      ) else (\n        total_distance /. gaps\n      )\n    in\n    let exact_match_score =\n      if quite_close_to_zero stats.exact_match_found_char_count then (\n        0.0\n      ) else (\n        1.0\n      )\n    in\n    let ci_exact_match_score =\n      if quite_close_to_zero stats.ci_exact_match_found_char_count then (\n        0.0\n      ) else (\n        1.0\n      )\n    in\n    let sub_match_score_s_in_f =\n      if quite_close_to_zero stats.sub_match_found_char_count then (\n        0.0\n      ) else (\n        stats.sub_match_search_word_in_found_word_char_count\n        /.\n        stats.sub_match_found_char_count\n      )\n    in\n    let sub_match_score_f_in_s =\n      if quite_close_to_zero stats.sub_match_search_char_count then (\n        0.0\n      ) else (\n        stats.sub_match_found_word_in_search_word_char_count\n        /.\n        stats.sub_match_search_char_count\n      )\n    in\n    let ci_sub_match_score_s_in_f =\n      if quite_close_to_zero stats.ci_sub_match_found_char_count then (\n        0.0\n      ) else (\n        stats.ci_sub_match_search_word_in_found_word_char_count\n        /.\n        stats.ci_sub_match_found_char_count\n      )\n    in\n    let ci_sub_match_score_f_in_s =\n      if quite_close_to_zero stats.ci_sub_match_search_char_count then (\n        0.0\n      ) else (\n        stats.ci_sub_match_found_word_in_search_word_char_count\n        /.\n        stats.ci_sub_match_search_char_count\n      )\n    in\n    let fuzzy_match_score =\n      if quite_close_to_zero stats.fuzzy_match_search_char_count then (\n        0.0\n      ) else (\n        1.0\n        -.\n        (stats.fuzzy_match_edit_distance\n         /.\n         stats.fuzzy_match_search_char_count)\n      )\n    in\n    let total_char_count =\n      stats.total_search_char_count +. stats.total_found_char_count\n    in\n    let case_sensitive_bonus_multiplier =\n      if\n        List.exists\n          (fun token ->\n             match ET.data token with\n             | `Explicit_spaces -> false\n             | `String search_word -> (\n                 not\n                   (String.equal\n                      (String.lowercase_ascii search_word)\n                      search_word))\n          )\n          (Search_phrase.enriched_tokens search_phrase)\n      then (\n        1.10\n      ) else (\n        1.00\n      )\n    in\n    let exact_match_weight =\n      case_sensitive_bonus_multiplier\n      *.\n      (stats.exact_match_found_char_count *. 2.0) /. total_char_count\n    in\n    let ci_exact_match_weight =\n      (stats.ci_exact_match_found_char_count *. 2.0) /. total_char_count\n    in\n    let f_in_s_penalty_multiplier = 0.8 in\n    let sub_match_weight_s_in_f =\n      case_sensitive_bonus_multiplier\n      *.\n      stats.sub_match_found_char_count /. stats.total_found_char_count\n    in\n    let sub_match_weight_f_in_s =\n      case_sensitive_bonus_multiplier\n      *.\n      f_in_s_penalty_multiplier\n      *.\n      (stats.sub_match_search_char_count /. stats.total_search_char_count)\n    in\n    let ci_sub_match_weight_s_in_f =\n      (stats.ci_sub_match_found_char_count /. stats.total_found_char_count)\n    in\n    let ci_sub_match_weight_f_in_s =\n      f_in_s_penalty_multiplier\n      *.\n      (stats.ci_sub_match_search_char_count /. stats.total_search_char_count)\n    in\n    let fuzzy_match_weight =\n      (stats.fuzzy_match_found_char_count *. 2.0) /. total_char_count\n    in\n    let distance_score =\n      if average_distance <= 1.0 then (\n        1.0\n      ) else (\n        1.0 -. (0.20 *. (average_distance /. (Int.to_float !Params.max_token_search_dist)))\n      )\n    in\n    (1.0 +. (0.10 *. search_phrase_length))\n    *.\n    (1.0 +. (0.10 *. (unique_match_count /. search_phrase_length)))\n    *.\n    (1.0 -. (0.10 *. (out_of_order_match_count /. search_phrase_length)))\n    *.\n    (1.00 +. (0.10 *. found_phrase_opening_closing_symbol_match_count))\n    *.\n    distance_score\n    *.\n    (\n      (exact_match_weight *. exact_match_score)\n      +.\n      (ci_exact_match_weight *. ci_exact_match_score)\n      +.\n      (sub_match_weight_s_in_f *. sub_match_score_s_in_f)\n      +.\n      (sub_match_weight_f_in_s *. sub_match_score_f_in_s)\n      +.\n      (ci_sub_match_weight_s_in_f *. ci_sub_match_score_s_in_f)\n      +.\n      (ci_sub_match_weight_f_in_s *. ci_sub_match_score_f_in_s)\n      +.\n      (fuzzy_match_weight *. fuzzy_match_score)\n    )\nend\n\nlet make search_phrase ~found_phrase ~found_phrase_opening_closing_symbol_match_count =\n  let len = List.length (Search_phrase.enriched_tokens search_phrase) in\n  if len = 0 then (\n    invalid_arg \"Search_result.make search_phrase is empty\"\n  ) else if len <> List.length found_phrase then (\n    invalid_arg \"length of found_phrase does not match length of search_phrase\"\n  ) else (\n    let score = Score.score search_phrase ~found_phrase ~found_phrase_opening_closing_symbol_match_count in\n    { score; search_phrase; found_phrase }\n  )\n\nlet search_phrase (t : t) =\n  t.search_phrase\n\nlet found_phrase (t : t) =\n  t.found_phrase\n\nlet score (t : t) =\n  t.score\n\nlet compare_relevance (t1 : t) (t2 : t) =\n  (* Order the more relevant result to the front. *)\n  if Float.abs (t1.score -. t2.score) < Params.float_compare_margin then (\n    (* If scores are within the comparison margin,\n       then order result that appears earlier to the front. *)\n    let t1_found_phrase_start_pos = (List.hd t1.found_phrase).found_word_pos in\n    let t2_found_phrase_start_pos = (List.hd t2.found_phrase).found_word_pos in\n    Int.compare t1_found_phrase_start_pos t2_found_phrase_start_pos\n  ) else (\n    (* Otherwise just order result with higher score to the front. *)\n    Float.compare t2.score t1.score\n  )\n"
  },
  {
    "path": "lib/search_result.mli",
    "content": "type indexed_found_word = {\n  found_word_pos : int;\n  found_word_ci : string;\n  found_word : string;\n}\n\ntype t\n\nval make :\n  Search_phrase.t ->\n  found_phrase:indexed_found_word list ->\n  found_phrase_opening_closing_symbol_match_count:int ->\n  t\n\nval search_phrase : t -> Search_phrase.t\n\nval found_phrase : t -> indexed_found_word list\n\nval score : t -> float\n\nval equal : t -> t -> bool\n\nval compare_relevance : t -> t -> int\n"
  },
  {
    "path": "lib/search_result_heap.ml",
    "content": "include CCHeap.Make (struct\n    type t = Search_result.t\n\n    let leq x y =\n      (Search_result.score x) <= (Search_result.score y)\n  end)\n"
  },
  {
    "path": "lib/sqlite3_utils.ml",
    "content": "include Sqlite3\n\nlet db_pool =\n  Eio.Pool.create\n    (* This is not ideal since validate is not called until next use of the\n       pool, so an idle DB connection could be held for a lot longer than\n       described here. But this seems to be the best we can do.\n    *)\n    ~validate:(fun (last_used, _db) ->\n        Unix.time () -. !last_used <= 30.0\n      )\n    ~dispose:(fun (_last_used, db) ->\n        (* Best-effort closing. *)\n        try\n          let try_count = ref 0 in\n          while !try_count < 10 && not (db_close db) do\n            Unix.sleepf 0.01;\n            incr try_count;\n          done\n        with\n        | _ -> ()\n      )\n    Task_pool.size\n    (fun () ->\n       (ref (Unix.time ()),\n        db_open\n          ~mutex:`FULL\n          (CCOption.get_exn_or \"Docfd_lib.Params.db_path uninitialized\" !Params.db_path)\n       )\n    )\n\nlet with_db : type a. ?db:db -> (db -> a) -> a =\n  fun ?db f ->\n  match db with\n  | None -> (\n      Eio.Pool.use db_pool ~never_block:true (fun (last_used, db) ->\n          last_used := Unix.time ();\n          f db\n        )\n    )\n  | Some db -> (\n      f db\n    )\n\nlet retry_if_busy (f : unit -> Sqlite3.Rc.t) =\n  let rec aux tries_left =\n    let r = f () in\n    if tries_left > 0 then (\n      match r with\n      | BUSY -> (\n          Unix.sleepf 0.1;\n          aux (tries_left - 1)\n        )\n      | _ -> r\n    ) else (\n      r\n    )\n  in\n  aux 50\n\nlet exec db s =\n  retry_if_busy (fun () -> Sqlite3.exec db s)\n  |> Sqlite3.Rc.check\n\nlet prepare db s =\n  Sqlite3.prepare db s\n\nlet bind_names stmt l =\n  retry_if_busy (fun () -> Sqlite3.bind_names stmt l)\n  |> Sqlite3.Rc.check\n\nlet reset stmt =\n  retry_if_busy (fun () -> Sqlite3.reset stmt)\n  |> Sqlite3.Rc.check\n\nlet step stmt =\n  match retry_if_busy (fun () -> Sqlite3.step stmt) with\n  | OK | DONE | ROW -> ()\n  | x -> Sqlite3.Rc.check x\n\nlet finalize stmt =\n  retry_if_busy (fun () -> Sqlite3.finalize stmt)\n  |> Sqlite3.Rc.check\n\nlet with_stmt : type a. ?db:db -> string -> ?names:((string * Sqlite3.Data.t) list) -> (Sqlite3.stmt -> a) -> a =\n  fun ?db s ?names f ->\n  with_db ?db (fun db ->\n      let stmt = prepare db s in\n      Option.iter\n        (fun names -> bind_names stmt names)\n        names;\n      let res = f stmt in\n      finalize stmt;\n      res\n    )\n\nlet step_stmt : type a. ?db:db -> string -> ?names:((string * Data.t) list) -> (stmt -> a) -> a =\n  fun ?db s ?names f ->\n  with_stmt ?db s ?names\n    (fun stmt ->\n       step stmt;\n       f stmt\n    )\n\nlet iter_stmt ?db s ?names (f : Data.t array -> unit) =\n  with_stmt ?db s ?names\n    (fun stmt ->\n       Rc.check (Sqlite3.iter stmt ~f)\n    )\n\nlet fold_stmt : type a. ?db:db -> string -> ?names:((string * Data.t) list) -> (a -> Sqlite3.Data.t array -> a) -> a -> a =\n  fun ?db s ?names f init ->\n  with_stmt ?db s ?names\n    (fun stmt ->\n       let rc, res = Sqlite3.fold stmt ~f ~init in\n       Sqlite3.Rc.check rc;\n       res\n    )\n"
  },
  {
    "path": "lib/stop_signal.ml",
    "content": "type t = {\n  mutable stop : bool;\n  cond : Eio.Condition.t;\n  mutex : Eio.Mutex.t;\n}\n\nlet make () =\n  {\n    stop = false;\n    cond = Eio.Condition.create ();\n    mutex = Eio.Mutex.create ();\n  }\n\nlet await (t : t) =\n  Eio.Mutex.use_ro t.mutex (fun () ->\n      while not t.stop do\n        Eio.Condition.await t.cond t.mutex\n      done\n    )\n\nlet broadcast (t : t) =\n  Eio.Mutex.use_rw ~protect:false t.mutex\n    (fun () -> t.stop <- true);\n  Eio.Condition.broadcast t.cond\n"
  },
  {
    "path": "lib/stop_signal.mli",
    "content": "type t\n\nval make : unit -> t\n\nval await : t -> unit\n\nval broadcast : t -> unit\n"
  },
  {
    "path": "lib/string_map.ml",
    "content": "include CCMap.Make (String)\n"
  },
  {
    "path": "lib/string_set.ml",
    "content": "include CCSet.Make (String)\n"
  },
  {
    "path": "lib/task_pool.ml",
    "content": "type t = Eio.Executor_pool.t\n\nlet size = max 1 (Domain.recommended_domain_count () - 1)\n\nlet make ~sw mgr =\n  Eio.Executor_pool.create ~sw ~domain_count:size mgr\n\nlet run (t : t) (f : unit -> 'a) : 'a =\n  Eio.Executor_pool.submit_exn t ~weight:1.0 f\n\nlet map_list : 'a 'b . t -> ('a -> 'b) -> 'a list -> 'b list =\n  fun t f l ->\n  Eio.Fiber.List.map ~max_fibers:size\n    (fun x ->\n       Eio.Fiber.yield ();\n       run t (fun () -> f x))\n    l\n\nlet filter_list : 'a 'b . t -> ('a -> bool) -> 'a list -> 'a list =\n  fun t f l ->\n  Eio.Fiber.List.filter ~max_fibers:size\n    (fun x ->\n       Eio.Fiber.yield ();\n       run t (fun () -> f x))\n    l\n\nlet filter_map_list : 'a 'b . t -> ('a -> 'b option) -> 'a list -> 'b list =\n  fun t f l ->\n  Eio.Fiber.List.filter_map ~max_fibers:size\n    (fun x ->\n       Eio.Fiber.yield ();\n       run t (fun () -> f x))\n    l\n"
  },
  {
    "path": "lib/task_pool.mli",
    "content": "type t\n\nval size : int\n\nval make : sw:Eio.Switch.t -> _ Eio.Domain_manager.t -> t\n\nval run : t -> (unit -> 'a) -> 'a\n\nval map_list : t -> ('a -> 'b) -> 'a list -> 'b list\n\nval filter_list : t -> ('a -> bool) -> 'a list -> 'a list\n\nval filter_map_list : t -> ('a -> 'b option) -> 'a list -> 'b list\n"
  },
  {
    "path": "lib/tokenization.ml",
    "content": "let chunk_tokens (s : (int * string) Seq.t) : (int * string) Seq.t =\n  let rec aux offset s =\n    match s () with\n    | Seq.Nil -> Seq.empty\n    | Seq.Cons ((pos, word), rest) -> (\n        let word_len = String.length word in\n        if word_len <= Params.max_token_size then (\n          fun () -> Seq.Cons ((pos + offset, word), aux offset rest)\n        ) else (\n          let up_to_limit =\n            String.sub word 0 Params.max_token_size\n          in\n          let rest_of_token =\n            String.sub word Params.max_token_size (word_len - Params.max_token_size)\n          in\n          fun () ->\n            Seq.Cons\n              ((pos + offset, up_to_limit),\n               (aux (offset + 1) (Seq.cons (pos, rest_of_token) rest)))\n        )\n      )\n  in\n  aux 0 s\n\ntype token =\n  | Space of string\n  | Text of string\n\nlet split_utf8_seg (s : string) : string list =\n  let open Angstrom in\n  let open Parser_components in\n  let token_p =\n    choice [\n      take_while1 is_alphanum;\n      utf_8_char;\n    ]\n  in\n  let tokens_p = many token_p in\n  match Angstrom.(parse_string ~consume:Consume.All) tokens_p s with\n  | Ok l -> l\n  | Error _ -> []\n\nlet tokenize_with_pos ~drop_spaces (s : string) : (int * string) Seq.t =\n  let segmenter = Uuseg.create `Word in\n  let s = Misc_utils.sanitize_string s in\n  let s_len = String.length s in\n  let acc : token Dynarray.t = Dynarray.create () in\n  let buf : Uchar.t Dynarray.t = Dynarray.create () in\n  let sbuf = Buffer.create 256 in\n  let flush_to_acc () =\n    if Dynarray.length buf > 0 then (\n      Dynarray.iter (Buffer.add_utf_8_uchar sbuf) buf;\n      if Uucp.White.is_white_space (Dynarray.get buf 0) then (\n        Dynarray.add_last acc (Space (Buffer.contents sbuf))\n      ) else (\n        Buffer.contents sbuf\n        |> split_utf8_seg\n        |> List.iter (fun s ->\n            Dynarray.add_last acc (Text s)\n          )\n      );\n      Dynarray.clear buf;\n      Buffer.clear sbuf\n    )\n  in\n  let rec add v =\n    match Uuseg.add segmenter v with\n    | `Uchar uc -> (\n        Dynarray.add_last buf uc;\n        add `Await\n      )\n    | `Boundary -> (\n        flush_to_acc ();\n        add `Await\n      )\n    | `Await | `End -> ()\n  in\n  let rec aux pos =\n    if pos >= s_len then (\n      add `End;\n      flush_to_acc ()\n    ) else (\n      let decode = String.get_utf_8_uchar s pos in\n      if Uchar.utf_decode_is_valid decode then (\n        let uchar = Uchar.utf_decode_uchar decode in\n        add (`Uchar uchar);\n        aux (pos + Uchar.utf_decode_length decode)\n      ) else (\n        aux (pos + 1)\n      )\n    )\n  in\n  aux 0;\n  Dynarray.to_seq acc\n  |> Seq.mapi (fun i x -> (i, x))\n  |> Seq.filter_map (fun ((i, token) : int * token) ->\n      match token with\n      | Text s -> Some (i, s)\n      | Space s ->\n        if drop_spaces then\n          None\n        else\n          Some (i, s)\n    )\n  |> chunk_tokens\n\nlet tokenize ~drop_spaces s =\n  tokenize_with_pos ~drop_spaces s\n  |> Seq.map snd\n"
  },
  {
    "path": "lib/word_db.ml",
    "content": "type t = {\n  lock : Eio.Mutex.t;\n  mutable size : int;\n  mutable size_written_to_db : int;\n  mutable word_of_id : string Int_map.t;\n  id_of_word : (string, int) Hashtbl.t;\n}\n\nlet t : t =\n  {\n    lock = Eio.Mutex.create ();\n    size = 0;\n    size_written_to_db = 0;\n    word_of_id = Int_map.empty;\n    id_of_word = Hashtbl.create 100_000;\n  }\n\nlet lock : type a. (unit -> a) -> a =\n  fun f ->\n  Eio.Mutex.use_rw ~protect:true t.lock f\n\nlet filter pool (f : string -> bool) : (int * string) Dynarray.t =\n  let word_of_id =\n    lock (fun () ->\n        t.word_of_id\n      )\n  in\n  let max_end_exc_seen = ref 0 in\n  let chunk_size = !Params.index_chunk_size * 10 in\n  let chunk_start_end_exc_ranges =\n    OSeq.(0 -- (t.size - 1) / chunk_size)\n    |> Seq.map (fun chunk_index ->\n        let start = chunk_index * chunk_size in\n        let end_exc =\n          min\n            ((chunk_index + 1) * chunk_size)\n            t.size\n        in\n        max_end_exc_seen := max !max_end_exc_seen end_exc;\n        (start, end_exc)\n      )\n    |> List.of_seq\n  in\n  assert (!max_end_exc_seen = t.size);\n  let batches =\n    chunk_start_end_exc_ranges\n    |> Task_pool.map_list pool (fun (start, end_exc) ->\n        let acc = Dynarray.create () in\n        for i=start to end_exc-1 do\n          let word = Int_map.find i word_of_id in\n          if f word then (\n            Dynarray.add_last acc (i, word)\n          )\n        done;\n        acc\n      )\n  in\n  let acc = Dynarray.create () in\n  List.iter (fun batch ->\n      Dynarray.append acc batch\n    ) batches;\n  acc\n\nlet add (word : string) : int =\n  lock (fun () ->\n      match Hashtbl.find_opt t.id_of_word word with\n      | Some id -> id\n      | None -> (\n          let id = t.size in\n          t.size <- t.size + 1;\n          t.word_of_id <- Int_map.add id word t.word_of_id;\n          Hashtbl.replace t.id_of_word word id;\n          id\n        )\n    )\n\nlet word_of_id i : string =\n  lock (fun () ->\n      Int_map.find i t.word_of_id\n    )\n\nlet id_of_word s : int option =\n  lock (fun () ->\n      Hashtbl.find_opt t.id_of_word s\n    )\n\nlet read_from_db () : unit =\n  let open Sqlite3_utils in\n  lock (fun () ->\n      with_db (fun db ->\n          t.word_of_id <- Int_map.empty;\n          Hashtbl.clear t.id_of_word;\n          iter_stmt ~db\n            {|\n  SELECT id, word\n  FROM word\n  ORDER by id\n  |}\n            ~names:[]\n            (fun data ->\n               let id = Data.to_int_exn data.(0) in\n               let word = Data.to_string_exn data.(1) in\n               t.word_of_id <- Int_map.add id word t.word_of_id;\n               Hashtbl.replace t.id_of_word word id;\n            )\n        );\n      t.size <- Int_map.cardinal t.word_of_id;\n      t.size_written_to_db <- t.size;\n    )\n\nlet write_to_db db ~already_in_transaction : unit =\n  let open Sqlite3_utils in\n  lock (fun () ->\n      if not already_in_transaction then (\n        step_stmt ~db \"BEGIN IMMEDIATE\" ignore;\n      );\n      let word_table_size =\n        step_stmt ~db\n          {|\n      SELECT COUNT(1) FROM word\n      |}\n          (fun stmt ->\n             Int64.to_int (column_int64 stmt 0)\n          )\n      in\n      if word_table_size <> t.size_written_to_db then (\n        Misc_utils.exit_with_error_msg\n          \"unexpected change in word table, likely due to indexing from another Docfd instance\";\n      );\n      with_stmt ~db\n        {|\n  INSERT INTO word\n  (id, word)\n  VALUES\n  (@id, @word)\n  ON CONFLICT(id) DO NOTHING\n  |}\n        (fun stmt ->\n           for id = t.size_written_to_db to t.size-1 do\n             let word = Int_map.find id t.word_of_id in\n             bind_names\n               stmt\n               [ (\"@id\", INT (Int64.of_int id))\n               ; (\"@word\", TEXT word)\n               ];\n             step stmt;\n             reset stmt;\n           done\n        );\n      if not already_in_transaction then (\n        step_stmt ~db \"COMMIT\" ignore;\n      );\n      t.size_written_to_db <- t.size;\n    )\n"
  },
  {
    "path": "lib/word_db.mli",
    "content": "type t\n\nval add : string -> int\n\nval filter : Task_pool.t -> (string -> bool) -> (int * string) Dynarray.t\n\nval word_of_id : int -> string\n\nval id_of_word : string -> int option\n\nval read_from_db : unit -> unit\n\nval write_to_db : Sqlite3.db -> already_in_transaction:bool -> unit\n"
  },
  {
    "path": "line-wrapping-tests.t/dune",
    "content": "(cram\n  (deps ../bin/docfd.exe))\n"
  },
  {
    "path": "line-wrapping-tests.t/long-words.txt",
    "content": "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n\nabcdefghijklmnopqrstuvwxyz\n01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\nabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\n\n\n\n\n\n\n\n\n\n\n0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n\nabcdefghijklmnopqrstuvwxyz\n01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\nabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\n"
  },
  {
    "path": "line-wrapping-tests.t/run.t",
    "content": "Word breaking:\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 long-words.txt --sample \"01 ab\" --search-result-print-text-width 80\n  $TESTCASE_ROOT/long-words.txt\n  1: 01234567890123456789012345678901234567890123456789012345678901234567890123456\n     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n     78901234567890123456789\n     ^^^^^^^^^^^^^^^^^^^^^^^\n  2: \n  3: abcdefghijklmnopqrstuvwxyz\n     ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  \n  16: 0123456789012345678901234567890123456789012345678901234567890123456789012345\n      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n      678901234567890123456789\n      ^^^^^^^^^^^^^^^^^^^^^^^^\n  17: \n  18: abcdefghijklmnopqrstuvwxyz\n      ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  \n  1: 01234567890123456789012345678901234567890123456789012345678901234567890123456\n     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n     78901234567890123456789\n     ^^^^^^^^^^^^^^^^^^^^^^^\n  2: \n  3: abcdefghijklmnopqrstuvwxyz\n  4: 01234567890123456789012345678901234567890123456789012345678901234567890123456\n     78901234567890123456789012345678901234567890123456789012345678901234567890123\n     4567890123456789012345678901234567890123456789\n  5: abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\n  6: \n  7: \n  8: \n  9: \n  10: \n  11: \n  12: \n  13: \n  14: \n  15: \n  16: 0123456789012345678901234567890123456789012345678901234567890123456789012345\n      678901234567890123456789\n  17: \n  18: abcdefghijklmnopqrstuvwxyz\n      ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  \n  3: abcdefghijklmnopqrstuvwxyz\n     ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  4: 01234567890123456789012345678901234567890123456789012345678901234567890123456\n     78901234567890123456789012345678901234567890123456789012345678901234567890123\n     4567890123456789012345678901234567890123456789\n  5: abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\n  6: \n  7: \n  8: \n  9: \n  10: \n  11: \n  12: \n  13: \n  14: \n  15: \n  16: 0123456789012345678901234567890123456789012345678901234567890123456789012345\n      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n      678901234567890123456789\n      ^^^^^^^^^^^^^^^^^^^^^^^^\n  \n  1: 01234567890123456789012345678901234567890123456789012345678901234567890123456\n     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n     78901234567890123456789\n     ^^^^^^^^^^^^^^^^^^^^^^^\n  2: \n  3: abcdefghijklmnopqrstuvwxyz\n  4: 01234567890123456789012345678901234567890123456789012345678901234567890123456\n     78901234567890123456789012345678901234567890123456789012345678901234567890123\n     4567890123456789012345678901234567890123456789\n  5: abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\n     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 long-words.txt --sample \"01 ab\" --search-result-print-text-width 20\n  $TESTCASE_ROOT/long-words.txt\n  1: 01234567890123456\n     ^^^^^^^^^^^^^^^^^\n     78901234567890123\n     ^^^^^^^^^^^^^^^^^\n     45678901234567890\n     ^^^^^^^^^^^^^^^^^\n     12345678901234567\n     ^^^^^^^^^^^^^^^^^\n     89012345678901234\n     ^^^^^^^^^^^^^^^^^\n     567890123456789\n     ^^^^^^^^^^^^^^^\n  2: \n  3: abcdefghijklmnopq\n     ^^^^^^^^^^^^^^^^^\n     rstuvwxyz\n     ^^^^^^^^^\n  \n  16: 0123456789012345\n      ^^^^^^^^^^^^^^^^\n      6789012345678901\n      ^^^^^^^^^^^^^^^^\n      2345678901234567\n      ^^^^^^^^^^^^^^^^\n      8901234567890123\n      ^^^^^^^^^^^^^^^^\n      4567890123456789\n      ^^^^^^^^^^^^^^^^\n      0123456789012345\n      ^^^^^^^^^^^^^^^^\n      6789\n      ^^^^\n  17: \n  18: abcdefghijklmnop\n      ^^^^^^^^^^^^^^^^\n      qrstuvwxyz\n      ^^^^^^^^^^\n  \n  1: 01234567890123456\n     ^^^^^^^^^^^^^^^^^\n     78901234567890123\n     ^^^^^^^^^^^^^^^^^\n     45678901234567890\n     ^^^^^^^^^^^^^^^^^\n     12345678901234567\n     ^^^^^^^^^^^^^^^^^\n     89012345678901234\n     ^^^^^^^^^^^^^^^^^\n     567890123456789\n     ^^^^^^^^^^^^^^^\n  2: \n  3: abcdefghijklmnopq\n     rstuvwxyz\n  4: 01234567890123456\n     78901234567890123\n     45678901234567890\n     12345678901234567\n     89012345678901234\n     56789012345678901\n     23456789012345678\n     90123456789012345\n     67890123456789012\n     34567890123456789\n     01234567890123456\n     7890123456789\n  5: abcdefghijklmnopq\n     rstuvwxyzabcdefgh\n     ijklmnopqrstuvwxy\n     z\n  6: \n  7: \n  8: \n  9: \n  10: \n  11: \n  12: \n  13: \n  14: \n  15: \n  16: 0123456789012345\n      6789012345678901\n      2345678901234567\n      8901234567890123\n      4567890123456789\n      0123456789012345\n      6789\n  17: \n  18: abcdefghijklmnop\n      ^^^^^^^^^^^^^^^^\n      qrstuvwxyz\n      ^^^^^^^^^^\n  \n  3: abcdefghijklmnopq\n     ^^^^^^^^^^^^^^^^^\n     rstuvwxyz\n     ^^^^^^^^^\n  4: 01234567890123456\n     78901234567890123\n     45678901234567890\n     12345678901234567\n     89012345678901234\n     56789012345678901\n     23456789012345678\n     90123456789012345\n     67890123456789012\n     34567890123456789\n     01234567890123456\n     7890123456789\n  5: abcdefghijklmnopq\n     rstuvwxyzabcdefgh\n     ijklmnopqrstuvwxy\n     z\n  6: \n  7: \n  8: \n  9: \n  10: \n  11: \n  12: \n  13: \n  14: \n  15: \n  16: 0123456789012345\n      ^^^^^^^^^^^^^^^^\n      6789012345678901\n      ^^^^^^^^^^^^^^^^\n      2345678901234567\n      ^^^^^^^^^^^^^^^^\n      8901234567890123\n      ^^^^^^^^^^^^^^^^\n      4567890123456789\n      ^^^^^^^^^^^^^^^^\n      0123456789012345\n      ^^^^^^^^^^^^^^^^\n      6789\n      ^^^^\n  \n  1: 01234567890123456\n     ^^^^^^^^^^^^^^^^^\n     78901234567890123\n     ^^^^^^^^^^^^^^^^^\n     45678901234567890\n     ^^^^^^^^^^^^^^^^^\n     12345678901234567\n     ^^^^^^^^^^^^^^^^^\n     89012345678901234\n     ^^^^^^^^^^^^^^^^^\n     567890123456789\n     ^^^^^^^^^^^^^^^\n  2: \n  3: abcdefghijklmnopq\n     rstuvwxyz\n  4: 01234567890123456\n     78901234567890123\n     45678901234567890\n     12345678901234567\n     89012345678901234\n     56789012345678901\n     23456789012345678\n     90123456789012345\n     67890123456789012\n     34567890123456789\n     01234567890123456\n     7890123456789\n  5: abcdefghijklmnopq\n     ^^^^^^^^^^^^^^^^^\n     rstuvwxyzabcdefgh\n     ^^^^^^^^^^^^^^^^^\n     ijklmnopqrstuvwxy\n     ^^^^^^^^^^^^^^^^^\n     z\n     ^\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 words.txt --sample \"01 ab\" --search-result-print-text-width 5\n  $TESTCASE_ROOT/words.txt\n  1: 01\n     ^^\n     23\n     ^^\n     45\n     ^^\n  2: \n  3: ab\n     ^^\n     cd\n     ^^\n     ef\n     ^^\n     g\n     ^\n  \n  15: 0\n      ^\n      1\n      ^\n      2\n      ^\n      3\n      ^\n      4\n      ^\n      5\n      ^\n  16: \n  17: a\n      ^\n      b\n      ^\n      c\n      ^\n      d\n      ^\n      e\n      ^\n      f\n      ^\n      g\n      ^\n  \n  1: 01\n     ^^\n     23\n     ^^\n     45\n     ^^\n  2: \n  3: ab\n     cd\n     ef\n     g\n  4: \n  5: \n  6: \n  7: \n  8: \n  9: \n  10: \n  11: \n  12: \n  13: \n  14: \n  15: 0\n      1\n      2\n      3\n      4\n      5\n  16: \n  17: a\n      ^\n      b\n      ^\n      c\n      ^\n      d\n      ^\n      e\n      ^\n      f\n      ^\n      g\n      ^\n  \n  3: ab\n     ^^\n     cd\n     ^^\n     ef\n     ^^\n     g\n     ^\n  4: \n  5: \n  6: \n  7: \n  8: \n  9: \n  10: \n  11: \n  12: \n  13: \n  14: \n  15: 0\n      ^\n      1\n      ^\n      2\n      ^\n      3\n      ^\n      4\n      ^\n      5\n      ^\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 words.txt --sample \"01 ab\" --search-result-print-text-width 1\n  $TESTCASE_ROOT/words.txt\n  1: 0\n     ^\n     1\n     ^\n     2\n     ^\n     3\n     ^\n     4\n     ^\n     5\n     ^\n  2: \n  3: a\n     ^\n     b\n     ^\n     c\n     ^\n     d\n     ^\n     e\n     ^\n     f\n     ^\n     g\n     ^\n  \n  15: 0\n      ^\n      1\n      ^\n      2\n      ^\n      3\n      ^\n      4\n      ^\n      5\n      ^\n  16: \n  17: a\n      ^\n      b\n      ^\n      c\n      ^\n      d\n      ^\n      e\n      ^\n      f\n      ^\n      g\n      ^\n  \n  1: 0\n     ^\n     1\n     ^\n     2\n     ^\n     3\n     ^\n     4\n     ^\n     5\n     ^\n  2: \n  3: a\n     b\n     c\n     d\n     e\n     f\n     g\n  4: \n  5: \n  6: \n  7: \n  8: \n  9: \n  10: \n  11: \n  12: \n  13: \n  14: \n  15: 0\n      1\n      2\n      3\n      4\n      5\n  16: \n  17: a\n      ^\n      b\n      ^\n      c\n      ^\n      d\n      ^\n      e\n      ^\n      f\n      ^\n      g\n      ^\n  \n  3: a\n     ^\n     b\n     ^\n     c\n     ^\n     d\n     ^\n     e\n     ^\n     f\n     ^\n     g\n     ^\n  4: \n  5: \n  6: \n  7: \n  8: \n  9: \n  10: \n  11: \n  12: \n  13: \n  14: \n  15: 0\n      ^\n      1\n      ^\n      2\n      ^\n      3\n      ^\n      4\n      ^\n      5\n      ^\n\nLine wrapping and word breaking:\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 sentences.txt --sample \"lorem\" --search-result-print-text-width 80\n  $TESTCASE_ROOT/sentences.txt\n  1:     Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod \n         ^^^^^\n     tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, \n     quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo \n     consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse \n     cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 sentences.txt --sample \"lorem\" --search-result-print-text-width 20\n  $TESTCASE_ROOT/sentences.txt\n  1:     Lorem ipsum \n         ^^^^^\n     dolor sit amet, \n     consectetur \n     adipiscing elit, \n     sed do eiusmod \n     tempor incididunt\n      ut labore et \n     dolore magna \n     aliqua. Ut enim \n     ad minim veniam, \n     quis nostrud \n     exercitation \n     ullamco laboris \n     nisi ut aliquip \n     ex ea commodo \n     consequat. Duis \n     aute irure dolor \n     in reprehenderit \n     in voluptate \n     velit esse cillum\n      dolore eu fugiat\n      nulla pariatur. \n     Excepteur sint \n     occaecat \n     cupidatat non \n     proident, sunt in\n      culpa qui \n     officia deserunt \n     mollit anim id \n     est laborum.\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 sentences.txt --sample \"lorem\" --search-result-print-text-width 10\n  $TESTCASE_ROOT/sentences.txt\n  1:     \n     Lorem \n     ^^^^^\n     ipsum \n     dolor \n     sit \n     amet, \n     consect\n     etur\n      \n     adipisc\n     ing\n      elit, \n     sed do \n     eiusmod\n      tempor\n      \n     incidid\n     unt\n      ut \n     labore \n     et \n     dolore \n     magna \n     aliqua.\n      Ut \n     enim ad\n      minim \n     veniam,\n      quis \n     nostrud\n      \n     exercit\n     ation\n      \n     ullamco\n      \n     laboris\n      nisi \n     ut \n     aliquip\n      ex ea \n     commodo\n      \n     consequ\n     at\n     . Duis \n     aute \n     irure \n     dolor \n     in \n     reprehe\n     nderit\n      in \n     volupta\n     te\n      velit \n     esse \n     cillum \n     dolore \n     eu \n     fugiat \n     nulla \n     pariatu\n     r\n     . \n     Excepte\n     ur\n      sint \n     occaeca\n     t\n      \n     cupidat\n     at\n      non \n     proiden\n     t\n     , sunt \n     in \n     culpa \n     qui \n     officia\n      \n     deserun\n     t\n      mollit\n      anim \n     id est \n     laborum\n     .\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 sentences.txt --sample \"laborum 0\" --search-result-print-text-width 80\n  $TESTCASE_ROOT/sentences.txt\n  1:     Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod \n     tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, \n     quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo \n     consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse \n     cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n                                                                      ^^^^^^^\n  2: 0 12 abcd efghi jkl mnopqrst uvwx yz 0123456 789012345\n     ^\n  \n  1:     Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod \n     tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, \n     quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo \n     consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse \n     cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n                                                                      ^^^^^^^\n  2: 0 12 abcd efghi jkl mnopqrst uvwx yz 0123456 789012345\n                                          ^^^^^^^\n  \n  1:     Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod \n     tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, \n     quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo \n     consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse \n     cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n      proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n                                                                      ^^^^^^^\n  2: 0 12 abcd efghi jkl mnopqrst uvwx yz 0123456 789012345\n                                                  ^^^^^^^^^\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 sentences.txt --sample \"laborum 0\" --search-result-print-text-width 20\n  $TESTCASE_ROOT/sentences.txt\n  1:     Lorem ipsum \n     dolor sit amet, \n     consectetur \n     adipiscing elit, \n     sed do eiusmod \n     tempor incididunt\n      ut labore et \n     dolore magna \n     aliqua. Ut enim \n     ad minim veniam, \n     quis nostrud \n     exercitation \n     ullamco laboris \n     nisi ut aliquip \n     ex ea commodo \n     consequat. Duis \n     aute irure dolor \n     in reprehenderit \n     in voluptate \n     velit esse cillum\n      dolore eu fugiat\n      nulla pariatur. \n     Excepteur sint \n     occaecat \n     cupidatat non \n     proident, sunt in\n      culpa qui \n     officia deserunt \n     mollit anim id \n     est laborum.\n         ^^^^^^^\n  2: 0 12 abcd efghi \n     ^\n     jkl mnopqrst uvwx\n      yz 0123456 \n     789012345\n  \n  1:     Lorem ipsum \n     dolor sit amet, \n     consectetur \n     adipiscing elit, \n     sed do eiusmod \n     tempor incididunt\n      ut labore et \n     dolore magna \n     aliqua. Ut enim \n     ad minim veniam, \n     quis nostrud \n     exercitation \n     ullamco laboris \n     nisi ut aliquip \n     ex ea commodo \n     consequat. Duis \n     aute irure dolor \n     in reprehenderit \n     in voluptate \n     velit esse cillum\n      dolore eu fugiat\n      nulla pariatur. \n     Excepteur sint \n     occaecat \n     cupidatat non \n     proident, sunt in\n      culpa qui \n     officia deserunt \n     mollit anim id \n     est laborum.\n         ^^^^^^^\n  2: 0 12 abcd efghi \n     jkl mnopqrst uvwx\n      yz 0123456 \n         ^^^^^^^\n     789012345\n  \n  1:     Lorem ipsum \n     dolor sit amet, \n     consectetur \n     adipiscing elit, \n     sed do eiusmod \n     tempor incididunt\n      ut labore et \n     dolore magna \n     aliqua. Ut enim \n     ad minim veniam, \n     quis nostrud \n     exercitation \n     ullamco laboris \n     nisi ut aliquip \n     ex ea commodo \n     consequat. Duis \n     aute irure dolor \n     in reprehenderit \n     in voluptate \n     velit esse cillum\n      dolore eu fugiat\n      nulla pariatur. \n     Excepteur sint \n     occaecat \n     cupidatat non \n     proident, sunt in\n      culpa qui \n     officia deserunt \n     mollit anim id \n     est laborum.\n         ^^^^^^^\n  2: 0 12 abcd efghi \n     jkl mnopqrst uvwx\n      yz 0123456 \n     789012345\n     ^^^^^^^^^\n  $ docfd --cache-dir .cache --search-result-print-snippet-min-size 0 sentences.txt --sample \"laborum 0\" --search-result-print-text-width 10\n  $TESTCASE_ROOT/sentences.txt\n  1:     \n     Lorem \n     ipsum \n     dolor \n     sit \n     amet, \n     consect\n     etur\n      \n     adipisc\n     ing\n      elit, \n     sed do \n     eiusmod\n      tempor\n      \n     incidid\n     unt\n      ut \n     labore \n     et \n     dolore \n     magna \n     aliqua.\n      Ut \n     enim ad\n      minim \n     veniam,\n      quis \n     nostrud\n      \n     exercit\n     ation\n      \n     ullamco\n      \n     laboris\n      nisi \n     ut \n     aliquip\n      ex ea \n     commodo\n      \n     consequ\n     at\n     . Duis \n     aute \n     irure \n     dolor \n     in \n     reprehe\n     nderit\n      in \n     volupta\n     te\n      velit \n     esse \n     cillum \n     dolore \n     eu \n     fugiat \n     nulla \n     pariatu\n     r\n     . \n     Excepte\n     ur\n      sint \n     occaeca\n     t\n      \n     cupidat\n     at\n      non \n     proiden\n     t\n     , sunt \n     in \n     culpa \n     qui \n     officia\n      \n     deserun\n     t\n      mollit\n      anim \n     id est \n     laborum\n     ^^^^^^^\n     .\n  2: 0 12 \n     ^\n     abcd \n     efghi \n     jkl \n     mnopqrs\n     t\n      uvwx \n     yz \n     0123456\n      \n     7890123\n     45\n  \n  1:     \n     Lorem \n     ipsum \n     dolor \n     sit \n     amet, \n     consect\n     etur\n      \n     adipisc\n     ing\n      elit, \n     sed do \n     eiusmod\n      tempor\n      \n     incidid\n     unt\n      ut \n     labore \n     et \n     dolore \n     magna \n     aliqua.\n      Ut \n     enim ad\n      minim \n     veniam,\n      quis \n     nostrud\n      \n     exercit\n     ation\n      \n     ullamco\n      \n     laboris\n      nisi \n     ut \n     aliquip\n      ex ea \n     commodo\n      \n     consequ\n     at\n     . Duis \n     aute \n     irure \n     dolor \n     in \n     reprehe\n     nderit\n      in \n     volupta\n     te\n      velit \n     esse \n     cillum \n     dolore \n     eu \n     fugiat \n     nulla \n     pariatu\n     r\n     . \n     Excepte\n     ur\n      sint \n     occaeca\n     t\n      \n     cupidat\n     at\n      non \n     proiden\n     t\n     , sunt \n     in \n     culpa \n     qui \n     officia\n      \n     deserun\n     t\n      mollit\n      anim \n     id est \n     laborum\n     ^^^^^^^\n     .\n  2: 0 12 \n     abcd \n     efghi \n     jkl \n     mnopqrs\n     t\n      uvwx \n     yz \n     0123456\n     ^^^^^^^\n      \n     7890123\n     45\n  \n  1:     \n     Lorem \n     ipsum \n     dolor \n     sit \n     amet, \n     consect\n     etur\n      \n     adipisc\n     ing\n      elit, \n     sed do \n     eiusmod\n      tempor\n      \n     incidid\n     unt\n      ut \n     labore \n     et \n     dolore \n     magna \n     aliqua.\n      Ut \n     enim ad\n      minim \n     veniam,\n      quis \n     nostrud\n      \n     exercit\n     ation\n      \n     ullamco\n      \n     laboris\n      nisi \n     ut \n     aliquip\n      ex ea \n     commodo\n      \n     consequ\n     at\n     . Duis \n     aute \n     irure \n     dolor \n     in \n     reprehe\n     nderit\n      in \n     volupta\n     te\n      velit \n     esse \n     cillum \n     dolore \n     eu \n     fugiat \n     nulla \n     pariatu\n     r\n     . \n     Excepte\n     ur\n      sint \n     occaeca\n     t\n      \n     cupidat\n     at\n      non \n     proiden\n     t\n     , sunt \n     in \n     culpa \n     qui \n     officia\n      \n     deserun\n     t\n      mollit\n      anim \n     id est \n     laborum\n     ^^^^^^^\n     .\n  2: 0 12 \n     abcd \n     efghi \n     jkl \n     mnopqrs\n     t\n      uvwx \n     yz \n     0123456\n      \n     7890123\n     ^^^^^^^\n     45\n     ^^\n"
  },
  {
    "path": "line-wrapping-tests.t/sentences.txt",
    "content": "    Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n0 12 abcd efghi jkl mnopqrst uvwx yz 0123456 789012345\n"
  },
  {
    "path": "line-wrapping-tests.t/words.txt",
    "content": "012345\n\nabcdefg\n\n\n\n\n\n\n\n\n\n\n\n012345\n\nabcdefg\n"
  },
  {
    "path": "match-type-tests.t/dune",
    "content": "(cram\n  (deps ../bin/docfd.exe))\n"
  },
  {
    "path": "match-type-tests.t/run.t",
    "content": "Exact match:\n  $ docfd --cache-dir .cache test.txt --sample \"'abc\"\n  [1]\n  $ docfd --cache-dir .cache test.txt --sample \"'abcd\"\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n     ^^^^\n  2: abcdef\n  3: ABCD\n  \n  1: abcd\n  2: abcdef\n  3: ABCD\n     ^^^^\n  4: ABCDEF\n  5: ABcd\n  \n  3: ABCD\n  4: ABCDEF\n  5: ABcd\n     ^^^^\n  6: ABcdEF\n  7: \n  \n  6: ABcdEF\n  7: \n  8: 'abcd\n      ^^^^\n  9: 'abcd'\n  10: ^efgh\n  \n  7: \n  8: 'abcd\n  9: 'abcd'\n      ^^^^\n  10: ^efgh\n  11: ^^efgh\n  $ docfd --cache-dir .cache test.txt --sample \"\\\\'abcd\"\n  $TESTCASE_ROOT/test.txt\n  6: ABcdEF\n  7: \n  8: 'abcd\n     ^^^^^\n  9: 'abcd'\n  10: ^efgh\n  \n  7: \n  8: 'abcd\n  9: 'abcd'\n     ^^^^^\n  10: ^efgh\n  11: ^^efgh\n  \n  6: ABcdEF\n  7: \n  8: 'abcd\n      ^^^^\n  9: 'abcd'\n     ^\n  10: ^efgh\n  11: ^^efgh\n  \n  7: \n  8: 'abcd\n  9: 'abcd'\n      ^^^^^\n  10: ^efgh\n  11: ^^efgh\n  \n  4: ABCDEF\n  5: ABcd\n  6: ABcdEF\n     ^^^^^^\n  7: \n  8: 'abcd\n     ^\n  9: 'abcd'\n  10: ^efgh\n  $ docfd --cache-dir .cache test.txt --sample \"'abcdef\"\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n  2: abcdef\n     ^^^^^^\n  3: ABCD\n  4: ABCDEF\n  \n  2: abcdef\n  3: ABCD\n  4: ABCDEF\n     ^^^^^^\n  5: ABcd\n  6: ABcdEF\n  \n  4: ABCDEF\n  5: ABcd\n  6: ABcdEF\n     ^^^^^^\n  7: \n  8: 'abcd\n  $ docfd --cache-dir .cache test.txt --sample \"''abcd\"\n  $TESTCASE_ROOT/test.txt\n  6: ABcdEF\n  7: \n  8: 'abcd\n     ^^^^^\n  9: 'abcd'\n  10: ^efgh\n  \n  7: \n  8: 'abcd\n  9: 'abcd'\n     ^^^^^\n  10: ^efgh\n  11: ^^efgh\n\nExact match smart case sensitivity:\n  $ docfd --cache-dir .cache test.txt --sample \"'ABCD\"\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n  2: abcdef\n  3: ABCD\n     ^^^^\n  4: ABCDEF\n  5: ABcd\n  $ docfd --cache-dir .cache test.txt --sample \"'ABcd\"\n  $TESTCASE_ROOT/test.txt\n  3: ABCD\n  4: ABCDEF\n  5: ABcd\n     ^^^^\n  6: ABcdEF\n  7: \n\nPrefix match:\n  $ docfd --cache-dir .cache test.txt --sample \"^bcd\"\n  [1]\n  $ docfd --cache-dir .cache test.txt --sample \"^abcd\"\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n     ^^^^\n  2: abcdef\n  3: ABCD\n  \n  1: abcd\n  2: abcdef\n  3: ABCD\n     ^^^^\n  4: ABCDEF\n  5: ABcd\n  \n  3: ABCD\n  4: ABCDEF\n  5: ABcd\n     ^^^^\n  6: ABcdEF\n  7: \n  \n  6: ABcdEF\n  7: \n  8: 'abcd\n      ^^^^\n  9: 'abcd'\n  10: ^efgh\n  \n  7: \n  8: 'abcd\n  9: 'abcd'\n      ^^^^\n  10: ^efgh\n  11: ^^efgh\n  $ docfd --cache-dir .cache test.txt --sample \"^abcdef\"\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n  2: abcdef\n     ^^^^^^\n  3: ABCD\n  4: ABCDEF\n  \n  2: abcdef\n  3: ABCD\n  4: ABCDEF\n     ^^^^^^\n  5: ABcd\n  6: ABcdEF\n  \n  4: ABCDEF\n  5: ABcd\n  6: ABcdEF\n     ^^^^^^\n  7: \n  8: 'abcd\n  $ docfd --cache-dir .cache test.txt --sample \"^'abcd\"\n  $TESTCASE_ROOT/test.txt\n  6: ABcdEF\n  7: \n  8: 'abcd\n     ^^^^^\n  9: 'abcd'\n  10: ^efgh\n  \n  7: \n  8: 'abcd\n  9: 'abcd'\n     ^^^^^\n  10: ^efgh\n  11: ^^efgh\n\nPrefix match smart case sensitivity:\n  $ docfd --cache-dir .cache test.txt --sample \"^ABCD\"\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n  2: abcdef\n  3: ABCD\n     ^^^^\n  4: ABCDEF\n  5: ABcd\n  \n  2: abcdef\n  3: ABCD\n  4: ABCDEF\n     ^^^^^^\n  5: ABcd\n  6: ABcdEF\n  $ docfd --cache-dir .cache test.txt --sample \"^ABcd\"\n  $TESTCASE_ROOT/test.txt\n  3: ABCD\n  4: ABCDEF\n  5: ABcd\n     ^^^^\n  6: ABcdEF\n  7: \n  \n  4: ABCDEF\n  5: ABcd\n  6: ABcdEF\n     ^^^^^^\n  7: \n  8: 'abcd\n\nSuffix match:\n  $ docfd --cache-dir .cache test.txt --sample 'bcd$'\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n     ^^^^\n  2: abcdef\n  3: ABCD\n  \n  1: abcd\n  2: abcdef\n  3: ABCD\n     ^^^^\n  4: ABCDEF\n  5: ABcd\n  \n  3: ABCD\n  4: ABCDEF\n  5: ABcd\n     ^^^^\n  6: ABcdEF\n  7: \n  \n  6: ABcdEF\n  7: \n  8: 'abcd\n      ^^^^\n  9: 'abcd'\n  10: ^efgh\n  \n  7: \n  8: 'abcd\n  9: 'abcd'\n      ^^^^\n  10: ^efgh\n  11: ^^efgh\n  $ docfd --cache-dir .cache test.txt --sample 'abcd$$'\n  $TESTCASE_ROOT/test.txt\n  13: efgh$$\n  14: \n  15: abcd$\n      ^^^^^\n  16: efgh$\n  17: \n  $ docfd --cache-dir .cache test.txt --sample 'ef$'\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n  2: abcdef\n     ^^^^^^\n  3: ABCD\n  4: ABCDEF\n  \n  2: abcdef\n  3: ABCD\n  4: ABCDEF\n     ^^^^^^\n  5: ABcd\n  6: ABcdEF\n  \n  4: ABCDEF\n  5: ABcd\n  6: ABcdEF\n     ^^^^^^\n  7: \n  8: 'abcd\n\nSuffix match smart case sensitivity:\n  $ docfd --cache-dir .cache test.txt --sample 'ABCD$'\n  $TESTCASE_ROOT/test.txt\n  1: abcd\n  2: abcdef\n  3: ABCD\n     ^^^^\n  4: ABCDEF\n  5: ABcd\n  $ docfd --cache-dir .cache test.txt --sample 'EF$'\n  $TESTCASE_ROOT/test.txt\n  2: abcdef\n  3: ABCD\n  4: ABCDEF\n     ^^^^^^\n  5: ABcd\n  6: ABcdEF\n  \n  4: ABCDEF\n  5: ABcd\n  6: ABcdEF\n     ^^^^^^\n  7: \n  8: 'abcd\n\nFuzzy match explicit spaces:\n  $ docfd --cache-dir .cache test.txt --sample 'hel~word'\n  $TESTCASE_ROOT/test.txt\n  16: efgh$\n  17: \n  18: hello world\n      ^^^^^^^^^^^\n  19: hello   world\n  20: \n  \n  17: \n  18: hello world\n  19: hello   world\n      ^^^^^^^^^^^^^\n  20: \n  21: Hello world\n  \n  19: hello   world\n  20: \n  21: Hello world\n      ^^^^^^^^^^^\n  22: \n  23: HELLO WORLD\n  \n  21: Hello world\n  22: \n  23: HELLO WORLD\n      ^^^^^^^^^^^\n  \n  16: efgh$\n  17: \n  18: hello world\n           ^^^^^^\n  19: hello   world\n      ^^^^^\n  20: \n  21: Hello world\n\nExact match explicit spaces:\n  $ docfd --cache-dir .cache test.txt --sample \"'hello~world\"\n  $TESTCASE_ROOT/test.txt\n  16: efgh$\n  17: \n  18: hello world\n      ^^^^^^^^^^^\n  19: hello   world\n  20: \n  \n  17: \n  18: hello world\n  19: hello   world\n      ^^^^^^^^^^^^^\n  20: \n  21: Hello world\n  \n  19: hello   world\n  20: \n  21: Hello world\n      ^^^^^^^^^^^\n  22: \n  23: HELLO WORLD\n  \n  21: Hello world\n  22: \n  23: HELLO WORLD\n      ^^^^^^^^^^^\n  $ docfd --cache-dir .cache test.txt --sample \"'Hello~world\"\n  $TESTCASE_ROOT/test.txt\n  19: hello   world\n  20: \n  21: Hello world\n      ^^^^^^^^^^^\n  22: \n  23: HELLO WORLD\n  $ docfd --cache-dir .cache test.txt --sample \"'Hello~World\"\n  [1]\n\nPrefix match explicit spaces:\n  $ docfd --cache-dir .cache test.txt --sample '^hello~wo'\n  $TESTCASE_ROOT/test.txt\n  16: efgh$\n  17: \n  18: hello world\n      ^^^^^^^^^^^\n  19: hello   world\n  20: \n  \n  17: \n  18: hello world\n  19: hello   world\n      ^^^^^^^^^^^^^\n  20: \n  21: Hello world\n  \n  19: hello   world\n  20: \n  21: Hello world\n      ^^^^^^^^^^^\n  22: \n  23: HELLO WORLD\n  \n  21: Hello world\n  22: \n  23: HELLO WORLD\n      ^^^^^^^^^^^\n  $ docfd --cache-dir .cache test.txt --sample '^ello~wo'\n  [1]\n\nSuffix match explicit spaces:\n  $ docfd --cache-dir .cache test.txt --sample 'lo~world$'\n  $TESTCASE_ROOT/test.txt\n  16: efgh$\n  17: \n  18: hello world\n      ^^^^^^^^^^^\n  19: hello   world\n  20: \n  \n  17: \n  18: hello world\n  19: hello   world\n      ^^^^^^^^^^^^^\n  20: \n  21: Hello world\n  \n  19: hello   world\n  20: \n  21: Hello world\n      ^^^^^^^^^^^\n  22: \n  23: HELLO WORLD\n  \n  21: Hello world\n  22: \n  23: HELLO WORLD\n      ^^^^^^^^^^^\n  $ docfd --cache-dir .cache test.txt --sample 'lo~worl$'\n  [1]\n"
  },
  {
    "path": "match-type-tests.t/test.txt",
    "content": "abcd\nabcdef\nABCD\nABCDEF\nABcd\nABcdEF\n\n'abcd\n'abcd'\n^efgh\n^^efgh\nefgh$\nefgh$$\n\nabcd$\nefgh$\n\nhello world\nhello   world\n\nHello world\n\nHELLO WORLD\n"
  },
  {
    "path": "misc-behavior-tests.t/abcd.txt",
    "content": "abcd\n"
  },
  {
    "path": "misc-behavior-tests.t/dune",
    "content": "(cram\n  (deps ../bin/docfd.exe))\n"
  },
  {
    "path": "misc-behavior-tests.t/run.t",
    "content": "Stdin temp file cleanup:\n  $ echo \"abcd\" | docfd --cache-dir .cache --search \"a\" | tail -n +2\n  1: abcd\n     ^^^^\n  $ ls /tmp/docfd-*\n  ls: cannot access '/tmp/docfd-*': No such file or directory\n  [2]\n\nStdin and path both specified:\n  $ echo \"0123\" | docfd --cache-dir .cache abcd.txt --search \"01\" # Should not print anything since stdin should be ignored.\n  [1]\n  $ echo \"0123\" | docfd --cache-dir .cache abcd.txt --search \"ab\"\n  $TESTCASE_ROOT/abcd.txt\n  1: abcd\n     ^^^^\n\n--underline:\n  $ docfd --cache-dir .cache --underline never abcd.txt --search \"ab|cd\"\n  $TESTCASE_ROOT/abcd.txt\n  1: abcd\n  \n  1: abcd\n  $ docfd --cache-dir .cache --underline always abcd.txt --search \"ab|cd\"\n  $TESTCASE_ROOT/abcd.txt\n  1: abcd\n     ^^^^\n  \n  1: abcd\n     ^^^^\n  $ docfd --cache-dir .cache --underline auto abcd.txt --search \"ab|cd\"\n  $TESTCASE_ROOT/abcd.txt\n  1: abcd\n     ^^^^\n  \n  1: abcd\n     ^^^^\n\n--color:\n  $ docfd --cache-dir .cache --color never abcd.txt --search \"ab|cd\"\n  $TESTCASE_ROOT/abcd.txt\n  1: abcd\n     ^^^^\n  \n  1: abcd\n     ^^^^\n  $ # The output below is messed up after passing through Dune, I do not know why.\n  $ docfd --cache-dir .cache --color always abcd.txt --search \"ab|cd\"\n  $TESTCASE_ROOT/abcd.txt1: abcd   ^^^^\n  1: abcd   ^^^^\n  $ docfd --cache-dir .cache --color auto abcd.txt --search \"ab|cd\"\n  $TESTCASE_ROOT/abcd.txt\n  1: abcd\n     ^^^^\n  \n  1: abcd\n     ^^^^\n"
  },
  {
    "path": "non-interactive-mode-return-code-tests.t/dune",
    "content": "(cram\n  (deps ../bin/docfd.exe))\n"
  },
  {
    "path": "non-interactive-mode-return-code-tests.t/run.t",
    "content": "Setup:\n  $ echo \"0123 abcd\" >> test0.txt\n  $ echo \"0123 efgh\" >> test1.txt\n\n--sample, text all files:\n  $ docfd --sample \"'0123\" .\n  $TESTCASE_ROOT/test1.txt\n  1: 0123 efgh\n     ^^^^\n  \n  $TESTCASE_ROOT/test0.txt\n  1: 0123 abcd\n     ^^^^\n  $ docfd --sample \"'0123\" . -l\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ docfd --sample \"'0123\" . --files-without-match\n  [1]\n\n--sample, text in only one file:\n  $ docfd --sample \"'abcd\" .\n  $TESTCASE_ROOT/test0.txt\n  1: 0123 abcd\n          ^^^^\n  $ docfd --sample \"'abcd\" . -l\n  $TESTCASE_ROOT/test0.txt\n  $ docfd --sample \"'abcd\" . --files-without-match\n  $TESTCASE_ROOT/test1.txt\n\n--sample, text not in any file:\n  $ docfd --sample \"'hello\" .\n  [1]\n  $ docfd --sample \"'hello\" . -l\n  [1]\n  $ docfd --sample \"'hello\" . --files-without-match\n  $TESTCASE_ROOT/test0.txt\n  $TESTCASE_ROOT/test1.txt\n\n--search, text all files:\n  $ docfd --search \"'0123\" .\n  $TESTCASE_ROOT/test1.txt\n  1: 0123 efgh\n     ^^^^\n  \n  $TESTCASE_ROOT/test0.txt\n  1: 0123 abcd\n     ^^^^\n  $ docfd --search \"'0123\" . -l\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ docfd --search \"'0123\" . --files-without-match\n  [1]\n\n--search, text in only one file:\n  $ docfd --search \"'abcd\" .\n  $TESTCASE_ROOT/test0.txt\n  1: 0123 abcd\n          ^^^^\n  $ docfd --search \"'abcd\" . -l\n  $TESTCASE_ROOT/test0.txt\n  $ docfd --search \"'abcd\" . --files-without-match\n  $TESTCASE_ROOT/test1.txt\n\n--search, text not in any file:\n  $ docfd --search \"'hello\" .\n  [1]\n  $ docfd --search \"'hello\" . -l\n  [1]\n  $ docfd --search \"'hello\" . --files-without-match\n  $TESTCASE_ROOT/test0.txt\n  $TESTCASE_ROOT/test1.txt\n"
  },
  {
    "path": "open-with-tests.t/dune",
    "content": "(cram\n  (deps ../bin/docfd.exe))\n"
  },
  {
    "path": "open-with-tests.t/run.t",
    "content": "Error case tests:\n  $ docfd --index-only --open-with pdf:term='okular {path}'\n  error: failed to parse pdf:term=okular {path}, invalid launch mode\n  [1]\n  $ docfd --index-only --open-with pdf='okular {path}'\n  error: failed to parse pdf=okular {path}, expected char :\n  [1]\n  $ docfd --index-only --open-with pdfterminal='okular {path}'\n  error: failed to parse pdfterminal=okular {path}, expected char :\n  [1]\n  $ docfd --index-only --open-with pdf:terminal='okular path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with pdf:terminal='okular {path'\n  error: failed to parse pdf:terminal=okular {path, expected char }\n  [1]\n  $ docfd --index-only --open-with pdf:terminal='okular {abc}'\n  error: failed to parse pdf:terminal=okular {abc}, invalid placeholder\n  [1]\n\nPDF parsing test, terminal launch mode:\n  $ docfd --index-only --open-with pdf:terminal='okular {path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with pdf:terminal='okular {page_num}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with pdf:terminal='okular {line_num}'\n  error: failed to parse pdf:terminal=okular {line_num}, line_num not available\n  [1]\n  $ docfd --index-only --open-with pdf:terminal='okular {search_word}'\n  Initializing in-memory index\n\nPDF parsing test, detached launch mode:\n  $ docfd --index-only --open-with pdf:detached='okular {path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with pdf:detached='okular {page_num}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with pdf:detached='okular {line_num}'\n  error: failed to parse pdf:detached=okular {line_num}, line_num not available\n  [1]\n  $ docfd --index-only --open-with pdf:detached='okular {search_word}'\n  Initializing in-memory index\n\nPandoc supported extensions parsing test, terminal launch mode:\n  $ docfd --index-only --open-with odt:terminal='xdg-open {path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with odt:terminal='xdg-open {page_num}'\n  error: failed to parse odt:terminal=xdg-open {page_num}, page_num not available\n  [1]\n  $ docfd --index-only --open-with odt:terminal='xdg-open {line_num}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with odt:terminal='xdg-open {search_word}'\n  error: failed to parse odt:terminal=xdg-open {search_word}, search_word not available\n  [1]\n\nPandoc supported extensions parsing test, detached launch mode:\n  $ docfd --index-only --open-with odt:detached='xdg-open {path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with odt:detached='xdg-open {page_num}'\n  error: failed to parse odt:detached=xdg-open {page_num}, page_num not available\n  [1]\n  $ docfd --index-only --open-with odt:detached='xdg-open {line_num}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with odt:detached='xdg-open {search_word}'\n  error: failed to parse odt:detached=xdg-open {search_word}, search_word not available\n  [1]\n\nText parsing test, terminal launch mode:\n  $ docfd --index-only --open-with txt:terminal='nano {path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with txt:terminal='nano {page_num}'\n  error: failed to parse txt:terminal=nano {page_num}, page_num not available\n  [1]\n  $ docfd --index-only --open-with txt:terminal='nano {line_num}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with txt:terminal='nano {search_word}'\n  error: failed to parse txt:terminal=nano {search_word}, search_word not available\n  [1]\n\nText parsing test, detached launch mode:\n  $ docfd --index-only --open-with txt:detached='nano {path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with txt:detached='nano {page_num}'\n  error: failed to parse txt:detached=nano {page_num}, page_num not available\n  [1]\n  $ docfd --index-only --open-with txt:detached='nano {line_num}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with txt:detached='nano {search_word}'\n  error: failed to parse txt:detached=nano {search_word}, search_word not available\n  [1]\n\nUnrecognized extensions parsing test, terminal launch mode:\n  $ docfd --index-only --open-with abc:terminal='nano {path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with abc:terminal='nano {page_num}'\n  error: failed to parse abc:terminal=nano {page_num}, page_num not available\n  [1]\n  $ docfd --index-only --open-with abc:terminal='nano {line_num}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with abc:terminal='nano {search_word}'\n  error: failed to parse abc:terminal=nano {search_word}, search_word not available\n  [1]\n\nUnrecognized extensions parsing test, detached launch mode:\n  $ docfd --index-only --open-with abc:detached='nano {path}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with abc:detached='nano {page_num}'\n  error: failed to parse abc:detached=nano {page_num}, page_num not available\n  [1]\n  $ docfd --index-only --open-with abc:detached='nano {line_num}'\n  Initializing in-memory index\n  $ docfd --index-only --open-with abc:detached='nano {search_word}'\n  error: failed to parse abc:detached=nano {search_word}, search_word not available\n  [1]\n"
  },
  {
    "path": "printing-tests.t/empty.txt",
    "content": ""
  },
  {
    "path": "printing-tests.t/run.t",
    "content": "--sample:\n  $ docfd --cache-dir .cache --sample abcd .\n  $TESTCASE_ROOT/test3.txt\n  1: abcd\n     ^^^^\n  \n  $TESTCASE_ROOT/test2.txt\n  1: hello\n  2: \n  3: abcd\n     ^^^^\n  4: \n  5: abcdefgh\n  \n  5: abcdefgh\n  6: \n  7: hello world abcd\n                 ^^^^\n  \n  3: abcd\n  4: \n  5: abcdefgh\n     ^^^^^^^^\n  6: \n  7: hello world abcd\n\n--search:\n  $ docfd --cache-dir .cache --search abcd .\n  $TESTCASE_ROOT/test3.txt\n  1: abcd\n     ^^^^\n  \n  $TESTCASE_ROOT/test2.txt\n  1: hello\n  2: \n  3: abcd\n     ^^^^\n  4: \n  5: abcdefgh\n  \n  5: abcdefgh\n  6: \n  7: hello world abcd\n                 ^^^^\n  \n  3: abcd\n  4: \n  5: abcdefgh\n     ^^^^^^^^\n  6: \n  7: hello world abcd\n\n-l/--files-with-match:\n  $ docfd --cache-dir .cache --sample abcd . -l\n  $TESTCASE_ROOT/test3.txt\n  $TESTCASE_ROOT/test2.txt\n  $ docfd --cache-dir .cache --sample abcd . --files-with-match\n  $TESTCASE_ROOT/test3.txt\n  $TESTCASE_ROOT/test2.txt\n\n--files-without-match:\n  $ docfd --cache-dir .cache --sample abcd . --files-without-match\n  $TESTCASE_ROOT/empty.txt\n  $TESTCASE_ROOT/test0.txt\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test4.txt\n"
  },
  {
    "path": "printing-tests.t/test0.txt",
    "content": "hello\n"
  },
  {
    "path": "printing-tests.t/test1.txt",
    "content": "hello\n"
  },
  {
    "path": "printing-tests.t/test2.txt",
    "content": "hello\n\nabcd\n\nabcdefgh\n\nhello world abcd\n"
  },
  {
    "path": "printing-tests.t/test3.txt",
    "content": "abcd\n"
  },
  {
    "path": "printing-tests.t/test4.txt",
    "content": "efgh\n"
  },
  {
    "path": "profiling/dune",
    "content": "(rule\n  (targets string_set.ml)\n  (deps ../lib/string_set.ml)\n  (action (copy# %{deps} %{targets}))\n  )\n\n(rule\n  (targets misc_utils.ml)\n  (deps ../lib/misc_utils.ml)\n  (action (copy# %{deps} %{targets}))\n  )\n\n(executable\n (flags     (-w \"+a-4-9-29-37-40-42-44-48-50-32-30-70@8\"))\n (name main)\n (libraries docfd_lib\n            containers\n            cmdliner\n            fmt\n            notty\n            notty.unix\n            nottui\n            lwd\n            oseq\n            eio\n            eio_main\n            digestif.ocaml\n            digestif\n )\n)\n"
  },
  {
    "path": "profiling/main.ml",
    "content": "open Docfd_lib\n\nlet lines = [\n  \"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer placerat lacus non cursus\";\n  \"tincidunt. Suspendisse viverra leo ac quam tincidunt, quis euismod neque tempus. \";\n  \"Vestibulum rutrum commodo tristique. Curabitur tristique dapibus dolor, vitae porttitor\";\n  \"est tristique quis. Sed urna ex, gravida vitae ipsum vel, fermentum viverra risus. \";\n  \"Maecenas et nulla iaculis, bibendum libero vitae, varius est. Fusce eros enim, placerat \";\n  \"quis magna eu, rutrum vulputate ante. Praesent non mi vel ipsum finibus lobortis. \";\n  \"Duis posuere auctor hendrerit. Nunc sodales egestas vestibulum. Quisque suscipit maximus \";\n  \"aliquam. Pellentesque tempor mi condimentum bibendum bibendum. Donec vitae accumsan quam, \";\n  \"nec vulputate lectus. Nulla ligula ipsum, dictum vel augue at, semper vestibulum ex. \";\n  \"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed arcu ligula, cursus nec \";\n  \"lacinia ut, lobortis in libero.\";\n  \"\";\n  \"Sed ultricies placerat urna, hendrerit ornare elit semper sit amet. Praesent \";\n  \"pretium blandit velit, eu imperdiet lectus tincidunt ut. Suspendisse eget eros \";\n  \"tellus. Nulla tristique vel libero non dapibus. Ut scelerisque sem sit amet \";\n  \"odio mattis vestibulum. Nam vitae commodo mi. Vestibulum consequat orci at tellus porta \";\n  \"placerat.\";\n  \"\";\n  \"In vel mi vestibulum felis accumsan congue eget efficitur tortor. Integer \";\n  \"quam purus, malesuada vel nisl at, posuere vestibulum augue. Curabitur velit \";\n  \"tortor, vestibulum id placerat eu, convallis at velit. Ut a lectus \";\n  \"quis erat tincidunt aliquet. Etiam ut erat magna. Maecenas quis commodo leo, \";\n  \"eleifend elementum ante. Nullam dapibus erat augue, a bibendum quam volutpat id. \";\n  \"Morbi in ullamcorper arcu. Fusce venenatis lacus purus, vel pellentesque mi \";\n  \"elementum a. Maecenas at mattis massa. Fusce ut elit tortor. Morbi rhoncus \";\n  \"molestie orci eu malesuada. Aliquam gravida rutrum sem, vitae condimentum magna \";\n  \"convallis pulvinar. Duis urna lacus, ultrices a ultrices pharetra, eleifend \";\n  \"in ante. Fusce id elementum dolor. Nullam ornare nisl ac ultrices lobortis.\";\n  \"\";\n  \"Proin ullamcorper vulputate enim sed facilisis. Praesent vel mi metus. \";\n  \"Fusce sagittis efficitur odio at condimentum. Nullam mollis lacinia consequat. \";\n  \"Integer vel ex sit amet nunc aliquam molestie et eu nibh. Nam leo nunc, laoreet \";\n  \"vitae iaculis sit amet, dapibus sit amet neque. Suspendisse eleifend, leo eget \";\n  \"tempor molestie, massa enim auctor dui, quis vehicula erat urna id felis. Vivamus \";\n  \"pharetra, sem non tempus ornare, risus tortor posuere tortor, eu pellentesque eros est \";\n  \"ac erat. Sed sit amet tellus nisl. Phasellus magna urna, tincidunt in sem \";\n  \"id, aliquam vulputate leo. Sed eleifend justo eu mauris egestas imperdiet. Fusce sagittis\";\n  \", turpis ac efficitur pulvinar, purus tellus gravida sem, eget accumsan nisl sapien a \";\n  \"ante. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac \";\n  \"turpis egestas. Nunc eget nibh orci. Cras facilisis facilisis sapien, \";\n  \"a vehicula lorem imperdiet vel. Proin vel nulla nisi.\";\n  \"\";\n  \"Aenean sit amet risus at lectus pellentesque pellentesque at eu quam. \";\n  \"Duis euismod porttitor ante quis lacinia. Cras sit amet vulputate nunc. \";\n  \"Integer sollicitudin vitae sapien finibus fermentum. Donec eu tellus \";\n  \"suscipit, dignissim turpis non, eleifend massa. Nullam quis ex nisi. \";\n  \"Quisque dignissim quis leo eu finibus.  \";\n]\n\nlet bench ~name ~cycle (f : unit -> 'a) =\n  let start_time = Sys.time () in\n  for _=0 to cycle-1 do\n    f () |> ignore\n  done;\n  let end_time = Sys.time () in\n  Printf.printf \"%s: time per cycle: %6fs\\n\" name\n    ((end_time -. start_time) /. (float_of_int cycle))\n\nlet main env =\n  Eio.Switch.run @@ fun sw ->\n  assert (Option.is_none (init ~db_path:\"test.db\" ~document_count_limit:1000));\n  let pool = Task_pool.make ~sw (Eio.Stdenv.domain_mgr env) in\n  let _raw = Index.Raw.of_lines pool (List.to_seq lines) in\n  Params'.max_fuzzy_edit_dist := 3;\n  let search_exp = Search_exp.parse \"vestibul rutru\" |> Option.get in\n  let s = \"PellentesquePellentesque\" in\n  for len=1 to 20 do\n    let limit = 2 in\n    bench ~name:(Fmt.str \"Spelll.of_string, limit: %d, len %2d:\" limit len) ~cycle:10 (fun () ->\n        Spelll.of_string ~limit:2 (String.sub s 0 len))\n  done;\n  for len=1 to 20 do\n    let limit = 1 in\n    bench ~name:(Fmt.str \"Spelll.of_string, limit: %d, len %2d:\" limit len) ~cycle:10 (fun () ->\n        Spelll.of_string ~limit:1 (String.sub s 0 len))\n  done;\n  bench ~name:\"Index.search\" ~cycle:1000 (fun () ->\n      Index.search pool (Stop_signal.make ()) ~search_scope:None search_exp);\n  ()\n\nlet () = Eio_main.run main\n"
  },
  {
    "path": "publish.sh",
    "content": "#!/usr/bin/env bash\n\nRETAG_CONFIRM_TEXT=\"retag-docfd\"\n\nopam_repo=\"$HOME/opam-repository\"\n\necho \"Checking if $opam_repo exists\"\n\nif [ ! -d \"$opam_repo\" ]; then\n  echo \"$opam_repo does not exist\"\n  exit 1\nfi\n\nver=$(cat CHANGELOG.md \\\n  | grep '## ' \\\n  | head -n 1 \\\n  | sed -n 's/^## \\s*\\(\\S*\\)$/\\1/p')\n\necho \"Detected version for Docfd:\" $ver\n\ngit_tag=\"$ver\"\n\necho \"Computed git tag for Docfd:\" $git_tag\n\nread -p \"Are the version and git tag correct [y/n]? \" ans\n\nif [[ $ans != \"y\" ]]; then\n  echo \"Publishing canceled\"\n  exit 0\nfi\n\necho \"Checking if $git_tag exists in repo already\"\n\nif [[ $(git tag -l \"$git_tag\") == \"\" ]]; then\n  echo \"Tagging commit\"\n  git tag \"$git_tag\"\nelse\n  read -p \"Tag already exists, retag [y/n]? \" ans\n\n  if [[ $ans == \"y\" ]]; then\n    read -p \"Type \\\"$RETAG_CONFIRM_TEXT\\\" to confirm: \" ans\n\n    if [[ $ans != \"$RETAG_CONFIRM_TEXT\" ]]; then\n      echo \"Publishing canceled\"\n      exit 0\n    fi\n\n    echo \"Removing tag\"\n    git tag -d \"$git_tag\"\n    git push --delete origin \"$git_tag\"\n\n    echo \"Tagging commit\"\n    git tag \"$git_tag\"\n  fi\nfi\n\necho \"Pushing all tags\"\n\ngit push --tags\n\narchive=\"$git_tag\".tar.gz\n\necho \"Archiving as $archive\"\n\nrm -f \"$archive\"\ngit archive --output=./\"$archive\" \"$git_tag\"\n\necho \"Hashing $archive\"\n\nhash_cmd=sha256sum\n\narchive_hash=$(\"$hash_cmd\" \"$archive\" | awk '{ print $1 }')\n\necho \"Hash from $hash_cmd:\" $archive_hash\n\npackages=(\n  \"docfd\"\n)\n\nfor package in ${packages[@]}; do\n  package_dir=\"$opam_repo\"/packages/\"$package\"/\"$package\".\"$ver\"\n  dest_opam=\"$package_dir\"/opam\n\n  echo \"Making directory $package_dir\"\n  mkdir -p \"$package_dir\"\n\n  echo \"Copying $package.opam over\"\n  cp \"$package.opam\" \"$dest_opam\"\n\n  echo \"Adding url section to $dest_opam\"\n  echo \"\nurl {\n  src:\n    \\\"https://github.com/darrenldl/docfd/releases/download/$git_tag/$archive\\\"\n  checksum:\n    \\\"sha256=$archive_hash\\\"\n}\n\" >> \"$dest_opam\"\ndone\n"
  },
  {
    "path": "run-container.sh",
    "content": "#!/usr/bin/env bash\n\npodman run -it \\\n  -v ~/docfd:/home/docfd \\\n  --workdir /home/docfd \\\n  --env VISUAL=nano \\\n  --rm \\\n  localhost/docfd \\\n  /bin/bash --login\n"
  },
  {
    "path": "script-tests.t/dune",
    "content": "(cram\n  (deps ../bin/docfd.exe))\n"
  },
  {
    "path": "script-tests.t/run.t",
    "content": "Setup:\n  $ echo \"abcd\" > test0.txt\n  $ echo \"efgh\" > test1.txt\n  $ echo \"hijk\" > test2.txt\n  $ echo \"0123\" > test3.txt\n  $ echo \"search: ^ab\" >> 0.docfd-script\n  $ echo \"search: 'xyz\" >> 1.docfd-script\n  $ tree\n  .\n  |-- 0.docfd-script\n  |-- 1.docfd-script\n  |-- dune -> ../../../../default/script-tests.t/dune\n  |-- test0.txt\n  |-- test1.txt\n  |-- test2.txt\n  `-- test3.txt\n  \n  0 directories, 7 files\n\nBasic:\n  $ docfd -l --script 0.docfd-script .\n  $TESTCASE_ROOT/test0.txt\n  $ docfd -l --script 1.docfd-script .\n  [1]\n"
  },
  {
    "path": "search-scope-narrowing-tests.t/dune",
    "content": "(cram\n  (deps ../bin/docfd.exe))\n"
  },
  {
    "path": "search-scope-narrowing-tests.t/run.t",
    "content": "Setup:\n  $ echo \"abcd\" >> test0.txt\n  $ echo \"efgh\" >> test0.txt\n  $ echo \"0123\" >> test0.txt\n  $ echo \"ijkl\" >> test0.txt\n  $ echo \"abcd\" >> test1.txt\n  $ echo \"efgh\" >> test1.txt\n  $ echo \"0123\" >> test1.txt\n  $ echo \"ijkl\" >> test1.txt\n  $ tree\n  .\n  |-- dune -> ../../../../default/search-scope-narrowing-tests.t/dune\n  |-- test0.txt\n  `-- test1.txt\n  \n  0 directories, 3 files\n\nSingle restriction:\n  $ # Case 0 for single restriction\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script test0.txt -l\n  $TESTCASE_ROOT/test0.txt\n  $ # Case 1 for single restriction\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"search: '0123\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script test0.txt -l\n  [1]\n  $ # Case 2 for single restriction\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"narrow level: 0\" >> test.docfd-script\n  $ echo \"search: '0123\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script test0.txt -l\n  $TESTCASE_ROOT/test0.txt\n\nChained restriction:\n  $ # Case 0 for chained restrictions\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"search: '0123\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script test0.txt -l\n  [1]\n  $ # Case 1 for chained restrictions\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"narrow level: 2\" >> test.docfd-script\n  $ echo \"search: '0123\" >> test.docfd-script\n  $ echo \"narrow level: 2\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script test0.txt -l\n  $TESTCASE_ROOT/test0.txt\n  $ # Case 2 for chained restrictions\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"narrow level: 2\" >> test.docfd-script\n  $ echo \"search: '0123\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script test0.txt -l\n  $TESTCASE_ROOT/test0.txt\n\nFile path filter + restrictions:\n  $ # Baseline case: \"clear filter\" after \"search\" should trigger a search for each file that has not been searched through yet\n  $ # So both documents should appear\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ # Baseline case quoted string using single quote\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:'test0.txt'\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ # Baseline case quoted string using double quote\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo 'filter: path-glob:\"test0.txt\"' >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ # Since there is no \"search\" after \"narrow\", both documents should still appear\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ # \"narrow\" + \"search\" after filtering should prevent test1.txt from appearing, even after we clear the filter\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test0.txt\n  $ # Similar to the above case, but the order of \"search\" and \"clear filter\" is swapped\n  $ # test1.txt still should not appear\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test0.txt\n  $ # Similar to the above case, but we also reset the search scope via \"narrow level: 0\"\n  $ # Since resetting the search scope does not refresh the search results, test1.txt should still not appear as there is not another \"search\"\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ echo \"narrow level: 0\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test0.txt\n  $ # Similar to the above case, but we search again after resetting search scope\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ echo \"narrow level: 0\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ # Simplified version of the above case where we skip the search before \"narrow level: 0\"\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ echo \"narrow level: 0\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ # Similar to the above case, but the order of \"clear filter\" and \"narrow level: 0\" is swapped\n  $ # Both documents should still appear\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"narrow level: 0\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n  $ # Similar to the above case, but the order of \"clear filter\" and \"search\" is swapped\n  $ # Both documents should still appear\n  $ echo \"\" > test.docfd-script\n  $ echo \"search: 'abcd\" >> test.docfd-script\n  $ echo \"filter: path-glob:test0.txt\" >> test.docfd-script\n  $ echo \"narrow level: 1\" >> test.docfd-script\n  $ echo \"narrow level: 0\" >> test.docfd-script\n  $ echo \"search: 'efgh\" >> test.docfd-script\n  $ echo \"clear filter\" >> test.docfd-script\n  $ docfd --tokens-per-search-scope-level 1 --script test.docfd-script -l .\n  $TESTCASE_ROOT/test1.txt\n  $TESTCASE_ROOT/test0.txt\n"
  },
  {
    "path": "tests/dune",
    "content": "(executable\n (flags     (-w \"+a-4-9-29-37-40-42-44-48-50-70@8\" -g))\n (name main)\n (libraries qcheck-core\n            qcheck-alcotest\n            alcotest\n            docfd_lib\n            eio\n            eio_main\n            )\n )\n"
  },
  {
    "path": "tests/main.ml",
    "content": "open Docfd_lib\n\nlet () =\n  Eio_main.run (fun env ->\n      Eio.Switch.run (fun sw ->\n          let _task_pool = Task_pool.make ~sw (Eio.Stdenv.domain_mgr env) in\n          let alco_suites =\n            [\n              (\"Search_exp_tests.Alco\", Search_exp_tests.Alco.suite);\n              (\"Utils_tests.Alco\", Utils_tests.Alco.suite);\n            ]\n          in\n          let qc_suites =\n            [\n            ]\n            |> List.map (fun (name, suite) ->\n                (name, List.map QCheck_alcotest.to_alcotest suite))\n          in\n          let suites = alco_suites @ qc_suites in\n          Alcotest.run \"docfd-lib\" suites\n        )\n    )\n"
  },
  {
    "path": "tests/search_exp_tests.ml",
    "content": "open Docfd_lib\nopen Test_utils\n\nmodule Alco = struct\n  let test_invalid_exp (s : string) =\n    Alcotest.(check bool)\n      \"true\"\n      true\n      (Option.is_none\n         (Search_exp.parse s))\n\n  let test_empty_phrase (s : string) =\n    let phrase = Search_phrase.parse s in\n    Alcotest.(check bool)\n      \"case0\"\n      true\n      (Search_phrase.is_empty phrase);\n    Alcotest.(check bool)\n      \"case1\"\n      true\n      (List.is_empty (Search_phrase.enriched_tokens phrase))\n\n  let test_empty_exp (s : string) =\n    let exp = Search_exp.parse s |> Option.get in\n    Alcotest.(check bool)\n      \"case0\"\n      true\n      (Search_exp.is_empty exp);\n    let flattened = Search_exp.flattened exp in\n    Alcotest.(check bool)\n      \"case1\"\n      true\n      (List.is_empty flattened\n       ||\n       List.for_all Search_phrase.is_empty flattened)\n\n  let at s : Search_phrase.annotated_token =\n    Search_phrase.{ data = `String s; group_id = 0 }\n\n  let atm m : Search_phrase.annotated_token =\n    Search_phrase.{ data = `Match_typ_marker m; group_id = 0 }\n\n  let ats : Search_phrase.annotated_token =\n    Search_phrase.{ data = `Explicit_spaces; group_id = 0 }\n\n  let et\n      ?(m : Search_phrase.match_typ = `Fuzzy)\n      string\n      is_linked_to_prev\n      is_linked_to_next\n    : Search_phrase.Enriched_token.t =\n    let automaton = Spelll.of_string ~limit:0 \"\" in\n    Search_phrase.Enriched_token.make\n      (`String string)\n      ~is_linked_to_prev\n      ~is_linked_to_next\n      automaton\n      m\n\n  let ets\n      ?(m : Search_phrase.match_typ = `Fuzzy)\n      is_linked_to_prev\n      is_linked_to_next\n    =\n    let automaton = Spelll.of_string ~limit:0 \"\" in\n    Search_phrase.Enriched_token.make\n      `Explicit_spaces\n      ~is_linked_to_prev\n      ~is_linked_to_next\n      automaton\n      m\n\n  let test_exp\n      ?(neg = false)\n      (s : string)\n      (l : (Search_phrase.annotated_token list * Search_phrase.Enriched_token.t list) list)\n    =\n    let neg' = neg in\n    let phrases =\n      l\n      |> List.map fst\n      |> List.map (fun l ->\n          List.to_seq l\n          |> Search_phrase.of_annotated_tokens)\n    in\n    let enriched_token_list_list =\n      List.map snd l\n    in\n    Alcotest.(check\n                (if neg' then (\n                    neg (list search_phrase_testable)\n                  ) else (\n                   list search_phrase_testable\n                 )))\n      (Fmt.str \"case0 of %S\" s)\n      (List.sort Search_phrase.compare phrases)\n      (Search_exp.parse s\n       |> Option.get\n       |> Search_exp.flattened\n       |> List.sort Search_phrase.compare\n      );\n    Alcotest.(check (list (list enriched_token_testable)))\n      (Fmt.str \"case1 of %S\" s)\n      enriched_token_list_list\n      (phrases\n       |> List.map Search_phrase.enriched_tokens\n      )\n\n  let corpus () =\n    test_empty_exp \"\";\n    test_empty_phrase \"\";\n    test_empty_exp \"    \";\n    test_empty_phrase \"    \";\n    test_empty_exp \"\\r\\n\";\n    test_empty_phrase \"\\r\\n\";\n    test_empty_exp \"\\t\";\n    test_empty_phrase \"\\t\";\n    test_empty_exp \"\\r\\n\\t\";\n    test_empty_phrase \"\\r\\n\\t\";\n    test_empty_exp \" \\r \\n \\t \";\n    test_empty_phrase \" \\r \\n \\t \";\n    test_empty_exp \"()\";\n    test_empty_exp \" () \";\n    test_empty_exp \"( )\";\n    test_empty_exp \" ( ) \";\n    test_empty_exp \" ( ) () \";\n    test_empty_exp \" ( ( ) ) () \";\n    test_empty_exp \" ( () ) (( )) \";\n    test_empty_exp \" ( () ) (( () )) \";\n    test_invalid_exp \" ( ) ( \";\n    test_invalid_exp \" ) ( \";\n    test_invalid_exp \" ( ( ) \";\n    test_invalid_exp \" ( ( ) \";\n    test_invalid_exp \"?\";\n    test_invalid_exp \"?  \";\n    test_exp \"\\\\?\"\n      [ ([ at \"?\" ],\n         [ et \"?\" false false ])\n      ];\n    test_exp \"hello_world\"\n      [ ([ at \"hello\"; at \"_\"; at \"world\" ],\n         [ et \"hello\" false true; et \"_\" true true; et \"world\" true false ])\n      ];\n    test_exp \"(hello)\"\n      [ ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"()hello\"\n      [ ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"hello()\"\n      [ ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"( ) hello\"\n      [ ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"hello ( )\"\n      [ ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"?hello\"\n      [ ([], [])\n      ; ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"(?hello)\"\n      [ ([], [])\n      ; ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"?(hello)\"\n      [ ([], [])\n      ; ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"?hello()\"\n      [ ([], [])\n      ; ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"?hello ()\"\n      [ ([], [])\n      ; ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"? hello\"\n      [ ([], [])\n      ; ([ at \"hello\" ],\n         [ et \"hello\" false false ])\n      ];\n    test_exp \"?hello world\"\n      [ ([ at \"world\" ],\n         [ et \"world\" false false ])\n      ; ([ at \"hello\"; at \" \"; at \"world\" ],\n         [ et \"hello\" false false; et \"world\" false false ])\n      ];\n    test_exp \"? hello world\"\n      [ ([ at \"world\" ],\n         [ et \"world\" false false ])\n      ; ([ at \"hello\"; at \" \"; at \"world\" ],\n         [ et \"hello\" false false; et \"world\" false false ])\n      ];\n    test_exp \"?(hello) world\"\n      [ ([ at \"world\" ],\n         [ et \"world\" false false ] )\n      ; ([ at \"hello\"; at \" \"; at \"world\" ],\n         [ et \"hello\" false false; et \"world\" false false ] )\n      ];\n    test_exp \"? (hello) world\"\n      [ ([ at \"world\" ],\n         [ et \"world\" false false ] )\n      ; ([ at \"hello\"; at \" \"; at \"world\" ],\n         [ et \"hello\" false false; et \"world\" false false ])\n      ];\n    test_exp \"?(hello world) abcd\"\n      [ ([ at \"abcd\" ],\n         [ et \"abcd\" false false ] )\n      ; ([ at \"hello\"; at \" \"; at \"world\"; at \" \"; at \"abcd\" ],\n         [ et \"hello\" false false; et \"world\" false false; et \"abcd\" false false ] )\n      ];\n    test_exp \"ab ?(hello world) cd\"\n      [ ([ at \"ab\"; at \" \"; at \"cd\" ],\n         [ et \"ab\" false false; et \"cd\" false false ])\n      ; ([ at \"ab\"; at \" \"; at \"hello\"; at \" \"; at \"world\"; at \" \"; at \"cd\" ],\n         [ et \"ab\" false false; et \"hello\" false false; et \"world\" false false; et \"cd\" false false ])\n      ];\n    test_exp \"ab ?hello world cd\"\n      [ ([ at \"ab\"; at \" \"; at \"world\"; at \" \"; at \"cd\" ],\n         [ et \"ab\" false false; et \"world\" false false; et \"cd\" false false ])\n      ; ([ at \"ab\"; at \" \"; at \"hello\"; at \" \"; at \"world\"; at \" \"; at \"cd\" ],\n         [ et \"ab\" false false; et \"hello\" false false; et \"world\" false false; et \"cd\" false false ])\n      ];\n    test_exp \"go (left | right)\"\n      [ ([ at \"go\"; at \" \"; at \"left\" ],\n         [ et \"go\" false false; et \"left\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\" ],\n         [ et \"go\" false false; et \"right\" false false ])\n      ];\n    test_exp \"go (?up | left | right)\"\n      [ ([ at \"go\" ],\n         [ et \"go\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\" ],\n         [ et \"go\" false false; et \"left\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\" ],\n         [ et \"go\" false false; et \"right\" false false ])];\n    test_exp \"(left | right) (up | down)\"\n      [ ([ at \"left\"; at \" \"; at \"up\" ],\n         [ et \"left\" false false; et \"up\" false false ])\n      ; ([ at \"left\"; at \" \"; at \"down\" ],\n         [ et \"left\" false false; et \"down\" false false ])\n      ; ([ at \"right\"; at \" \"; at \"up\" ],\n         [ et \"right\" false false; et \"up\" false false ])\n      ; ([ at \"right\"; at \" \"; at \"down\" ],\n         [ et \"right\" false false; et \"down\" false false ])\n      ];\n    test_exp \"((a|b)(c|d)) (e | f)\"\n      [ ([ at \"a\"; at \" \"; at \"c\"; at \" \"; at \"e\" ],\n         [ et \"a\" false false; et \"c\" false false; et \"e\" false false ])\n      ; ([ at \"a\"; at \" \"; at \"c\"; at \" \"; at \"f\" ],\n         [ et \"a\" false false; et \"c\" false false; et \"f\" false false ])\n      ; ([ at \"a\"; at \" \"; at \"d\"; at \" \"; at \"e\" ],\n         [ et \"a\" false false; et \"d\" false false; et \"e\" false false ])\n      ; ([ at \"a\"; at \" \"; at \"d\"; at \" \"; at \"f\" ],\n         [ et \"a\" false false; et \"d\" false false; et \"f\" false false ])\n      ; ([ at \"b\"; at \" \"; at \"c\"; at \" \"; at \"e\" ],\n         [ et \"b\" false false; et \"c\" false false; et \"e\" false false ])\n      ; ([ at \"b\"; at \" \"; at \"c\"; at \" \"; at \"f\" ],\n         [ et \"b\" false false; et \"c\" false false; et \"f\" false false ])\n      ; ([ at \"b\"; at \" \"; at \"d\"; at \" \"; at \"e\" ],\n         [ et \"b\" false false; et \"d\" false false; et \"e\" false false ])\n      ; ([ at \"b\"; at \" \"; at \"d\"; at \" \"; at \"f\" ],\n         [ et \"b\" false false; et \"d\" false false; et \"f\" false false ])\n      ];\n    test_exp \"(?left | right) (up | down)\"\n      [ ([ at \"up\" ],\n         [ et \"up\" false false ])\n      ; ([ at \"down\" ],\n         [ et \"down\" false false ])\n      ; ([ at \"left\"; at \" \"; at \"up\" ],\n         [ et \"left\" false false; et \"up\" false false ])\n      ; ([ at \"left\"; at \" \"; at \"down\" ],\n         [ et \"left\" false false; et \"down\" false false ])\n      ; ([ at \"right\"; at \" \"; at \"up\" ],\n         [ et \"right\" false false; et \"up\" false false ])\n      ; ([ at \"right\"; at \" \"; at \"down\" ],\n         [ et \"right\" false false; et \"down\" false false ])\n      ];\n    test_exp \"go (left | right) or ( up | down )\"\n      [ ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"down\" false false ])\n      ];\n    test_exp \"and/or\"\n      [ ([ at \"and\"; at \"/\"; at \"or\" ],\n         [ et \"and\" false true; et \"/\" true true; et \"or\" true false ])\n      ];\n    test_exp ~neg:true \"and/or\"\n      [ ([ at \"and\"; at \" \"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false false; et \"/\" false false; et \"or\" false false ])\n      ];\n    test_exp ~neg:true \"and/or\"\n      [ ([ at \"and\"; at \" \"; at \"/\"; at \"or\" ],\n         [ et \"and\" false false; et \"/\" false true; et \"or\" true false ])\n      ];\n    test_exp ~neg:true \"and/or\"\n      [ ([ at \"and\"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false true; et \"/\" true false; et \"or\" false false ])\n      ];\n    test_exp \"and / or\"\n      [ ([ at \"and\"; at \" \"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false false; et \"/\" false false; et \"or\" false false ])\n      ];\n    test_exp ~neg:true \"and / or\"\n      [ ([ at \"and\"; at \"/\"; at \"or\" ],\n         [ et \"and\" false true; et \"/\" true true; et \"or\" true false ])\n      ];\n    test_exp ~neg:true \"and / or\"\n      [ ([ at \"and\"; at \" \"; at \"/\"; at \"or\" ],\n         [ et \"and\" false false; et \"/\" false true; et \"or\" true false ])\n      ];\n    test_exp ~neg:true \"and / or\"\n      [ ([ at \"and\"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false true; et \"/\" true false; et \"or\" false false ])\n      ];\n    test_exp \"(and)/ or\"\n      [ ([ at \"and\"; at \" \"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false false; et \"/\" false false; et \"or\" false false ])\n      ];\n    test_exp ~neg:true \"(and)/ or\"\n      [ ([ at \"and\"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false true; et \"/\" true false; et \"or\" false false ])\n      ];\n    test_exp \"and(/) or\"\n      [ ([ at \"and\"; at \" \"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false false; et \"/\" false false; et \"or\" false false ])\n      ];\n    test_exp ~neg:true \"and(/) or\"\n      [ ([ at \"and\"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false true; et \"/\" true false; et \"or\" false false ])\n      ];\n    test_exp \"and/(or)\"\n      [ ([ at \"and\"; at \"/\"; at \" \"; at \"or\" ],\n         [ et \"and\" false true; et \"/\" true false; et \"or\" false false ])\n      ];\n    test_exp ~neg:true \"and/(or)\"\n      [ ([ at \"and\"; at \"/\"; at \"or\" ],\n         [ et \"and\" false true; et \"/\" true true; et \"or\" true false ])\n      ];\n    test_exp \"go (left | right) and/or ( up | down )\"\n      [ ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"and\"; at \"/\"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"and\" false true; et \"/\" true true; et \"or\" true false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"and\"; at \"/\"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"and\" false true; et \"/\" true true; et \"or\" true false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"and\"; at \"/\"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"and\" false true; et \"/\" true true; et \"or\" true false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"and\"; at \"/\"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"and\" false true; et \"/\" true true; et \"or\" true false; et \"down\" false false ])\n      ];\n    test_exp \"go (left | right) and / or ( up | down )\"\n      [ ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"and\"; at \" \"; at \"/\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"and\" false false; et \"/\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"and\"; at \" \"; at \"/\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"and\" false false; et \"/\" false false; et \"or\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"and\"; at \" \"; at \"/\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"and\" false false; et \"/\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"and\"; at \" \"; at \"/\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"and\" false false; et \"/\" false false; et \"or\" false false; et \"down\" false false ])\n      ];\n    test_exp \"go ?(left | right) ( up | down )\"\n      [ ([ at \"go\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"down\" false false ])\n      ];\n    test_exp \"go ?((left | right) or) ( up | down )\"\n      [ ([ at \"go\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"down\" false false ])\n      ];\n    test_exp \"go ?(?(left | right) or) ( up | down )\"\n      [ ([ at \"go\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"or\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"down\" false false ])\n      ];\n    test_exp \"go ?(?(left | right) or) or ( ?up | down )\"\n      [ ([ at \"go\"; at \" \"; at \"or\" ],\n         [ et \"go\" false false; et \"or\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"or\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"or\"; at \" \"; at \"or\" ],\n         [ et \"go\" false false; et \"or\" false false; et \"or\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"or\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"or\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"or\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"or\" false false; et \"or\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"or\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"or\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"left\"; at \" \"; at \"or\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"left\" false false; et \"or\" false false; et \"or\" false false; et \"down\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"or\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"or\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"or\"; at \" \"; at \"up\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"or\" false false; et \"up\" false false ])\n      ; ([ at \"go\"; at \" \"; at \"right\"; at \" \"; at \"or\"; at \" \"; at \"or\"; at \" \"; at \"down\" ],\n         [ et \"go\" false false; et \"right\" false false; et \"or\" false false; et \"or\" false false; et \"down\" false false ])\n      ];\n    test_exp \"- - -\"\n      [ ([ at \"-\"; at \" \"; at \"-\"; at \" \"; at \"-\" ],\n         [ et \"-\" false false; et \"-\" false false; et \"-\" false false ])\n      ];\n    test_exp \"-- -\"\n      [ ([ at \"-\"; at \"-\"; at \" \"; at \"-\" ],\n         [ et \"-\" false true; et \"-\" true false; et \"-\" false false ])\n      ];\n    test_exp \"\\\\'abcd\"\n      [ ([ at \"'\"; at \"abcd\" ],\n         [ et \"'\" false true; et \"abcd\" true false ])\n      ];\n    test_exp \"'abcd\"\n      [ ([ atm `Exact; at \"abcd\" ],\n         [ et ~m:`Exact \"abcd\" false false ])\n      ];\n    test_exp \"' abcd\"\n      [ ([ atm `Exact; at \" \"; at \"abcd\" ],\n         [ et \"'\" false false; et \"abcd\" false false ])\n      ];\n    test_exp \"' abcd\"\n      [ ([ at \"'\"; at \" \"; at \"abcd\" ],\n         [ et \"'\" false false; et \"abcd\" false false ])\n      ];\n    test_exp \"\\\\^abcd\"\n      [ ([ at \"^\"; at \"abcd\" ],\n         [ et \"^\" false true; et \"abcd\" true false ])\n      ];\n    test_exp \"^abcd\"\n      [ ([ atm `Prefix; at \"abcd\" ],\n         [ et ~m:`Prefix \"abcd\" false false ])\n      ];\n    test_exp \"^ abcd\"\n      [ ([ at \"^\"; at \" \"; at \"abcd\" ],\n         [ et \"^\" false false; et \"abcd\" false false ])\n      ];\n    test_exp \"^ abcd\"\n      [ ([ atm `Prefix; at \" \"; at \"abcd\" ],\n         [ et \"^\" false false; et \"abcd\" false false ])\n      ];\n    test_exp \"abcd$\"\n      [ ([ at \"abcd\"; atm `Suffix ],\n         [ et ~m:`Suffix \"abcd\" false false ])\n      ];\n    test_exp \"abcd $\"\n      [ ([ at \"abcd\"; at \" \"; at \"$\" ],\n         [ et \"abcd\" false false; et \"$\" false false ])\n      ];\n    test_exp \"abcd $\"\n      [ ([ at \"abcd\"; at \" \"; atm `Suffix ],\n         [ et \"abcd\" false false; et \"$\" false false ])\n      ];\n    test_exp \"''abcd\"\n      [ ([ atm `Exact; atm `Exact; at \"abcd\" ],\n         [ et ~m:`Exact \"'\" false true; et ~m:`Exact \"abcd\" true false ])\n      ];\n    test_exp \"^^abcd\"\n      [ ([ atm `Prefix; at \"^\"; at \"abcd\" ],\n         [ et ~m:`Exact \"^\" false true; et ~m:`Prefix \"abcd\" true false ])\n      ];\n    test_exp \"abcd$$\"\n      [ ([ at \"abcd\"; at \"$\"; atm `Suffix ],\n         [ et ~m:`Suffix \"abcd\" false true; et ~m:`Exact \"$\" true false ])\n      ];\n    test_exp \"abcd$$\"\n      [ ([ at \"abcd\"; atm `Suffix; atm `Suffix ],\n         [ et ~m:`Suffix \"abcd\" false true; et ~m:`Exact \"$\" true false ])\n      ];\n    test_exp \"'ab~cd\"\n      [ ([ atm `Exact; at \"ab\"; ats; at \"cd\" ],\n         [ et ~m:`Exact \"ab\" false true\n         ; ets ~m:`Exact true true\n         ; et ~m:`Exact \"cd\" true false ])\n      ];\n    test_exp \"^ab~cd$\"\n      [ ([ atm `Prefix; at \"ab\"; ats; at \"cd\"; atm `Suffix ],\n         [ et ~m:`Exact \"ab\" false true\n         ; ets ~m:`Exact true true\n         ; et ~m:`Exact \"cd\" true true\n         ; et ~m:`Prefix \"$\" true false\n         ])\n      ];\n    test_exp \"ab~cd$\"\n      [ ([ at \"ab\"; ats; at \"cd\"; atm `Suffix ],\n         [ et ~m:`Suffix \"ab\" false true\n         ; ets ~m:`Exact true true\n         ; et ~m:`Exact \"cd\" true false\n         ])\n      ];\n    test_exp \" ~cd$\"\n      [ ([ ats; at \"cd\"; atm `Suffix ],\n         [ ets ~m:`Suffix false true\n         ; et ~m:`Exact \"cd\" true false\n         ])\n      ];\n    test_exp \"'^abcd efgh$$ ij$kl$\"\n      [ ([ atm `Exact\n         ; atm `Prefix\n         ; at \"abcd\"\n         ; at \" \"\n         ; at \"efgh\"\n         ; atm `Suffix\n         ; atm `Suffix\n         ; at \" \"\n         ; at \"ij\"\n         ; atm `Suffix\n         ; at \"kl\"\n         ; atm `Suffix\n         ],\n         [ et ~m:`Exact \"^\" false true\n         ; et ~m:`Exact \"abcd\" true false\n         ; et ~m:`Suffix \"efgh\" false true\n         ; et ~m:`Exact \"$\" true false\n         ; et ~m:`Suffix \"ij\" false true\n         ; et ~m:`Exact \"$\" true true\n         ; et ~m:`Exact \"kl\" true false\n         ])\n      ];\n    test_exp \"'^abcd efgh$$ ij$kl$\"\n      [ ([ atm `Exact\n         ; at \"^\"\n         ; at \"abcd\"\n         ; at \" \"\n         ; at \"efgh\"\n         ; at \"$\"\n         ; atm `Suffix\n         ; at \" \"\n         ; at \"ij\"\n         ; at \"$\"\n         ; at \"kl\"\n         ; atm `Suffix\n         ],\n         [ et ~m:`Exact \"^\" false true\n         ; et ~m:`Exact \"abcd\" true false\n         ; et ~m:`Suffix \"efgh\" false true\n         ; et ~m:`Exact \"$\" true false\n         ; et ~m:`Suffix \"ij\" false true\n         ; et ~m:`Exact \"$\" true true\n         ; et ~m:`Exact \"kl\" true false\n         ])\n      ];\n    ()\n\n  let suite =\n    [\n      Alcotest.test_case \"corpus\" `Quick corpus;\n    ]\nend\n"
  },
  {
    "path": "tests/test_utils.ml",
    "content": "open Docfd_lib\n\nlet enriched_token_testable : (module Alcotest.TESTABLE with type t = Search_phrase.Enriched_token.t) =\n  (module Search_phrase.Enriched_token)\n\nlet search_phrase_testable : (module Alcotest.TESTABLE with type t = Search_phrase.t) =\n  (module Search_phrase)\n"
  },
  {
    "path": "tests/utils_tests.ml",
    "content": "open Docfd_lib\n\nmodule Alco = struct\n  let normalize_path_to_absolute_corpus () =\n    let test expected input =\n      Alcotest.(check string)\n        (Printf.sprintf \"%s becomes %s\" input expected)\n        expected\n        (Misc_utils'.normalize_path_to_absolute input)\n    in\n    let cwd = Sys.getcwd () in\n    let test' expected input =\n      test (if expected = \"\" then cwd else Filename.concat cwd expected) input\n    in\n    test \"/\" \"/..\";\n    test \"/\" \"/..\";\n    test \"/\" \"/abcd/..\";\n    test \"/\" \"/abcd/..\";\n    test \"/abc\" \"/abc/\";\n    test \"/abc\" \"/abc/def/..\";\n    test \"/abc\" \"/abc//\";\n    test \"/abc/def\" \"/abc//def\";\n    test \"/abc/def\" \"/abc/./def\";\n    test \"/abc/def\" \"/abc/.///def/.\";\n    test \"/def\" \"/abc/.//../def/.\";\n    test' \"\" \"abc/..\";\n    test' \"def\" \"abc/../def\";\n    test' \"abc/def\" \"abc////.//./././/def\";\n    ()\n\n  let normalize_glob_to_absolute_corpus () =\n    let test expected input =\n      Alcotest.(check string)\n        (Printf.sprintf \"%s becomes %s\" input expected)\n        expected\n        (Misc_utils'.normalize_glob_to_absolute input)\n    in\n    let cwd = Sys.getcwd () in\n    let test' expected input =\n      test (if expected = \"\" then cwd else Filename.concat cwd expected) input\n    in\n    test \"/\" \"/..\";\n    test \"/\" \"/..\";\n    test \"/\" \"/abcd/..\";\n    test \"/\" \"/abcd/..\";\n    test \"/abc\" \"/abc/\";\n    test \"/abc\" \"/abc/def/..\";\n    test \"/abc\" \"/abc/*/..\";\n    test \"/abc/**/..\" \"/abc/**/..\";\n    test \"/abc/**/def\" \"/abc/**/def\";\n    test \"/**/def/*/..\" \"/abc/../**/def/*/..\";\n    test \"/abc/**/def\" \"/abc/.////**/def\";\n    test \"/abc\" \"/abc//\";\n    test \"/abc/def\" \"/abc//def\";\n    test \"/abc/def\" \"/abc/./def\";\n    test \"/abc/def\" \"/abc/.///def/.\";\n    test \"/def\" \"/abc/.//../def/.\";\n    test' \"\" \"abc/..\";\n    test' \"def\" \"abc/../def\";\n    test' \"abc/def\" \"abc////.//./././/def\";\n    test' \"abc/**/../def\" \"abc/**/../def\";\n    test' \"abc/**/../def\" \"abc/*/../**/../def\";\n    test' \"abc/*/def\" \"abc/*/*/../def\";\n    ()\n\n  let suite =\n    [\n      Alcotest.test_case\n        \"normalize_path_to_absolute_corpus\"\n        `Quick\n        normalize_path_to_absolute_corpus;\n      Alcotest.test_case\n        \"normalize_glob_to_absolute_corpus\"\n        `Quick\n        normalize_glob_to_absolute_corpus;\n    ]\nend\n"
  },
  {
    "path": "update-version-string.py",
    "content": "import os\nimport re\n\nml_path = \"bin/version_string.ml\"\n\nversion = os.environ.get('DOCFD_VERSION_OVERRIDE')\n\nif version is None or version == \"\":\n    with open(\"CHANGELOG.md\") as f:\n        for line in f:\n            if line.startswith(\"## \") and not (\"future release\" in line.lower()):\n                version = line.split(\" \")[1].strip()\n                break\n\nprint(f\"Detected version for Docfd: {version}\")\n\nprint(f\"Writing to {ml_path}\")\n\nwith open(ml_path, \"w\") as f:\n    f.write(f\"let s = \\\"{version}\\\"\")\n    f.write(\"\\n\")\n"
  }
]