[
  {
    "path": ".github/workflows/cover.yml",
    "content": "on:\n  push:\n    branches:\n      - master\n  pull_request:\nname: Coverage\njobs:\n  coverage:\n    runs-on: ubuntu-latest\n    steps:\n      - name: Install Go\n        uses: actions/setup-go@v5\n        with:\n          go-version: '1.25.x'\n      - name: Checkout code\n        uses: actions/checkout@v2\n      - name: Clean environment\n        run: |\n          go clean -cache -testcache -modcache\n          rm -f profile.cov\n      - name: Test\n        run: |\n          go test -coverprofile=profile.cov ./...\n      - name: Remove non-GO entries from coverage profile\n        run: |\n          grep -E 'mode|\\.go' profile.cov > profile_go.cov\n      - name: Send coverage\n        uses: shogo82148/actions-goveralls@v1\n        with:\n          path-to-profile: profile_go.cov\n"
  },
  {
    "path": ".github/workflows/tests.yml",
    "content": "on:\n  push:\n    branches:\n    - master\n  pull_request:\nname: Tests\njobs:\n  test:\n    strategy:\n      matrix:\n        go-version: [1.23.x, 1.24.x, 1.25.x]\n        platform: [ubuntu-latest, macos-latest, windows-latest]\n    runs-on: ${{ matrix.platform }}\n    steps:\n    - name: Install Go\n      uses: actions/setup-go@v5\n      with:\n        go-version: ${{ matrix.go-version }}\n    - name: Checkout code\n      uses: actions/checkout@v2\n    - name: Test\n      run: |\n        go version\n        go test -race ./...\n"
  },
  {
    "path": ".gitignore",
    "content": "#*\n*.sublime-*\n*~\n.#*\n.project\n.settings\n**/.idea/\n**/*.iml\n.DS_Store\nquery_string.y.go.tmp\n/analysis/token_filters/cld2/cld2-read-only\n/analysis/token_filters/cld2/libcld2_full.a\n/cmd/bleve/bleve\nvendor/**\n!vendor/manifest\n/y.output\n/search/query/y.output\n*.test\ntags\ngo.sum\n"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "# Contributing to Bleve\n\nWe look forward to your contributions, but ask that you first review these guidelines.\n\n## Sign the CLA\n\nAs Bleve is a Couchbase project we require contributors accept the [Couchbase Contributor License Agreement](http://review.couchbase.org/static/individual_agreement.html). To sign this agreement log into the Couchbase [code review tool](http://review.couchbase.org/). The Bleve project does not use this code review tool but it is still used to track acceptance of the contributor license agreements.\n\n## Submitting a Pull Request\n\nAll types of contributions are welcome, but please keep the following in mind:\n\n- If you're planning a large change, you should really discuss it in a github issue or on the google group first. This helps avoid duplicate effort and spending time on something that may not be merged.\n- Existing tests should continue to pass, new tests for the contribution are nice to have.\n- All code should have gone through `go fmt`\n- All code should pass `go vet`\n"
  },
  {
    "path": "LICENSE",
    "content": "\n                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License."
  },
  {
    "path": "README.md",
    "content": "# ![bleve](docs/bleve.png) bleve\n\n[![Tests](https://github.com/blevesearch/bleve/actions/workflows/tests.yml/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve/actions/workflows/tests.yml?query=event%3Apush+branch%3Amaster)\n[![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg)](https://coveralls.io/github/blevesearch/bleve)\n[![Go Reference](https://pkg.go.dev/badge/github.com/blevesearch/bleve/v2.svg)](https://pkg.go.dev/github.com/blevesearch/bleve/v2)\n[![Join the chat](https://badges.gitter.im/join_chat.svg)](https://app.gitter.im/#/room/#blevesearch_bleve:gitter.im)\n[![Go Report Card](https://goreportcard.com/badge/github.com/blevesearch/bleve/v2)](https://goreportcard.com/report/github.com/blevesearch/bleve/v2)\n[![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge)\n[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)\n\nA modern indexing + search library in GO\n\n## Features\n\n* Index any GO data structure or JSON\n* Intelligent defaults backed up by powerful configuration ([scorch](https://github.com/blevesearch/bleve/blob/master/index/scorch/README.md))\n* Supported field types:\n  * `text`, `number`, `datetime`, `boolean`, `geopoint`, `geoshape`, `IP`, `vector`\n* Supported query types:\n  * `term`, `phrase`, `match`, `match_phrase`, `prefix`, `regexp`, `wildcard`, `fuzzy`\n  * term range, numeric range, date range, boolean field\n  * compound queries: `conjuncts`, `disjuncts`, boolean (`must`/`should`/`must_not`)\n  * [query string syntax](http://www.blevesearch.com/docs/Query-String-Query/)\n  * [geo spatial search](https://github.com/blevesearch/bleve/blob/master/geo/README.md)\n  * approximate k-nearest neighbors via [vector search](https://github.com/blevesearch/bleve/blob/master/docs/vectors.md)\n  * [synonym search](https://github.com/blevesearch/bleve/blob/master/docs/synonyms.md)\n  * [hierarchical nested search](https://github.com/blevesearch/bleve/blob/master/docs/hierarchy.md)\n* [tf-idf](https://github.com/blevesearch/bleve/blob/master/docs/scoring.md#tf-idf) / [bm25](https://github.com/blevesearch/bleve/blob/master/docs/scoring.md#bm25) scoring models\n* Hybrid search: exact + semantic\n  * Supports [RRF (Reciprocal Rank Fusion) and RSF (Relative Score Fusion)](docs/score_fusion.md)\n* [Result pagination](https://github.com/blevesearch/bleve/blob/master/docs/pagination.md)\n* Query time boosting\n* Search result match highlighting with document fragments\n* Aggregations/faceting support:\n  * terms facet\n  * numeric range facet\n  * date range facet\n\n## Indexing\n\n```go\nmessage := struct {\n    Id   string\n    From string\n    Body string\n}{\n    Id:   \"example\",\n    From: \"xyz@couchbase.com\",\n    Body: \"bleve indexing is easy\",\n}\n\nmapping := bleve.NewIndexMapping()\nindex, err := bleve.New(\"example.bleve\", mapping)\nif err != nil {\n    panic(err)\n}\nindex.Index(message.Id, message)\n```\n\n## Querying\n\n```go\nindex, _ := bleve.Open(\"example.bleve\")\nquery := bleve.NewQueryStringQuery(\"bleve\")\nsearchRequest := bleve.NewSearchRequest(query)\nsearchResult, _ := index.Search(searchRequest)\n```\n\n## Command Line Interface\n\nTo install the CLI for the latest release of bleve, run:\n\n```bash\ngo install github.com/blevesearch/bleve/v2/cmd/bleve@latest\n```\n\n```text\n$ bleve --help\nBleve is a command-line tool to interact with a bleve index.\n\nUsage:\n  bleve [command]\n\nAvailable Commands:\n  bulk        bulk loads from newline delimited JSON files\n  check       checks the contents of the index\n  count       counts the number documents in the index\n  create      creates a new index\n  dictionary  prints the term dictionary for the specified field in the index\n  dump        dumps the contents of the index\n  fields      lists the fields in this index\n  help        Help about any command\n  index       adds the files to the index\n  mapping     prints the mapping used for this index\n  query       queries the index\n  registry    registry lists the bleve components compiled into this executable\n  scorch      command-line tool to interact with a scorch index\n\nFlags:\n  -h, --help   help for bleve\n\nUse \"bleve [command] --help\" for more information about a command.\n```\n\n## Text Analysis\n\nBleve includes general-purpose analyzers (customizable) as well as pre-built text analyzers for the following languages:\n\nArabic (ar), Bulgarian (bg), Catalan (ca), Chinese-Japanese-Korean (cjk), Kurdish (ckb), Danish (da), German (de), Greek (el), English (en), Spanish - Castilian (es), Basque (eu), Persian (fa), Finnish (fi), French (fr), Gaelic (ga), Spanish - Galician (gl), Hindi (hi), Croatian (hr), Hungarian (hu), Armenian (hy), Indonesian (id, in), Italian (it), Dutch (nl), Norwegian (no), Polish (pl), Portuguese (pt), Romanian (ro), Russian (ru), Swedish (sv), Turkish (tr)\n\n## Text Analysis Wizard\n\n[bleveanalysis.couchbase.com](https://bleveanalysis.couchbase.com)\n\n## Discussion/Issues\n\nDiscuss usage/development of bleve and/or report issues here:\n\n* [Github issues](https://github.com/blevesearch/bleve/issues)\n* [Google group](https://groups.google.com/forum/#!forum/bleve)\n\n## License\n\nApache License Version 2.0\n"
  },
  {
    "path": "SECURITY.md",
    "content": "# Security Policy\n\n## Supported Versions\n\nWe support the latest release (for example, bleve v2.5.x).\n\n## Reporting a Vulnerability\n\nAll security issues for this project should be reported via email to [security@couchbase.com](mailto:security@couchbase.com) and [fts-team@couchbase.com](mailto:fts-team@couchbase.com).\n\nThis mail will be delivered to the owners of this project.\n\n- To ensure your report is NOT marked as spam, please include the word \"security/vulnerability\" along with the project name (blevesearch/bleve) in the subject of the email.\n- Please be as descriptive as possible while explaining the issue, and a testcase highlighting the issue is always welcome.\n\nYour email will be acknowledged at the soonest possible.\n"
  },
  {
    "path": "analysis/analyzer/custom/custom.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage custom\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"custom\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\n\tvar err error\n\tvar charFilters []analysis.CharFilter\n\tcharFiltersValue, ok := config[\"char_filters\"]\n\tif ok {\n\t\tswitch charFiltersValue := charFiltersValue.(type) {\n\t\tcase []string:\n\t\t\tcharFilters, err = getCharFilters(charFiltersValue, cache)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\tcase []interface{}:\n\t\t\tcharFiltersNames, err := convertInterfaceSliceToStringSlice(charFiltersValue, \"char filter\")\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tcharFilters, err = getCharFilters(charFiltersNames, cache)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"unsupported type for char_filters, must be slice\")\n\t\t}\n\t}\n\n\tvar tokenizerName string\n\ttokenizerValue, ok := config[\"tokenizer\"]\n\tif ok {\n\t\ttokenizerName, ok = tokenizerValue.(string)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"must specify tokenizer as string\")\n\t\t}\n\t} else {\n\t\treturn nil, fmt.Errorf(\"must specify tokenizer\")\n\t}\n\n\ttokenizer, err := cache.TokenizerNamed(tokenizerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar tokenFilters []analysis.TokenFilter\n\ttokenFiltersValue, ok := config[\"token_filters\"]\n\tif ok {\n\t\tswitch tokenFiltersValue := tokenFiltersValue.(type) {\n\t\tcase []string:\n\t\t\ttokenFilters, err = getTokenFilters(tokenFiltersValue, cache)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\tcase []interface{}:\n\t\t\ttokenFiltersNames, err := convertInterfaceSliceToStringSlice(tokenFiltersValue, \"token filter\")\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\ttokenFilters, err = getTokenFilters(tokenFiltersNames, cache)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"unsupported type for token_filters, must be slice\")\n\t\t}\n\t}\n\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t}\n\tif charFilters != nil {\n\t\trv.CharFilters = charFilters\n\t}\n\tif tokenFilters != nil {\n\t\trv.TokenFilters = tokenFilters\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(Name, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc getCharFilters(charFilterNames []string, cache *registry.Cache) ([]analysis.CharFilter, error) {\n\tcharFilters := make([]analysis.CharFilter, len(charFilterNames))\n\tfor i, charFilterName := range charFilterNames {\n\t\tcharFilter, err := cache.CharFilterNamed(charFilterName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tcharFilters[i] = charFilter\n\t}\n\n\treturn charFilters, nil\n}\n\nfunc getTokenFilters(tokenFilterNames []string, cache *registry.Cache) ([]analysis.TokenFilter, error) {\n\ttokenFilters := make([]analysis.TokenFilter, len(tokenFilterNames))\n\tfor i, tokenFilterName := range tokenFilterNames {\n\t\ttokenFilter, err := cache.TokenFilterNamed(tokenFilterName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ttokenFilters[i] = tokenFilter\n\t}\n\n\treturn tokenFilters, nil\n}\n\nfunc convertInterfaceSliceToStringSlice(interfaceSlice []interface{}, objType string) ([]string, error) {\n\tstringSlice := make([]string, len(interfaceSlice))\n\tfor i, interfaceObj := range interfaceSlice {\n\t\tstringObj, ok := interfaceObj.(string)\n\t\tif ok {\n\t\t\tstringSlice[i] = stringObj\n\t\t} else {\n\t\t\treturn nil, fmt.Errorf(\"%s name must be a string\", objType)\n\t\t}\n\t}\n\n\treturn stringSlice, nil\n}\n"
  },
  {
    "path": "analysis/analyzer/keyword/keyword.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage keyword\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/single\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"keyword\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tkeywordTokenizer, err := cache.TokenizerNamed(single.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: keywordTokenizer,\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(Name, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/analyzer/simple/simple.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage simple\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"simple\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(letter.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(Name, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/analyzer/standard/standard.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage standard\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/lang/en\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"standard\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopEnFilter, err := cache.TokenFilterNamed(en.StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopEnFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(Name, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/analyzer/web/web.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage web\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/lang/en\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/web\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"web\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(web.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopEnFilter, err := cache.TokenFilterNamed(en.StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopEnFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(Name, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/benchmark_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage analysis_test\n\nimport (\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc BenchmarkAnalysis(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\n\t\tcache := registry.NewCache()\n\t\tanalyzer, err := cache.AnalyzerNamed(standard.Name)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\n\t\tts := analyzer.Analyze(bleveWikiArticle)\n\t\tfreqs := analysis.TokenFrequency(ts, nil, index.IncludeTermVectors)\n\t\tif len(freqs) != 511 {\n\t\t\tb.Errorf(\"expected %d freqs, got %d\", 511, len(freqs))\n\t\t}\n\t}\n}\n\nvar bleveWikiArticle = []byte(`Boiling liquid expanding vapor explosion\nFrom Wikipedia, the free encyclopedia\nSee also: Boiler explosion and Steam explosion\n\nFlames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire.\n\nThis article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013)\nA boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1]\nContents  [hide]\n1 Mechanism\n1.1 Water example\n1.2 BLEVEs without chemical reactions\n2 Fires\n3 Incidents\n4 Safety measures\n5 See also\n6 References\n7 External links\nMechanism[edit]\n\nThis section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013)\nThere are three characteristics of liquids which are relevant to the discussion of a BLEVE:\nIf a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed).\nThe boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example.\nWhen a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did.\nTypically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, propane, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures.\nIf the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.\nWater example[edit]\nImagine, for example, a tank of pressurized liquid water held at 204.4 °C (400 °F). This tank would normally be pressurized to 1.7 MPa (250 psi) above atmospheric (\"gauge\") pressure. If the tank containing the water were to rupture, there would for a slight moment exist a volume of liquid water which would be\nat atmospheric pressure, and\n204.4 °C (400 °F).\nAt atmospheric pressure the boiling point of water is 100 °C (212 °F) - liquid water at atmospheric pressure cannot exist at temperatures higher than 100 °C (212 °F). At that moment, the water would boil and turn to vapour explosively, and the 204.4 °C (400 °F) liquid water turned to gas would take up a lot more volume than it did as liquid, causing a vapour explosion. Such explosions can happen when the superheated water of a steam engine escapes through a crack in a boiler, causing a boiler explosion.\nBLEVEs without chemical reactions[edit]\nIt is important to note that a BLEVE need not be a chemical explosion—nor does there need to be a fire—however if a flammable substance is subject to a BLEVE it may also be subject to intense heating, either from an external source of heat which may have caused the vessel to rupture in the first place or from an internal source of localized heating such as skin friction. This heating can cause a flammable substance to ignite, adding a secondary explosion caused by the primary BLEVE. While blast effects of any BLEVE can be devastating, a flammable substance such as propane can add significantly to the danger.\nBleve explosion.svg\nWhile the term BLEVE is most often used to describe the results of a container of flammable liquid rupturing due to fire, a BLEVE can occur even with a non-flammable substance such as water,[2] liquid nitrogen,[3] liquid helium or other refrigerants or cryogens, and therefore is not usually considered a type of chemical explosion.\nFires[edit]\nBLEVEs can be caused by an external fire near the storage vessel causing heating of the contents and pressure build-up. While tanks are often designed to withstand great pressure, constant heating can cause the metal to weaken and eventually fail. If the tank is being heated in an area where there is no liquid, it may rupture faster without the liquid to absorb the heat. Gas containers are usually equipped with relief valves that vent off excess pressure, but the tank can still fail if the pressure is not released quickly enough.[1] Relief valves are sized to release pressure fast enough to prevent the pressure from increasing beyond the strength of the vessel, but not so fast as to be the cause of an explosion. An appropriately sized relief valve will allow the liquid inside to boil slowly, maintaining a constant pressure in the vessel until all the liquid has boiled and the vessel empties.\nIf the substance involved is flammable, it is likely that the resulting cloud of the substance will ignite after the BLEVE has occurred, forming a fireball and possibly a fuel-air explosion, also termed a vapor cloud explosion (VCE). If the materials are toxic, a large area will be contaminated.[4]\nIncidents[edit]\nThe term \"BLEVE\" was coined by three researchers at Factory Mutual, in the analysis of an accident there in 1957 involving a chemical reactor vessel.[5]\nIn August 1959 the Kansas City Fire Department suffered its largest ever loss of life in the line of duty, when a 25,000 gallon (95,000 litre) gas tank exploded during a fire on Southwest Boulevard killing five firefighters. This was the first time BLEVE was used to describe a burning fuel tank.[citation needed]\nLater incidents included the Cheapside Street Whisky Bond Fire in Glasgow, Scotland in 1960; Feyzin, France in 1966; Crescent City, Illinois in 1970; Kingman, Arizona in 1973; a liquid nitrogen tank rupture[6] at Air Products and Chemicals and Mobay Chemical Company at New Martinsville, West Virginia on January 31, 1978 [1];Texas City, Texas in 1978; Murdock, Illinois in 1983; San Juan Ixhuatepec, Mexico City in 1984; and Toronto, Ontario in 2008.\nSafety measures[edit]\n[icon]\tThis section requires expansion. (July 2013)\nSome fire mitigation measures are listed under liquefied petroleum gas.\nSee also[edit]\nBoiler explosion\nExpansion ratio\nExplosive boiling or phase explosion\nRapid phase transition\nViareggio train derailment\n2008 Toronto explosions\nGas carriers\nLos Alfaques Disaster\nLac-Mégantic derailment\nReferences[edit]\n^ Jump up to: a b Kletz, Trevor (March 1990). Critical Aspects of Safety and Loss Prevention. London: Butterworth–Heinemann. pp. 43–45. ISBN 0-408-04429-2.\nJump up ^ \"Temperature Pressure Relief Valves on Water Heaters: test, inspect, replace, repair guide\". Inspect-ny.com. Retrieved 2011-07-12.\nJump up ^ Liquid nitrogen BLEVE demo\nJump up ^ \"Chemical Process Safety\" (PDF). Retrieved 2011-07-12.\nJump up ^ David F. Peterson, BLEVE: Facts, Risk Factors, and Fallacies, Fire Engineering magazine (2002).\nJump up ^ \"STATE EX REL. VAPOR CORP. v. NARICK\". Supreme Court of Appeals of West Virginia. 1984-07-12. Retrieved 2014-03-16.\nExternal links[edit]\n\tLook up boiling liquid expanding vapor explosion in Wiktionary, the free dictionary.\n\tWikimedia Commons has media related to BLEVE.\nBLEVE Demo on YouTube — video of a controlled BLEVE demo\nhuge explosions on YouTube — video of propane and isobutane BLEVEs from a train derailment at Murdock, Illinois (3 September 1983)\nPropane BLEVE on YouTube — video of BLEVE from the Toronto propane depot fire\nMoscow Ring Road Accident on YouTube - Dozens of LPG tank BLEVEs after a road accident in Moscow\nKingman, AZ BLEVE — An account of the 5 July 1973 explosion in Kingman, with photographs\nPropane Tank Explosions — Description of circumstances required to cause a propane tank BLEVE.\nAnalysis of BLEVE Events at DOE Sites - Details physics and mathematics of BLEVEs.\nHID - SAFETY REPORT ASSESSMENT GUIDE: Whisky Maturation Warehouses - The liquor is aged in wooden barrels that can suffer BLEVE.\nCategories: ExplosivesFirefightingFireTypes of fireGas technologiesIndustrial fires and explosions`)\n"
  },
  {
    "path": "analysis/char/asciifolding/asciifolding.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// converted to Go from Lucene's AsciiFoldingFilter\n// https://lucene.apache.org/core/4_0_0/analyzers-common/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.html\n\npackage asciifolding\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"asciifolding\"\n\ntype AsciiFoldingFilter struct{}\n\nfunc New() *AsciiFoldingFilter {\n\treturn &AsciiFoldingFilter{}\n}\n\nfunc (s *AsciiFoldingFilter) Filter(input []byte) []byte {\n\tif len(input) == 0 {\n\t\treturn input\n\t}\n\n\tin := []rune(string(input))\n\tlength := len(in)\n\n\t// Worst-case length required if all runes fold to 4 runes\n\tout := make([]rune, length, length*4)\n\n\tout = foldToASCII(in, 0, out, 0, length)\n\treturn []byte(string(out))\n}\n\nfunc AsciiFoldingFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) {\n\treturn New(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterCharFilter(Name, AsciiFoldingFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\n// Converts characters above ASCII to their ASCII equivalents.\n// For example, accents are removed from accented characters.\nfunc foldToASCII(input []rune, inputPos int, output []rune, outputPos int, length int) []rune {\n\tend := inputPos + length\n\tfor pos := inputPos; pos < end; pos++ {\n\t\tc := input[pos]\n\n\t\t// Quick test: if it's not in range then just keep current character\n\t\tif c < '\\u0080' {\n\t\t\toutput[outputPos] = c\n\t\t\toutputPos++\n\t\t} else {\n\t\t\tswitch c {\n\t\t\tcase '\\u00C0': // À [LATIN CAPITAL LETTER A WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00C1': // Á [LATIN CAPITAL LETTER A WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00C2': //  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00C3': // à [LATIN CAPITAL LETTER A WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00C4': // Ä [LATIN CAPITAL LETTER A WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00C5': // Å [LATIN CAPITAL LETTER A WITH RING ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0100': // Ā [LATIN CAPITAL LETTER A WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0102': // Ă [LATIN CAPITAL LETTER A WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0104': // Ą [LATIN CAPITAL LETTER A WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u018F': // Ə http://en.wikipedia.org/wiki/Schwa [LATIN CAPITAL LETTER SCHWA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01CD': // Ǎ [LATIN CAPITAL LETTER A WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01DE': // Ǟ [LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E0': // Ǡ [LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01FA': // Ǻ [LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0200': // Ȁ [LATIN CAPITAL LETTER A WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0202': // Ȃ [LATIN CAPITAL LETTER A WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0226': // Ȧ [LATIN CAPITAL LETTER A WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u023A': // Ⱥ [LATIN CAPITAL LETTER A WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D00': // ᴀ [LATIN LETTER SMALL CAPITAL A]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E00': // Ḁ [LATIN CAPITAL LETTER A WITH RING BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA0': // Ạ [LATIN CAPITAL LETTER A WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA2': // Ả [LATIN CAPITAL LETTER A WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA4': // Ấ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA6': // Ầ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA8': // Ẩ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EAA': // Ẫ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EAC': // Ậ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EAE': // Ắ [LATIN CAPITAL LETTER A WITH BREVE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB0': // Ằ [LATIN CAPITAL LETTER A WITH BREVE AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB2': // Ẳ [LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB4': // Ẵ [LATIN CAPITAL LETTER A WITH BREVE AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24B6': // Ⓐ [CIRCLED LATIN CAPITAL LETTER A]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF21': // Ａ [FULLWIDTH LATIN CAPITAL LETTER A]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB6': // Ặ [LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW]\n\t\t\t\toutput[outputPos] = 'A'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00E0': // à [LATIN SMALL LETTER A WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00E1': // á [LATIN SMALL LETTER A WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00E2': // â [LATIN SMALL LETTER A WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00E3': // ã [LATIN SMALL LETTER A WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00E4': // ä [LATIN SMALL LETTER A WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00E5': // å [LATIN SMALL LETTER A WITH RING ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0101': // ā [LATIN SMALL LETTER A WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0103': // ă [LATIN SMALL LETTER A WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0105': // ą [LATIN SMALL LETTER A WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01CE': // ǎ [LATIN SMALL LETTER A WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01DF': // ǟ [LATIN SMALL LETTER A WITH DIAERESIS AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E1': // ǡ [LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01FB': // ǻ [LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0201': // ȁ [LATIN SMALL LETTER A WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0203': // ȃ [LATIN SMALL LETTER A WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0227': // ȧ [LATIN SMALL LETTER A WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0250': // ɐ [LATIN SMALL LETTER TURNED A]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0259': // ə [LATIN SMALL LETTER SCHWA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u025A': // ɚ [LATIN SMALL LETTER SCHWA WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D8F': // ᶏ [LATIN SMALL LETTER A WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D95': // ᶕ [LATIN SMALL LETTER SCHWA WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E01': // ạ [LATIN SMALL LETTER A WITH RING BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E9A': // ả [LATIN SMALL LETTER A WITH RIGHT HALF RING]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA1': // ạ [LATIN SMALL LETTER A WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA3': // ả [LATIN SMALL LETTER A WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA5': // ấ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA7': // ầ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EA9': // ẩ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EAB': // ẫ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EAD': // ậ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EAF': // ắ [LATIN SMALL LETTER A WITH BREVE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB1': // ằ [LATIN SMALL LETTER A WITH BREVE AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB3': // ẳ [LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB5': // ẵ [LATIN SMALL LETTER A WITH BREVE AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB7': // ặ [LATIN SMALL LETTER A WITH BREVE AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2090': // ₐ [LATIN SUBSCRIPT SMALL LETTER A]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2094': // ₔ [LATIN SUBSCRIPT SMALL LETTER SCHWA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D0': // ⓐ [CIRCLED LATIN SMALL LETTER A]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C65': // ⱥ [LATIN SMALL LETTER A WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C6F': // Ɐ [LATIN CAPITAL LETTER TURNED A]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF41': // ａ [FULLWIDTH LATIN SMALL LETTER A]\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA732': // Ꜳ [LATIN CAPITAL LETTER AA]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'A'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'A'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00C6': // Æ [LATIN CAPITAL LETTER AE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E2': // Ǣ [LATIN CAPITAL LETTER AE WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01FC': // Ǽ [LATIN CAPITAL LETTER AE WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D01': // ᴁ [LATIN LETTER SMALL CAPITAL AE]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'A'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'E'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA734': // Ꜵ [LATIN CAPITAL LETTER AO]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'A'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'O'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA736': // Ꜷ [LATIN CAPITAL LETTER AU]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'A'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'U'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA738': // Ꜹ [LATIN CAPITAL LETTER AV]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA73A': // Ꜻ [LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'A'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'V'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA73C': // Ꜽ [LATIN CAPITAL LETTER AY]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'A'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'Y'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u249C': // ⒜ [PARENTHESIZED LATIN SMALL LETTER A]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA733': // ꜳ [LATIN SMALL LETTER AA]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00E6': // æ [LATIN SMALL LETTER AE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E3': // ǣ [LATIN SMALL LETTER AE WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01FD': // ǽ [LATIN SMALL LETTER AE WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D02': // ᴂ [LATIN SMALL LETTER TURNED AE]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'e'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA735': // ꜵ [LATIN SMALL LETTER AO]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'o'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA737': // ꜷ [LATIN SMALL LETTER AU]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'u'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA739': // ꜹ [LATIN SMALL LETTER AV]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA73B': // ꜻ [LATIN SMALL LETTER AV WITH HORIZONTAL BAR]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'v'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA73D': // ꜽ [LATIN SMALL LETTER AY]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'a'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'y'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0181': // Ɓ [LATIN CAPITAL LETTER B WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0182': // Ƃ [LATIN CAPITAL LETTER B WITH TOPBAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0243': // Ƀ [LATIN CAPITAL LETTER B WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0299': // ʙ [LATIN LETTER SMALL CAPITAL B]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D03': // ᴃ [LATIN LETTER SMALL CAPITAL BARRED B]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E02': // Ḃ [LATIN CAPITAL LETTER B WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E04': // Ḅ [LATIN CAPITAL LETTER B WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E06': // Ḇ [LATIN CAPITAL LETTER B WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24B7': // Ⓑ [CIRCLED LATIN CAPITAL LETTER B]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF22': // Ｂ [FULLWIDTH LATIN CAPITAL LETTER B]\n\t\t\t\toutput[outputPos] = 'B'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0180': // ƀ [LATIN SMALL LETTER B WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0183': // ƃ [LATIN SMALL LETTER B WITH TOPBAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0253': // ɓ [LATIN SMALL LETTER B WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D6C': // ᵬ [LATIN SMALL LETTER B WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D80': // ᶀ [LATIN SMALL LETTER B WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E03': // ḃ [LATIN SMALL LETTER B WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E05': // ḅ [LATIN SMALL LETTER B WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E07': // ḇ [LATIN SMALL LETTER B WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D1': // ⓑ [CIRCLED LATIN SMALL LETTER B]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF42': // ｂ [FULLWIDTH LATIN SMALL LETTER B]\n\t\t\t\toutput[outputPos] = 'b'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u249D': // ⒝ [PARENTHESIZED LATIN SMALL LETTER B]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'b'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00C7': // Ç [LATIN CAPITAL LETTER C WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0106': // Ć [LATIN CAPITAL LETTER C WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0108': // Ĉ [LATIN CAPITAL LETTER C WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u010A': // Ċ [LATIN CAPITAL LETTER C WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u010C': // Č [LATIN CAPITAL LETTER C WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0187': // Ƈ [LATIN CAPITAL LETTER C WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u023B': // Ȼ [LATIN CAPITAL LETTER C WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0297': // ʗ [LATIN LETTER STRETCHED C]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D04': // ᴄ [LATIN LETTER SMALL CAPITAL C]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E08': // Ḉ [LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24B8': // Ⓒ [CIRCLED LATIN CAPITAL LETTER C]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF23': // Ｃ [FULLWIDTH LATIN CAPITAL LETTER C]\n\t\t\t\toutput[outputPos] = 'C'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00E7': // ç [LATIN SMALL LETTER C WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0107': // ć [LATIN SMALL LETTER C WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0109': // ĉ [LATIN SMALL LETTER C WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u010B': // ċ [LATIN SMALL LETTER C WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u010D': // č [LATIN SMALL LETTER C WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0188': // ƈ [LATIN SMALL LETTER C WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u023C': // ȼ [LATIN SMALL LETTER C WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0255': // ɕ [LATIN SMALL LETTER C WITH CURL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E09': // ḉ [LATIN SMALL LETTER C WITH CEDILLA AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2184': // ↄ [LATIN SMALL LETTER REVERSED C]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D2': // ⓒ [CIRCLED LATIN SMALL LETTER C]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA73E': // Ꜿ [LATIN CAPITAL LETTER REVERSED C WITH DOT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA73F': // ꜿ [LATIN SMALL LETTER REVERSED C WITH DOT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF43': // ｃ [FULLWIDTH LATIN SMALL LETTER C]\n\t\t\t\toutput[outputPos] = 'c'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u249E': // ⒞ [PARENTHESIZED LATIN SMALL LETTER C]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'c'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00D0': // Ð [LATIN CAPITAL LETTER ETH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u010E': // Ď [LATIN CAPITAL LETTER D WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0110': // Đ [LATIN CAPITAL LETTER D WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0189': // Ɖ [LATIN CAPITAL LETTER AFRICAN D]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u018A': // Ɗ [LATIN CAPITAL LETTER D WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u018B': // Ƌ [LATIN CAPITAL LETTER D WITH TOPBAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D05': // ᴅ [LATIN LETTER SMALL CAPITAL D]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D06': // ᴆ [LATIN LETTER SMALL CAPITAL ETH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E0A': // Ḋ [LATIN CAPITAL LETTER D WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E0C': // Ḍ [LATIN CAPITAL LETTER D WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E0E': // Ḏ [LATIN CAPITAL LETTER D WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E10': // Ḑ [LATIN CAPITAL LETTER D WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E12': // Ḓ [LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24B9': // Ⓓ [CIRCLED LATIN CAPITAL LETTER D]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA779': // Ꝺ [LATIN CAPITAL LETTER INSULAR D]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF24': // Ｄ [FULLWIDTH LATIN CAPITAL LETTER D]\n\t\t\t\toutput[outputPos] = 'D'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00F0': // ð [LATIN SMALL LETTER ETH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u010F': // ď [LATIN SMALL LETTER D WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0111': // đ [LATIN SMALL LETTER D WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u018C': // ƌ [LATIN SMALL LETTER D WITH TOPBAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0221': // ȡ [LATIN SMALL LETTER D WITH CURL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0256': // ɖ [LATIN SMALL LETTER D WITH TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0257': // ɗ [LATIN SMALL LETTER D WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D6D': // ᵭ [LATIN SMALL LETTER D WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D81': // ᶁ [LATIN SMALL LETTER D WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D91': // ᶑ [LATIN SMALL LETTER D WITH HOOK AND TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E0B': // ḋ [LATIN SMALL LETTER D WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E0D': // ḍ [LATIN SMALL LETTER D WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E0F': // ḏ [LATIN SMALL LETTER D WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E11': // ḑ [LATIN SMALL LETTER D WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E13': // ḓ [LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D3': // ⓓ [CIRCLED LATIN SMALL LETTER D]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA77A': // ꝺ [LATIN SMALL LETTER INSULAR D]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF44': // ｄ [FULLWIDTH LATIN SMALL LETTER D]\n\t\t\t\toutput[outputPos] = 'd'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01C4': // DŽ [LATIN CAPITAL LETTER DZ WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F1': // DZ [LATIN CAPITAL LETTER DZ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'D'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'Z'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01C5': // Dž [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F2': // Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'D'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'z'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u249F': // ⒟ [PARENTHESIZED LATIN SMALL LETTER D]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'd'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0238': // ȸ [LATIN SMALL LETTER DB DIGRAPH]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'd'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'b'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01C6': // dž [LATIN SMALL LETTER DZ WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F3': // dz [LATIN SMALL LETTER DZ]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u02A3': // ʣ [LATIN SMALL LETTER DZ DIGRAPH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u02A5': // ʥ [LATIN SMALL LETTER DZ DIGRAPH WITH CURL]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'd'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'z'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00C8': // È [LATIN CAPITAL LETTER E WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00C9': // É [LATIN CAPITAL LETTER E WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00CA': // Ê [LATIN CAPITAL LETTER E WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00CB': // Ë [LATIN CAPITAL LETTER E WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0112': // Ē [LATIN CAPITAL LETTER E WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0114': // Ĕ [LATIN CAPITAL LETTER E WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0116': // Ė [LATIN CAPITAL LETTER E WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0118': // Ę [LATIN CAPITAL LETTER E WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u011A': // Ě [LATIN CAPITAL LETTER E WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u018E': // Ǝ [LATIN CAPITAL LETTER REVERSED E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0190': // Ɛ [LATIN CAPITAL LETTER OPEN E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0204': // Ȅ [LATIN CAPITAL LETTER E WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0206': // Ȇ [LATIN CAPITAL LETTER E WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0228': // Ȩ [LATIN CAPITAL LETTER E WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0246': // Ɇ [LATIN CAPITAL LETTER E WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D07': // ᴇ [LATIN LETTER SMALL CAPITAL E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E14': // Ḕ [LATIN CAPITAL LETTER E WITH MACRON AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E16': // Ḗ [LATIN CAPITAL LETTER E WITH MACRON AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E18': // Ḙ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E1A': // Ḛ [LATIN CAPITAL LETTER E WITH TILDE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E1C': // Ḝ [LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB8': // Ẹ [LATIN CAPITAL LETTER E WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EBA': // Ẻ [LATIN CAPITAL LETTER E WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EBC': // Ẽ [LATIN CAPITAL LETTER E WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EBE': // Ế [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC0': // Ề [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC2': // Ể [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC4': // Ễ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC6': // Ệ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24BA': // Ⓔ [CIRCLED LATIN CAPITAL LETTER E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C7B': // ⱻ [LATIN LETTER SMALL CAPITAL TURNED E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF25': // Ｅ [FULLWIDTH LATIN CAPITAL LETTER E]\n\t\t\t\toutput[outputPos] = 'E'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00E8': // è [LATIN SMALL LETTER E WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00E9': // é [LATIN SMALL LETTER E WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00EA': // ê [LATIN SMALL LETTER E WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00EB': // ë [LATIN SMALL LETTER E WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0113': // ē [LATIN SMALL LETTER E WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0115': // ĕ [LATIN SMALL LETTER E WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0117': // ė [LATIN SMALL LETTER E WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0119': // ę [LATIN SMALL LETTER E WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u011B': // ě [LATIN SMALL LETTER E WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01DD': // ǝ [LATIN SMALL LETTER TURNED E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0205': // ȅ [LATIN SMALL LETTER E WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0207': // ȇ [LATIN SMALL LETTER E WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0229': // ȩ [LATIN SMALL LETTER E WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0247': // ɇ [LATIN SMALL LETTER E WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0258': // ɘ [LATIN SMALL LETTER REVERSED E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u025B': // ɛ [LATIN SMALL LETTER OPEN E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u025C': // ɜ [LATIN SMALL LETTER REVERSED OPEN E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u025D': // ɝ [LATIN SMALL LETTER REVERSED OPEN E WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u025E': // ɞ [LATIN SMALL LETTER CLOSED REVERSED OPEN E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u029A': // ʚ [LATIN SMALL LETTER CLOSED OPEN E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D08': // ᴈ [LATIN SMALL LETTER TURNED OPEN E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D92': // ᶒ [LATIN SMALL LETTER E WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D93': // ᶓ [LATIN SMALL LETTER OPEN E WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D94': // ᶔ [LATIN SMALL LETTER REVERSED OPEN E WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E15': // ḕ [LATIN SMALL LETTER E WITH MACRON AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E17': // ḗ [LATIN SMALL LETTER E WITH MACRON AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E19': // ḙ [LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E1B': // ḛ [LATIN SMALL LETTER E WITH TILDE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E1D': // ḝ [LATIN SMALL LETTER E WITH CEDILLA AND BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EB9': // ẹ [LATIN SMALL LETTER E WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EBB': // ẻ [LATIN SMALL LETTER E WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EBD': // ẽ [LATIN SMALL LETTER E WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EBF': // ế [LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC1': // ề [LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC3': // ể [LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC5': // ễ [LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC7': // ệ [LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2091': // ₑ [LATIN SUBSCRIPT SMALL LETTER E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D4': // ⓔ [CIRCLED LATIN SMALL LETTER E]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C78': // ⱸ [LATIN SMALL LETTER E WITH NOTCH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF45': // ｅ [FULLWIDTH LATIN SMALL LETTER E]\n\t\t\t\toutput[outputPos] = 'e'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A0': // ⒠ [PARENTHESIZED LATIN SMALL LETTER E]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'e'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0191': // Ƒ [LATIN CAPITAL LETTER F WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E1E': // Ḟ [LATIN CAPITAL LETTER F WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24BB': // Ⓕ [CIRCLED LATIN CAPITAL LETTER F]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA730': // ꜰ [LATIN LETTER SMALL CAPITAL F]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA77B': // Ꝼ [LATIN CAPITAL LETTER INSULAR F]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA7FB': // ꟻ [LATIN EPIGRAPHIC LETTER REVERSED F]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF26': // Ｆ [FULLWIDTH LATIN CAPITAL LETTER F]\n\t\t\t\toutput[outputPos] = 'F'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0192': // ƒ [LATIN SMALL LETTER F WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D6E': // ᵮ [LATIN SMALL LETTER F WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D82': // ᶂ [LATIN SMALL LETTER F WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E1F': // ḟ [LATIN SMALL LETTER F WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E9B': // ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D5': // ⓕ [CIRCLED LATIN SMALL LETTER F]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA77C': // ꝼ [LATIN SMALL LETTER INSULAR F]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF46': // ｆ [FULLWIDTH LATIN SMALL LETTER F]\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A1': // ⒡ [PARENTHESIZED LATIN SMALL LETTER F]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFB00': // ff [LATIN SMALL LIGATURE FF]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFB03': // ffi [LATIN SMALL LIGATURE FFI]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'i'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFB04': // ffl [LATIN SMALL LIGATURE FFL]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFB01': // fi [LATIN SMALL LIGATURE FI]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'i'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFB02': // fl [LATIN SMALL LIGATURE FL]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'f'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u011C': // Ĝ [LATIN CAPITAL LETTER G WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u011E': // Ğ [LATIN CAPITAL LETTER G WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0120': // Ġ [LATIN CAPITAL LETTER G WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0122': // Ģ [LATIN CAPITAL LETTER G WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0193': // Ɠ [LATIN CAPITAL LETTER G WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E4': // Ǥ [LATIN CAPITAL LETTER G WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E5': // ǥ [LATIN SMALL LETTER G WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E6': // Ǧ [LATIN CAPITAL LETTER G WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E7': // ǧ [LATIN SMALL LETTER G WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F4': // Ǵ [LATIN CAPITAL LETTER G WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0262': // ɢ [LATIN LETTER SMALL CAPITAL G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u029B': // ʛ [LATIN LETTER SMALL CAPITAL G WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E20': // Ḡ [LATIN CAPITAL LETTER G WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24BC': // Ⓖ [CIRCLED LATIN CAPITAL LETTER G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA77D': // Ᵹ [LATIN CAPITAL LETTER INSULAR G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA77E': // Ꝿ [LATIN CAPITAL LETTER TURNED INSULAR G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF27': // Ｇ [FULLWIDTH LATIN CAPITAL LETTER G]\n\t\t\t\toutput[outputPos] = 'G'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u011D': // ĝ [LATIN SMALL LETTER G WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u011F': // ğ [LATIN SMALL LETTER G WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0121': // ġ [LATIN SMALL LETTER G WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0123': // ģ [LATIN SMALL LETTER G WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F5': // ǵ [LATIN SMALL LETTER G WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0260': // ɠ [LATIN SMALL LETTER G WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0261': // ɡ [LATIN SMALL LETTER SCRIPT G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D77': // ᵷ [LATIN SMALL LETTER TURNED G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D79': // ᵹ [LATIN SMALL LETTER INSULAR G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D83': // ᶃ [LATIN SMALL LETTER G WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E21': // ḡ [LATIN SMALL LETTER G WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D6': // ⓖ [CIRCLED LATIN SMALL LETTER G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA77F': // ꝿ [LATIN SMALL LETTER TURNED INSULAR G]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF47': // ｇ [FULLWIDTH LATIN SMALL LETTER G]\n\t\t\t\toutput[outputPos] = 'g'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A2': // ⒢ [PARENTHESIZED LATIN SMALL LETTER G]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'g'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0124': // Ĥ [LATIN CAPITAL LETTER H WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0126': // Ħ [LATIN CAPITAL LETTER H WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u021E': // Ȟ [LATIN CAPITAL LETTER H WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u029C': // ʜ [LATIN LETTER SMALL CAPITAL H]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E22': // Ḣ [LATIN CAPITAL LETTER H WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E24': // Ḥ [LATIN CAPITAL LETTER H WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E26': // Ḧ [LATIN CAPITAL LETTER H WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E28': // Ḩ [LATIN CAPITAL LETTER H WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E2A': // Ḫ [LATIN CAPITAL LETTER H WITH BREVE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24BD': // Ⓗ [CIRCLED LATIN CAPITAL LETTER H]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C67': // Ⱨ [LATIN CAPITAL LETTER H WITH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C75': // Ⱶ [LATIN CAPITAL LETTER HALF H]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF28': // Ｈ [FULLWIDTH LATIN CAPITAL LETTER H]\n\t\t\t\toutput[outputPos] = 'H'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0125': // ĥ [LATIN SMALL LETTER H WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0127': // ħ [LATIN SMALL LETTER H WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u021F': // ȟ [LATIN SMALL LETTER H WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0265': // ɥ [LATIN SMALL LETTER TURNED H]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0266': // ɦ [LATIN SMALL LETTER H WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u02AE': // ʮ [LATIN SMALL LETTER TURNED H WITH FISHHOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u02AF': // ʯ [LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E23': // ḣ [LATIN SMALL LETTER H WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E25': // ḥ [LATIN SMALL LETTER H WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E27': // ḧ [LATIN SMALL LETTER H WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E29': // ḩ [LATIN SMALL LETTER H WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E2B': // ḫ [LATIN SMALL LETTER H WITH BREVE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E96': // ẖ [LATIN SMALL LETTER H WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D7': // ⓗ [CIRCLED LATIN SMALL LETTER H]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C68': // ⱨ [LATIN SMALL LETTER H WITH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C76': // ⱶ [LATIN SMALL LETTER HALF H]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF48': // ｈ [FULLWIDTH LATIN SMALL LETTER H]\n\t\t\t\toutput[outputPos] = 'h'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01F6': // Ƕ http://en.wikipedia.org/wiki/Hwair [LATIN CAPITAL LETTER HWAIR]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'H'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'V'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A3': // ⒣ [PARENTHESIZED LATIN SMALL LETTER H]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'h'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0195': // ƕ [LATIN SMALL LETTER HV]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'h'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'v'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00CC': // Ì [LATIN CAPITAL LETTER I WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00CD': // Í [LATIN CAPITAL LETTER I WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00CE': // Î [LATIN CAPITAL LETTER I WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00CF': // Ï [LATIN CAPITAL LETTER I WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0128': // Ĩ [LATIN CAPITAL LETTER I WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u012A': // Ī [LATIN CAPITAL LETTER I WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u012C': // Ĭ [LATIN CAPITAL LETTER I WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u012E': // Į [LATIN CAPITAL LETTER I WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0130': // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0196': // Ɩ [LATIN CAPITAL LETTER IOTA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0197': // Ɨ [LATIN CAPITAL LETTER I WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01CF': // Ǐ [LATIN CAPITAL LETTER I WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0208': // Ȉ [LATIN CAPITAL LETTER I WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u020A': // Ȋ [LATIN CAPITAL LETTER I WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u026A': // ɪ [LATIN LETTER SMALL CAPITAL I]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D7B': // ᵻ [LATIN SMALL CAPITAL LETTER I WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E2C': // Ḭ [LATIN CAPITAL LETTER I WITH TILDE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E2E': // Ḯ [LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC8': // Ỉ [LATIN CAPITAL LETTER I WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ECA': // Ị [LATIN CAPITAL LETTER I WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24BE': // Ⓘ [CIRCLED LATIN CAPITAL LETTER I]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA7FE': // ꟾ [LATIN EPIGRAPHIC LETTER I LONGA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF29': // Ｉ [FULLWIDTH LATIN CAPITAL LETTER I]\n\t\t\t\toutput[outputPos] = 'I'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00EC': // ì [LATIN SMALL LETTER I WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00ED': // í [LATIN SMALL LETTER I WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00EE': // î [LATIN SMALL LETTER I WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00EF': // ï [LATIN SMALL LETTER I WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0129': // ĩ [LATIN SMALL LETTER I WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u012B': // ī [LATIN SMALL LETTER I WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u012D': // ĭ [LATIN SMALL LETTER I WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u012F': // į [LATIN SMALL LETTER I WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0131': // ı [LATIN SMALL LETTER DOTLESS I]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D0': // ǐ [LATIN SMALL LETTER I WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0209': // ȉ [LATIN SMALL LETTER I WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u020B': // ȋ [LATIN SMALL LETTER I WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0268': // ɨ [LATIN SMALL LETTER I WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D09': // ᴉ [LATIN SMALL LETTER TURNED I]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D62': // ᵢ [LATIN SUBSCRIPT SMALL LETTER I]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D7C': // ᵼ [LATIN SMALL LETTER IOTA WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D96': // ᶖ [LATIN SMALL LETTER I WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E2D': // ḭ [LATIN SMALL LETTER I WITH TILDE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E2F': // ḯ [LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EC9': // ỉ [LATIN SMALL LETTER I WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ECB': // ị [LATIN SMALL LETTER I WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2071': // ⁱ [SUPERSCRIPT LATIN SMALL LETTER I]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D8': // ⓘ [CIRCLED LATIN SMALL LETTER I]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF49': // ｉ [FULLWIDTH LATIN SMALL LETTER I]\n\t\t\t\toutput[outputPos] = 'i'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0132': // IJ [LATIN CAPITAL LIGATURE IJ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'I'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'J'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A4': // ⒤ [PARENTHESIZED LATIN SMALL LETTER I]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'i'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0133': // ij [LATIN SMALL LIGATURE IJ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'i'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'j'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0134': // Ĵ [LATIN CAPITAL LETTER J WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0248': // Ɉ [LATIN CAPITAL LETTER J WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D0A': // ᴊ [LATIN LETTER SMALL CAPITAL J]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24BF': // Ⓙ [CIRCLED LATIN CAPITAL LETTER J]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF2A': // Ｊ [FULLWIDTH LATIN CAPITAL LETTER J]\n\t\t\t\toutput[outputPos] = 'J'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0135': // ĵ [LATIN SMALL LETTER J WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F0': // ǰ [LATIN SMALL LETTER J WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0237': // ȷ [LATIN SMALL LETTER DOTLESS J]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0249': // ɉ [LATIN SMALL LETTER J WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u025F': // ɟ [LATIN SMALL LETTER DOTLESS J WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0284': // ʄ [LATIN SMALL LETTER DOTLESS J WITH STROKE AND HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u029D': // ʝ [LATIN SMALL LETTER J WITH CROSSED-TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24D9': // ⓙ [CIRCLED LATIN SMALL LETTER J]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C7C': // ⱼ [LATIN SUBSCRIPT SMALL LETTER J]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF4A': // ｊ [FULLWIDTH LATIN SMALL LETTER J]\n\t\t\t\toutput[outputPos] = 'j'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A5': // ⒥ [PARENTHESIZED LATIN SMALL LETTER J]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'j'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0136': // Ķ [LATIN CAPITAL LETTER K WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0198': // Ƙ [LATIN CAPITAL LETTER K WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E8': // Ǩ [LATIN CAPITAL LETTER K WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D0B': // ᴋ [LATIN LETTER SMALL CAPITAL K]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E30': // Ḱ [LATIN CAPITAL LETTER K WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E32': // Ḳ [LATIN CAPITAL LETTER K WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E34': // Ḵ [LATIN CAPITAL LETTER K WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C0': // Ⓚ [CIRCLED LATIN CAPITAL LETTER K]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C69': // Ⱪ [LATIN CAPITAL LETTER K WITH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA740': // Ꝁ [LATIN CAPITAL LETTER K WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA742': // Ꝃ [LATIN CAPITAL LETTER K WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA744': // Ꝅ [LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF2B': // Ｋ [FULLWIDTH LATIN CAPITAL LETTER K]\n\t\t\t\toutput[outputPos] = 'K'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0137': // ķ [LATIN SMALL LETTER K WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0199': // ƙ [LATIN SMALL LETTER K WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01E9': // ǩ [LATIN SMALL LETTER K WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u029E': // ʞ [LATIN SMALL LETTER TURNED K]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D84': // ᶄ [LATIN SMALL LETTER K WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E31': // ḱ [LATIN SMALL LETTER K WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E33': // ḳ [LATIN SMALL LETTER K WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E35': // ḵ [LATIN SMALL LETTER K WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24DA': // ⓚ [CIRCLED LATIN SMALL LETTER K]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C6A': // ⱪ [LATIN SMALL LETTER K WITH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA741': // ꝁ [LATIN SMALL LETTER K WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA743': // ꝃ [LATIN SMALL LETTER K WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA745': // ꝅ [LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF4B': // ｋ [FULLWIDTH LATIN SMALL LETTER K]\n\t\t\t\toutput[outputPos] = 'k'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A6': // ⒦ [PARENTHESIZED LATIN SMALL LETTER K]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'k'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0139': // Ĺ [LATIN CAPITAL LETTER L WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u013B': // Ļ [LATIN CAPITAL LETTER L WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u013D': // Ľ [LATIN CAPITAL LETTER L WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u013F': // Ŀ [LATIN CAPITAL LETTER L WITH MIDDLE DOT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0141': // Ł [LATIN CAPITAL LETTER L WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u023D': // Ƚ [LATIN CAPITAL LETTER L WITH BAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u029F': // ʟ [LATIN LETTER SMALL CAPITAL L]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D0C': // ᴌ [LATIN LETTER SMALL CAPITAL L WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E36': // Ḷ [LATIN CAPITAL LETTER L WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E38': // Ḹ [LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E3A': // Ḻ [LATIN CAPITAL LETTER L WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E3C': // Ḽ [LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C1': // Ⓛ [CIRCLED LATIN CAPITAL LETTER L]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C60': // Ⱡ [LATIN CAPITAL LETTER L WITH DOUBLE BAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C62': // Ɫ [LATIN CAPITAL LETTER L WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA746': // Ꝇ [LATIN CAPITAL LETTER BROKEN L]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA748': // Ꝉ [LATIN CAPITAL LETTER L WITH HIGH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA780': // Ꞁ [LATIN CAPITAL LETTER TURNED L]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF2C': // Ｌ [FULLWIDTH LATIN CAPITAL LETTER L]\n\t\t\t\toutput[outputPos] = 'L'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u013A': // ĺ [LATIN SMALL LETTER L WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u013C': // ļ [LATIN SMALL LETTER L WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u013E': // ľ [LATIN SMALL LETTER L WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0140': // ŀ [LATIN SMALL LETTER L WITH MIDDLE DOT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0142': // ł [LATIN SMALL LETTER L WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u019A': // ƚ [LATIN SMALL LETTER L WITH BAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0234': // ȴ [LATIN SMALL LETTER L WITH CURL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u026B': // ɫ [LATIN SMALL LETTER L WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u026C': // ɬ [LATIN SMALL LETTER L WITH BELT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u026D': // ɭ [LATIN SMALL LETTER L WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D85': // ᶅ [LATIN SMALL LETTER L WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E37': // ḷ [LATIN SMALL LETTER L WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E39': // ḹ [LATIN SMALL LETTER L WITH DOT BELOW AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E3B': // ḻ [LATIN SMALL LETTER L WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E3D': // ḽ [LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24DB': // ⓛ [CIRCLED LATIN SMALL LETTER L]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C61': // ⱡ [LATIN SMALL LETTER L WITH DOUBLE BAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA747': // ꝇ [LATIN SMALL LETTER BROKEN L]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA749': // ꝉ [LATIN SMALL LETTER L WITH HIGH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA781': // ꞁ [LATIN SMALL LETTER TURNED L]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF4C': // ｌ [FULLWIDTH LATIN SMALL LETTER L]\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01C7': // LJ [LATIN CAPITAL LETTER LJ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'L'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'J'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u1EFA': // Ỻ [LATIN CAPITAL LETTER MIDDLE-WELSH LL]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'L'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'L'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01C8': // Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'L'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'j'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A7': // ⒧ [PARENTHESIZED LATIN SMALL LETTER L]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01C9': // lj [LATIN SMALL LETTER LJ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'j'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u1EFB': // ỻ [LATIN SMALL LETTER MIDDLE-WELSH LL]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u02AA': // ʪ [LATIN SMALL LETTER LS DIGRAPH]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 's'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u02AB': // ʫ [LATIN SMALL LETTER LZ DIGRAPH]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'l'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'z'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u019C': // Ɯ [LATIN CAPITAL LETTER TURNED M]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D0D': // ᴍ [LATIN LETTER SMALL CAPITAL M]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E3E': // Ḿ [LATIN CAPITAL LETTER M WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E40': // Ṁ [LATIN CAPITAL LETTER M WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E42': // Ṃ [LATIN CAPITAL LETTER M WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C2': // Ⓜ [CIRCLED LATIN CAPITAL LETTER M]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C6E': // Ɱ [LATIN CAPITAL LETTER M WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA7FD': // ꟽ [LATIN EPIGRAPHIC LETTER INVERTED M]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA7FF': // ꟿ [LATIN EPIGRAPHIC LETTER ARCHAIC M]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF2D': // Ｍ [FULLWIDTH LATIN CAPITAL LETTER M]\n\t\t\t\toutput[outputPos] = 'M'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u026F': // ɯ [LATIN SMALL LETTER TURNED M]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0270': // ɰ [LATIN SMALL LETTER TURNED M WITH LONG LEG]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0271': // ɱ [LATIN SMALL LETTER M WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D6F': // ᵯ [LATIN SMALL LETTER M WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D86': // ᶆ [LATIN SMALL LETTER M WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E3F': // ḿ [LATIN SMALL LETTER M WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E41': // ṁ [LATIN SMALL LETTER M WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E43': // ṃ [LATIN SMALL LETTER M WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24DC': // ⓜ [CIRCLED LATIN SMALL LETTER M]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF4D': // ｍ [FULLWIDTH LATIN SMALL LETTER M]\n\t\t\t\toutput[outputPos] = 'm'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A8': // ⒨ [PARENTHESIZED LATIN SMALL LETTER M]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'm'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00D1': // Ñ [LATIN CAPITAL LETTER N WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0143': // Ń [LATIN CAPITAL LETTER N WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0145': // Ņ [LATIN CAPITAL LETTER N WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0147': // Ň [LATIN CAPITAL LETTER N WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u014A': // Ŋ http://en.wikipedia.org/wiki/Eng_(letter) [LATIN CAPITAL LETTER ENG]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u019D': // Ɲ [LATIN CAPITAL LETTER N WITH LEFT HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F8': // Ǹ [LATIN CAPITAL LETTER N WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0220': // Ƞ [LATIN CAPITAL LETTER N WITH LONG RIGHT LEG]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0274': // ɴ [LATIN LETTER SMALL CAPITAL N]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D0E': // ᴎ [LATIN LETTER SMALL CAPITAL REVERSED N]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E44': // Ṅ [LATIN CAPITAL LETTER N WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E46': // Ṇ [LATIN CAPITAL LETTER N WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E48': // Ṉ [LATIN CAPITAL LETTER N WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E4A': // Ṋ [LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C3': // Ⓝ [CIRCLED LATIN CAPITAL LETTER N]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF2E': // Ｎ [FULLWIDTH LATIN CAPITAL LETTER N]\n\t\t\t\toutput[outputPos] = 'N'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00F1': // ñ [LATIN SMALL LETTER N WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0144': // ń [LATIN SMALL LETTER N WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0146': // ņ [LATIN SMALL LETTER N WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0148': // ň [LATIN SMALL LETTER N WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0149': // ʼn [LATIN SMALL LETTER N PRECEDED BY APOSTROPHE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u014B': // ŋ http://en.wikipedia.org/wiki/Eng_(letter) [LATIN SMALL LETTER ENG]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u019E': // ƞ [LATIN SMALL LETTER N WITH LONG RIGHT LEG]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F9': // ǹ [LATIN SMALL LETTER N WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0235': // ȵ [LATIN SMALL LETTER N WITH CURL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0272': // ɲ [LATIN SMALL LETTER N WITH LEFT HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0273': // ɳ [LATIN SMALL LETTER N WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D70': // ᵰ [LATIN SMALL LETTER N WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D87': // ᶇ [LATIN SMALL LETTER N WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E45': // ṅ [LATIN SMALL LETTER N WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E47': // ṇ [LATIN SMALL LETTER N WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E49': // ṉ [LATIN SMALL LETTER N WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E4B': // ṋ [LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u207F': // ⁿ [SUPERSCRIPT LATIN SMALL LETTER N]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24DD': // ⓝ [CIRCLED LATIN SMALL LETTER N]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF4E': // ｎ [FULLWIDTH LATIN SMALL LETTER N]\n\t\t\t\toutput[outputPos] = 'n'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01CA': // NJ [LATIN CAPITAL LETTER NJ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'N'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'J'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01CB': // Nj [LATIN CAPITAL LETTER N WITH SMALL LETTER J]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'N'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'j'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24A9': // ⒩ [PARENTHESIZED LATIN SMALL LETTER N]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'n'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01CC': // nj [LATIN SMALL LETTER NJ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'n'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'j'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00D2': // Ò [LATIN CAPITAL LETTER O WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00D3': // Ó [LATIN CAPITAL LETTER O WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00D4': // Ô [LATIN CAPITAL LETTER O WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00D5': // Õ [LATIN CAPITAL LETTER O WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00D6': // Ö [LATIN CAPITAL LETTER O WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00D8': // Ø [LATIN CAPITAL LETTER O WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u014C': // Ō [LATIN CAPITAL LETTER O WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u014E': // Ŏ [LATIN CAPITAL LETTER O WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0150': // Ő [LATIN CAPITAL LETTER O WITH DOUBLE ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0186': // Ɔ [LATIN CAPITAL LETTER OPEN O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u019F': // Ɵ [LATIN CAPITAL LETTER O WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01A0': // Ơ [LATIN CAPITAL LETTER O WITH HORN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D1': // Ǒ [LATIN CAPITAL LETTER O WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01EA': // Ǫ [LATIN CAPITAL LETTER O WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01EC': // Ǭ [LATIN CAPITAL LETTER O WITH OGONEK AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01FE': // Ǿ [LATIN CAPITAL LETTER O WITH STROKE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u020C': // Ȍ [LATIN CAPITAL LETTER O WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u020E': // Ȏ [LATIN CAPITAL LETTER O WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u022A': // Ȫ [LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u022C': // Ȭ [LATIN CAPITAL LETTER O WITH TILDE AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u022E': // Ȯ [LATIN CAPITAL LETTER O WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0230': // Ȱ [LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D0F': // ᴏ [LATIN LETTER SMALL CAPITAL O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D10': // ᴐ [LATIN LETTER SMALL CAPITAL OPEN O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E4C': // Ṍ [LATIN CAPITAL LETTER O WITH TILDE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E4E': // Ṏ [LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E50': // Ṑ [LATIN CAPITAL LETTER O WITH MACRON AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E52': // Ṓ [LATIN CAPITAL LETTER O WITH MACRON AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ECC': // Ọ [LATIN CAPITAL LETTER O WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ECE': // Ỏ [LATIN CAPITAL LETTER O WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED0': // Ố [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED2': // Ồ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED4': // Ổ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED6': // Ỗ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED8': // Ộ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EDA': // Ớ [LATIN CAPITAL LETTER O WITH HORN AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EDC': // Ờ [LATIN CAPITAL LETTER O WITH HORN AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EDE': // Ở [LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE0': // Ỡ [LATIN CAPITAL LETTER O WITH HORN AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE2': // Ợ [LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C4': // Ⓞ [CIRCLED LATIN CAPITAL LETTER O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA74A': // Ꝋ [LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA74C': // Ꝍ [LATIN CAPITAL LETTER O WITH LOOP]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF2F': // Ｏ [FULLWIDTH LATIN CAPITAL LETTER O]\n\t\t\t\toutput[outputPos] = 'O'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00F2': // ò [LATIN SMALL LETTER O WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00F3': // ó [LATIN SMALL LETTER O WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00F4': // ô [LATIN SMALL LETTER O WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00F5': // õ [LATIN SMALL LETTER O WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00F6': // ö [LATIN SMALL LETTER O WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00F8': // ø [LATIN SMALL LETTER O WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u014D': // ō [LATIN SMALL LETTER O WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u014F': // ŏ [LATIN SMALL LETTER O WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0151': // ő [LATIN SMALL LETTER O WITH DOUBLE ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01A1': // ơ [LATIN SMALL LETTER O WITH HORN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D2': // ǒ [LATIN SMALL LETTER O WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01EB': // ǫ [LATIN SMALL LETTER O WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01ED': // ǭ [LATIN SMALL LETTER O WITH OGONEK AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01FF': // ǿ [LATIN SMALL LETTER O WITH STROKE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u020D': // ȍ [LATIN SMALL LETTER O WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u020F': // ȏ [LATIN SMALL LETTER O WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u022B': // ȫ [LATIN SMALL LETTER O WITH DIAERESIS AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u022D': // ȭ [LATIN SMALL LETTER O WITH TILDE AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u022F': // ȯ [LATIN SMALL LETTER O WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0231': // ȱ [LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0254': // ɔ [LATIN SMALL LETTER OPEN O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0275': // ɵ [LATIN SMALL LETTER BARRED O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D16': // ᴖ [LATIN SMALL LETTER TOP HALF O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D17': // ᴗ [LATIN SMALL LETTER BOTTOM HALF O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D97': // ᶗ [LATIN SMALL LETTER OPEN O WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E4D': // ṍ [LATIN SMALL LETTER O WITH TILDE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E4F': // ṏ [LATIN SMALL LETTER O WITH TILDE AND DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E51': // ṑ [LATIN SMALL LETTER O WITH MACRON AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E53': // ṓ [LATIN SMALL LETTER O WITH MACRON AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ECD': // ọ [LATIN SMALL LETTER O WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ECF': // ỏ [LATIN SMALL LETTER O WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED1': // ố [LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED3': // ồ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED5': // ổ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED7': // ỗ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1ED9': // ộ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EDB': // ớ [LATIN SMALL LETTER O WITH HORN AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EDD': // ờ [LATIN SMALL LETTER O WITH HORN AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EDF': // ở [LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE1': // ỡ [LATIN SMALL LETTER O WITH HORN AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE3': // ợ [LATIN SMALL LETTER O WITH HORN AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2092': // ₒ [LATIN SUBSCRIPT SMALL LETTER O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24DE': // ⓞ [CIRCLED LATIN SMALL LETTER O]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C7A': // ⱺ [LATIN SMALL LETTER O WITH LOW RING INSIDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA74B': // ꝋ [LATIN SMALL LETTER O WITH LONG STROKE OVERLAY]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA74D': // ꝍ [LATIN SMALL LETTER O WITH LOOP]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF4F': // ｏ [FULLWIDTH LATIN SMALL LETTER O]\n\t\t\t\toutput[outputPos] = 'o'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0152': // Œ [LATIN CAPITAL LIGATURE OE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0276': // ɶ [LATIN LETTER SMALL CAPITAL OE]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'O'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'E'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA74E': // Ꝏ [LATIN CAPITAL LETTER OO]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'O'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'O'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0222': // Ȣ http://en.wikipedia.org/wiki/OU [LATIN CAPITAL LETTER OU]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D15': // ᴕ [LATIN LETTER SMALL CAPITAL OU]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'O'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'U'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24AA': // ⒪ [PARENTHESIZED LATIN SMALL LETTER O]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'o'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0153': // œ [LATIN SMALL LIGATURE OE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D14': // ᴔ [LATIN SMALL LETTER TURNED OE]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'o'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'e'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA74F': // ꝏ [LATIN SMALL LETTER OO]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'o'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'o'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0223': // ȣ http://en.wikipedia.org/wiki/OU [LATIN SMALL LETTER OU]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'o'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'u'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01A4': // Ƥ [LATIN CAPITAL LETTER P WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D18': // ᴘ [LATIN LETTER SMALL CAPITAL P]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E54': // Ṕ [LATIN CAPITAL LETTER P WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E56': // Ṗ [LATIN CAPITAL LETTER P WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C5': // Ⓟ [CIRCLED LATIN CAPITAL LETTER P]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C63': // Ᵽ [LATIN CAPITAL LETTER P WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA750': // Ꝑ [LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA752': // Ꝓ [LATIN CAPITAL LETTER P WITH FLOURISH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA754': // Ꝕ [LATIN CAPITAL LETTER P WITH SQUIRREL TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF30': // Ｐ [FULLWIDTH LATIN CAPITAL LETTER P]\n\t\t\t\toutput[outputPos] = 'P'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01A5': // ƥ [LATIN SMALL LETTER P WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D71': // ᵱ [LATIN SMALL LETTER P WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D7D': // ᵽ [LATIN SMALL LETTER P WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D88': // ᶈ [LATIN SMALL LETTER P WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E55': // ṕ [LATIN SMALL LETTER P WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E57': // ṗ [LATIN SMALL LETTER P WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24DF': // ⓟ [CIRCLED LATIN SMALL LETTER P]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA751': // ꝑ [LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA753': // ꝓ [LATIN SMALL LETTER P WITH FLOURISH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA755': // ꝕ [LATIN SMALL LETTER P WITH SQUIRREL TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA7FC': // ꟼ [LATIN EPIGRAPHIC LETTER REVERSED P]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF50': // ｐ [FULLWIDTH LATIN SMALL LETTER P]\n\t\t\t\toutput[outputPos] = 'p'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24AB': // ⒫ [PARENTHESIZED LATIN SMALL LETTER P]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'p'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u024A': // Ɋ [LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C6': // Ⓠ [CIRCLED LATIN CAPITAL LETTER Q]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA756': // Ꝗ [LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA758': // Ꝙ [LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF31': // Ｑ [FULLWIDTH LATIN CAPITAL LETTER Q]\n\t\t\t\toutput[outputPos] = 'Q'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0138': // ĸ http://en.wikipedia.org/wiki/Kra_(letter) [LATIN SMALL LETTER KRA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u024B': // ɋ [LATIN SMALL LETTER Q WITH HOOK TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u02A0': // ʠ [LATIN SMALL LETTER Q WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E0': // ⓠ [CIRCLED LATIN SMALL LETTER Q]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA757': // ꝗ [LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA759': // ꝙ [LATIN SMALL LETTER Q WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF51': // ｑ [FULLWIDTH LATIN SMALL LETTER Q]\n\t\t\t\toutput[outputPos] = 'q'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24AC': // ⒬ [PARENTHESIZED LATIN SMALL LETTER Q]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'q'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0239': // ȹ [LATIN SMALL LETTER QP DIGRAPH]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'q'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'p'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0154': // Ŕ [LATIN CAPITAL LETTER R WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0156': // Ŗ [LATIN CAPITAL LETTER R WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0158': // Ř [LATIN CAPITAL LETTER R WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0210': // Ȓ [LATIN CAPITAL LETTER R WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0212': // Ȓ [LATIN CAPITAL LETTER R WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u024C': // Ɍ [LATIN CAPITAL LETTER R WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0280': // ʀ [LATIN LETTER SMALL CAPITAL R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0281': // ʁ [LATIN LETTER SMALL CAPITAL INVERTED R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D19': // ᴙ [LATIN LETTER SMALL CAPITAL REVERSED R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D1A': // ᴚ [LATIN LETTER SMALL CAPITAL TURNED R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E58': // Ṙ [LATIN CAPITAL LETTER R WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E5A': // Ṛ [LATIN CAPITAL LETTER R WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E5C': // Ṝ [LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E5E': // Ṟ [LATIN CAPITAL LETTER R WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C7': // Ⓡ [CIRCLED LATIN CAPITAL LETTER R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C64': // Ɽ [LATIN CAPITAL LETTER R WITH TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA75A': // Ꝛ [LATIN CAPITAL LETTER R ROTUNDA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA782': // Ꞃ [LATIN CAPITAL LETTER INSULAR R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF32': // Ｒ [FULLWIDTH LATIN CAPITAL LETTER R]\n\t\t\t\toutput[outputPos] = 'R'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0155': // ŕ [LATIN SMALL LETTER R WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0157': // ŗ [LATIN SMALL LETTER R WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0159': // ř [LATIN SMALL LETTER R WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0211': // ȑ [LATIN SMALL LETTER R WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0213': // ȓ [LATIN SMALL LETTER R WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u024D': // ɍ [LATIN SMALL LETTER R WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u027C': // ɼ [LATIN SMALL LETTER R WITH LONG LEG]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u027D': // ɽ [LATIN SMALL LETTER R WITH TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u027E': // ɾ [LATIN SMALL LETTER R WITH FISHHOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u027F': // ɿ [LATIN SMALL LETTER REVERSED R WITH FISHHOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D63': // ᵣ [LATIN SUBSCRIPT SMALL LETTER R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D72': // ᵲ [LATIN SMALL LETTER R WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D73': // ᵳ [LATIN SMALL LETTER R WITH FISHHOOK AND MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D89': // ᶉ [LATIN SMALL LETTER R WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E59': // ṙ [LATIN SMALL LETTER R WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E5B': // ṛ [LATIN SMALL LETTER R WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E5D': // ṝ [LATIN SMALL LETTER R WITH DOT BELOW AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E5F': // ṟ [LATIN SMALL LETTER R WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E1': // ⓡ [CIRCLED LATIN SMALL LETTER R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA75B': // ꝛ [LATIN SMALL LETTER R ROTUNDA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA783': // ꞃ [LATIN SMALL LETTER INSULAR R]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF52': // ｒ [FULLWIDTH LATIN SMALL LETTER R]\n\t\t\t\toutput[outputPos] = 'r'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24AD': // ⒭ [PARENTHESIZED LATIN SMALL LETTER R]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'r'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u015A': // Ś [LATIN CAPITAL LETTER S WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u015C': // Ŝ [LATIN CAPITAL LETTER S WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u015E': // Ş [LATIN CAPITAL LETTER S WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0160': // Š [LATIN CAPITAL LETTER S WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0218': // Ș [LATIN CAPITAL LETTER S WITH COMMA BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E60': // Ṡ [LATIN CAPITAL LETTER S WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E62': // Ṣ [LATIN CAPITAL LETTER S WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E64': // Ṥ [LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E66': // Ṧ [LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E68': // Ṩ [LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C8': // Ⓢ [CIRCLED LATIN CAPITAL LETTER S]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA731': // ꜱ [LATIN LETTER SMALL CAPITAL S]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA785': // ꞅ [LATIN SMALL LETTER INSULAR S]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF33': // Ｓ [FULLWIDTH LATIN CAPITAL LETTER S]\n\t\t\t\toutput[outputPos] = 'S'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u015B': // ś [LATIN SMALL LETTER S WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u015D': // ŝ [LATIN SMALL LETTER S WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u015F': // ş [LATIN SMALL LETTER S WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0161': // š [LATIN SMALL LETTER S WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u017F': // ſ http://en.wikipedia.org/wiki/Long_S [LATIN SMALL LETTER LONG S]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0219': // ș [LATIN SMALL LETTER S WITH COMMA BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u023F': // ȿ [LATIN SMALL LETTER S WITH SWASH TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0282': // ʂ [LATIN SMALL LETTER S WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D74': // ᵴ [LATIN SMALL LETTER S WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D8A': // ᶊ [LATIN SMALL LETTER S WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E61': // ṡ [LATIN SMALL LETTER S WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E63': // ṣ [LATIN SMALL LETTER S WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E65': // ṥ [LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E67': // ṧ [LATIN SMALL LETTER S WITH CARON AND DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E69': // ṩ [LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E9C': // ẜ [LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E9D': // ẝ [LATIN SMALL LETTER LONG S WITH HIGH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E2': // ⓢ [CIRCLED LATIN SMALL LETTER S]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA784': // Ꞅ [LATIN CAPITAL LETTER INSULAR S]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF53': // ｓ [FULLWIDTH LATIN SMALL LETTER S]\n\t\t\t\toutput[outputPos] = 's'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u1E9E': // ẞ [LATIN CAPITAL LETTER SHARP S]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'S'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'S'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24AE': // ⒮ [PARENTHESIZED LATIN SMALL LETTER S]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 's'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00DF': // ß [LATIN SMALL LETTER SHARP S]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 's'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 's'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFB06': // st [LATIN SMALL LIGATURE ST]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 's'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 't'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0162': // Ţ [LATIN CAPITAL LETTER T WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0164': // Ť [LATIN CAPITAL LETTER T WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0166': // Ŧ [LATIN CAPITAL LETTER T WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01AC': // Ƭ [LATIN CAPITAL LETTER T WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01AE': // Ʈ [LATIN CAPITAL LETTER T WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u021A': // Ț [LATIN CAPITAL LETTER T WITH COMMA BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u023E': // Ⱦ [LATIN CAPITAL LETTER T WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D1B': // ᴛ [LATIN LETTER SMALL CAPITAL T]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E6A': // Ṫ [LATIN CAPITAL LETTER T WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E6C': // Ṭ [LATIN CAPITAL LETTER T WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E6E': // Ṯ [LATIN CAPITAL LETTER T WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E70': // Ṱ [LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24C9': // Ⓣ [CIRCLED LATIN CAPITAL LETTER T]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA786': // Ꞇ [LATIN CAPITAL LETTER INSULAR T]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF34': // Ｔ [FULLWIDTH LATIN CAPITAL LETTER T]\n\t\t\t\toutput[outputPos] = 'T'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0163': // ţ [LATIN SMALL LETTER T WITH CEDILLA]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0165': // ť [LATIN SMALL LETTER T WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0167': // ŧ [LATIN SMALL LETTER T WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01AB': // ƫ [LATIN SMALL LETTER T WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01AD': // ƭ [LATIN SMALL LETTER T WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u021B': // ț [LATIN SMALL LETTER T WITH COMMA BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0236': // ȶ [LATIN SMALL LETTER T WITH CURL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0287': // ʇ [LATIN SMALL LETTER TURNED T]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0288': // ʈ [LATIN SMALL LETTER T WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D75': // ᵵ [LATIN SMALL LETTER T WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E6B': // ṫ [LATIN SMALL LETTER T WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E6D': // ṭ [LATIN SMALL LETTER T WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E6F': // ṯ [LATIN SMALL LETTER T WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E71': // ṱ [LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E97': // ẗ [LATIN SMALL LETTER T WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E3': // ⓣ [CIRCLED LATIN SMALL LETTER T]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C66': // ⱦ [LATIN SMALL LETTER T WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF54': // ｔ [FULLWIDTH LATIN SMALL LETTER T]\n\t\t\t\toutput[outputPos] = 't'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00DE': // Þ [LATIN CAPITAL LETTER THORN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA766': // Ꝧ [LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'T'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'H'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA728': // Ꜩ [LATIN CAPITAL LETTER TZ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'T'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'Z'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24AF': // ⒯ [PARENTHESIZED LATIN SMALL LETTER T]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 't'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u02A8': // ʨ [LATIN SMALL LETTER TC DIGRAPH WITH CURL]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 't'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'c'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00FE': // þ [LATIN SMALL LETTER THORN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D7A': // ᵺ [LATIN SMALL LETTER TH WITH STRIKETHROUGH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA767': // ꝧ [LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 't'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'h'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u02A6': // ʦ [LATIN SMALL LETTER TS DIGRAPH]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 't'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 's'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA729': // ꜩ [LATIN SMALL LETTER TZ]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 't'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'z'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00D9': // Ù [LATIN CAPITAL LETTER U WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00DA': // Ú [LATIN CAPITAL LETTER U WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00DB': // Û [LATIN CAPITAL LETTER U WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00DC': // Ü [LATIN CAPITAL LETTER U WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0168': // Ũ [LATIN CAPITAL LETTER U WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u016A': // Ū [LATIN CAPITAL LETTER U WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u016C': // Ŭ [LATIN CAPITAL LETTER U WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u016E': // Ů [LATIN CAPITAL LETTER U WITH RING ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0170': // Ű [LATIN CAPITAL LETTER U WITH DOUBLE ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0172': // Ų [LATIN CAPITAL LETTER U WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01AF': // Ư [LATIN CAPITAL LETTER U WITH HORN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D3': // Ǔ [LATIN CAPITAL LETTER U WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D5': // Ǖ [LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D7': // Ǘ [LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D9': // Ǚ [LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01DB': // Ǜ [LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0214': // Ȕ [LATIN CAPITAL LETTER U WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0216': // Ȗ [LATIN CAPITAL LETTER U WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0244': // Ʉ [LATIN CAPITAL LETTER U BAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D1C': // ᴜ [LATIN LETTER SMALL CAPITAL U]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D7E': // ᵾ [LATIN SMALL CAPITAL LETTER U WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E72': // Ṳ [LATIN CAPITAL LETTER U WITH DIAERESIS BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E74': // Ṵ [LATIN CAPITAL LETTER U WITH TILDE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E76': // Ṷ [LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E78': // Ṹ [LATIN CAPITAL LETTER U WITH TILDE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E7A': // Ṻ [LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE4': // Ụ [LATIN CAPITAL LETTER U WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE6': // Ủ [LATIN CAPITAL LETTER U WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE8': // Ứ [LATIN CAPITAL LETTER U WITH HORN AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EEA': // Ừ [LATIN CAPITAL LETTER U WITH HORN AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EEC': // Ử [LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EEE': // Ữ [LATIN CAPITAL LETTER U WITH HORN AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF0': // Ự [LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24CA': // Ⓤ [CIRCLED LATIN CAPITAL LETTER U]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF35': // Ｕ [FULLWIDTH LATIN CAPITAL LETTER U]\n\t\t\t\toutput[outputPos] = 'U'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00F9': // ù [LATIN SMALL LETTER U WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00FA': // ú [LATIN SMALL LETTER U WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00FB': // û [LATIN SMALL LETTER U WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00FC': // ü [LATIN SMALL LETTER U WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0169': // ũ [LATIN SMALL LETTER U WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u016B': // ū [LATIN SMALL LETTER U WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u016D': // ŭ [LATIN SMALL LETTER U WITH BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u016F': // ů [LATIN SMALL LETTER U WITH RING ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0171': // ű [LATIN SMALL LETTER U WITH DOUBLE ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0173': // ų [LATIN SMALL LETTER U WITH OGONEK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01B0': // ư [LATIN SMALL LETTER U WITH HORN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D4': // ǔ [LATIN SMALL LETTER U WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D6': // ǖ [LATIN SMALL LETTER U WITH DIAERESIS AND MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01D8': // ǘ [LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01DA': // ǚ [LATIN SMALL LETTER U WITH DIAERESIS AND CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01DC': // ǜ [LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0215': // ȕ [LATIN SMALL LETTER U WITH DOUBLE GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0217': // ȗ [LATIN SMALL LETTER U WITH INVERTED BREVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0289': // ʉ [LATIN SMALL LETTER U BAR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D64': // ᵤ [LATIN SUBSCRIPT SMALL LETTER U]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D99': // ᶙ [LATIN SMALL LETTER U WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E73': // ṳ [LATIN SMALL LETTER U WITH DIAERESIS BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E75': // ṵ [LATIN SMALL LETTER U WITH TILDE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E77': // ṷ [LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E79': // ṹ [LATIN SMALL LETTER U WITH TILDE AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E7B': // ṻ [LATIN SMALL LETTER U WITH MACRON AND DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE5': // ụ [LATIN SMALL LETTER U WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE7': // ủ [LATIN SMALL LETTER U WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EE9': // ứ [LATIN SMALL LETTER U WITH HORN AND ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EEB': // ừ [LATIN SMALL LETTER U WITH HORN AND GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EED': // ử [LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EEF': // ữ [LATIN SMALL LETTER U WITH HORN AND TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF1': // ự [LATIN SMALL LETTER U WITH HORN AND DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E4': // ⓤ [CIRCLED LATIN SMALL LETTER U]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF55': // ｕ [FULLWIDTH LATIN SMALL LETTER U]\n\t\t\t\toutput[outputPos] = 'u'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24B0': // ⒰ [PARENTHESIZED LATIN SMALL LETTER U]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'u'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u1D6B': // ᵫ [LATIN SMALL LETTER UE]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'u'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'e'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u01B2': // Ʋ [LATIN CAPITAL LETTER V WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0245': // Ʌ [LATIN CAPITAL LETTER TURNED V]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D20': // ᴠ [LATIN LETTER SMALL CAPITAL V]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E7C': // Ṽ [LATIN CAPITAL LETTER V WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E7E': // Ṿ [LATIN CAPITAL LETTER V WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EFC': // Ỽ [LATIN CAPITAL LETTER MIDDLE-WELSH V]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24CB': // Ⓥ [CIRCLED LATIN CAPITAL LETTER V]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA75E': // Ꝟ [LATIN CAPITAL LETTER V WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA768': // Ꝩ [LATIN CAPITAL LETTER VEND]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF36': // Ｖ [FULLWIDTH LATIN CAPITAL LETTER V]\n\t\t\t\toutput[outputPos] = 'V'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u028B': // ʋ [LATIN SMALL LETTER V WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u028C': // ʌ [LATIN SMALL LETTER TURNED V]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D65': // ᵥ [LATIN SUBSCRIPT SMALL LETTER V]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D8C': // ᶌ [LATIN SMALL LETTER V WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E7D': // ṽ [LATIN SMALL LETTER V WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E7F': // ṿ [LATIN SMALL LETTER V WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E5': // ⓥ [CIRCLED LATIN SMALL LETTER V]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C71': // ⱱ [LATIN SMALL LETTER V WITH RIGHT HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C74': // ⱴ [LATIN SMALL LETTER V WITH CURL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA75F': // ꝟ [LATIN SMALL LETTER V WITH DIAGONAL STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF56': // ｖ [FULLWIDTH LATIN SMALL LETTER V]\n\t\t\t\toutput[outputPos] = 'v'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA760': // Ꝡ [LATIN CAPITAL LETTER VY]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'V'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'Y'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24B1': // ⒱ [PARENTHESIZED LATIN SMALL LETTER V]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'v'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uA761': // ꝡ [LATIN SMALL LETTER VY]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = 'v'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'y'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0174': // Ŵ [LATIN CAPITAL LETTER W WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01F7': // Ƿ http://en.wikipedia.org/wiki/Wynn [LATIN CAPITAL LETTER WYNN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D21': // ᴡ [LATIN LETTER SMALL CAPITAL W]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E80': // Ẁ [LATIN CAPITAL LETTER W WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E82': // Ẃ [LATIN CAPITAL LETTER W WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E84': // Ẅ [LATIN CAPITAL LETTER W WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E86': // Ẇ [LATIN CAPITAL LETTER W WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E88': // Ẉ [LATIN CAPITAL LETTER W WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24CC': // Ⓦ [CIRCLED LATIN CAPITAL LETTER W]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C72': // Ⱳ [LATIN CAPITAL LETTER W WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF37': // Ｗ [FULLWIDTH LATIN CAPITAL LETTER W]\n\t\t\t\toutput[outputPos] = 'W'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0175': // ŵ [LATIN SMALL LETTER W WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01BF': // ƿ http://en.wikipedia.org/wiki/Wynn [LATIN LETTER WYNN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u028D': // ʍ [LATIN SMALL LETTER TURNED W]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E81': // ẁ [LATIN SMALL LETTER W WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E83': // ẃ [LATIN SMALL LETTER W WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E85': // ẅ [LATIN SMALL LETTER W WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E87': // ẇ [LATIN SMALL LETTER W WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E89': // ẉ [LATIN SMALL LETTER W WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E98': // ẘ [LATIN SMALL LETTER W WITH RING ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E6': // ⓦ [CIRCLED LATIN SMALL LETTER W]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C73': // ⱳ [LATIN SMALL LETTER W WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF57': // ｗ [FULLWIDTH LATIN SMALL LETTER W]\n\t\t\t\toutput[outputPos] = 'w'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24B2': // ⒲ [PARENTHESIZED LATIN SMALL LETTER W]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'w'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u1E8A': // Ẋ [LATIN CAPITAL LETTER X WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E8C': // Ẍ [LATIN CAPITAL LETTER X WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24CD': // Ⓧ [CIRCLED LATIN CAPITAL LETTER X]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF38': // Ｘ [FULLWIDTH LATIN CAPITAL LETTER X]\n\t\t\t\toutput[outputPos] = 'X'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u1D8D': // ᶍ [LATIN SMALL LETTER X WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E8B': // ẋ [LATIN SMALL LETTER X WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E8D': // ẍ [LATIN SMALL LETTER X WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2093': // ₓ [LATIN SUBSCRIPT SMALL LETTER X]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E7': // ⓧ [CIRCLED LATIN SMALL LETTER X]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF58': // ｘ [FULLWIDTH LATIN SMALL LETTER X]\n\t\t\t\toutput[outputPos] = 'x'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24B3': // ⒳ [PARENTHESIZED LATIN SMALL LETTER X]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'x'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00DD': // Ý [LATIN CAPITAL LETTER Y WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0176': // Ŷ [LATIN CAPITAL LETTER Y WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0178': // Ÿ [LATIN CAPITAL LETTER Y WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01B3': // Ƴ [LATIN CAPITAL LETTER Y WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0232': // Ȳ [LATIN CAPITAL LETTER Y WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u024E': // Ɏ [LATIN CAPITAL LETTER Y WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u028F': // ʏ [LATIN LETTER SMALL CAPITAL Y]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E8E': // Ẏ [LATIN CAPITAL LETTER Y WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF2': // Ỳ [LATIN CAPITAL LETTER Y WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF4': // Ỵ [LATIN CAPITAL LETTER Y WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF6': // Ỷ [LATIN CAPITAL LETTER Y WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF8': // Ỹ [LATIN CAPITAL LETTER Y WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EFE': // Ỿ [LATIN CAPITAL LETTER Y WITH LOOP]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24CE': // Ⓨ [CIRCLED LATIN CAPITAL LETTER Y]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF39': // Ｙ [FULLWIDTH LATIN CAPITAL LETTER Y]\n\t\t\t\toutput[outputPos] = 'Y'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00FD': // ý [LATIN SMALL LETTER Y WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00FF': // ÿ [LATIN SMALL LETTER Y WITH DIAERESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0177': // ŷ [LATIN SMALL LETTER Y WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01B4': // ƴ [LATIN SMALL LETTER Y WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0233': // ȳ [LATIN SMALL LETTER Y WITH MACRON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u024F': // ɏ [LATIN SMALL LETTER Y WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u028E': // ʎ [LATIN SMALL LETTER TURNED Y]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E8F': // ẏ [LATIN SMALL LETTER Y WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E99': // ẙ [LATIN SMALL LETTER Y WITH RING ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF3': // ỳ [LATIN SMALL LETTER Y WITH GRAVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF5': // ỵ [LATIN SMALL LETTER Y WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF7': // ỷ [LATIN SMALL LETTER Y WITH HOOK ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EF9': // ỹ [LATIN SMALL LETTER Y WITH TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1EFF': // ỿ [LATIN SMALL LETTER Y WITH LOOP]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E8': // ⓨ [CIRCLED LATIN SMALL LETTER Y]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF59': // ｙ [FULLWIDTH LATIN SMALL LETTER Y]\n\t\t\t\toutput[outputPos] = 'y'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24B4': // ⒴ [PARENTHESIZED LATIN SMALL LETTER Y]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'y'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u0179': // Ź [LATIN CAPITAL LETTER Z WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u017B': // Ż [LATIN CAPITAL LETTER Z WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u017D': // Ž [LATIN CAPITAL LETTER Z WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01B5': // Ƶ [LATIN CAPITAL LETTER Z WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u021C': // Ȝ http://en.wikipedia.org/wiki/Yogh [LATIN CAPITAL LETTER YOGH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0224': // Ȥ [LATIN CAPITAL LETTER Z WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D22': // ᴢ [LATIN LETTER SMALL CAPITAL Z]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E90': // Ẑ [LATIN CAPITAL LETTER Z WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E92': // Ẓ [LATIN CAPITAL LETTER Z WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E94': // Ẕ [LATIN CAPITAL LETTER Z WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24CF': // Ⓩ [CIRCLED LATIN CAPITAL LETTER Z]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C6B': // Ⱬ [LATIN CAPITAL LETTER Z WITH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA762': // Ꝣ [LATIN CAPITAL LETTER VISIGOTHIC Z]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF3A': // Ｚ [FULLWIDTH LATIN CAPITAL LETTER Z]\n\t\t\t\toutput[outputPos] = 'Z'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u017A': // ź [LATIN SMALL LETTER Z WITH ACUTE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u017C': // ż [LATIN SMALL LETTER Z WITH DOT ABOVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u017E': // ž [LATIN SMALL LETTER Z WITH CARON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u01B6': // ƶ [LATIN SMALL LETTER Z WITH STROKE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u021D': // ȝ http://en.wikipedia.org/wiki/Yogh [LATIN SMALL LETTER YOGH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0225': // ȥ [LATIN SMALL LETTER Z WITH HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0240': // ɀ [LATIN SMALL LETTER Z WITH SWASH TAIL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0290': // ʐ [LATIN SMALL LETTER Z WITH RETROFLEX HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u0291': // ʑ [LATIN SMALL LETTER Z WITH CURL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D76': // ᵶ [LATIN SMALL LETTER Z WITH MIDDLE TILDE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1D8E': // ᶎ [LATIN SMALL LETTER Z WITH PALATAL HOOK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E91': // ẑ [LATIN SMALL LETTER Z WITH CIRCUMFLEX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E93': // ẓ [LATIN SMALL LETTER Z WITH DOT BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u1E95': // ẕ [LATIN SMALL LETTER Z WITH LINE BELOW]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24E9': // ⓩ [CIRCLED LATIN SMALL LETTER Z]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2C6C': // ⱬ [LATIN SMALL LETTER Z WITH DESCENDER]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uA763': // ꝣ [LATIN SMALL LETTER VISIGOTHIC Z]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF5A': // ｚ [FULLWIDTH LATIN SMALL LETTER Z]\n\t\t\t\toutput[outputPos] = 'z'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u24B5': // ⒵ [PARENTHESIZED LATIN SMALL LETTER Z]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = 'z'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2070': // ⁰ [SUPERSCRIPT ZERO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2080': // ₀ [SUBSCRIPT ZERO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24EA': // ⓪ [CIRCLED DIGIT ZERO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24FF': // ⓿ [NEGATIVE CIRCLED DIGIT ZERO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF10': // ０ [FULLWIDTH DIGIT ZERO]\n\t\t\t\toutput[outputPos] = '0'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00B9': // ¹ [SUPERSCRIPT ONE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2081': // ₁ [SUBSCRIPT ONE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2460': // ① [CIRCLED DIGIT ONE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F5': // ⓵ [DOUBLE CIRCLED DIGIT ONE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2776': // ❶ [DINGBAT NEGATIVE CIRCLED DIGIT ONE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2780': // ➀ [DINGBAT CIRCLED SANS-SERIF DIGIT ONE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u278A': // ➊ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ONE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF11': // １ [FULLWIDTH DIGIT ONE]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2488': // ⒈ [DIGIT ONE FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2474': // ⑴ [PARENTHESIZED DIGIT ONE]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00B2': // ² [SUPERSCRIPT TWO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2082': // ₂ [SUBSCRIPT TWO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2461': // ② [CIRCLED DIGIT TWO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F6': // ⓶ [DOUBLE CIRCLED DIGIT TWO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2777': // ❷ [DINGBAT NEGATIVE CIRCLED DIGIT TWO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2781': // ➁ [DINGBAT CIRCLED SANS-SERIF DIGIT TWO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u278B': // ➋ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT TWO]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF12': // ２ [FULLWIDTH DIGIT TWO]\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2489': // ⒉ [DIGIT TWO FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2475': // ⑵ [PARENTHESIZED DIGIT TWO]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00B3': // ³ [SUPERSCRIPT THREE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2083': // ₃ [SUBSCRIPT THREE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2462': // ③ [CIRCLED DIGIT THREE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F7': // ⓷ [DOUBLE CIRCLED DIGIT THREE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2778': // ❸ [DINGBAT NEGATIVE CIRCLED DIGIT THREE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2782': // ➂ [DINGBAT CIRCLED SANS-SERIF DIGIT THREE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u278C': // ➌ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT THREE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF13': // ３ [FULLWIDTH DIGIT THREE]\n\t\t\t\toutput[outputPos] = '3'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u248A': // ⒊ [DIGIT THREE FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '3'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2476': // ⑶ [PARENTHESIZED DIGIT THREE]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '3'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2074': // ⁴ [SUPERSCRIPT FOUR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2084': // ₄ [SUBSCRIPT FOUR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2463': // ④ [CIRCLED DIGIT FOUR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F8': // ⓸ [DOUBLE CIRCLED DIGIT FOUR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2779': // ❹ [DINGBAT NEGATIVE CIRCLED DIGIT FOUR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2783': // ➃ [DINGBAT CIRCLED SANS-SERIF DIGIT FOUR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u278D': // ➍ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FOUR]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF14': // ４ [FULLWIDTH DIGIT FOUR]\n\t\t\t\toutput[outputPos] = '4'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u248B': // ⒋ [DIGIT FOUR FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '4'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2477': // ⑷ [PARENTHESIZED DIGIT FOUR]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '4'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2075': // ⁵ [SUPERSCRIPT FIVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2085': // ₅ [SUBSCRIPT FIVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2464': // ⑤ [CIRCLED DIGIT FIVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F9': // ⓹ [DOUBLE CIRCLED DIGIT FIVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u277A': // ❺ [DINGBAT NEGATIVE CIRCLED DIGIT FIVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2784': // ➄ [DINGBAT CIRCLED SANS-SERIF DIGIT FIVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u278E': // ➎ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FIVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF15': // ５ [FULLWIDTH DIGIT FIVE]\n\t\t\t\toutput[outputPos] = '5'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u248C': // ⒌ [DIGIT FIVE FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '5'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2478': // ⑸ [PARENTHESIZED DIGIT FIVE]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '5'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2076': // ⁶ [SUPERSCRIPT SIX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2086': // ₆ [SUBSCRIPT SIX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2465': // ⑥ [CIRCLED DIGIT SIX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24FA': // ⓺ [DOUBLE CIRCLED DIGIT SIX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u277B': // ❻ [DINGBAT NEGATIVE CIRCLED DIGIT SIX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2785': // ➅ [DINGBAT CIRCLED SANS-SERIF DIGIT SIX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u278F': // ➏ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SIX]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF16': // ６ [FULLWIDTH DIGIT SIX]\n\t\t\t\toutput[outputPos] = '6'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u248D': // ⒍ [DIGIT SIX FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '6'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2479': // ⑹ [PARENTHESIZED DIGIT SIX]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '6'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2077': // ⁷ [SUPERSCRIPT SEVEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2087': // ₇ [SUBSCRIPT SEVEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2466': // ⑦ [CIRCLED DIGIT SEVEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24FB': // ⓻ [DOUBLE CIRCLED DIGIT SEVEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u277C': // ❼ [DINGBAT NEGATIVE CIRCLED DIGIT SEVEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2786': // ➆ [DINGBAT CIRCLED SANS-SERIF DIGIT SEVEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2790': // ➐ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SEVEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF17': // ７ [FULLWIDTH DIGIT SEVEN]\n\t\t\t\toutput[outputPos] = '7'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u248E': // ⒎ [DIGIT SEVEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '7'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u247A': // ⑺ [PARENTHESIZED DIGIT SEVEN]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '7'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2078': // ⁸ [SUPERSCRIPT EIGHT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2088': // ₈ [SUBSCRIPT EIGHT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2467': // ⑧ [CIRCLED DIGIT EIGHT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24FC': // ⓼ [DOUBLE CIRCLED DIGIT EIGHT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u277D': // ❽ [DINGBAT NEGATIVE CIRCLED DIGIT EIGHT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2787': // ➇ [DINGBAT CIRCLED SANS-SERIF DIGIT EIGHT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2791': // ➑ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT EIGHT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF18': // ８ [FULLWIDTH DIGIT EIGHT]\n\t\t\t\toutput[outputPos] = '8'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u248F': // ⒏ [DIGIT EIGHT FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '8'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u247B': // ⑻ [PARENTHESIZED DIGIT EIGHT]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '8'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2079': // ⁹ [SUPERSCRIPT NINE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2089': // ₉ [SUBSCRIPT NINE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2468': // ⑨ [CIRCLED DIGIT NINE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24FD': // ⓽ [DOUBLE CIRCLED DIGIT NINE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u277E': // ❾ [DINGBAT NEGATIVE CIRCLED DIGIT NINE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2788': // ➈ [DINGBAT CIRCLED SANS-SERIF DIGIT NINE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2792': // ➒ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT NINE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF19': // ９ [FULLWIDTH DIGIT NINE]\n\t\t\t\toutput[outputPos] = '9'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2490': // ⒐ [DIGIT NINE FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '9'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u247C': // ⑼ [PARENTHESIZED DIGIT NINE]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '9'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2469': // ⑩ [CIRCLED NUMBER TEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24FE': // ⓾ [DOUBLE CIRCLED NUMBER TEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u277F': // ❿ [DINGBAT NEGATIVE CIRCLED NUMBER TEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2789': // ➉ [DINGBAT CIRCLED SANS-SERIF NUMBER TEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2793': // ➓ [DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '0'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2491': // ⒑ [NUMBER TEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '0'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u247D': // ⑽ [PARENTHESIZED NUMBER TEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '0'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u246A': // ⑪ [CIRCLED NUMBER ELEVEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24EB': // ⓫ [NEGATIVE CIRCLED NUMBER ELEVEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2492': // ⒒ [NUMBER ELEVEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u247E': // ⑾ [PARENTHESIZED NUMBER ELEVEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u246B': // ⑫ [CIRCLED NUMBER TWELVE]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24EC': // ⓬ [NEGATIVE CIRCLED NUMBER TWELVE]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2493': // ⒓ [NUMBER TWELVE FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u247F': // ⑿ [PARENTHESIZED NUMBER TWELVE]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u246C': // ⑬ [CIRCLED NUMBER THIRTEEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24ED': // ⓭ [NEGATIVE CIRCLED NUMBER THIRTEEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '3'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2494': // ⒔ [NUMBER THIRTEEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '3'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2480': // ⒀ [PARENTHESIZED NUMBER THIRTEEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '3'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u246D': // ⑭ [CIRCLED NUMBER FOURTEEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24EE': // ⓮ [NEGATIVE CIRCLED NUMBER FOURTEEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '4'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2495': // ⒕ [NUMBER FOURTEEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '4'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2481': // ⒁ [PARENTHESIZED NUMBER FOURTEEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '4'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u246E': // ⑮ [CIRCLED NUMBER FIFTEEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24EF': // ⓯ [NEGATIVE CIRCLED NUMBER FIFTEEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '5'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2496': // ⒖ [NUMBER FIFTEEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '5'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2482': // ⒂ [PARENTHESIZED NUMBER FIFTEEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '5'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u246F': // ⑯ [CIRCLED NUMBER SIXTEEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F0': // ⓰ [NEGATIVE CIRCLED NUMBER SIXTEEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '6'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2497': // ⒗ [NUMBER SIXTEEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '6'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2483': // ⒃ [PARENTHESIZED NUMBER SIXTEEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '6'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2470': // ⑰ [CIRCLED NUMBER SEVENTEEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F1': // ⓱ [NEGATIVE CIRCLED NUMBER SEVENTEEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '7'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2498': // ⒘ [NUMBER SEVENTEEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '7'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2484': // ⒄ [PARENTHESIZED NUMBER SEVENTEEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '7'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2471': // ⑱ [CIRCLED NUMBER EIGHTEEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F2': // ⓲ [NEGATIVE CIRCLED NUMBER EIGHTEEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '8'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2499': // ⒙ [NUMBER EIGHTEEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '8'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2485': // ⒅ [PARENTHESIZED NUMBER EIGHTEEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '8'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2472': // ⑲ [CIRCLED NUMBER NINETEEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F3': // ⓳ [NEGATIVE CIRCLED NUMBER NINETEEN]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '9'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u249A': // ⒚ [NUMBER NINETEEN FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '9'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2486': // ⒆ [PARENTHESIZED NUMBER NINETEEN]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '1'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '9'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2473': // ⑳ [CIRCLED NUMBER TWENTY]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u24F4': // ⓴ [NEGATIVE CIRCLED NUMBER TWENTY]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '0'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u249B': // ⒛ [NUMBER TWENTY FULL STOP]\n\t\t\t\toutput = output[:(len(output) + 2)]\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '0'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2487': // ⒇ [PARENTHESIZED NUMBER TWENTY]\n\t\t\t\toutput = output[:(len(output) + 3)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '2'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '0'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u00AB': // « [LEFT-POINTING DOUBLE ANGLE QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u00BB': // » [RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u201C': // “ [LEFT DOUBLE QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u201D': // ” [RIGHT DOUBLE QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u201E': // „ [DOUBLE LOW-9 QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2033': // ″ [DOUBLE PRIME]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2036': // ‶ [REVERSED DOUBLE PRIME]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u275D': // ❝ [HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u275E': // ❞ [HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u276E': // ❮ [HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u276F': // ❯ [HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF02': // ＂ [FULLWIDTH QUOTATION MARK]\n\t\t\t\toutput[outputPos] = '\"'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2018': // ‘ [LEFT SINGLE QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2019': // ’ [RIGHT SINGLE QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u201A': // ‚ [SINGLE LOW-9 QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u201B': // ‛ [SINGLE HIGH-REVERSED-9 QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2032': // ′ [PRIME]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2035': // ‵ [REVERSED PRIME]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2039': // ‹ [SINGLE LEFT-POINTING ANGLE QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u203A': // › [SINGLE RIGHT-POINTING ANGLE QUOTATION MARK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u275B': // ❛ [HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u275C': // ❜ [HEAVY SINGLE COMMA QUOTATION MARK ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF07': // ＇ [FULLWIDTH APOSTROPHE]\n\t\t\t\toutput[outputPos] = '\\''\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2010': // ‐ [HYPHEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2011': // ‑ [NON-BREAKING HYPHEN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2012': // ‒ [FIGURE DASH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2013': // – [EN DASH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2014': // — [EM DASH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u207B': // ⁻ [SUPERSCRIPT MINUS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u208B': // ₋ [SUBSCRIPT MINUS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF0D': // － [FULLWIDTH HYPHEN-MINUS]\n\t\t\t\toutput[outputPos] = '-'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2045': // ⁅ [LEFT SQUARE BRACKET WITH QUILL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2772': // ❲ [LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF3B': // ［ [FULLWIDTH LEFT SQUARE BRACKET]\n\t\t\t\toutput[outputPos] = '['\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2046': // ⁆ [RIGHT SQUARE BRACKET WITH QUILL]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2773': // ❳ [LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF3D': // ］ [FULLWIDTH RIGHT SQUARE BRACKET]\n\t\t\t\toutput[outputPos] = ']'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u207D': // ⁽ [SUPERSCRIPT LEFT PARENTHESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u208D': // ₍ [SUBSCRIPT LEFT PARENTHESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2768': // ❨ [MEDIUM LEFT PARENTHESIS ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u276A': // ❪ [MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF08': // （ [FULLWIDTH LEFT PARENTHESIS]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2E28': // ⸨ [LEFT DOUBLE PARENTHESIS]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '('\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u207E': // ⁾ [SUPERSCRIPT RIGHT PARENTHESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u208E': // ₎ [SUBSCRIPT RIGHT PARENTHESIS]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2769': // ❩ [MEDIUM RIGHT PARENTHESIS ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u276B': // ❫ [MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF09': // ） [FULLWIDTH RIGHT PARENTHESIS]\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2E29': // ⸩ [RIGHT DOUBLE PARENTHESIS]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = ')'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u276C': // ❬ [MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2770': // ❰ [HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF1C': // ＜ [FULLWIDTH LESS-THAN SIGN]\n\t\t\t\toutput[outputPos] = '<'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u276D': // ❭ [MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u2771': // ❱ [HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF1E': // ＞ [FULLWIDTH GREATER-THAN SIGN]\n\t\t\t\toutput[outputPos] = '>'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2774': // ❴ [MEDIUM LEFT CURLY BRACKET ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF5B': // ｛ [FULLWIDTH LEFT CURLY BRACKET]\n\t\t\t\toutput[outputPos] = '{'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2775': // ❵ [MEDIUM RIGHT CURLY BRACKET ORNAMENT]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF5D': // ｝ [FULLWIDTH RIGHT CURLY BRACKET]\n\t\t\t\toutput[outputPos] = '}'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u207A': // ⁺ [SUPERSCRIPT PLUS SIGN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u208A': // ₊ [SUBSCRIPT PLUS SIGN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF0B': // ＋ [FULLWIDTH PLUS SIGN]\n\t\t\t\toutput[outputPos] = '+'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u207C': // ⁼ [SUPERSCRIPT EQUALS SIGN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\u208C': // ₌ [SUBSCRIPT EQUALS SIGN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF1D': // ＝ [FULLWIDTH EQUALS SIGN]\n\t\t\t\toutput[outputPos] = '='\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF01': // ！ [FULLWIDTH EXCLAMATION MARK]\n\t\t\t\toutput[outputPos] = '!'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u203C': // ‼ [DOUBLE EXCLAMATION MARK]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '!'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '!'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2049': // ⁉ [EXCLAMATION QUESTION MARK]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '!'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '?'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF03': // ＃ [FULLWIDTH NUMBER SIGN]\n\t\t\t\toutput[outputPos] = '#'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF04': // ＄ [FULLWIDTH DOLLAR SIGN]\n\t\t\t\toutput[outputPos] = '$'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2052': // ⁒ [COMMERCIAL MINUS SIGN]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF05': // ％ [FULLWIDTH PERCENT SIGN]\n\t\t\t\toutput[outputPos] = '%'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF06': // ＆ [FULLWIDTH AMPERSAND]\n\t\t\t\toutput[outputPos] = '&'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u204E': // ⁎ [LOW ASTERISK]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF0A': // ＊ [FULLWIDTH ASTERISK]\n\t\t\t\toutput[outputPos] = '*'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF0C': // ， [FULLWIDTH COMMA]\n\t\t\t\toutput[outputPos] = ','\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF0E': // ． [FULLWIDTH FULL STOP]\n\t\t\t\toutput[outputPos] = '.'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2044': // ⁄ [FRACTION SLASH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF0F': // ／ [FULLWIDTH SOLIDUS]\n\t\t\t\toutput[outputPos] = '/'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF1A': // ： [FULLWIDTH COLON]\n\t\t\t\toutput[outputPos] = ':'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u204F': // ⁏ [REVERSED SEMICOLON]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF1B': // ； [FULLWIDTH SEMICOLON]\n\t\t\t\toutput[outputPos] = ';'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF1F': // ？ [FULLWIDTH QUESTION MARK]\n\t\t\t\toutput[outputPos] = '?'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2047': // ⁇ [DOUBLE QUESTION MARK]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '?'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '?'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2048': // ⁈ [QUESTION EXCLAMATION MARK]\n\t\t\t\toutput = output[:(len(output) + 1)]\n\t\t\t\toutput[outputPos] = '?'\n\t\t\t\toutputPos++\n\t\t\t\toutput[outputPos] = '!'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF20': // ＠ [FULLWIDTH COMMERCIAL AT]\n\t\t\t\toutput[outputPos] = '@'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF3C': // ＼ [FULLWIDTH REVERSE SOLIDUS]\n\t\t\t\toutput[outputPos] = '\\\\'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2038': // ‸ [CARET]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF3E': // ＾ [FULLWIDTH CIRCUMFLEX ACCENT]\n\t\t\t\toutput[outputPos] = '^'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\uFF3F': // ＿ [FULLWIDTH LOW LINE]\n\t\t\t\toutput[outputPos] = '_'\n\t\t\t\toutputPos++\n\n\t\t\tcase '\\u2053': // ⁓ [SWUNG DASH]\n\t\t\t\tfallthrough\n\t\t\tcase '\\uFF5E': // ～ [FULLWIDTH TILDE]\n\t\t\t\toutput[outputPos] = '~'\n\t\t\t\toutputPos++\n\n\t\t\tdefault:\n\t\t\t\toutput[outputPos] = c\n\t\t\t\toutputPos++\n\t\t\t}\n\t\t}\n\t}\n\treturn output\n}\n"
  },
  {
    "path": "analysis/char/asciifolding/asciifolding_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage asciifolding\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestAsciiFoldingFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput []byte\n\t}{\n\t\t{\n\t\t\t// empty input passes\n\t\t\tinput:  []byte(``),\n\t\t\toutput: []byte(``),\n\t\t},\n\t\t{\n\t\t\t// no modification for plain ASCII\n\t\t\tinput:  []byte(`The quick brown fox jumps over the lazy dog`),\n\t\t\toutput: []byte(`The quick brown fox jumps over the lazy dog`),\n\t\t},\n\t\t{\n\t\t\t// Umlauts are folded to plain ASCII\n\t\t\tinput:  []byte(`The quick bröwn fox jümps over the läzy dog`),\n\t\t\toutput: []byte(`The quick brown fox jumps over the lazy dog`),\n\t\t},\n\t\t{\n\t\t\t// composite unicode runes are folded to more than one ASCII rune\n\t\t\tinput:  []byte(`ÆꜴ`),\n\t\t\toutput: []byte(`AEAO`),\n\t\t},\n\t\t{\n\t\t\t// apples from https://issues.couchbase.com/browse/MB-33486\n\t\t\tinput:  []byte(`Ápple Àpple Äpple Âpple Ãpple Åpple`),\n\t\t\toutput: []byte(`Apple Apple Apple Apple Apple Apple`),\n\t\t},\n\t\t{\n\t\t\t// Fix ASCII folding of \\u24A2\n\t\t\tinput:  []byte(`⒢`),\n\t\t\toutput: []byte(`(g)`),\n\t\t},\n\t\t{\n\t\t\t// Test folding of \\u2053 (SWUNG DASH)\n\t\t\tinput:  []byte(`a⁓b`),\n\t\t\toutput: []byte(`a~b`),\n\t\t},\n\t\t{\n\t\t\t// Test folding of \\uFF5E (FULLWIDTH TILDE)\n\t\t\tinput:  []byte(`c～d`),\n\t\t\toutput: []byte(`c~d`),\n\t\t},\n\t\t{\n\t\t\t// Test folding of \\uFF3F (FULLWIDTH LOW LINE) - case before tilde\n\t\t\tinput:  []byte(`e＿f`),\n\t\t\toutput: []byte(`e_f`),\n\t\t},\n\t\t{\n\t\t\t// Test mix including tilde and default fallthrough (using a character not explicitly folded)\n\t\t\tinput:  []byte(`a⁓b✅c～d`),\n\t\t\toutput: []byte(`a~b✅c~d`),\n\t\t},\n\t\t{\n\t\t\t// Test start of 'A' fallthrough block\n\t\t\tinput:  []byte(`ÀBC`),\n\t\t\toutput: []byte(`ABC`),\n\t\t},\n\t\t{\n\t\t\t// Test end of 'A' fallthrough block\n\t\t\tinput:  []byte(`DEFẶ`),\n\t\t\toutput: []byte(`DEFA`),\n\t\t},\n\t\t{\n\t\t\t// Test start of 'AE' fallthrough block\n\t\t\tinput:  []byte(`Æ`),\n\t\t\toutput: []byte(`AE`),\n\t\t},\n\t\t{\n\t\t\t// Test end of 'AE' fallthrough block\n\t\t\tinput:  []byte(`ᴁ`),\n\t\t\toutput: []byte(`AE`),\n\t\t},\n\t\t{\n\t\t\t// Test 'DZ' multi-rune output\n\t\t\tinput:  []byte(`DŽebra`),\n\t\t\toutput: []byte(`DZebra`),\n\t\t},\n\t\t{\n\t\t\t// Test start of 'a' fallthrough block\n\t\t\tinput:  []byte(`àbc`),\n\t\t\toutput: []byte(`abc`),\n\t\t},\n\t\t{\n\t\t\t// Test end of 'a' fallthrough block\n\t\t\tinput:  []byte(`defａ`),\n\t\t\toutput: []byte(`defa`),\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tfilter := New()\n\t\tt.Run(fmt.Sprintf(\"on %s\", test.input), func(t *testing.T) {\n\t\t\toutput := filter.Filter(test.input)\n\t\t\tif !reflect.DeepEqual(output, test.output) {\n\t\t\t\tt.Errorf(\"\\nExpected:\\n`%s`\\ngot:\\n`%s`\\n\", string(test.output), string(output))\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "analysis/char/html/html.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage html\n\nimport (\n\t\"bytes\"\n\t\"regexp\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"html\"\n\nvar htmlCharFilterRegexp = regexp.MustCompile(`</?[!\\w]+((\\s+\\w+(\\s*=\\s*(?:\".*?\"|'.*?'|[^'\">\\s]+))?)+\\s*|\\s*)/?>`)\n\ntype CharFilter struct {\n\tr           *regexp.Regexp\n\treplacement []byte\n}\n\nfunc New() *CharFilter {\n\treturn &CharFilter{\n\t\tr:           htmlCharFilterRegexp,\n\t\treplacement: []byte(\" \"),\n\t}\n}\n\nfunc (s *CharFilter) Filter(input []byte) []byte {\n\treturn s.r.ReplaceAllFunc(\n\t\tinput, func(in []byte) []byte {\n\t\t\treturn bytes.Repeat(s.replacement, len(in))\n\t\t})\n}\n\nfunc CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) {\n\treturn New(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterCharFilter(Name, CharFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/char/regexp/regexp.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage regexp\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"regexp\"\n\ntype CharFilter struct {\n\tr           *regexp.Regexp\n\treplacement []byte\n}\n\nfunc New(r *regexp.Regexp, replacement []byte) *CharFilter {\n\treturn &CharFilter{\n\t\tr:           r,\n\t\treplacement: replacement,\n\t}\n}\n\nfunc (s *CharFilter) Filter(input []byte) []byte {\n\treturn s.r.ReplaceAll(input, s.replacement)\n}\n\nfunc CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) {\n\tregexpStr, ok := config[\"regexp\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify regexp\")\n\t}\n\tr, err := regexp.Compile(regexpStr)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unable to build regexp char filter: %v\", err)\n\t}\n\treplaceBytes := []byte(\" \")\n\treplaceStr, ok := config[\"replace\"].(string)\n\tif ok {\n\t\treplaceBytes = []byte(replaceStr)\n\t}\n\treturn New(r, replaceBytes), nil\n}\n\nfunc init() {\n\terr := registry.RegisterCharFilter(Name, CharFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/char/regexp/regexp_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage regexp\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"testing\"\n)\n\nfunc TestRegexpCharFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tregexStr string\n\t\treplace  []byte\n\t\tinput    []byte\n\t\toutput   []byte\n\t}{\n\t\t{\n\t\t\tregexStr: `</?[!\\w]+((\\s+\\w+(\\s*=\\s*(?:\".*?\"|'.*?'|[^'\">\\s]+))?)+\\s*|\\s*)/?>`,\n\t\t\treplace:  []byte{' '},\n\t\t\tinput:    []byte(`<html>test</html>`),\n\t\t\toutput:   []byte(` test `),\n\t\t},\n\t\t{\n\t\t\tregexStr: `\\x{200C}`,\n\t\t\treplace:  []byte{' '},\n\t\t\tinput:    []byte(\"water\\u200Cunder\\u200Cthe\\u200Cbridge\"),\n\t\t\toutput:   []byte(\"water under the bridge\"),\n\t\t},\n\t\t{\n\t\t\tregexStr: `([a-z])\\s+(\\d)`,\n\t\t\treplace:  []byte(`$1-$2`),\n\t\t\tinput:    []byte(`temp 1`),\n\t\t\toutput:   []byte(`temp-1`),\n\t\t},\n\t\t{\n\t\t\tregexStr: `foo.?`,\n\t\t\treplace:  []byte(`X`),\n\t\t\tinput:    []byte(`seafood, fool`),\n\t\t\toutput:   []byte(`seaX, X`),\n\t\t},\n\t\t{\n\t\t\tregexStr: `def`,\n\t\t\treplace:  []byte(`_`),\n\t\t\tinput:    []byte(`abcdefghi`),\n\t\t\toutput:   []byte(`abc_ghi`),\n\t\t},\n\t\t{\n\t\t\tregexStr: `456`,\n\t\t\treplace:  []byte(`000000`),\n\t\t\tinput:    []byte(`123456789`),\n\t\t\toutput:   []byte(`123000000789`),\n\t\t},\n\t\t{\n\t\t\tregexStr: `“|”`,\n\t\t\treplace:  []byte(`\"`),\n\t\t\tinput:    []byte(`“hello”`),\n\t\t\toutput:   []byte(`\"hello\"`),\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"match %s replace %s\", test.regexStr, string(test.replace)), func(t *testing.T) {\n\t\t\tregex := regexp.MustCompile(test.regexStr)\n\t\t\tfilter := New(regex, test.replace)\n\n\t\t\toutput := filter.Filter(test.input)\n\t\t\tif !reflect.DeepEqual(test.output, output) {\n\t\t\t\tt.Errorf(\"Expected: `%s`, Got: `%s`\\n\", string(test.output), string(output))\n\t\t\t}\n\t\t})\n\n\t}\n}\n"
  },
  {
    "path": "analysis/char/zerowidthnonjoiner/zerowidthnonjoiner.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage zerowidthnonjoiner\n\nimport (\n\t\"regexp\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\tregexpCharFilter \"github.com/blevesearch/bleve/v2/analysis/char/regexp\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"zero_width_spaces\"\n\nvar zeroWidthNonJoinerRegexp = regexp.MustCompile(`\\x{200C}`)\n\nfunc CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) {\n\treplaceBytes := []byte(\" \")\n\treturn regexpCharFilter.New(zeroWidthNonJoinerRegexp, replaceBytes), nil\n}\n\nfunc init() {\n\terr := registry.RegisterCharFilter(Name, CharFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/flexible/flexible.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage flexible\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"flexiblego\"\n\ntype DateTimeParser struct {\n\tlayouts []string\n}\n\nfunc New(layouts []string) *DateTimeParser {\n\treturn &DateTimeParser{\n\t\tlayouts: layouts,\n\t}\n}\n\nfunc (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {\n\tfor _, layout := range p.layouts {\n\t\trv, err := time.Parse(layout, input)\n\t\tif err == nil {\n\t\t\treturn rv, layout, nil\n\t\t}\n\t}\n\treturn time.Time{}, \"\", analysis.ErrInvalidDateTime\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\tlayouts, ok := config[\"layouts\"].([]interface{})\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify layouts\")\n\t}\n\tvar layoutStrs []string\n\tfor _, layout := range layouts {\n\t\tlayoutStr, ok := layout.(string)\n\t\tif ok {\n\t\t\tlayoutStrs = append(layoutStrs, layoutStr)\n\t\t}\n\t}\n\treturn New(layoutStrs), nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/flexible/flexible_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage flexible\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestFlexibleDateTimeParser(t *testing.T) {\n\ttestLocation := time.FixedZone(\"\", -8*60*60)\n\n\trfc3339NoTimezone := \"2006-01-02T15:04:05\"\n\trfc3339NoTimezoneNoT := \"2006-01-02 15:04:05\"\n\trfc3339NoTime := \"2006-01-02\"\n\n\tdateOptionalTimeParser := New(\n\t\t[]string{\n\t\t\ttime.RFC3339Nano,\n\t\t\ttime.RFC3339,\n\t\t\trfc3339NoTimezone,\n\t\t\trfc3339NoTimezoneNoT,\n\t\t\trfc3339NoTime,\n\t\t})\n\n\ttests := []struct {\n\t\tinput          string\n\t\texpectedTime   time.Time\n\t\texpectedLayout string\n\t\texpectedError  error\n\t}{\n\t\t{\n\t\t\tinput:          \"2014-08-03\",\n\t\t\texpectedTime:   time.Date(2014, 8, 3, 0, 0, 0, 0, time.UTC),\n\t\t\texpectedLayout: rfc3339NoTime,\n\t\t\texpectedError:  nil,\n\t\t},\n\t\t{\n\t\t\tinput:          \"2014-08-03T15:59:30\",\n\t\t\texpectedTime:   time.Date(2014, 8, 3, 15, 59, 30, 0, time.UTC),\n\t\t\texpectedLayout: rfc3339NoTimezone,\n\t\t\texpectedError:  nil,\n\t\t},\n\t\t{\n\t\t\tinput:          \"2014-08-03 15:59:30\",\n\t\t\texpectedTime:   time.Date(2014, 8, 3, 15, 59, 30, 0, time.UTC),\n\t\t\texpectedLayout: rfc3339NoTimezoneNoT,\n\t\t\texpectedError:  nil,\n\t\t},\n\t\t{\n\t\t\tinput:          \"2014-08-03T15:59:30-08:00\",\n\t\t\texpectedTime:   time.Date(2014, 8, 3, 15, 59, 30, 0, testLocation),\n\t\t\texpectedLayout: time.RFC3339Nano,\n\t\t\texpectedError:  nil,\n\t\t},\n\t\t{\n\n\t\t\tinput:          \"2014-08-03T15:59:30.999999999-08:00\",\n\t\t\texpectedTime:   time.Date(2014, 8, 3, 15, 59, 30, 999999999, testLocation),\n\t\t\texpectedLayout: time.RFC3339Nano,\n\t\t\texpectedError:  nil,\n\t\t},\n\t\t{\n\t\t\tinput:          \"not a date time\",\n\t\t\texpectedTime:   time.Time{},\n\t\t\texpectedLayout: \"\",\n\t\t\texpectedError:  analysis.ErrInvalidDateTime,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.input, func(t *testing.T) {\n\t\t\tactualTime, actualLayout, actualErr := dateOptionalTimeParser.ParseDateTime(test.input)\n\t\t\tif actualErr != test.expectedError {\n\t\t\t\tt.Fatalf(\"expected error %#v, got %#v\", test.expectedError, actualErr)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(actualTime, test.expectedTime) {\n\t\t\t\tt.Errorf(\"expected time %v, got %v\", test.expectedTime, actualTime)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(actualLayout, test.expectedLayout) {\n\t\t\t\tt.Errorf(\"expected layout %v, got %v\", test.expectedLayout, actualLayout)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/iso/iso.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage iso\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"isostyle\"\n\nvar textLiteralDelimiter byte = '\\'' // single quote\n\n// ISO style date strings are represented in\n// https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html\n//\n// Some format specifiers are not specified in go time package, such as:\n// - 'V' for timezone name, like 'Europe/Berlin' or 'America/New_York'.\n// - 'Q' for quarter of year, like Q3 or 3rd Quarter.\n// - 'zzzz' for full name of timezone like \"Japan Standard Time\" or \"Eastern Standard Time\".\n// - 'O' for localized zone-offset, like GMT+8 or GMT+08:00.\n// - '[]' for optional section of the format.\n// - 'G' for era, like AD or BC.\n// - 'W' for week of month.\n// - 'D' for day of year.\n// So date strings with these date elements cannot be parsed.\nvar timeElementToLayout = map[byte]map[int]string{\n\t'M': {\n\t\t4: \"January\", // MMMM = full month name\n\t\t3: \"Jan\",     // MMM = short month name\n\t\t2: \"01\",      // MM = month of year (2 digits) (01-12)\n\t\t1: \"1\",       // M = month of year (1 digit) (1-12)\n\t},\n\t'd': {\n\t\t2: \"02\", // dd = day of month (2 digits) (01-31)\n\t\t1: \"2\",  // d = day of month (1 digit) (1-31)\n\t},\n\t'a': {\n\t\t2: \"pm\", // aa = pm/am\n\t\t1: \"PM\", // a = PM/AM\n\t},\n\t'H': {\n\t\t2: \"15\", // HH = hour (24 hour clock) (2 digits)\n\t\t1: \"15\", // H = hour (24 hour clock) (1 digit)\n\t},\n\t'm': {\n\t\t2: \"04\", // mm = minute (2 digits)\n\t\t1: \"4\",  // m = minute (1 digit)\n\t},\n\t's': {\n\t\t2: \"05\", // ss = seconds (2 digits)\n\t\t1: \"5\",  // s = seconds (1 digit)\n\t},\n\n\t// timezone offsets from UTC below\n\t'X': {\n\t\t5: \"Z07:00:00\", // XXXXX = timezone offset (+-hh:mm:ss)\n\t\t4: \"Z070000\",   // XXXX = timezone offset (+-hhmmss)\n\t\t3: \"Z07:00\",    // XXX = timezone offset (+-hh:mm)\n\t\t2: \"Z0700\",     // XX = timezone offset (+-hhmm)\n\t\t1: \"Z07\",       // X = timezone offset (+-hh)\n\t},\n\t'x': {\n\t\t5: \"-07:00:00\", // xxxxx = timezone offset (+-hh:mm:ss)\n\t\t4: \"-070000\",   // xxxx = timezone offset (+-hhmmss)\n\t\t3: \"-07:00\",    // xxx = timezone offset (+-hh:mm)\n\t\t2: \"-0700\",     // xx = timezone offset (+-hhmm)\n\t\t1: \"-07\",       // x = timezone offset (+-hh)\n\t},\n}\n\ntype DateTimeParser struct {\n\tlayouts []string\n}\n\nfunc New(layouts []string) *DateTimeParser {\n\treturn &DateTimeParser{\n\t\tlayouts: layouts,\n\t}\n}\n\nfunc (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {\n\tfor _, layout := range p.layouts {\n\t\trv, err := time.Parse(layout, input)\n\t\tif err == nil {\n\t\t\treturn rv, layout, nil\n\t\t}\n\t}\n\treturn time.Time{}, \"\", analysis.ErrInvalidDateTime\n}\n\nfunc letterCounter(layout string, idx int) int {\n\tcount := 1\n\tfor idx+count < len(layout) {\n\t\tif layout[idx+count] == layout[idx] {\n\t\t\tcount++\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn count\n}\n\nfunc invalidFormatError(character byte, count int) error {\n\treturn fmt.Errorf(\"invalid format string, unknown format specifier: %s\", strings.Repeat(string(character), count))\n}\n\nfunc parseISOString(layout string) (string, error) {\n\tvar dateTimeLayout strings.Builder\n\n\tfor idx := 0; idx < len(layout); {\n\t\t// check if the character is a text literal delimiter (')\n\t\tif layout[idx] == textLiteralDelimiter {\n\t\t\tif idx+1 < len(layout) && layout[idx+1] == textLiteralDelimiter {\n\t\t\t\t// if the next character is also a text literal delimiter, then\n\t\t\t\t// copy the character as is\n\t\t\t\tdateTimeLayout.WriteByte(textLiteralDelimiter)\n\t\t\t\tidx += 2\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// find the next text literal delimiter\n\t\t\tfor idx++; idx < len(layout); idx++ {\n\t\t\t\tif layout[idx] == textLiteralDelimiter {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tdateTimeLayout.WriteByte(layout[idx])\n\t\t\t}\n\t\t\t// idx can either be equal to len(layout) if the text literal delimiter is not found\n\t\t\t// after the first text literal delimiter or it will be equal to the index of the\n\t\t\t// second text literal delimiter\n\t\t\tif idx == len(layout) {\n\t\t\t\t// text literal delimiter not found error\n\t\t\t\treturn \"\", fmt.Errorf(\"invalid format string, expected text literal delimiter: %s\", string(textLiteralDelimiter))\n\t\t\t}\n\t\t\t// increment idx to skip the second text literal delimiter\n\t\t\tidx++\n\t\t\tcontinue\n\t\t}\n\t\t// check if character is a letter in english alphabet - a-zA-Z which are reserved\n\t\t// for format specifiers\n\t\tif (layout[idx] >= 'a' && layout[idx] <= 'z') || (layout[idx] >= 'A' && layout[idx] <= 'Z') {\n\t\t\t// find the number of times the character occurs consecutively\n\t\t\tcount := letterCounter(layout, idx)\n\t\t\tcharacter := layout[idx]\n\t\t\t// first check the table\n\t\t\tif layout, ok := timeElementToLayout[character][count]; ok {\n\t\t\t\tdateTimeLayout.WriteString(layout)\n\t\t\t} else {\n\t\t\t\tswitch character {\n\t\t\t\tcase 'y', 'u', 'Y':\n\t\t\t\t\t// year\n\t\t\t\t\tif count == 2 {\n\t\t\t\t\t\tdateTimeLayout.WriteString(\"06\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\tformat := fmt.Sprintf(\"%%0%ds\", count)\n\t\t\t\t\t\tdateTimeLayout.WriteString(fmt.Sprintf(format, \"2006\"))\n\t\t\t\t\t}\n\t\t\t\tcase 'h', 'K':\n\t\t\t\t\t// hour (1-12)\n\t\t\t\t\tswitch count {\n\t\t\t\t\tcase 2:\n\t\t\t\t\t\t// hh, KK -> 03\n\t\t\t\t\t\tdateTimeLayout.WriteString(\"03\")\n\t\t\t\t\tcase 1:\n\t\t\t\t\t\t// h, K -> 3\n\t\t\t\t\t\tdateTimeLayout.WriteString(\"3\")\n\t\t\t\t\tdefault:\n\t\t\t\t\t\t// e.g., hhh\n\t\t\t\t\t\treturn \"\", invalidFormatError(character, count)\n\t\t\t\t\t}\n\t\t\t\tcase 'E':\n\t\t\t\t\t// day of week\n\t\t\t\t\tif count == 4 {\n\t\t\t\t\t\tdateTimeLayout.WriteString(\"Monday\") // EEEE -> Monday\n\t\t\t\t\t} else if count <= 3 {\n\t\t\t\t\t\tdateTimeLayout.WriteString(\"Mon\") // E, EE, EEE -> Mon\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn \"\", invalidFormatError(character, count) // e.g., EEEEE\n\t\t\t\t\t}\n\t\t\t\tcase 'S':\n\t\t\t\t\t// fraction of second\n\t\t\t\t\t// .SSS = millisecond\n\t\t\t\t\t// .SSSSSS = microsecond\n\t\t\t\t\t// .SSSSSSSSS = nanosecond\n\t\t\t\t\tif count > 9 {\n\t\t\t\t\t\treturn \"\", invalidFormatError(character, count)\n\t\t\t\t\t}\n\t\t\t\t\tdateTimeLayout.WriteString(strings.Repeat(string('0'), count))\n\t\t\t\tcase 'z':\n\t\t\t\t\t// timezone id\n\t\t\t\t\tif count < 5 {\n\t\t\t\t\t\tdateTimeLayout.WriteString(\"MST\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn \"\", invalidFormatError(character, count)\n\t\t\t\t\t}\n\t\t\t\tdefault:\n\t\t\t\t\treturn \"\", invalidFormatError(character, count)\n\t\t\t\t}\n\t\t\t}\n\t\t\tidx += count\n\t\t} else {\n\t\t\t// copy the character as is\n\t\t\tdateTimeLayout.WriteByte(layout[idx])\n\t\t\tidx++\n\t\t}\n\t}\n\treturn dateTimeLayout.String(), nil\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\tlayouts, ok := config[\"layouts\"].([]interface{})\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify layouts\")\n\t}\n\tvar layoutStrs []string\n\tfor _, layout := range layouts {\n\t\tlayoutStr, ok := layout.(string)\n\t\tif ok {\n\t\t\tlayout, err := parseISOString(layoutStr)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tlayoutStrs = append(layoutStrs, layout)\n\t\t}\n\t}\n\treturn New(layoutStrs), nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/iso/iso_test.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage iso\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n)\n\nfunc TestConversionFromISOStyle(t *testing.T) {\n\ttests := []struct {\n\t\tinput  string\n\t\toutput string\n\t\terr    error\n\t}{\n\t\t{\n\t\t\tinput:  \"yyyy-MM-dd\",\n\t\t\toutput: \"2006-01-02\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"uuu/M''''dd'T'HH:m:ss.SSS\",\n\t\t\toutput: \"2006/1''02T15:4:05.000\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"YYYY-MM-dd'T'H:mm:ss zzz\",\n\t\t\toutput: \"2006-01-02T15:04:05 MST\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"MMMM dd yyyy', 'HH:mm:ss.SSS\",\n\t\t\toutput: \"January 02 2006, 15:04:05.000\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"h 'o'''' clock' a, XXX\",\n\t\t\toutput: \"3 o' clock PM, Z07:00\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"YYYY-MM-dd'T'HH:mm:ss'Z'\",\n\t\t\toutput: \"2006-01-02T15:04:05Z\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"E MMM d H:mm:ss z Y\",\n\t\t\toutput: \"Mon Jan 2 15:04:05 MST 2006\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"E MMM DD H:m:s z Y\",\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: DD\"),\n\t\t},\n\t\t{\n\t\t\tinput:  \"E MMM''''' H:m:s z Y\",\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, expected text literal delimiter: '\"),\n\t\t},\n\t\t{\n\t\t\tinput:  \"MMMMM dd yyyy', 'HH:mm:ss.SSS\",\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: MMMMM\"),\n\t\t},\n\t\t{\n\t\t\tinput:  \"yy\", // year (2 digits)\n\t\t\toutput: \"06\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"yyyyy\", // year (5 digits, padded)\n\t\t\toutput: \"02006\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"h\", // hour 1-12 (1 digit)\n\t\t\toutput: \"3\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"hh\", // hour 1-12 (2 digits)\n\t\t\toutput: \"03\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"KK\", // hour 1-12 (2 digits, alt)\n\t\t\toutput: \"03\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"hhh\", // invalid hour count\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: hhh\"),\n\t\t},\n\t\t{\n\t\t\tinput:  \"E\", // Day of week (short)\n\t\t\toutput: \"Mon\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"EEE\", // Day of week (short)\n\t\t\toutput: \"Mon\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"EEEE\", // Day of week (long)\n\t\t\toutput: \"Monday\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"EEEEE\", // Day of week (long)\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: EEEEE\"),\n\t\t},\n\t\t{\n\t\t\tinput:  \"S\", // Fraction of second (1 digit)\n\t\t\toutput: \"0\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"SSSSSSSSS\", // Fraction of second (9 digits)\n\t\t\toutput: \"000000000\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"SSSSSSSSSS\", // Invalid fraction of second count\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: SSSSSSSSSS\"),\n\t\t},\n\t\t{\n\t\t\tinput:  \"z\", // Timezone name (short)\n\t\t\toutput: \"MST\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tinput:  \"zzz\", // Timezone name (short) - Corrected expectation\n\t\t\toutput: \"MST\", // Should output MST\n\t\t\terr:    nil,   // Should not produce an error\n\t\t},\n\t\t{\n\t\t\tinput:  \"zzzz\", // Timezone name (long) - Corrected expectation\n\t\t\toutput: \"MST\",  // Should output MST\n\t\t\terr:    nil,    // Should not produce an error\n\t\t},\n\t\t{\n\t\t\tinput:  \"G\", // Era designator (unsupported)\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: G\"),\n\t\t},\n\t\t{\n\t\t\tinput:  \"W\", // Week of month (unsupported)\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: W\"),\n\t\t},\n\t}\n\tfor i, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"test %d: %s\", i, test.input), func(t *testing.T) {\n\t\t\tout, err := parseISOString(test.input)\n\t\t\t// Check error matching\n\t\t\tif (err != nil && test.err == nil) || (err == nil && test.err != nil) || (err != nil && test.err != nil && err.Error() != test.err.Error()) {\n\t\t\t\tt.Fatalf(\"expected error %v, got error %v\", test.err, err)\n\t\t\t}\n\t\t\t// Check output matching only if no error was expected/occurred\n\t\t\tif err == nil && test.err == nil && out != test.output {\n\t\t\t\tt.Fatalf(\"expected output '%v', got '%v'\", test.output, out)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/optional/optional.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage optional\n\nimport (\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/flexible\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"dateTimeOptional\"\n\nconst rfc3339NoTimezone = \"2006-01-02T15:04:05\"\nconst rfc3339NoTimezoneNoT = \"2006-01-02 15:04:05\"\nconst rfc3339Offset = \"2006-01-02 15:04:05 -0700\"\nconst rfc3339NoTime = \"2006-01-02\"\n\nvar layouts = []string{\n\ttime.RFC3339Nano,\n\ttime.RFC3339,\n\trfc3339NoTimezone,\n\trfc3339NoTimezoneNoT,\n\trfc3339Offset,\n\trfc3339NoTime,\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\treturn flexible.New(layouts), nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/percent/percent.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage percent\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"percentstyle\"\n\nvar formatDelimiter byte = '%'\n\n// format specifiers as per strftime in the C standard library\n// https://man7.org/linux/man-pages/man3/strftime.3.html\nvar formatSpecifierToLayout = map[byte]string{\n\tformatDelimiter: string(formatDelimiter), // %% = % (literal %)\n\t'a':             \"Mon\",                   // %a = short weekday name\n\t'A':             \"Monday\",                // %A = full weekday name\n\t'd':             \"02\",                    // %d = day of month (2 digits) (01-31)\n\t'e':             \"2\",                     // %e = day of month (1 digit) (1-31)\n\t'b':             \"Jan\",                   // %b = short month name\n\t'B':             \"January\",               // %B = full month name\n\t'm':             \"01\",                    // %m = month of year (2 digits) (01-12)\n\t'y':             \"06\",                    // %y = year without century\n\t'Y':             \"2006\",                  // %Y = year with century\n\t'H':             \"15\",                    // %H = hour (24 hour clock) (2 digits)\n\t'I':             \"03\",                    // %I = hour (12 hour clock) (2 digits)\n\t'l':             \"3\",                     // %l = hour (12 hour clock) (1 digit)\n\t'p':             \"PM\",                    // %p = PM/AM\n\t'P':             \"pm\",                    // %P = pm/am (lowercase)\n\t'M':             \"04\",                    // %M = minute (2 digits)\n\t'S':             \"05\",                    // %S = seconds (2 digits)\n\t'f':             \"999999\",                // .%f = fraction of seconds - up to microseconds (6 digits) - deci/milli/micro\n\t'Z':             \"MST\",                   // %Z = timezone name (GMT, JST, UTC etc)\n\t// %z is present in timezone options\n\n\t// some additional options not in strftime to support additional options such as\n\t// disallow 0 padding in minute and seconds, nanosecond precision, etc\n\t'o': \"1\",         // %o = month of year (1 digit) (1-12)\n\t'i': \"4\",         // %i = minute (1 digit)\n\t's': \"5\",         // %s = seconds (1 digit)\n\t'N': \"999999999\", // .%N = fraction of seconds - up to microseconds (9 digits) - milli/micro/nano\n}\n\n// some additional options for timezone\n// such as allowing colon in timezone offset and specifying the seconds\n// timezone offsets are from UTC\nvar timezoneOptions = map[string]string{\n\t\"z\":   \"Z0700\",     // %z = timezone offset in +-hhmm / +-(2 digit hour)(2 digit minute) +0500, -0600 etc\n\t\"z:M\": \"Z07:00\",    // %z:M = timezone offset(+-hh:mm) / +-(2 digit hour):(2 digit minute) +05:00, -06:00 etc\n\t\"z:S\": \"Z07:00:00\", // %z:M = timezone offset(+-hh:mm:ss) / +-(2 digit hour):(2 digit minute):(2 digit second) +05:20:00, -06:30:00 etc\n\t\"zH\":  \"Z07\",       // %zH = timezone offset(+-hh) / +-(2 digit hour) +05, -06 etc\n\t\"zS\":  \"Z070000\",   // %zS = timezone offset(+-hhmmss) / +-(2 digit hour)(2 digit minute)(2 digit second) +052000, -063000 etc\n}\n\ntype DateTimeParser struct {\n\tlayouts []string\n}\n\nfunc New(layouts []string) *DateTimeParser {\n\treturn &DateTimeParser{\n\t\tlayouts: layouts,\n\t}\n}\n\nfunc checkTZOptions(formatString string, idx int) (string, int) {\n\t// idx points to '%'\n\t// We know formatString[idx+1] == 'z'\n\tnextIdx := idx + 2 // Index of the character immediately after 'z'\n\n\t// Default values assume only '%z' is present\n\tlayout := timezoneOptions[\"z\"]\n\tfinalIdx := nextIdx // Index after '%z'\n\n\tif nextIdx < len(formatString) {\n\t\tswitch formatString[nextIdx] {\n\t\tcase ':':\n\t\t\t// Check for modifier after the colon ':'\n\t\t\tcolonModifierIdx := nextIdx + 1\n\t\t\tif colonModifierIdx < len(formatString) {\n\t\t\t\tswitch formatString[colonModifierIdx] {\n\t\t\t\tcase 'M':\n\t\t\t\t\t// Found %z:M\n\t\t\t\t\tlayout = timezoneOptions[\"z:M\"]\n\t\t\t\t\tfinalIdx = colonModifierIdx + 1 // Index after %z:M\n\t\t\t\tcase 'S':\n\t\t\t\t\t// Found %z:S\n\t\t\t\t\tlayout = timezoneOptions[\"z:S\"]\n\t\t\t\t\tfinalIdx = colonModifierIdx + 1 // Index after %z:S\n\t\t\t\t\t// default: If %z: is followed by something else, or just %z: at the end.\n\t\t\t\t\t// Keep the default layout (\"z\") and finalIdx (idx + 2).\n\t\t\t\t\t// The ':' will be treated as a literal by the main loop.\n\t\t\t\t}\n\t\t\t}\n\t\t\t// else: %z: is at the very end of the string.\n\t\t\t// Keep the default layout (\"z\") and finalIdx (idx + 2).\n\t\t\t// The ':' will be treated as a literal by the main loop.\n\n\t\tcase 'H':\n\t\t\t// Found %zH\n\t\t\tlayout = timezoneOptions[\"zH\"]\n\t\t\tfinalIdx = nextIdx + 1 // Index after %zH\n\t\tcase 'S':\n\t\t\t// Found %zS\n\t\t\tlayout = timezoneOptions[\"zS\"]\n\t\t\tfinalIdx = nextIdx + 1 // Index after %zS\n\n\t\t\t// default: If %z is followed by something other than ':', 'H', or 'S'.\n\t\t\t// Keep the default layout (\"z\") and finalIdx (idx + 2).\n\t\t\t// The character formatString[nextIdx] will be handled by the main loop.\n\t\t}\n\t}\n\t// else: %z is at the very end of the string.\n\t// Keep the default layout (\"z\") and finalIdx (idx + 2).\n\n\treturn layout, finalIdx\n}\n\nfunc parseFormatString(formatString string) (string, error) {\n\tvar dateTimeLayout strings.Builder\n\t// iterate over the format string and replace the format specifiers with\n\t// the corresponding golang constants\n\tfor idx := 0; idx < len(formatString); {\n\t\t// check if the character is a format delimiter (%)\n\t\tif formatString[idx] == formatDelimiter {\n\t\t\t// check if there is a character after the format delimiter (%)\n\t\t\tif idx+1 >= len(formatString) {\n\t\t\t\treturn \"\", fmt.Errorf(\"invalid format string, expected character after %s\", string(formatDelimiter))\n\t\t\t}\n\t\t\tformatSpecifier := formatString[idx+1]\n\t\t\tif layout, ok := formatSpecifierToLayout[formatSpecifier]; ok {\n\t\t\t\tdateTimeLayout.WriteString(layout)\n\t\t\t\tidx += 2\n\t\t\t} else if formatSpecifier == 'z' {\n\t\t\t\t// did not find a valid specifier\n\t\t\t\t// check if it is for timezone\n\t\t\t\tvar tzLayout string\n\t\t\t\ttzLayout, idx = checkTZOptions(formatString, idx)\n\t\t\t\tdateTimeLayout.WriteString(tzLayout)\n\t\t\t} else {\n\t\t\t\treturn \"\", fmt.Errorf(\"invalid format string, unknown format specifier: %s\", string(formatSpecifier))\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\t// copy the character as is\n\t\tdateTimeLayout.WriteByte(formatString[idx])\n\t\tidx++\n\t}\n\treturn dateTimeLayout.String(), nil\n}\n\nfunc (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {\n\tfor _, layout := range p.layouts {\n\t\trv, err := time.Parse(layout, input)\n\t\tif err == nil {\n\t\t\treturn rv, layout, nil\n\t\t}\n\t}\n\treturn time.Time{}, \"\", analysis.ErrInvalidDateTime\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\tlayouts, ok := config[\"layouts\"].([]interface{})\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify layouts\")\n\t}\n\n\tlayoutStrs := make([]string, 0, len(layouts))\n\tfor _, layout := range layouts {\n\t\tlayoutStr, ok := layout.(string)\n\t\tif ok {\n\t\t\tlayout, err := parseFormatString(layoutStr)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tlayoutStrs = append(layoutStrs, layout)\n\t\t}\n\t}\n\n\treturn New(layoutStrs), nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/percent/percent_test.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage percent\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestConversionFromPercentStyle(t *testing.T) {\n\ttests := []struct {\n\t\tname   string // Added name field\n\t\tinput  string\n\t\toutput string\n\t\terr    error\n\t}{\n\t\t{\n\t\t\tname:   \"basic YMD\",\n\t\t\tinput:  \"%Y-%m-%d\",\n\t\t\toutput: \"2006-01-02\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"YMD with double percent and literal T\",\n\t\t\tinput:  \"%Y/%m%%%%%dT%H%M:%S\",\n\t\t\toutput: \"2006/01%%02T1504:05\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"YMD T HMS Z z\",\n\t\t\tinput:  \"%Y-%m-%dT%H:%M:%S %Z%z\",\n\t\t\toutput: \"2006-01-02T15:04:05 MSTZ0700\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"Full month, padded day/hour, am/pm, z:M\",\n\t\t\tinput:  \"%B %e, %Y %l:%i %P %z:M\",\n\t\t\toutput: \"January 2, 2006 3:4 pm Z07:00\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"Long format with literals and timezone literal :S\",\n\t\t\tinput:  \"Hour %H Minute %Mseconds %S.%N Timezone:%Z:S, Weekday %a; Day %d Month %b, Year %y\",\n\t\t\toutput: \"Hour 15 Minute 04seconds 05.999999999 Timezone:MST:S, Weekday Mon; Day 02 Month Jan, Year 06\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"YMD T HMS with nanoseconds\",\n\t\t\tinput:  \"%Y-%m-%dT%H:%M:%S.%N\",\n\t\t\toutput: \"2006-01-02T15:04:05.999999999\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"HMS Z z\",\n\t\t\tinput:  \"%H:%M:%S %Z %z\",\n\t\t\toutput: \"15:04:05 MST Z0700\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"HMS Z z literal colon\",\n\t\t\tinput:  \"%H:%M:%S %Z %z:\",\n\t\t\toutput: \"15:04:05 MST Z0700:\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"HMS Z z:M\",\n\t\t\tinput:  \"%H:%M:%S %Z %z:M\",\n\t\t\toutput: \"15:04:05 MST Z07:00\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"HMS Z z:S\",\n\t\t\tinput:  \"%H:%M:%S %Z %z:S\",\n\t\t\toutput: \"15:04:05 MST Z07:00:00\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"HMS Z z: literal A\",\n\t\t\tinput:  \"%H:%M:%S %Z %z:A\",\n\t\t\toutput: \"15:04:05 MST Z0700:A\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"HMS Z z literal M\",\n\t\t\tinput:  \"%H:%M:%S %Z %zM\",\n\t\t\toutput: \"15:04:05 MST Z0700M\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"HMS Z zH\",\n\t\t\tinput:  \"%H:%M:%S %Z %zH\",\n\t\t\toutput: \"15:04:05 MST Z07\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"HMS Z zS\",\n\t\t\tinput:  \"%H:%M:%S %Z %zS\",\n\t\t\toutput: \"15:04:05 MST Z070000\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"Complex combination z zS z: zH\",\n\t\t\tinput:  \"%H:%M:%S %Z %z%Z %zS%z:%zH\",\n\t\t\toutput: \"15:04:05 MST Z0700MST Z070000Z0700:Z07\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"z at end\",\n\t\t\tinput:  \"%Y-%m-%d %z\",\n\t\t\toutput: \"2006-01-02 Z0700\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"z: at end\",\n\t\t\tinput:  \"%Y-%m-%d %z:\",\n\t\t\toutput: \"2006-01-02 Z0700:\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"zH at end\",\n\t\t\tinput:  \"%Y-%m-%d %zH\",\n\t\t\toutput: \"2006-01-02 Z07\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"zS at end\",\n\t\t\tinput:  \"%Y-%m-%d %zS\",\n\t\t\toutput: \"2006-01-02 Z070000\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"z:M at end\",\n\t\t\tinput:  \"%Y-%m-%d %z:M\",\n\t\t\toutput: \"2006-01-02 Z07:00\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"z:S at end\",\n\t\t\tinput:  \"%Y-%m-%d %z:S\",\n\t\t\toutput: \"2006-01-02 Z07:00:00\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"z followed by literal X\",\n\t\t\tinput:  \"%Y-%m-%d %zX\",\n\t\t\toutput: \"2006-01-02 Z0700X\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"z: followed by literal X\",\n\t\t\tinput:  \"%Y-%m-%d %z:X\",\n\t\t\toutput: \"2006-01-02 Z0700:X\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"Invalid specifier T\",\n\t\t\tinput:  \"%Y-%m-%d%T%H:%M:%S %ZM\",\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: T\"),\n\t\t},\n\t\t{\n\t\t\tname:   \"Ends with %\",\n\t\t\tinput:  \"%Y-%m-%dT%H:%M:%S %ZM%\",\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, expected character after %%\"),\n\t\t},\n\t\t{\n\t\t\tname:   \"Just %\",\n\t\t\tinput:  \"%\",\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, expected character after %%\"),\n\t\t},\n\t\t{\n\t\t\tname:   \"Just %%\",\n\t\t\tinput:  \"%%\",\n\t\t\toutput: \"%\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"Unknown specifier x\",\n\t\t\tinput:  \"%x\",\n\t\t\toutput: \"\",\n\t\t\terr:    fmt.Errorf(\"invalid format string, unknown format specifier: x\"),\n\t\t},\n\t\t{\n\t\t\tname:   \"Literal prefix\",\n\t\t\tinput:  \"literal %Y\",\n\t\t\toutput: \"literal 2006\",\n\t\t\terr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:   \"Literal suffix\",\n\t\t\tinput:  \"%Y literal\",\n\t\t\toutput: \"2006 literal\",\n\t\t\terr:    nil,\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tout, err := parseFormatString(test.input)\n\n\t\t\t// Enhanced Error Check:\n\t\t\texpectedErrStr := \"\"\n\t\t\tif test.err != nil {\n\t\t\t\texpectedErrStr = test.err.Error()\n\t\t\t}\n\t\t\tactualErrStr := \"\"\n\t\t\tif err != nil {\n\t\t\t\tactualErrStr = err.Error()\n\t\t\t}\n\n\t\t\tif expectedErrStr != actualErrStr {\n\t\t\t\t// Provide more detailed output if errors don't match as strings\n\t\t\t\tt.Fatalf(\"error mismatch:\\nExpected error: %q\\nGot error     : %q\", expectedErrStr, actualErrStr)\n\t\t\t}\n\n\t\t\t// Original error presence check (redundant if string check passes, but safe to keep)\n\t\t\tif (err != nil && test.err == nil) || (err == nil && test.err != nil) {\n\t\t\t\tt.Fatalf(\"presence mismatch: expected error %v, got error %v\", test.err, err)\n\t\t\t}\n\n\t\t\t// Check output matching only if no error was expected/occurred\n\t\t\tif err == nil && test.err == nil && out != test.output {\n\t\t\t\tt.Fatalf(\"output mismatch: expected '%v', got '%v'\", test.output, out)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestDateTimeParser_ParseDateTime(t *testing.T) {\n\t// Pre-create some parsers with known Go layouts\n\tparser1 := New([]string{\"2006-01-02\", \"01/02/2006\"}) // YYYY-MM-DD, MM/DD/YYYY\n\tparser2 := New([]string{\"15:04:05\"})                 // HH:MM:SS\n\tparserEmpty := New([]string{})                       // No layouts\n\n\t// Define expected time values\n\ttime1, _ := time.Parse(\"2006-01-02\", \"2023-10-27\")\n\ttime2, _ := time.Parse(\"01/02/2006\", \"10/27/2023\")\n\ttime3, _ := time.Parse(\"15:04:05\", \"14:30:00\")\n\n\ttests := []struct {\n\t\tname         string\n\t\tparser       *DateTimeParser\n\t\tinput        string\n\t\texpectTime   time.Time\n\t\texpectLayout string\n\t\texpectErr    error\n\t}{\n\t\t{\n\t\t\tname:         \"match first layout\",\n\t\t\tparser:       parser1,\n\t\t\tinput:        \"2023-10-27\",\n\t\t\texpectTime:   time1,\n\t\t\texpectLayout: \"2006-01-02\",\n\t\t\texpectErr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:         \"match second layout\",\n\t\t\tparser:       parser1,\n\t\t\tinput:        \"10/27/2023\",\n\t\t\texpectTime:   time2,\n\t\t\texpectLayout: \"01/02/2006\",\n\t\t\texpectErr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:         \"no matching layout\",\n\t\t\tparser:       parser1,\n\t\t\tinput:        \"14:30:00\", // Matches parser2's layout, not parser1's\n\t\t\texpectTime:   time.Time{},\n\t\t\texpectLayout: \"\",\n\t\t\texpectErr:    analysis.ErrInvalidDateTime,\n\t\t},\n\t\t{\n\t\t\tname:         \"match only layout\",\n\t\t\tparser:       parser2,\n\t\t\tinput:        \"14:30:00\",\n\t\t\texpectTime:   time3,\n\t\t\texpectLayout: \"15:04:05\",\n\t\t\texpectErr:    nil,\n\t\t},\n\t\t{\n\t\t\tname:         \"invalid date format for layout\",\n\t\t\tparser:       parser1,\n\t\t\tinput:        \"27-10-2023\", // Wrong separators\n\t\t\texpectTime:   time.Time{},\n\t\t\texpectLayout: \"\",\n\t\t\texpectErr:    analysis.ErrInvalidDateTime, // time.Parse fails on all, returns ErrInvalidDateTime\n\t\t},\n\t\t{\n\t\t\tname:         \"empty input\",\n\t\t\tparser:       parser1,\n\t\t\tinput:        \"\",\n\t\t\texpectTime:   time.Time{},\n\t\t\texpectLayout: \"\",\n\t\t\texpectErr:    analysis.ErrInvalidDateTime,\n\t\t},\n\t\t{\n\t\t\tname:         \"parser with no layouts\",\n\t\t\tparser:       parserEmpty,\n\t\t\tinput:        \"2023-10-27\",\n\t\t\texpectTime:   time.Time{},\n\t\t\texpectLayout: \"\",\n\t\t\texpectErr:    analysis.ErrInvalidDateTime,\n\t\t},\n\t\t{\n\t\t\tname:         \"not a date string\",\n\t\t\tparser:       parser1,\n\t\t\tinput:        \"hello world\",\n\t\t\texpectTime:   time.Time{},\n\t\t\texpectLayout: \"\",\n\t\t\texpectErr:    analysis.ErrInvalidDateTime,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tgotTime, gotLayout, gotErr := test.parser.ParseDateTime(test.input)\n\n\t\t\t// Check error\n\t\t\tif !reflect.DeepEqual(gotErr, test.expectErr) {\n\t\t\t\tt.Fatalf(\"error mismatch:\\nExpected: %v\\nGot:      %v\", test.expectErr, gotErr)\n\t\t\t}\n\n\t\t\t// Check time only if no error expected\n\t\t\tif test.expectErr == nil {\n\t\t\t\tif !gotTime.Equal(test.expectTime) {\n\t\t\t\t\tt.Errorf(\"time mismatch:\\nExpected: %v\\nGot:      %v\", test.expectTime, gotTime)\n\t\t\t\t}\n\t\t\t\tif gotLayout != test.expectLayout {\n\t\t\t\t\tt.Errorf(\"layout mismatch:\\nExpected: %q\\nGot:      %q\", test.expectLayout, gotLayout)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestDateTimeParserConstructor(t *testing.T) {\n\ttests := []struct {\n\t\tname          string\n\t\tconfig        map[string]interface{}\n\t\texpectLayouts []string // Expected Go layouts after parsing\n\t\texpectErr     error\n\t}{\n\t\t{\n\t\t\tname: \"valid config with multiple layouts\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"layouts\": []interface{}{\"%Y-%m-%d\", \"%H:%M:%S %Z\"},\n\t\t\t},\n\t\t\texpectLayouts: []string{\"2006-01-02\", \"15:04:05 MST\"},\n\t\t\texpectErr:     nil,\n\t\t},\n\t\t{\n\t\t\tname: \"valid config with single layout\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"layouts\": []interface{}{\"%Y/%m/%d %z:M\"},\n\t\t\t},\n\t\t\texpectLayouts: []string{\"2006/01/02 Z07:00\"},\n\t\t\texpectErr:     nil,\n\t\t},\n\t\t{\n\t\t\tname: \"valid config with complex layout\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"layouts\": []interface{}{\"%a, %d %b %Y %H:%M:%S %zH\"},\n\t\t\t},\n\t\t\texpectLayouts: []string{\"Mon, 02 Jan 2006 15:04:05 Z07\"},\n\t\t\texpectErr:     nil,\n\t\t},\n\t\t{\n\t\t\tname: \"config missing layouts key\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"other_key\": \"value\",\n\t\t\t},\n\t\t\texpectLayouts: nil,\n\t\t\texpectErr:     fmt.Errorf(\"must specify layouts\"),\n\t\t},\n\t\t{\n\t\t\tname: \"config layouts not a slice\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"layouts\": \"not-a-slice\", // Value is a string\n\t\t\t},\n\t\t\texpectLayouts: nil,\n\t\t\t// Update the expected error message\n\t\t\texpectErr: fmt.Errorf(\"must specify layouts\"),\n\t\t},\n\t\t{\n\t\t\tname: \"config layouts contains non-string\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"layouts\": []interface{}{\"%Y-%m-%d\", 123},\n\t\t\t},\n\t\t\t// Should process the valid string, ignore the int\n\t\t\texpectLayouts: []string{\"2006-01-02\"},\n\t\t\texpectErr:     nil,\n\t\t},\n\t\t{\n\t\t\tname: \"config layouts contains invalid percent format\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"layouts\": []interface{}{\"%Y-%m-%d\", \"%x\"}, // %x is invalid\n\t\t\t},\n\t\t\texpectLayouts: nil,\n\t\t\texpectErr:     fmt.Errorf(\"invalid format string, unknown format specifier: x\"),\n\t\t},\n\t\t{\n\t\t\tname: \"config layouts contains format ending in %\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"layouts\": []interface{}{\"%Y-%m-%d\", \"%H:%M:%\"},\n\t\t\t},\n\t\t\texpectLayouts: nil,\n\t\t\texpectErr:     fmt.Errorf(\"invalid format string, expected character after %%\"),\n\t\t},\n\t\t{\n\t\t\tname: \"config with empty layouts slice\",\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"layouts\": []interface{}{},\n\t\t\t},\n\t\t\texpectLayouts: []string{}, // Expect an empty slice, not nil\n\t\t\texpectErr:     nil,\n\t\t},\n\t\t{\n\t\t\tname:          \"nil config\",\n\t\t\tconfig:        nil,\n\t\t\texpectLayouts: nil,\n\t\t\texpectErr:     fmt.Errorf(\"must specify layouts\"),\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// Cache is not used by this constructor, so nil is fine\n\t\t\tparserIntf, err := DateTimeParserConstructor(test.config, nil)\n\n\t\t\t// Check error\n\t\t\t// Use string comparison for errors as they might be created differently\n\t\t\texpectedErrStr := \"\"\n\t\t\tif test.expectErr != nil {\n\t\t\t\texpectedErrStr = test.expectErr.Error()\n\t\t\t}\n\t\t\tactualErrStr := \"\"\n\t\t\tif err != nil {\n\t\t\t\tactualErrStr = err.Error()\n\t\t\t}\n\t\t\tif expectedErrStr != actualErrStr {\n\t\t\t\tt.Fatalf(\"error mismatch:\\nExpected: %q\\nGot:      %q\", expectedErrStr, actualErrStr)\n\t\t\t}\n\n\t\t\t// Check layouts only if no error expected\n\t\t\tif test.expectErr == nil {\n\t\t\t\t// Type assert to access the layouts field\n\t\t\t\tparser, ok := parserIntf.(*DateTimeParser)\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Fatalf(\"constructor did not return a *DateTimeParser\")\n\t\t\t\t}\n\t\t\t\tif !reflect.DeepEqual(parser.layouts, test.expectLayouts) {\n\t\t\t\t\tt.Errorf(\"layouts mismatch:\\nExpected: %v\\nGot:      %v\", test.expectLayouts, parser.layouts)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/sanitized/sanitized.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sanitized\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"sanitizedgo\"\n\nvar validMagicNumbers = map[string]struct{}{\n\t\"2006\":    {},\n\t\"06\":      {}, // Year\n\t\"01\":      {},\n\t\"1\":       {},\n\t\"_1\":      {},\n\t\"January\": {},\n\t\"Jan\":     {}, // Month\n\t\"02\":      {},\n\t\"2\":       {},\n\t\"_2\":      {},\n\t\"__2\":     {},\n\t\"002\":     {},\n\t\"Monday\":  {},\n\t\"Mon\":     {}, // Day\n\t\"15\":      {},\n\t\"3\":       {},\n\t\"03\":      {}, // Hour\n\t\"4\":       {},\n\t\"04\":      {}, // Minute\n\t\"5\":       {},\n\t\"05\":      {}, // Second\n\t\"0700\":    {},\n\t\"070000\":  {},\n\t\"07\":      {},\n\t\"00\":      {},\n\t\"\":        {},\n}\n\nvar layoutSplitRegex = regexp.MustCompile(\"[\\\\+\\\\-= :T,Z\\\\.<>;\\\\?!`~@#$%\\\\^&\\\\*|'\\\"\\\\(\\\\){}\\\\[\\\\]/\\\\\\\\]\")\n\nvar layoutStripRegex = regexp.MustCompile(`PM|pm|\\.9+|\\.0+|MST`)\n\ntype DateTimeParser struct {\n\tlayouts []string\n}\n\nfunc New(layouts []string) *DateTimeParser {\n\treturn &DateTimeParser{\n\t\tlayouts: layouts,\n\t}\n}\n\nfunc (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {\n\tfor _, layout := range p.layouts {\n\t\trv, err := time.Parse(layout, input)\n\t\tif err == nil {\n\t\t\treturn rv, layout, nil\n\t\t}\n\t}\n\treturn time.Time{}, \"\", analysis.ErrInvalidDateTime\n}\n\n// date time layouts must be a combination of constants specified in golang time package\n// https://pkg.go.dev/time#pkg-constants\n// this validation verifies that only these constants are used in the custom layout\n// for compatibility with the golang time package\nfunc validateLayout(layout string) bool {\n\t// first we strip out commonly used constants\n\t// such as \"PM\" which can be present in the layout\n\t// right after a time component, e.g. 03:04PM;\n\t// because regex split cannot separate \"03:04PM\" into\n\t// \"03:04\" and \"PM\". We also strip out \".9+\" and \".0+\"\n\t// which represent fractional seconds.\n\tlayout = layoutStripRegex.ReplaceAllString(layout, \"\")\n\t// then we split the layout by non-constant characters\n\t// which is a regex and verify that each split is a valid magic number\n\tsplit := layoutSplitRegex.Split(layout, -1)\n\tfor i := range split {\n\t\t_, found := validMagicNumbers[split[i]]\n\t\tif !found {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\tlayouts, ok := config[\"layouts\"].([]interface{})\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify layouts\")\n\t}\n\tvar layoutStrs []string\n\tfor _, layout := range layouts {\n\t\tlayoutStr, ok := layout.(string)\n\t\tif ok {\n\t\t\tif !validateLayout(layoutStr) {\n\t\t\t\treturn nil, fmt.Errorf(\"invalid datetime parser layout: %s,\"+\n\t\t\t\t\t\" please refer to https://pkg.go.dev/time#pkg-constants for supported\"+\n\t\t\t\t\t\" layouts\", layoutStr)\n\t\t\t}\n\t\t\tlayoutStrs = append(layoutStrs, layoutStr)\n\t\t}\n\t}\n\treturn New(layoutStrs), nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/sanitized/sanitized_test.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sanitized\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestLayoutValidatorRegex(t *testing.T) {\n\tsplitRegexTests := []struct {\n\t\tinput  string\n\t\toutput []string\n\t}{\n\t\t{\n\t\t\tinput:  \"2014-08-03\",\n\t\t\toutput: []string{\"2014\", \"08\", \"03\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"2014-08-03T15:59:30\",\n\t\t\toutput: []string{\"2014\", \"08\", \"03\", \"15\", \"59\", \"30\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"2014.08-03 15/59`30\",\n\t\t\toutput: []string{\"2014\", \"08\", \"03\", \"15\", \"59\", \"30\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"2014/08/03T15:59:30Z08:00\",\n\t\t\toutput: []string{\"2014\", \"08\", \"03\", \"15\", \"59\", \"30\", \"08\", \"00\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"2014\\\\08|03T15=59.30.999999999+08*00\",\n\t\t\toutput: []string{\"2014\", \"08\", \"03\", \"15\", \"59\", \"30\", \"999999999\", \"08\", \"00\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"2006-01-02T15:04:05.999999999Z07:00\",\n\t\t\toutput: []string{\"2006\", \"01\", \"02\", \"15\", \"04\", \"05\", \"999999999\", \"07\", \"00\"},\n\t\t},\n\t\t{\n\t\t\tinput: \"A-B C:DTE,FZG.H<I>J;K?L!M`N~O@P#Q$R%S^U&V*W|X'Y\\\"A(B)C{D}E[F]G/H\\\\I+J=L\",\n\t\t\toutput: []string{\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\", \"K\", \"L\", \"M\", \"N\", \"O\", \"P\",\n\t\t\t\t\"Q\", \"R\", \"S\", \"U\", \"V\", \"W\", \"X\", \"Y\", \"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\", \"L\"},\n\t\t},\n\t}\n\tregex := layoutSplitRegex\n\tfor _, test := range splitRegexTests {\n\t\tt.Run(test.input, func(t *testing.T) {\n\t\t\tactualOutput := regex.Split(test.input, -1)\n\t\t\tif !reflect.DeepEqual(actualOutput, test.output) {\n\t\t\t\tt.Fatalf(\"expected output %v, got %v\", test.output, actualOutput)\n\t\t\t}\n\t\t})\n\t}\n\n\tstripRegexTests := []struct {\n\t\tinput  string\n\t\toutput string\n\t}{\n\t\t{\n\t\t\tinput:  \"3PM\",\n\t\t\toutput: \"3\",\n\t\t},\n\t\t{\n\t\t\tinput:  \"3.0PM\",\n\t\t\toutput: \"3\",\n\t\t},\n\t\t{\n\t\t\tinput:  \"3.9AM\",\n\t\t\toutput: \"3AM\",\n\t\t},\n\t\t{\n\t\t\tinput:  \"3.999999999pm\",\n\t\t\toutput: \"3\",\n\t\t},\n\t\t{\n\t\t\tinput:  \"2006-01-02T15:04:05.999999999Z07:00MST\",\n\t\t\toutput: \"2006-01-02T15:04:05Z07:00\",\n\t\t},\n\t\t{\n\t\t\tinput:  \"Jan _2 15:04:05.0000000+07:00MST\",\n\t\t\toutput: \"Jan _2 15:04:05+07:00\",\n\t\t},\n\t\t{\n\t\t\tinput:  \"15:04:05.99PM+07:00MST\",\n\t\t\toutput: \"15:04:05+07:00\",\n\t\t},\n\t}\n\tregex = layoutStripRegex\n\tfor _, test := range stripRegexTests {\n\t\tt.Run(test.input, func(t *testing.T) {\n\t\t\tactualOutput := layoutStripRegex.ReplaceAllString(test.input, \"\")\n\t\t\tif !reflect.DeepEqual(actualOutput, test.output) {\n\t\t\t\tt.Fatalf(\"expected output %v, got %v\", test.output, actualOutput)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/timestamp/microseconds/microseconds.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage microseconds\n\nimport (\n\t\"math\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"unix_micro\"\n\ntype DateTimeParser struct {\n}\n\nvar minBound int64 = math.MinInt64 / 1000\nvar maxBound int64 = math.MaxInt64 / 1000\n\nfunc (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {\n\t// unix timestamp is milliseconds since UNIX epoch\n\ttimestamp, err := strconv.ParseInt(input, 10, 64)\n\tif err != nil {\n\t\treturn time.Time{}, \"\", analysis.ErrInvalidTimestampString\n\t}\n\tif timestamp < minBound || timestamp > maxBound {\n\t\treturn time.Time{}, \"\", analysis.ErrInvalidTimestampRange\n\t}\n\treturn time.UnixMicro(timestamp), Name, nil\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\treturn &DateTimeParser{}, nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/timestamp/milliseconds/milliseconds.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage milliseconds\n\nimport (\n\t\"math\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"unix_milli\"\n\ntype DateTimeParser struct {\n}\n\nvar minBound int64 = math.MinInt64 / 1000000\nvar maxBound int64 = math.MaxInt64 / 1000000\n\nfunc (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {\n\t// unix timestamp is milliseconds since UNIX epoch\n\ttimestamp, err := strconv.ParseInt(input, 10, 64)\n\tif err != nil {\n\t\treturn time.Time{}, \"\", analysis.ErrInvalidTimestampString\n\t}\n\tif timestamp < minBound || timestamp > maxBound {\n\t\treturn time.Time{}, \"\", analysis.ErrInvalidTimestampRange\n\t}\n\treturn time.UnixMilli(timestamp), Name, nil\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\treturn &DateTimeParser{}, nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/timestamp/nanoseconds/nanoseconds.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage nanoseconds\n\nimport (\n\t\"math\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"unix_nano\"\n\ntype DateTimeParser struct {\n}\n\nvar minBound int64 = math.MinInt64\nvar maxBound int64 = math.MaxInt64\n\nfunc (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {\n\t// unix timestamp is milliseconds since UNIX epoch\n\ttimestamp, err := strconv.ParseInt(input, 10, 64)\n\tif err != nil {\n\t\treturn time.Time{}, \"\", analysis.ErrInvalidTimestampString\n\t}\n\tif timestamp < minBound || timestamp > maxBound {\n\t\treturn time.Time{}, \"\", analysis.ErrInvalidTimestampRange\n\t}\n\treturn time.Unix(0, timestamp), Name, nil\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\treturn &DateTimeParser{}, nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/datetime/timestamp/seconds/seconds.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage seconds\n\nimport (\n\t\"math\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"unix_sec\"\n\ntype DateTimeParser struct {\n}\n\nvar minBound int64 = math.MinInt64 / 1000000000\nvar maxBound int64 = math.MaxInt64 / 1000000000\n\nfunc (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {\n\t// unix timestamp is seconds since UNIX epoch\n\ttimestamp, err := strconv.ParseInt(input, 10, 64)\n\tif err != nil {\n\t\treturn time.Time{}, \"\", analysis.ErrInvalidTimestampString\n\t}\n\tif timestamp < minBound || timestamp > maxBound {\n\t\treturn time.Time{}, \"\", analysis.ErrInvalidTimestampRange\n\t}\n\treturn time.Unix(timestamp, 0), Name, nil\n}\n\nfunc DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {\n\treturn &DateTimeParser{}, nil\n}\n\nfunc init() {\n\terr := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/freq.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage analysis\n\nimport (\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies {\n\trv := make(map[string]*index.TokenFreq, len(tokens))\n\n\tif options.IncludeTermVectors() {\n\t\ttls := make([]index.TokenLocation, len(tokens))\n\t\ttlNext := 0\n\n\t\tfor _, token := range tokens {\n\t\t\ttls[tlNext] = index.TokenLocation{\n\t\t\t\tArrayPositions: arrayPositions,\n\t\t\t\tStart:          token.Start,\n\t\t\t\tEnd:            token.End,\n\t\t\t\tPosition:       token.Position,\n\t\t\t}\n\n\t\t\tcurr, ok := rv[string(token.Term)]\n\t\t\tif ok {\n\t\t\t\tcurr.Locations = append(curr.Locations, &tls[tlNext])\n\t\t\t} else {\n\t\t\t\tcurr = &index.TokenFreq{\n\t\t\t\t\tTerm:      token.Term,\n\t\t\t\t\tLocations: []*index.TokenLocation{&tls[tlNext]},\n\t\t\t\t}\n\t\t\t\trv[string(token.Term)] = curr\n\t\t\t}\n\n\t\t\tif !options.SkipFreqNorm() {\n\t\t\t\tcurr.SetFrequency(curr.Frequency() + 1)\n\t\t\t}\n\n\t\t\ttlNext++\n\t\t}\n\t} else {\n\t\tfor _, token := range tokens {\n\t\t\tcurr, exists := rv[string(token.Term)]\n\t\t\tif !exists {\n\t\t\t\tcurr = &index.TokenFreq{\n\t\t\t\t\tTerm: token.Term,\n\t\t\t\t}\n\t\t\t\trv[string(token.Term)] = curr\n\t\t\t}\n\n\t\t\tif !options.SkipFreqNorm() {\n\t\t\t\tcurr.SetFrequency(curr.Frequency() + 1)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn rv\n}\n"
  },
  {
    "path": "analysis/freq_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage analysis\n\nimport (\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestTokenFrequency(t *testing.T) {\n\ttokens := TokenStream{\n\t\t&Token{\n\t\t\tTerm:     []byte(\"water\"),\n\t\t\tPosition: 1,\n\t\t\tStart:    0,\n\t\t\tEnd:      5,\n\t\t},\n\t\t&Token{\n\t\t\tTerm:     []byte(\"water\"),\n\t\t\tPosition: 2,\n\t\t\tStart:    6,\n\t\t\tEnd:      11,\n\t\t},\n\t}\n\texpectedResult := index.TokenFrequencies{\n\t\t\"water\": &index.TokenFreq{\n\t\t\tTerm: []byte(\"water\"),\n\t\t\tLocations: []*index.TokenLocation{\n\t\t\t\t{\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      11,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\texpectedResult[\"water\"].SetFrequency(2)\n\tresult := TokenFrequency(tokens, nil, index.IncludeTermVectors)\n\tif !reflect.DeepEqual(result, expectedResult) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedResult, result)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ar/analyzer_ar.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ar\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"ar\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnormalizeFilter := unicodenorm.MustNewUnicodeNormalizeFilter(unicodenorm.NFKC)\n\tstopArFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnormalizeArFilter, err := cache.TokenFilterNamed(NormalizeName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerArFilter, err := cache.TokenFilterNamed(StemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tnormalizeFilter,\n\t\t\tstopArFilter,\n\t\t\tnormalizeArFilter,\n\t\t\tstemmerArFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ar/analyzer_ar_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ar\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestArabicAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: []byte(\"كبير\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"كبير\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// feminine marker\n\t\t{\n\t\t\tinput: []byte(\"كبيرة\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"كبير\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"مشروب\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"مشروب\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// plural -at\n\t\t{\n\t\t\tinput: []byte(\"مشروبات\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"مشروب\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// plural -in\n\t\t{\n\t\t\tinput: []byte(\"أمريكيين\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"امريك\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// singular with bare alif\n\t\t{\n\t\t\tinput: []byte(\"امريكي\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"امريك\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"كتاب\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"كتاب\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// definite article\n\t\t{\n\t\t\tinput: []byte(\"الكتاب\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"كتاب\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"ما ملكت أيمانكم\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ملكت\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    5,\n\t\t\t\t\tEnd:      13,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ايمانكم\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    14,\n\t\t\t\t\tEnd:      28,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stopwords\n\t\t{\n\t\t\tinput: []byte(\"الذين ملكت أيمانكم\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ملكت\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    11,\n\t\t\t\t\tEnd:      19,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ايمانكم\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    20,\n\t\t\t\t\tEnd:      34,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// presentation form normalization\n\t\t{\n\t\t\tinput: []byte(\"ﺍﻟﺴﻼﻢ\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"سلام\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.output, actual)\n\t\t\tt.Errorf(\"expected % x, got % x\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ar/arabic_normalize.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ar\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst NormalizeName = \"normalize_ar\"\n\nconst (\n\tAlef           = '\\u0627'\n\tAlefMadda      = '\\u0622'\n\tAlefHamzaAbove = '\\u0623'\n\tAlefHamzaBelow = '\\u0625'\n\tYeh            = '\\u064A'\n\tDotlessYeh     = '\\u0649'\n\tTehMarbuta     = '\\u0629'\n\tHeh            = '\\u0647'\n\tTatweel        = '\\u0640'\n\tFathatan       = '\\u064B'\n\tDammatan       = '\\u064C'\n\tKasratan       = '\\u064D'\n\tFatha          = '\\u064E'\n\tDamma          = '\\u064F'\n\tKasra          = '\\u0650'\n\tShadda         = '\\u0651'\n\tSukun          = '\\u0652'\n)\n\ntype ArabicNormalizeFilter struct {\n}\n\nfunc NewArabicNormalizeFilter() *ArabicNormalizeFilter {\n\treturn &ArabicNormalizeFilter{}\n}\n\nfunc (s *ArabicNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := normalize(token.Term)\n\t\ttoken.Term = term\n\t}\n\treturn input\n}\n\nfunc normalize(input []byte) []byte {\n\trunes := bytes.Runes(input)\n\tfor i := 0; i < len(runes); i++ {\n\t\tswitch runes[i] {\n\t\tcase AlefMadda, AlefHamzaAbove, AlefHamzaBelow:\n\t\t\trunes[i] = Alef\n\t\tcase DotlessYeh:\n\t\t\trunes[i] = Yeh\n\t\tcase TehMarbuta:\n\t\t\trunes[i] = Heh\n\t\tcase Tatweel, Kasratan, Dammatan, Fathatan, Fatha, Damma, Kasra, Shadda, Sukun:\n\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\ti--\n\t\t}\n\t}\n\treturn analysis.BuildTermFromRunes(runes)\n}\n\nfunc NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewArabicNormalizeFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ar/arabic_normalize_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ar\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestArabicNormalizeFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// AlifMadda\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"آجن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"اجن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// AlifHamzaAbove\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"أحمد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"احمد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// AlifHamzaBelow\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"إعاذ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"اعاذ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// AlifMaksura\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"بنى\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"بني\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// TehMarbuta\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"فاطمة\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"فاطمه\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Tatweel\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"روبرـــــت\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"روبرت\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Fatha\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"مَبنا\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"مبنا\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Kasra\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"علِي\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"علي\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Damma\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"بُوات\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"بوات\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Fathatan\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ولداً\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ولدا\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Kasratan\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ولدٍ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ولد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Dammatan\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ولدٌ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ولد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Sukun\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"نلْسون\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"نلسون\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Shaddah\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"هتميّ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"هتمي\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// empty\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tarabicNormalizeFilter := NewArabicNormalizeFilter()\n\tfor _, test := range tests {\n\t\tactual := arabicNormalizeFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected % x, got % x\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ar/stemmer_ar.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ar\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StemmerName = \"stemmer_ar\"\n\n// These were obtained from org.apache.lucene.analysis.ar.ArabicStemmer\nvar prefixes = [][]rune{\n\t[]rune(\"ال\"),\n\t[]rune(\"وال\"),\n\t[]rune(\"بال\"),\n\t[]rune(\"كال\"),\n\t[]rune(\"فال\"),\n\t[]rune(\"لل\"),\n\t[]rune(\"و\"),\n}\nvar suffixes = [][]rune{\n\t[]rune(\"ها\"),\n\t[]rune(\"ان\"),\n\t[]rune(\"ات\"),\n\t[]rune(\"ون\"),\n\t[]rune(\"ين\"),\n\t[]rune(\"يه\"),\n\t[]rune(\"ية\"),\n\t[]rune(\"ه\"),\n\t[]rune(\"ة\"),\n\t[]rune(\"ي\"),\n}\n\ntype ArabicStemmerFilter struct{}\n\nfunc NewArabicStemmerFilter() *ArabicStemmerFilter {\n\treturn &ArabicStemmerFilter{}\n}\n\nfunc (s *ArabicStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := stem(token.Term)\n\t\ttoken.Term = term\n\t}\n\treturn input\n}\n\nfunc canStemPrefix(input, prefix []rune) bool {\n\t// Wa- prefix requires at least 3 characters.\n\tif len(prefix) == 1 && len(input) < 4 {\n\t\treturn false\n\t}\n\t// Other prefixes require only 2.\n\tif len(input)-len(prefix) < 2 {\n\t\treturn false\n\t}\n\tfor i := range prefix {\n\t\tif prefix[i] != input[i] {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc canStemSuffix(input, suffix []rune) bool {\n\t// All suffixes require at least 2 characters after stemming.\n\tif len(input)-len(suffix) < 2 {\n\t\treturn false\n\t}\n\tstemEnd := len(input) - len(suffix)\n\tfor i := range suffix {\n\t\tif suffix[i] != input[stemEnd+i] {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc stem(input []byte) []byte {\n\trunes := bytes.Runes(input)\n\t// Strip a single prefix.\n\tfor _, p := range prefixes {\n\t\tif canStemPrefix(runes, p) {\n\t\t\trunes = runes[len(p):]\n\t\t\tbreak\n\t\t}\n\t}\n\t// Strip off multiple suffixes, in their order in the suffixes array.\n\tfor _, s := range suffixes {\n\t\tif canStemSuffix(runes, s) {\n\t\t\trunes = runes[:len(runes)-len(s)]\n\t\t}\n\t}\n\treturn analysis.BuildTermFromRunes(runes)\n}\n\nfunc StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewArabicStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ar/stemmer_ar_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ar\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestArabicStemmerFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// AlPrefix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"الحسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"حسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// WalPrefix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"والحسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"حسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// BalPrefix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"بالحسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"حسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// KalPrefix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"كالحسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"حسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// FalPrefix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"فالحسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"حسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// LlPrefix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"للاخر\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"اخر\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// WaPrefix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"وحسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"حسن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// AhSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"زوجها\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"زوج\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// AnSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهدان\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// AtSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهدات\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// WnSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهدون\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// YnSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهدين\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// YhSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهديه\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// YpSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهدية\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// HSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// PSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهدة\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// YSuffix\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهدي\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// ComboPrefSuf\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"وساهدون\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// ComboSuf\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهدهات\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ساهد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Shouldn't Stem\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"الو\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"الو\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// NonArabic\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"English\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"English\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"سلام\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"سلام\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"السلام\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"سلام\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"سلامة\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"سلام\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"السلامة\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"سلام\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"الوصل\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"وصل\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"والصل\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"صل\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Empty\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tarabicStemmerFilter := NewArabicStemmerFilter()\n\tfor _, test := range tests {\n\t\tactual := arabicStemmerFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected % x, got % x\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ar/stop_filter_ar.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ar\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ar/stop_words_ar.go",
    "content": "package ar\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_ar\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis\n// ` was changed to ' to allow for literal string\n\nvar ArabicStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license.\n# See http://members.unine.ch/jacques.savoy/clef/index.html.\n# Also see http://www.opensource.org/licenses/bsd-license.html\n# Cleaned on October 11, 2009 (not normalized, so use before normalization)\n# This means that when modifying this list, you might need to add some \n# redundant entries, for example containing forms with both أ and ا\nمن\nومن\nمنها\nمنه\nفي\nوفي\nفيها\nفيه\nو\nف\nثم\nاو\nأو\nب\nبها\nبه\nا\nأ\nاى\nاي\nأي\nأى\nلا\nولا\nالا\nألا\nإلا\nلكن\nما\nوما\nكما\nفما\nعن\nمع\nاذا\nإذا\nان\nأن\nإن\nانها\nأنها\nإنها\nانه\nأنه\nإنه\nبان\nبأن\nفان\nفأن\nوان\nوأن\nوإن\nالتى\nالتي\nالذى\nالذي\nالذين\nالى\nالي\nإلى\nإلي\nعلى\nعليها\nعليه\nاما\nأما\nإما\nايضا\nأيضا\nكل\nوكل\nلم\nولم\nلن\nولن\nهى\nهي\nهو\nوهى\nوهي\nوهو\nفهى\nفهي\nفهو\nانت\nأنت\nلك\nلها\nله\nهذه\nهذا\nتلك\nذلك\nهناك\nكانت\nكان\nيكون\nتكون\nوكانت\nوكان\nغير\nبعض\nقد\nنحو\nبين\nبينما\nمنذ\nضمن\nحيث\nالان\nالآن\nخلال\nبعد\nقبل\nحتى\nعند\nعندما\nلدى\nجميع\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(ArabicStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/bg/stop_filter_bg.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bg\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/bg/stop_words_bg.go",
    "content": "package bg\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_bg\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar BulgarianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license.\n# See http://members.unine.ch/jacques.savoy/clef/index.html.\n# Also see http://www.opensource.org/licenses/bsd-license.html\nа\nаз\nако\nала\nбе\nбез\nбеше\nби\nбил\nбила\nбили\nбило\nблизо\nбъдат\nбъде\nбяха\nв\nвас\nваш\nваша\nвероятно\nвече\nвзема\nви\nвие\nвинаги\nвсе\nвсеки\nвсички\nвсичко\nвсяка\nвъв\nвъпреки\nвърху\nг\nги\nглавно\nго\nд\nда\nдали\nдо\nдокато\nдокога\nдори\nдосега\nдоста\nе\nедва\nедин\nето\nза\nзад\nзаедно\nзаради\nзасега\nзатова\nзащо\nзащото\nи\nиз\nили\nим\nима\nимат\nиска\nй\nказа\nкак\nкаква\nкакво\nкакто\nкакъв\nкато\nкога\nкогато\nкоето\nкоито\nкой\nкойто\nколко\nкоято\nкъде\nкъдето\nкъм\nли\nм\nме\nмежду\nмен\nми\nмнозина\nмога\nмогат\nможе\nмоля\nмомента\nму\nн\nна\nнад\nназад\nнай\nнаправи\nнапред\nнапример\nнас\nне\nнего\nнея\nни\nние\nникой\nнито\nно\nнякои\nнякой\nняма\nобаче\nоколо\nосвен\nособено\nот\nотгоре\nотново\nоще\nпак\nпо\nповече\nповечето\nпод\nпоне\nпоради\nпосле\nпочти\nправи\nпред\nпреди\nпрез\nпри\nпък\nпърво\nс\nса\nсамо\nсе\nсега\nси\nскоро\nслед\nсме\nспоред\nсред\nсрещу\nсте\nсъм\nсъс\nсъщо\nт\nтази\nтака\nтакива\nтакъв\nтам\nтвой\nте\nтези\nти\nтн\nто\nтова\nтогава\nтози\nтой\nтолкова\nточно\nтрябва\nтук\nтъй\nтя\nтях\nу\nхаресва\nч\nче\nчесто\nчрез\nще\nщом\nя\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(BulgarianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ca/articles_ca.go",
    "content": "package ca\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst ArticlesName = \"articles_ca\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis\n\nvar CatalanArticles = []byte(`\nd\nl\nm\nn\ns\nt\n`)\n\nfunc ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(CatalanArticles)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ca/elision_ca.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ca\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/elision\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst ElisionName = \"elision_ca\"\n\nfunc ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tarticlesTokenMap, err := cache.TokenMapNamed(ArticlesName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building elision filter: %v\", err)\n\t}\n\treturn elision.NewElisionFilter(articlesTokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ca/elision_ca_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ca\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestFrenchElision(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"l'Institut\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"d'Estudis\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Institut\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Estudis\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\telisionFilter, err := cache.TokenFilterNamed(ElisionName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := elisionFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ca/stop_filter_ca.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ca\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ca/stop_words_ca.go",
    "content": "package ca\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_ca\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar CatalanStopWords = []byte(`# Catalan stopwords from http://github.com/vcl/cue.language (Apache 2 Licensed)\na\nabans\nací\nah\naixí\naixò\nal\nals\naleshores\nalgun\nalguna\nalgunes\nalguns\nalhora\nallà\nallí\nallò\naltra\naltre\naltres\namb\nambdós\nambdues\napa\naquell\naquella\naquelles\naquells\naquest\naquesta\naquestes\naquests\naquí\nbaix\ncada\ncadascú\ncadascuna\ncadascunes\ncadascuns\ncom\ncontra\nd'un\nd'una\nd'unes\nd'uns\ndalt\nde\ndel\ndels\ndes\ndesprés\ndins\ndintre\ndonat\ndoncs\ndurant\ne\neh\nel\nels\nem\nen\nencara\nens\nentre\nérem\neren\néreu\nes\nés\nesta\nestà\nestàvem\nestaven\nestàveu\nesteu\net\netc\nets\nfins\nfora\ngairebé\nha\nhan\nhas\nhavia\nhe\nhem\nheu\nhi \nho\ni\nigual\niguals\nja\nl'hi\nla\nles\nli\nli'n\nllavors\nm'he\nma\nmal\nmalgrat\nmateix\nmateixa\nmateixes\nmateixos\nme\nmentre\nmés\nmeu\nmeus\nmeva\nmeves\nmolt\nmolta\nmoltes\nmolts\nmon\nmons\nn'he\nn'hi\nne\nni\nno\nnogensmenys\nnomés\nnosaltres\nnostra\nnostre\nnostres\no\noh\noi\non\npas\npel\npels\nper\nperò\nperquè\npoc \npoca\npocs\npoques\npotser\npropi\nqual\nquals\nquan\nquant \nque\nquè\nquelcom\nqui\nquin\nquina\nquines\nquins\ns'ha\ns'han\nsa\nsemblant\nsemblants\nses\nseu \nseus\nseva\nseva\nseves\nsi\nsobre\nsobretot\nsóc\nsolament\nsols\nson \nsón\nsons \nsota\nsou\nt'ha\nt'han\nt'he\nta\ntal\ntambé\ntampoc\ntan\ntant\ntanta\ntantes\nteu\nteus\nteva\nteves\nton\ntons\ntot\ntota\ntotes\ntots\nun\nuna\nunes\nuns\nus\nva\nvaig\nvam\nvan\nvas\nveu\nvosaltres\nvostra\nvostre\nvostres\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(CatalanStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/cjk/analyzer_cjk.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cjk\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"cjk\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\twidthFilter, err := cache.TokenFilterNamed(WidthName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbigramFilter, err := cache.TokenFilterNamed(BigramName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\twidthFilter,\n\t\t\ttoLowerFilter,\n\t\t\tbigramFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/cjk/analyzer_cjk_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cjk\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestCJKAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: []byte(\"こんにちは世界\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こん\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"んに\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"にち\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ちは\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は世\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世界\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"一二三四五六七八九十\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"一二\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"二三\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"三四\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"四五\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"五六\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"六七\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"七八\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"八九\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 8,\n\t\t\t\t\tStart:    21,\n\t\t\t\t\tEnd:      27,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"九十\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 9,\n\t\t\t\t\tStart:    24,\n\t\t\t\t\tEnd:      30,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"一 二三四 五六七八九 十\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"一\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"二三\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    4,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"三四\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    7,\n\t\t\t\t\tEnd:      13,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"五六\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    14,\n\t\t\t\t\tEnd:      20,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"六七\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    17,\n\t\t\t\t\tEnd:      23,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"七八\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    20,\n\t\t\t\t\tEnd:      26,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"八九\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    23,\n\t\t\t\t\tEnd:      29,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"十\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 8,\n\t\t\t\t\tStart:    30,\n\t\t\t\t\tEnd:      33,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"abc defgh ijklmn opqrstu vwxy z\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"abc\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"defgh\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    4,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ijklmn\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    10,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"opqrstu\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    17,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"vwxy\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    25,\n\t\t\t\t\tEnd:      29,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"z\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    30,\n\t\t\t\t\tEnd:      31,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"あい\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"あい\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"あい   \"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"あい\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"test\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"test   \"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"あいtest\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"あい\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"testあい    \"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"あい\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    4,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"あいうえおabcかきくけこ\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"あい\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"いう\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"うえ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"えお\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"abc\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"かき\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"きく\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    21,\n\t\t\t\t\tEnd:      27,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"くけ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 8,\n\t\t\t\t\tStart:    24,\n\t\t\t\t\tEnd:      30,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"けこ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 9,\n\t\t\t\t\tStart:    27,\n\t\t\t\t\tEnd:      33,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"あいうえおabんcかきくけ こ\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"あい\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"いう\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"うえ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"えお\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ab\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      17,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    17,\n\t\t\t\t\tEnd:      20,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"c\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    20,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"かき\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 8,\n\t\t\t\t\tStart:    21,\n\t\t\t\t\tEnd:      27,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"きく\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 9,\n\t\t\t\t\tStart:    24,\n\t\t\t\t\tEnd:      30,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"くけ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 10,\n\t\t\t\t\tStart:    27,\n\t\t\t\t\tEnd:      33,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 11,\n\t\t\t\t\tStart:    34,\n\t\t\t\t\tEnd:      37,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"一 روبرت موير\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"一\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"روبرت\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    4,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"موير\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      23,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"一 رُوبرت موير\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"一\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"رُوبرت\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    4,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"موير\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    17,\n\t\t\t\t\tEnd:      25,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"𩬅艱鍟䇹愯瀛\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"𩬅艱\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      7,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"艱鍟\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    4,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"鍟䇹\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    7,\n\t\t\t\t\tEnd:      13,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"䇹愯\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    10,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"愯瀛\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    13,\n\t\t\t\t\tEnd:      19,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"一\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"一\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"一丁丂\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"一丁\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"丁丂\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfor _, test := range tests {\n\t\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.output, actual)\n\t\t}\n\t}\n}\n\nfunc BenchmarkCJKAnalyzer(b *testing.B) {\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tfor i := 0; i < b.N; i++ {\n\t\tanalyzer.Analyze(bleveWikiArticleJapanese)\n\t}\n}\n\nvar bleveWikiArticleJapanese = []byte(`加圧容器に貯蔵されている液体物質は、その時の気液平衡状態にあるが、火災により容器が加熱されていると容器内の液体は、その物質の大気圧のもとでの沸点より十分に高い温度まで加熱され、圧力も高くなる。この状態で容器が破裂すると容器内部の圧力は瞬間的に大気圧にまで低下する。\nこの時に容器内の平衡状態が破られ、液体は突沸し、気体になることで爆発現象を起こす。液化石油ガスなどでは、さらに拡散して空気と混ざったガスが自由空間蒸気雲爆発を起こす。液化石油ガスなどの常温常圧で気体になる物を高い圧力で液化して収納している容器、あるいは、そのような液体を輸送するためのパイプラインや配管などが火災などによって破壊されたときに起きる。\nブリーブという現象が明らかになったのは、フランス・リヨンの郊外にあるフェザンという町のフェザン製油所（ウニオン・ド・ゼネラル・ド・ペトロール）で大規模な爆発火災事故が発生したときだと言われている。\n中身の液体が高温高圧の水である場合には「水蒸気爆発」と呼ばれる。`)\n"
  },
  {
    "path": "analysis/lang/cjk/cjk_bigram.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cjk\n\nimport (\n\t\"bytes\"\n\t\"container/ring\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst BigramName = \"cjk_bigram\"\n\ntype CJKBigramFilter struct {\n\toutputUnigram bool\n}\n\nfunc NewCJKBigramFilter(outputUnigram bool) *CJKBigramFilter {\n\treturn &CJKBigramFilter{\n\t\toutputUnigram: outputUnigram,\n\t}\n}\n\nfunc (s *CJKBigramFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tr := ring.New(2)\n\titemsInRing := 0\n\tpos := 1\n\toutputPos := 1\n\n\trv := make(analysis.TokenStream, 0, len(input))\n\n\tfor _, tokout := range input {\n\t\tif tokout.Type == analysis.Ideographic {\n\t\t\trunes := bytes.Runes(tokout.Term)\n\t\t\tsofar := 0\n\t\t\tfor _, run := range runes {\n\t\t\t\trlen := utf8.RuneLen(run)\n\t\t\t\ttoken := &analysis.Token{\n\t\t\t\t\tTerm:     tokout.Term[sofar : sofar+rlen],\n\t\t\t\t\tStart:    tokout.Start + sofar,\n\t\t\t\t\tEnd:      tokout.Start + sofar + rlen,\n\t\t\t\t\tPosition: pos,\n\t\t\t\t\tType:     tokout.Type,\n\t\t\t\t\tKeyWord:  tokout.KeyWord,\n\t\t\t\t}\n\t\t\t\tpos++\n\t\t\t\tsofar += rlen\n\t\t\t\tif itemsInRing > 0 {\n\t\t\t\t\t// if items already buffered\n\t\t\t\t\t// check to see if this is aligned\n\t\t\t\t\tcurr := r.Value.(*analysis.Token)\n\t\t\t\t\tif token.Start-curr.End != 0 {\n\t\t\t\t\t\t// not aligned flush\n\t\t\t\t\t\tflushToken := s.flush(r, &itemsInRing, outputPos)\n\t\t\t\t\t\tif flushToken != nil {\n\t\t\t\t\t\t\toutputPos++\n\t\t\t\t\t\t\trv = append(rv, flushToken)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// now we can add this token to the buffer\n\t\t\t\tr = r.Next()\n\t\t\t\tr.Value = token\n\t\t\t\tif itemsInRing < 2 {\n\t\t\t\t\titemsInRing++\n\t\t\t\t}\n\t\t\t\tbuiltUnigram := false\n\t\t\t\tif itemsInRing > 1 && s.outputUnigram {\n\t\t\t\t\tunigram := s.buildUnigram(r, &itemsInRing, outputPos)\n\t\t\t\t\tif unigram != nil {\n\t\t\t\t\t\tbuiltUnigram = true\n\t\t\t\t\t\trv = append(rv, unigram)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tbigramToken := s.outputBigram(r, &itemsInRing, outputPos)\n\t\t\t\tif bigramToken != nil {\n\t\t\t\t\trv = append(rv, bigramToken)\n\t\t\t\t\toutputPos++\n\t\t\t\t}\n\n\t\t\t\t// prev token should be removed if unigram was built\n\t\t\t\tif builtUnigram {\n\t\t\t\t\titemsInRing--\n\t\t\t\t}\n\t\t\t}\n\n\t\t} else {\n\t\t\t// flush anything already buffered\n\t\t\tflushToken := s.flush(r, &itemsInRing, outputPos)\n\t\t\tif flushToken != nil {\n\t\t\t\trv = append(rv, flushToken)\n\t\t\t\toutputPos++\n\t\t\t}\n\t\t\t// output this token as is\n\t\t\ttokout.Position = outputPos\n\t\t\trv = append(rv, tokout)\n\t\t\toutputPos++\n\t\t}\n\t}\n\n\t// deal with possible trailing unigram\n\tif itemsInRing == 1 || s.outputUnigram {\n\t\tif itemsInRing == 2 {\n\t\t\tr = r.Next()\n\t\t}\n\t\tunigram := s.buildUnigram(r, &itemsInRing, outputPos)\n\t\tif unigram != nil {\n\t\t\trv = append(rv, unigram)\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (s *CJKBigramFilter) flush(r *ring.Ring, itemsInRing *int, pos int) *analysis.Token {\n\tvar rv *analysis.Token\n\tif *itemsInRing == 1 {\n\t\trv = s.buildUnigram(r, itemsInRing, pos)\n\t}\n\tr.Value = nil\n\t*itemsInRing = 0\n\n\treturn rv\n}\n\nfunc (s *CJKBigramFilter) outputBigram(r *ring.Ring, itemsInRing *int, pos int) *analysis.Token {\n\tif *itemsInRing == 2 {\n\t\tthisShingleRing := r.Move(-1)\n\t\tshingledBytes := make([]byte, 0)\n\n\t\t// do first token\n\t\tprev := thisShingleRing.Value.(*analysis.Token)\n\t\tshingledBytes = append(shingledBytes, prev.Term...)\n\n\t\t// do second token\n\t\tthisShingleRing = thisShingleRing.Next()\n\t\tcurr := thisShingleRing.Value.(*analysis.Token)\n\t\tshingledBytes = append(shingledBytes, curr.Term...)\n\n\t\ttoken := analysis.Token{\n\t\t\tType:     analysis.Double,\n\t\t\tTerm:     shingledBytes,\n\t\t\tPosition: pos,\n\t\t\tStart:    prev.Start,\n\t\t\tEnd:      curr.End,\n\t\t}\n\t\treturn &token\n\t}\n\n\treturn nil\n}\n\nfunc (s *CJKBigramFilter) buildUnigram(r *ring.Ring, itemsInRing *int, pos int) *analysis.Token {\n\tswitch *itemsInRing {\n\tcase 2:\n\t\tthisShingleRing := r.Move(-1)\n\t\t// do first token\n\t\tprev := thisShingleRing.Value.(*analysis.Token)\n\t\ttoken := analysis.Token{\n\t\t\tType:     analysis.Single,\n\t\t\tTerm:     prev.Term,\n\t\t\tPosition: pos,\n\t\t\tStart:    prev.Start,\n\t\t\tEnd:      prev.End,\n\t\t}\n\t\treturn &token\n\tcase 1:\n\t\t// do first token\n\t\tprev := r.Value.(*analysis.Token)\n\t\ttoken := analysis.Token{\n\t\t\tType:     analysis.Single,\n\t\t\tTerm:     prev.Term,\n\t\t\tPosition: pos,\n\t\t\tStart:    prev.Start,\n\t\t\tEnd:      prev.End,\n\t\t}\n\t\treturn &token\n\t}\n\n\treturn nil\n}\n\nfunc CJKBigramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\toutputUnigram := false\n\toutVal, ok := config[\"output_unigram\"].(bool)\n\tif ok {\n\t\toutputUnigram = outVal\n\t}\n\treturn NewCJKBigramFilter(outputUnigram), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(BigramName, CJKBigramFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/cjk/cjk_bigram_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cjk\n\nimport (\n\t\"container/ring\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\n// Helper function to create a token\nfunc makeToken(term string, start, end, pos int) *analysis.Token {\n\treturn &analysis.Token{\n\t\tTerm:     []byte(term),\n\t\tStart:    start,\n\t\tEnd:      end,\n\t\tPosition: pos, // Note: buildUnigram uses the 'pos' argument, not the token's original pos\n\t\tType:     analysis.Ideographic,\n\t}\n}\n\nfunc TestCJKBigramFilter_buildUnigram(t *testing.T) {\n\tfilter := NewCJKBigramFilter(false)\n\n\ttests := []struct {\n\t\tname        string\n\t\tringSetup   func() (*ring.Ring, int) // Function to set up the ring and itemsInRing\n\t\tinputPos    int                      // Position to pass to buildUnigram\n\t\texpectToken *analysis.Token\n\t}{\n\t\t{\n\t\t\tname: \"itemsInRing == 2\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\ttoken1 := makeToken(\"一\", 0, 3, 1) // Original pos 1\n\t\t\t\ttoken2 := makeToken(\"二\", 3, 6, 2) // Original pos 2\n\t\t\t\tr.Value = token1\n\t\t\t\tr = r.Next()\n\t\t\t\tr.Value = token2\n\t\t\t\t// r currently points to token2, r.Move(-1) points to token1\n\t\t\t\treturn r, 2\n\t\t\t},\n\t\t\tinputPos: 10, // Expected output position\n\t\t\texpectToken: &analysis.Token{\n\t\t\t\tType:     analysis.Single,\n\t\t\t\tTerm:     []byte(\"一\"),\n\t\t\t\tPosition: 10, // Should use inputPos\n\t\t\t\tStart:    0,\n\t\t\t\tEnd:      3,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"itemsInRing == 1 (ring points to the single item)\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\ttoken1 := makeToken(\"三\", 6, 9, 3)\n\t\t\t\tr.Value = token1\n\t\t\t\t// r points to token1\n\t\t\t\treturn r, 1\n\t\t\t},\n\t\t\tinputPos: 11,\n\t\t\texpectToken: &analysis.Token{\n\t\t\t\tType:     analysis.Single,\n\t\t\t\tTerm:     []byte(\"三\"),\n\t\t\t\tPosition: 11, // Should use inputPos\n\t\t\t\tStart:    6,\n\t\t\t\tEnd:      9,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"itemsInRing == 1 (ring points to nil, next is the single item)\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\ttoken1 := makeToken(\"四\", 9, 12, 4)\n\t\t\t\tr = r.Next() // r points to nil initially\n\t\t\t\tr.Value = token1\n\t\t\t\t// r points to token1\n\t\t\t\treturn r, 1\n\t\t\t},\n\t\t\tinputPos: 12,\n\t\t\texpectToken: &analysis.Token{\n\t\t\t\tType:     analysis.Single,\n\t\t\t\tTerm:     []byte(\"四\"),\n\t\t\t\tPosition: 12, // Should use inputPos\n\t\t\t\tStart:    9,\n\t\t\t\tEnd:      12,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"itemsInRing == 0\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\t// Ring is empty\n\t\t\t\treturn r, 0\n\t\t\t},\n\t\t\tinputPos:    13,\n\t\t\texpectToken: nil, // Expect nil when itemsInRing is not 1 or 2\n\t\t},\n\t\t{\n\t\t\tname: \"itemsInRing > 2 (should behave like 0)\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\ttoken1 := makeToken(\"五\", 12, 15, 5)\n\t\t\t\ttoken2 := makeToken(\"六\", 15, 18, 6)\n\t\t\t\tr.Value = token1\n\t\t\t\tr = r.Next()\n\t\t\t\tr.Value = token2\n\t\t\t\t// Simulate incorrect itemsInRing count\n\t\t\t\treturn r, 3\n\t\t\t},\n\t\t\tinputPos:    14,\n\t\t\texpectToken: nil, // Expect nil when itemsInRing is not 1 or 2\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tringPtr, itemsInRing := tt.ringSetup()\n\t\t\titemsInRingCopy := itemsInRing // Pass a pointer to a copy\n\n\t\t\tgotToken := filter.buildUnigram(ringPtr, &itemsInRingCopy, tt.inputPos)\n\n\t\t\tif !reflect.DeepEqual(gotToken, tt.expectToken) {\n\t\t\t\tt.Errorf(\"buildUnigram() got = %v, want %v\", gotToken, tt.expectToken)\n\t\t\t}\n\n\t\t\t// Check if itemsInRing was modified (it shouldn't be by buildUnigram)\n\t\t\tif itemsInRingCopy != itemsInRing {\n\t\t\t\tt.Errorf(\"buildUnigram() modified itemsInRing, got = %d, want %d\", itemsInRingCopy, itemsInRing)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestCJKBigramFilter_outputBigram(t *testing.T) {\n\t// Create a filter instance (outputUnigram value doesn't matter for outputBigram)\n\tfilter := NewCJKBigramFilter(false)\n\n\ttests := []struct {\n\t\tname        string\n\t\tringSetup   func() (*ring.Ring, int) // Function to set up the ring and itemsInRing\n\t\tinputPos    int                      // Position to pass to outputBigram\n\t\texpectToken *analysis.Token\n\t}{\n\t\t{\n\t\t\tname: \"itemsInRing == 2\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\ttoken1 := makeToken(\"一\", 0, 3, 1) // Original pos 1\n\t\t\t\ttoken2 := makeToken(\"二\", 3, 6, 2) // Original pos 2\n\t\t\t\tr.Value = token1\n\t\t\t\tr = r.Next()\n\t\t\t\tr.Value = token2\n\t\t\t\t// r currently points to token2, r.Move(-1) points to token1\n\t\t\t\treturn r, 2\n\t\t\t},\n\t\t\tinputPos: 10, // Expected output position\n\t\t\texpectToken: &analysis.Token{\n\t\t\t\tType:     analysis.Double,\n\t\t\t\tTerm:     []byte(\"一二\"), // Combined term\n\t\t\t\tPosition: 10,           // Should use inputPos\n\t\t\t\tStart:    0,            // Start of first token\n\t\t\t\tEnd:      6,            // End of second token\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"itemsInRing == 2 with different terms\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\ttoken1 := makeToken(\"你好\", 0, 6, 1)\n\t\t\t\ttoken2 := makeToken(\"世界\", 6, 12, 2)\n\t\t\t\tr.Value = token1\n\t\t\t\tr = r.Next()\n\t\t\t\tr.Value = token2\n\t\t\t\treturn r, 2\n\t\t\t},\n\t\t\tinputPos: 5,\n\t\t\texpectToken: &analysis.Token{\n\t\t\t\tType:     analysis.Double,\n\t\t\t\tTerm:     []byte(\"你好世界\"),\n\t\t\t\tPosition: 5,\n\t\t\t\tStart:    0,\n\t\t\t\tEnd:      12,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"itemsInRing == 1\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\ttoken1 := makeToken(\"三\", 6, 9, 3)\n\t\t\t\tr.Value = token1\n\t\t\t\treturn r, 1\n\t\t\t},\n\t\t\tinputPos:    11,\n\t\t\texpectToken: nil, // Expect nil when itemsInRing is not 2\n\t\t},\n\t\t{\n\t\t\tname: \"itemsInRing == 0\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\t// Ring is empty\n\t\t\t\treturn r, 0\n\t\t\t},\n\t\t\tinputPos:    13,\n\t\t\texpectToken: nil, // Expect nil when itemsInRing is not 2\n\t\t},\n\t\t{\n\t\t\tname: \"itemsInRing > 2 (should behave like 0)\",\n\t\t\tringSetup: func() (*ring.Ring, int) {\n\t\t\t\tr := ring.New(2)\n\t\t\t\ttoken1 := makeToken(\"五\", 12, 15, 5)\n\t\t\t\ttoken2 := makeToken(\"六\", 15, 18, 6)\n\t\t\t\tr.Value = token1\n\t\t\t\tr = r.Next()\n\t\t\t\tr.Value = token2\n\t\t\t\t// Simulate incorrect itemsInRing count\n\t\t\t\treturn r, 3\n\t\t\t},\n\t\t\tinputPos:    14,\n\t\t\texpectToken: nil, // Expect nil when itemsInRing is not 2\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tringPtr, itemsInRing := tt.ringSetup()\n\t\t\titemsInRingCopy := itemsInRing // Pass a pointer to a copy\n\n\t\t\tgotToken := filter.outputBigram(ringPtr, &itemsInRingCopy, tt.inputPos)\n\n\t\t\tif !reflect.DeepEqual(gotToken, tt.expectToken) {\n\t\t\t\tt.Errorf(\"outputBigram() got = %v, want %v\", gotToken, tt.expectToken)\n\t\t\t}\n\n\t\t\t// Check if itemsInRing was modified (it shouldn't be by outputBigram)\n\t\t\tif itemsInRingCopy != itemsInRing {\n\t\t\t\tt.Errorf(\"outputBigram() modified itemsInRing, got = %d, want %d\", itemsInRingCopy, itemsInRing)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestCJKBigramFilter(t *testing.T) {\n\ttests := []struct {\n\t\toutputUnigram bool\n\t\tinput         analysis.TokenStream\n\t\toutput        analysis.TokenStream\n\t}{\n\t\t// first test that non-adjacent terms are not combined\n\t\t{\n\t\t\toutputUnigram: false,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    5,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    5,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\toutputUnigram: false,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"に\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ち\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"界\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こん\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"んに\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"にち\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ちは\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は世\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世界\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\toutputUnigram: true,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"に\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ち\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"界\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こん\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"んに\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"に\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"にち\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ち\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ちは\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は世\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世界\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"界\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t// Assuming that `、` is removed by unicode tokenizer from `こんにちは、世界`\n\t\t\toutputUnigram: true,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"に\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ち\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"界\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 8,\n\t\t\t\t\tStart:    21,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こん\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"んに\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"に\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"にち\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ち\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ちは\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世界\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"界\"),\n\t\t\t\t\tType:     analysis.Single,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    21,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\toutputUnigram: false,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"に\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ち\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"は\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"cat\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"界\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 8,\n\t\t\t\t\tStart:    21,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"こん\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"んに\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"にち\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ちは\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"cat\"),\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"世界\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\toutputUnigram: false,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"パイプライン\"),\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"パイ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"イプ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"プラ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ライ\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"イン\"),\n\t\t\t\t\tType:     analysis.Double,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tcjkBigramFilter := NewCJKBigramFilter(test.outputUnigram)\n\t\tactual := cjkBigramFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/cjk/cjk_width.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cjk\n\nimport (\n\t\"bytes\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst WidthName = \"cjk_width\"\n\ntype CJKWidthFilter struct{}\n\nfunc NewCJKWidthFilter() *CJKWidthFilter {\n\treturn &CJKWidthFilter{}\n}\n\nfunc (s *CJKWidthFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\truneCount := utf8.RuneCount(token.Term)\n\t\trunes := bytes.Runes(token.Term)\n\t\tfor i := 0; i < runeCount; i++ {\n\t\t\tch := runes[i]\n\t\t\tif ch >= 0xFF01 && ch <= 0xFF5E {\n\t\t\t\t// fullwidth ASCII variants\n\t\t\t\trunes[i] -= 0xFEE0\n\t\t\t} else if ch >= 0xFF65 && ch <= 0xFF9F {\n\t\t\t\t// halfwidth Katakana variants\n\t\t\t\tif (ch == 0xFF9E || ch == 0xFF9F) && i > 0 && combine(runes, i, ch) {\n\t\t\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\t\t\ti--\n\t\t\t\t\truneCount = len(runes)\n\t\t\t\t} else {\n\t\t\t\t\trunes[i] = kanaNorm[ch-0xFF65]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\ttoken.Term = analysis.BuildTermFromRunes(runes)\n\t}\n\n\treturn input\n}\n\nvar kanaNorm = []rune{\n\t0x30fb, 0x30f2, 0x30a1, 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5,\n\t0x30e7, 0x30c3, 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab,\n\t0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd,\n\t0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd,\n\t0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, 0x30e0,\n\t0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec,\n\t0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309A,\n}\n\nvar kanaCombineVoiced = []rune{\n\t78, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n\t0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1,\n\t0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n}\nvar kanaCombineHalfVoiced = []rune{\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2,\n\t0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n}\n\nfunc combine(text []rune, pos int, r rune) bool {\n\tprev := text[pos-1]\n\tif prev >= 0x30A6 && prev <= 0x30FD {\n\t\tif r == 0xFF9F {\n\t\t\ttext[pos-1] += kanaCombineHalfVoiced[prev-0x30A6]\n\t\t} else {\n\t\t\ttext[pos-1] += kanaCombineVoiced[prev-0x30A6]\n\t\t}\n\t\treturn text[pos-1] != prev\n\t}\n\treturn false\n}\n\nfunc CJKWidthFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewCJKWidthFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(WidthName, CJKWidthFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/cjk/cjk_width_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cjk\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestCJKWidthFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Ｔｅｓｔ\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"１２３４\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Test\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"1234\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ｶﾀｶﾅ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"カタカナ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ｳﾞｨｯﾂ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ヴィッツ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ﾊﾟﾅｿﾆｯｸ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"パナソニック\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tcjkWidthFilter := NewCJKWidthFilter()\n\t\tactual := cjkWidthFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ckb/analyzer_ckb.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ckb\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst AnalyzerName = \"ckb\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnormCkbFilter, err := cache.TokenFilterNamed(NormalizeName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopCkbFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerCkbFilter, err := cache.TokenFilterNamed(StemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\tnormCkbFilter,\n\t\t\ttoLowerFilter,\n\t\t\tstopCkbFilter,\n\t\t\tstemmerCkbFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ckb/analyzer_ckb_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ckb\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSoraniAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stop word removal\n\t\t{\n\t\t\tinput: []byte(\"ئەم پیاوە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پیاو\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    7,\n\t\t\t\t\tEnd:      17,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"پیاوە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پیاو\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"پیاو\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پیاو\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ckb/sorani_normalize.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ckb\n\nimport (\n\t\"bytes\"\n\t\"unicode\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst NormalizeName = \"normalize_ckb\"\n\nconst (\n\tYeh        = '\\u064A'\n\tDotlessYeh = '\\u0649'\n\tFarsiYeh   = '\\u06CC'\n\n\tKaf   = '\\u0643'\n\tKeheh = '\\u06A9'\n\n\tHeh            = '\\u0647'\n\tAe             = '\\u06D5'\n\tZwnj           = '\\u200C'\n\tHehDoachashmee = '\\u06BE'\n\tTehMarbuta     = '\\u0629'\n\n\tReh       = '\\u0631'\n\tRreh      = '\\u0695'\n\tRrehAbove = '\\u0692'\n\n\tTatweel  = '\\u0640'\n\tFathatan = '\\u064B'\n\tDammatan = '\\u064C'\n\tKasratan = '\\u064D'\n\tFatha    = '\\u064E'\n\tDamma    = '\\u064F'\n\tKasra    = '\\u0650'\n\tShadda   = '\\u0651'\n\tSukun    = '\\u0652'\n)\n\ntype SoraniNormalizeFilter struct {\n}\n\nfunc NewSoraniNormalizeFilter() *SoraniNormalizeFilter {\n\treturn &SoraniNormalizeFilter{}\n}\n\nfunc (s *SoraniNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := normalize(token.Term)\n\t\ttoken.Term = term\n\t}\n\treturn input\n}\n\nfunc normalize(input []byte) []byte {\n\trunes := bytes.Runes(input)\n\tfor i := 0; i < len(runes); i++ {\n\t\tswitch runes[i] {\n\t\tcase Yeh, DotlessYeh:\n\t\t\trunes[i] = FarsiYeh\n\t\tcase Kaf:\n\t\t\trunes[i] = Keheh\n\t\tcase Zwnj:\n\t\t\tif i > 0 && runes[i-1] == Heh {\n\t\t\t\trunes[i-1] = Ae\n\t\t\t}\n\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\ti--\n\t\tcase Heh:\n\t\t\tif i == len(runes)-1 {\n\t\t\t\trunes[i] = Ae\n\t\t\t}\n\t\tcase TehMarbuta:\n\t\t\trunes[i] = Ae\n\t\tcase HehDoachashmee:\n\t\t\trunes[i] = Heh\n\t\tcase Reh:\n\t\t\tif i == 0 {\n\t\t\t\trunes[i] = Rreh\n\t\t\t}\n\t\tcase RrehAbove:\n\t\t\trunes[i] = Rreh\n\t\tcase Tatweel, Kasratan, Dammatan, Fathatan, Fatha, Damma, Kasra, Shadda, Sukun:\n\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\ti--\n\t\tdefault:\n\t\t\tif unicode.In(runes[i], unicode.Cf) {\n\t\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\t\ti--\n\t\t\t}\n\t\t}\n\t}\n\treturn analysis.BuildTermFromRunes(runes)\n}\n\nfunc NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewSoraniNormalizeFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ckb/sorani_normalize_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ckb\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestSoraniNormalizeFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// test Y\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u064A\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06CC\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0649\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06CC\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06CC\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06CC\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test K\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0643\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06A9\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06A9\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06A9\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test H\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0647\\u200C\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06D5\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0647\\u200C\\u06A9\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06D5\\u06A9\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06BE\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0647\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0629\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u06D5\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test final H\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0647\\u0647\\u0647\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0647\\u0647\\u06D5\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test RR\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0692\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0695\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test initial RR\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0631\\u0631\\u0631\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0695\\u0631\\u0631\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test remove\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0640\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u064B\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u064C\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u064D\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u064E\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u064F\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0650\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0651\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0652\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u200C\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// empty\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tsoraniNormalizeFilter := NewSoraniNormalizeFilter()\n\tfor _, test := range tests {\n\t\tactual := soraniNormalizeFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected % x, got % x\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ckb/sorani_stemmer_filter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ckb\n\nimport (\n\t\"bytes\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StemmerName = \"stemmer_ckb\"\n\ntype SoraniStemmerFilter struct {\n}\n\nfunc NewSoraniStemmerFilter() *SoraniStemmerFilter {\n\treturn &SoraniStemmerFilter{}\n}\n\nfunc (s *SoraniStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\t// if not protected keyword, stem it\n\t\tif !token.KeyWord {\n\t\t\tstemmed := stem(token.Term)\n\t\t\ttoken.Term = stemmed\n\t\t}\n\t}\n\treturn input\n}\n\nfunc stem(input []byte) []byte {\n\tinputLen := utf8.RuneCount(input)\n\n\t// postposition\n\tif inputLen > 5 && bytes.HasSuffix(input, []byte(\"دا\")) {\n\t\tinput = truncateRunes(input, 2)\n\t\tinputLen = utf8.RuneCount(input)\n\t} else if inputLen > 4 && bytes.HasSuffix(input, []byte(\"نا\")) {\n\t\tinput = truncateRunes(input, 1)\n\t\tinputLen = utf8.RuneCount(input)\n\t} else if inputLen > 6 && bytes.HasSuffix(input, []byte(\"ەوە\")) {\n\t\tinput = truncateRunes(input, 3)\n\t\tinputLen = utf8.RuneCount(input)\n\t}\n\n\t// possessive pronoun\n\tif inputLen > 6 &&\n\t\t(bytes.HasSuffix(input, []byte(\"مان\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"یان\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"تان\"))) {\n\t\tinput = truncateRunes(input, 3)\n\t\tinputLen = utf8.RuneCount(input)\n\t}\n\n\t// indefinite singular ezafe\n\tif inputLen > 6 && bytes.HasSuffix(input, []byte(\"ێکی\")) {\n\t\treturn truncateRunes(input, 3)\n\t} else if inputLen > 7 && bytes.HasSuffix(input, []byte(\"یەکی\")) {\n\t\treturn truncateRunes(input, 4)\n\t}\n\n\tif inputLen > 5 && bytes.HasSuffix(input, []byte(\"ێک\")) {\n\t\t// indefinite singular\n\t\treturn truncateRunes(input, 2)\n\t} else if inputLen > 6 && bytes.HasSuffix(input, []byte(\"یەک\")) {\n\t\t// indefinite singular\n\t\treturn truncateRunes(input, 3)\n\t} else if inputLen > 6 && bytes.HasSuffix(input, []byte(\"ەکە\")) {\n\t\t// definite singular\n\t\treturn truncateRunes(input, 3)\n\t} else if inputLen > 5 && bytes.HasSuffix(input, []byte(\"کە\")) {\n\t\t// definite singular\n\t\treturn truncateRunes(input, 2)\n\t} else if inputLen > 7 && bytes.HasSuffix(input, []byte(\"ەکان\")) {\n\t\t// definite plural\n\t\treturn truncateRunes(input, 4)\n\t} else if inputLen > 6 && bytes.HasSuffix(input, []byte(\"کان\")) {\n\t\t// definite plural\n\t\treturn truncateRunes(input, 3)\n\t} else if inputLen > 7 && bytes.HasSuffix(input, []byte(\"یانی\")) {\n\t\t// indefinite plural ezafe\n\t\treturn truncateRunes(input, 4)\n\t} else if inputLen > 6 && bytes.HasSuffix(input, []byte(\"انی\")) {\n\t\t// indefinite plural ezafe\n\t\treturn truncateRunes(input, 3)\n\t} else if inputLen > 6 && bytes.HasSuffix(input, []byte(\"یان\")) {\n\t\t// indefinite plural\n\t\treturn truncateRunes(input, 3)\n\t} else if inputLen > 5 && bytes.HasSuffix(input, []byte(\"ان\")) {\n\t\t// indefinite plural\n\t\treturn truncateRunes(input, 2)\n\t} else if inputLen > 7 && bytes.HasSuffix(input, []byte(\"یانە\")) {\n\t\t// demonstrative plural\n\t\treturn truncateRunes(input, 4)\n\t} else if inputLen > 6 && bytes.HasSuffix(input, []byte(\"انە\")) {\n\t\t// demonstrative plural\n\t\treturn truncateRunes(input, 3)\n\t} else if inputLen > 5 && (bytes.HasSuffix(input, []byte(\"ایە\")) || bytes.HasSuffix(input, []byte(\"ەیە\"))) {\n\t\t// demonstrative singular\n\t\treturn truncateRunes(input, 2)\n\t} else if inputLen > 4 && bytes.HasSuffix(input, []byte(\"ە\")) {\n\t\t// demonstrative singular\n\t\treturn truncateRunes(input, 1)\n\t} else if inputLen > 4 && bytes.HasSuffix(input, []byte(\"ی\")) {\n\t\t// absolute singular ezafe\n\t\treturn truncateRunes(input, 1)\n\t}\n\treturn input\n}\n\nfunc truncateRunes(input []byte, num int) []byte {\n\trunes := bytes.Runes(input)\n\trunes = runes[:len(runes)-num]\n\tout := buildTermFromRunes(runes)\n\treturn out\n}\n\nfunc buildTermFromRunes(runes []rune) []byte {\n\trv := make([]byte, 0, len(runes)*4)\n\tfor _, r := range runes {\n\t\truneBytes := make([]byte, utf8.RuneLen(r))\n\t\tutf8.EncodeRune(runeBytes, r)\n\t\trv = append(rv, runeBytes...)\n\t}\n\treturn rv\n}\n\nfunc StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewSoraniStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ckb/sorani_stemmer_filter_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ckb\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/single\"\n)\n\nfunc TestSoraniStemmerFilter(t *testing.T) {\n\n\t// in order to match the lucene tests\n\t// we will test with an analyzer, not just the stemmer\n\tanalyzer := analysis.DefaultAnalyzer{\n\t\tTokenizer: single.NewSingleTokenTokenizer(),\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\tNewSoraniNormalizeFilter(),\n\t\t\tNewSoraniStemmerFilter(),\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{ // -ek\n\t\t\tinput: []byte(\"پیاوێک\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پیاو\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -yek\n\t\t\tinput: []byte(\"دەرگایەک\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"دەرگا\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -aka\n\t\t\tinput: []byte(\"پیاوەكە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پیاو\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -ka\n\t\t\tinput: []byte(\"دەرگاكە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"دەرگا\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -a\n\t\t\tinput: []byte(\"کتاویە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"کتاوی\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -ya\n\t\t\tinput: []byte(\"دەرگایە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"دەرگا\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -An\n\t\t\tinput: []byte(\"پیاوان\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پیاو\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -yAn\n\t\t\tinput: []byte(\"دەرگایان\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"دەرگا\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -akAn\n\t\t\tinput: []byte(\"پیاوەکان\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پیاو\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -kAn\n\t\t\tinput: []byte(\"دەرگاکان\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"دەرگا\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -Ana\n\t\t\tinput: []byte(\"پیاوانە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پیاو\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -yAna\n\t\t\tinput: []byte(\"دەرگایانە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"دەرگا\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // Ezafe singular\n\t\t\tinput: []byte(\"هۆتیلی\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"هۆتیل\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // Ezafe indefinite\n\t\t\tinput: []byte(\"هۆتیلێکی\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"هۆتیل\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // Ezafe plural\n\t\t\tinput: []byte(\"هۆتیلانی\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"هۆتیل\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -awa\n\t\t\tinput: []byte(\"دوورەوە\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"دوور\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -dA\n\t\t\tinput: []byte(\"نیوەشەودا\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"نیوەشەو\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -A\n\t\t\tinput: []byte(\"سۆرانا\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"سۆران\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -mAn\n\t\t\tinput: []byte(\"پارەمان\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پارە\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -tAn\n\t\t\tinput: []byte(\"پارەتان\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پارە\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // -yAn\n\t\t\tinput: []byte(\"پارەیان\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"پارە\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{ // empty\n\t\t\tinput: []byte(\"\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      0,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"for input %s(% x)\", test.input, test.input)\n\t\t\tt.Errorf(\"\\texpected:\")\n\t\t\tfor _, token := range test.output {\n\t\t\t\tt.Errorf(\"\\t\\t%v %s(% x)\", token, token.Term, token.Term)\n\t\t\t}\n\t\t\tt.Errorf(\"\\tactual:\")\n\t\t\tfor _, token := range actual {\n\t\t\t\tt.Errorf(\"\\t\\t%v %s(% x)\", token, token.Term, token.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ckb/stop_filter_ckb.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ckb\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ckb/stop_words_ckb.go",
    "content": "package ckb\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_ckb\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar SoraniStopWords = []byte(`# set of kurdish stopwords\n# note these have been normalized with our scheme (e represented with U+06D5, etc)\n# constructed from:\n# * Fig 5 of \"Building A Test Collection For Sorani Kurdish\" (Esmaili et al)\n# * \"Sorani Kurdish: A Reference Grammar with selected readings\" (Thackston)\n# * Corpus-based analysis of 77M word Sorani collection: wikipedia, news, blogs, etc\n\n# and\nو\n# which\nکە\n# of\nی\n# made/did\nکرد\n# that/which\nئەوەی\n# on/head\nسەر\n# two\nدوو\n# also\nهەروەها\n# from/that\nلەو\n# makes/does\nدەکات\n# some\nچەند\n# every\nهەر\n\n# demonstratives\n# that\nئەو\n# this\nئەم\n\n# personal pronouns\n# I\nمن\n# we\nئێمە\n# you\nتۆ\n# you\nئێوە\n# he/she/it\nئەو\n# they\nئەوان\n\n# prepositions\n# to/with/by\nبە\nپێ\n# without\nبەبێ\n# along with/while/during\nبەدەم\n# in the opinion of\nبەلای\n# according to\nبەپێی\n# before\nبەرلە\n# in the direction of\nبەرەوی\n# in front of/toward\nبەرەوە\n# before/in the face of\nبەردەم\n# without\nبێ\n# except for\nبێجگە\n# for\nبۆ\n# on/in\nدە\nتێ\n# with\nدەگەڵ\n# after\nدوای\n# except for/aside from\nجگە\n# in/from\nلە\nلێ\n# in front of/before/because of\nلەبەر\n# between/among\nلەبەینی\n# concerning/about\nلەبابەت\n# concerning\nلەبارەی\n# instead of\nلەباتی\n# beside\nلەبن\n# instead of\nلەبرێتی\n# behind\nلەدەم\n# with/together with\nلەگەڵ\n# by\nلەلایەن\n# within\nلەناو\n# between/among\nلەنێو\n# for the sake of\nلەپێناوی\n# with respect to\nلەرەوی\n# by means of/for\nلەرێ\n# for the sake of\nلەرێگا\n# on/on top of/according to\nلەسەر\n# under\nلەژێر\n# between/among\nناو\n# between/among\nنێوان\n# after\nپاش\n# before\nپێش\n# like\nوەک\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(SoraniStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/cs/stop_filter_cs.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cs\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/cs/stop_words_cs.go",
    "content": "package cs\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_cs\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar CzechStopWords = []byte(`a\ns\nk\no\ni\nu\nv\nz\ndnes\ncz\ntímto\nbudeš\nbudem\nbyli\njseš\nmůj\nsvým\nta\ntomto\ntohle\ntuto\ntyto\njej\nzda\nproč\nmáte\ntato\nkam\ntohoto\nkdo\nkteří\nmi\nnám\ntom\ntomuto\nmít\nnic\nproto\nkterou\nbyla\ntoho\nprotože\nasi\nho\nnaši\nnapište\nre\ncož\ntím\ntakže\nsvých\njejí\nsvými\njste\naj\ntu\ntedy\nteto\nbylo\nkde\nke\npravé\nji\nnad\nnejsou\nči\npod\ntéma\nmezi\npřes\nty\npak\nvám\nani\nkdyž\nvšak\nneg\njsem\ntento\nčlánku\nčlánky\naby\njsme\npřed\npta\njejich\nbyl\nještě\naž\nbez\ntaké\npouze\nprvní\nvaše\nkterá\nnás\nnový\ntipy\npokud\nmůže\nstrana\njeho\nsvé\njiné\nzprávy\nnové\nnení\nvás\njen\npodle\nzde\nuž\nbýt\nvíce\nbude\njiž\nnež\nkterý\nby\nkteré\nco\nnebo\nten\ntak\nmá\npři\nod\npo\njsou\njak\ndalší\nale\nsi\nse\nve\nto\njako\nza\nzpět\nze\ndo\npro\nje\nna\natd\natp\njakmile\npřičemž\njá\non\nona\nono\noni\nony\nmy\nvy\njí\nji\nmě\nmne\njemu\ntomu\ntěm\ntěmu\nněmu\nněmuž\njehož\njíž\njelikož\njež\njakož\nnačež\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(CzechStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/da/analyzer_da.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage da\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst AnalyzerName = \"da\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopDaFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerDaFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopDaFilter,\n\t\t\tstemmerDaFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/da/analyzer_da_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage da\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestDanishAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"undersøg\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"undersøg\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"undersøgelse\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"undersøg\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      13,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"på\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/da/stemmer_da.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage da\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/danish\"\n)\n\nconst SnowballStemmerName = \"stemmer_da_snowball\"\n\ntype DanishStemmerFilter struct {\n}\n\nfunc NewDanishStemmerFilter() *DanishStemmerFilter {\n\treturn &DanishStemmerFilter{}\n}\n\nfunc (s *DanishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tdanish.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc DanishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewDanishStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, DanishStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/da/stop_filter_da.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage da\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/da/stop_words_da.go",
    "content": "package da\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_da\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar DanishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | A Danish stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\n | This is a ranked list (commonest to rarest) of stopwords derived from\n | a large text sample.\n\n\nog           | and\ni            | in\njeg          | I\ndet          | that (dem. pronoun)/it (pers. pronoun)\nat           | that (in front of a sentence)/to (with infinitive)\nen           | a/an\nden          | it (pers. pronoun)/that (dem. pronoun)\ntil          | to/at/for/until/against/by/of/into, more\ner           | present tense of \"to be\"\nsom          | who, as\npå           | on/upon/in/on/at/to/after/of/with/for, on\nde           | they\nmed          | with/by/in, along\nhan          | he\naf           | of/by/from/off/for/in/with/on, off\nfor          | at/for/to/from/by/of/ago, in front/before, because\nikke         | not\nder          | who/which, there/those\nvar          | past tense of \"to be\"\nmig          | me/myself\nsig          | oneself/himself/herself/itself/themselves\nmen          | but\net           | a/an/one, one (number), someone/somebody/one\nhar          | present tense of \"to have\"\nom           | round/about/for/in/a, about/around/down, if\nvi           | we\nmin          | my\nhavde        | past tense of \"to have\"\nham          | him\nhun          | she\nnu           | now\nover         | over/above/across/by/beyond/past/on/about, over/past\nda           | then, when/as/since\nfra          | from/off/since, off, since\ndu           | you\nud           | out\nsin          | his/her/its/one's\ndem          | them\nos           | us/ourselves\nop           | up\nman          | you/one\nhans         | his\nhvor         | where\neller        | or\nhvad         | what\nskal         | must/shall etc.\nselv         | myself/youself/herself/ourselves etc., even\nher          | here\nalle         | all/everyone/everybody etc.\nvil          | will (verb)\nblev         | past tense of \"to stay/to remain/to get/to become\"\nkunne        | could\nind          | in\nnår          | when\nvære         | present tense of \"to be\"\ndog          | however/yet/after all\nnoget        | something\nville        | would\njo           | you know/you see (adv), yes\nderes        | their/theirs\nefter        | after/behind/according to/for/by/from, later/afterwards\nned          | down\nskulle       | should\ndenne        | this\nend          | than\ndette        | this\nmit          | my/mine\nogså         | also\nunder        | under/beneath/below/during, below/underneath\nhave         | have\ndig          | you\nanden        | other\nhende        | her\nmine         | my\nalt          | everything\nmeget        | much/very, plenty of\nsit          | his, her, its, one's\nsine         | his, her, its, one's\nvor          | our\nmod          | against\ndisse        | these\nhvis         | if\ndin          | your/yours\nnogle        | some\nhos          | by/at\nblive        | be/become\nmange        | many\nad           | by/through\nbliver       | present tense of \"to be/to become\"\nhendes       | her/hers\nværet        | be\nthi          | for (conj)\njer          | you\nsådan        | such, like this/like that\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(DanishStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/analyzer_de.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage de\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst AnalyzerName = \"de\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopDeFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnormalizeDeFilter, err := cache.TokenFilterNamed(NormalizeName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlightStemmerDeFilter, err := cache.TokenFilterNamed(LightStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopDeFilter,\n\t\t\tnormalizeDeFilter,\n\t\t\tlightStemmerDeFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/analyzer_de_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage de\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestGermanAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: []byte(\"Tisch\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"tisch\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Tische\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"tisch\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Tischen\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"tisch\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      7,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// german specials\n\t\t{\n\t\t\tinput: []byte(\"Schaltflächen\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"schaltflach\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Schaltflaechen\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"schaltflach\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// tests added by marty to increase coverage\n\t\t{\n\t\t\tinput: []byte(\"Blechern\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"blech\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Klecks\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"kleck\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Mindestens\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"mindest\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Kugelfest\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"kugelf\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Baldigst\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"baldig\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/german_normalize.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage de\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst NormalizeName = \"normalize_de\"\n\nconst (\n\tN = 0 /* ordinary state */\n\tV = 1 /* stops 'u' from entering umlaut state */\n\tU = 2 /* umlaut state, allows e-deletion */\n)\n\ntype GermanNormalizeFilter struct {\n}\n\nfunc NewGermanNormalizeFilter() *GermanNormalizeFilter {\n\treturn &GermanNormalizeFilter{}\n}\n\nfunc (s *GermanNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := normalize(token.Term)\n\t\ttoken.Term = term\n\t}\n\treturn input\n}\n\nfunc normalize(input []byte) []byte {\n\tstate := N\n\trunes := bytes.Runes(input)\n\tfor i := 0; i < len(runes); i++ {\n\t\tswitch runes[i] {\n\t\tcase 'a', 'o':\n\t\t\tstate = U\n\t\tcase 'u':\n\t\t\tif state == N {\n\t\t\t\tstate = U\n\t\t\t} else {\n\t\t\t\tstate = V\n\t\t\t}\n\t\tcase 'e':\n\t\t\tif state == U {\n\t\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\t\ti--\n\t\t\t}\n\t\t\tstate = V\n\t\tcase 'i', 'q', 'y':\n\t\t\tstate = V\n\t\tcase 'ä':\n\t\t\trunes[i] = 'a'\n\t\t\tstate = V\n\t\tcase 'ö':\n\t\t\trunes[i] = 'o'\n\t\t\tstate = V\n\t\tcase 'ü':\n\t\t\trunes[i] = 'u'\n\t\t\tstate = V\n\t\tcase 'ß':\n\t\t\trunes[i] = 's'\n\t\t\ti++\n\t\t\trunes = analysis.InsertRune(runes, i, 's')\n\t\t\tstate = N\n\t\tdefault:\n\t\t\tstate = N\n\t\t}\n\t}\n\treturn analysis.BuildTermFromRunes(runes)\n}\n\nfunc NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewGermanNormalizeFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/german_normalize_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage de\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestGermanNormalizeFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// Tests that a/o/u + e is equivalent to the umlaut form\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Schaltflächen\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Schaltflachen\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Schaltflaechen\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Schaltflachen\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Tests the specific heuristic that ue is not folded after a vowel or q.\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"dauer\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"dauer\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Tests german specific folding of sharp-s\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"weißbier\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"weissbier\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// empty\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tgermanNormalizeFilter := NewGermanNormalizeFilter()\n\tfor _, test := range tests {\n\t\tactual := germanNormalizeFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected %s(% x), got %s(% x)\", test.output[0].Term, test.output[0].Term, actual[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/light_stemmer_de.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage de\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst LightStemmerName = \"stemmer_de_light\"\n\ntype GermanLightStemmerFilter struct {\n}\n\nfunc NewGermanLightStemmerFilter() *GermanLightStemmerFilter {\n\treturn &GermanLightStemmerFilter{}\n}\n\nfunc (s *GermanLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\trunes := bytes.Runes(token.Term)\n\t\trunes = stem(runes)\n\t\ttoken.Term = analysis.BuildTermFromRunes(runes)\n\t}\n\treturn input\n}\n\nfunc stem(input []rune) []rune {\n\n\tfor i, r := range input {\n\t\tswitch r {\n\t\tcase 'ä', 'à', 'á', 'â':\n\t\t\tinput[i] = 'a'\n\t\tcase 'ö', 'ò', 'ó', 'ô':\n\t\t\tinput[i] = 'o'\n\t\tcase 'ï', 'ì', 'í', 'î':\n\t\t\tinput[i] = 'i'\n\t\tcase 'ü', 'ù', 'ú', 'û':\n\t\t\tinput[i] = 'u'\n\t\t}\n\t}\n\n\tinput = step1(input)\n\treturn step2(input)\n}\n\nfunc stEnding(ch rune) bool {\n\tswitch ch {\n\tcase 'b', 'd', 'f', 'g', 'h', 'k', 'l', 'm', 'n', 't':\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc step1(s []rune) []rune {\n\tl := len(s)\n\tif l > 5 && s[l-3] == 'e' && s[l-2] == 'r' && s[l-1] == 'n' {\n\t\treturn s[:l-3]\n\t}\n\n\tif l > 4 && s[l-2] == 'e' {\n\t\tswitch s[l-1] {\n\t\tcase 'm', 'n', 'r', 's':\n\t\t\treturn s[:l-2]\n\t\t}\n\t}\n\n\tif l > 3 && s[l-1] == 'e' {\n\t\treturn s[:l-1]\n\t}\n\n\tif l > 3 && s[l-1] == 's' && stEnding(s[l-2]) {\n\t\treturn s[:l-1]\n\t}\n\n\treturn s\n}\n\nfunc step2(s []rune) []rune {\n\tl := len(s)\n\tif l > 5 && s[l-3] == 'e' && s[l-2] == 's' && s[l-1] == 't' {\n\t\treturn s[:l-3]\n\t}\n\n\tif l > 4 && s[l-2] == 'e' && (s[l-1] == 'r' || s[l-1] == 'n') {\n\t\treturn s[:l-2]\n\t}\n\n\tif l > 4 && s[l-2] == 's' && s[l-1] == 't' && stEnding(s[l-3]) {\n\t\treturn s[:l-2]\n\t}\n\n\treturn s\n}\n\nfunc GermanLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewGermanLightStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(LightStemmerName, GermanLightStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/stemmer_de_snowball.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage de\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/german\"\n)\n\nconst SnowballStemmerName = \"stemmer_de_snowball\"\n\ntype GermanStemmerFilter struct {\n}\n\nfunc NewGermanStemmerFilter() *GermanStemmerFilter {\n\treturn &GermanStemmerFilter{}\n}\n\nfunc (s *GermanStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tgerman.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc GermanStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewGermanStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, GermanStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/stemmer_de_test.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage de\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSnowballGermanStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abzuschrecken\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abzuschreck\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abzuwarten\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abzuwart\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"zwirnfabrik\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"zwirnfabr\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"zyniker\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"zynik\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/stop_filter_de.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage de\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/de/stop_words_de.go",
    "content": "package de\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_de\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar GermanStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | A German stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\n | The number of forms in this list is reduced significantly by passing it\n | through the German stemmer.\n\n\naber           |  but\n\nalle           |  all\nallem\nallen\naller\nalles\n\nals            |  than, as\nalso           |  so\nam             |  an + dem\nan             |  at\n\nander          |  other\nandere\nanderem\nanderen\nanderer\nanderes\nanderm\nandern\nanderr\nanders\n\nauch           |  also\nauf            |  on\naus            |  out of\nbei            |  by\nbin            |  am\nbis            |  until\nbist           |  art\nda             |  there\ndamit          |  with it\ndann           |  then\n\nder            |  the\nden\ndes\ndem\ndie\ndas\n\ndaß            |  that\n\nderselbe       |  the same\nderselben\ndenselben\ndesselben\ndemselben\ndieselbe\ndieselben\ndasselbe\n\ndazu           |  to that\n\ndein           |  thy\ndeine\ndeinem\ndeinen\ndeiner\ndeines\n\ndenn           |  because\n\nderer          |  of those\ndessen         |  of him\n\ndich           |  thee\ndir            |  to thee\ndu             |  thou\n\ndies           |  this\ndiese\ndiesem\ndiesen\ndieser\ndieses\n\n\ndoch           |  (several meanings)\ndort           |  (over) there\n\n\ndurch          |  through\n\nein            |  a\neine\neinem\neinen\neiner\neines\n\neinig          |  some\neinige\neinigem\neinigen\neiniger\neiniges\n\neinmal         |  once\n\ner             |  he\nihn            |  him\nihm            |  to him\n\nes             |  it\netwas          |  something\n\neuer           |  your\neure\neurem\neuren\neurer\neures\n\nfür            |  for\ngegen          |  towards\ngewesen        |  p.p. of sein\nhab            |  have\nhabe           |  have\nhaben          |  have\nhat            |  has\nhatte          |  had\nhatten         |  had\nhier           |  here\nhin            |  there\nhinter         |  behind\n\nich            |  I\nmich           |  me\nmir            |  to me\n\n\nihr            |  you, to her\nihre\nihrem\nihren\nihrer\nihres\neuch           |  to you\n\nim             |  in + dem\nin             |  in\nindem          |  while\nins            |  in + das\nist            |  is\n\njede           |  each, every\njedem\njeden\njeder\njedes\n\njene           |  that\njenem\njenen\njener\njenes\n\njetzt          |  now\nkann           |  can\n\nkein           |  no\nkeine\nkeinem\nkeinen\nkeiner\nkeines\n\nkönnen         |  can\nkönnte         |  could\nmachen         |  do\nman            |  one\n\nmanche         |  some, many a\nmanchem\nmanchen\nmancher\nmanches\n\nmein           |  my\nmeine\nmeinem\nmeinen\nmeiner\nmeines\n\nmit            |  with\nmuss           |  must\nmusste         |  had to\nnach           |  to(wards)\nnicht          |  not\nnichts         |  nothing\nnoch           |  still, yet\nnun            |  now\nnur            |  only\nob             |  whether\noder           |  or\nohne           |  without\nsehr           |  very\n\nsein           |  his\nseine\nseinem\nseinen\nseiner\nseines\n\nselbst         |  self\nsich           |  herself\n\nsie            |  they, she\nihnen          |  to them\n\nsind           |  are\nso             |  so\n\nsolche         |  such\nsolchem\nsolchen\nsolcher\nsolches\n\nsoll           |  shall\nsollte         |  should\nsondern        |  but\nsonst          |  else\nüber           |  over\num             |  about, around\nund            |  and\n\nuns            |  us\nunse\nunsem\nunsen\nunser\nunses\n\nunter          |  under\nviel           |  much\nvom            |  von + dem\nvon            |  from\nvor            |  before\nwährend        |  while\nwar            |  was\nwaren          |  were\nwarst          |  wast\nwas            |  what\nweg            |  away, off\nweil           |  because\nweiter         |  further\n\nwelche         |  which\nwelchem\nwelchen\nwelcher\nwelches\n\nwenn           |  when\nwerde          |  will\nwerden         |  will\nwie            |  how\nwieder         |  again\nwill           |  want\nwir            |  we\nwird           |  will\nwirst          |  willst\nwo             |  where\nwollen         |  want\nwollte         |  wanted\nwürde          |  would\nwürden         |  would\nzu             |  to\nzum            |  zu + dem\nzur            |  zu + der\nzwar           |  indeed\nzwischen       |  between\n\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(GermanStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/el/stop_filter_el.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage el\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/el/stop_words_el.go",
    "content": "package el\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_el\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar GreekStopWords = []byte(`# Lucene Greek Stopwords list\n# Note: by default this file is used after GreekLowerCaseFilter,\n# so when modifying this file use 'σ' instead of 'ς' \nο\nη\nτο\nοι\nτα\nτου\nτησ\nτων\nτον\nτην\nκαι \nκι\nκ\nειμαι\nεισαι\nειναι\nειμαστε\nειστε\nστο\nστον\nστη\nστην\nμα\nαλλα\nαπο\nγια\nπροσ\nμε\nσε\nωσ\nπαρα\nαντι\nκατα\nμετα\nθα\nνα\nδε\nδεν\nμη\nμην\nεπι\nενω\nεαν\nαν\nτοτε\nπου\nπωσ\nποιοσ\nποια\nποιο\nποιοι\nποιεσ\nποιων\nποιουσ\nαυτοσ\nαυτη\nαυτο\nαυτοι\nαυτων\nαυτουσ\nαυτεσ\nαυτα\nεκεινοσ\nεκεινη\nεκεινο\nεκεινοι\nεκεινεσ\nεκεινα\nεκεινων\nεκεινουσ\nοπωσ\nομωσ\nισωσ\nοσο\nοτι\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(GreekStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/en/analyzer_en.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package en implements an analyzer with reasonable defaults for processing\n// English text.\n//\n// It strips possessive suffixes ('s), transforms tokens to lower case,\n// removes stopwords from a built-in list, and applies porter stemming.\n//\n// The built-in stopwords list is defined in EnglishStopWords.\npackage en\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/porter\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"en\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tpossEnFilter, err := cache.TokenFilterNamed(PossessiveName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopEnFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerEnFilter, err := cache.TokenFilterNamed(porter.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\tpossEnFilter,\n\t\t\ttoLowerFilter,\n\t\t\tstopEnFilter,\n\t\t\tstemmerEnFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/en/analyzer_en_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage en\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestEnglishAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"books\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"book\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"book\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"book\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word removal\n\t\t{\n\t\t\tinput:  []byte(\"the\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t\t// possessive removal\n\t\t{\n\t\t\tinput: []byte(\"steven's\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"steven\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"steven\\u2019s\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"steven\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"steven\\uFF07s\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"steven\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/en/plural_stemmer.go",
    "content": "/*\n\tThis code was ported from the Open Search Project\n\thttps://github.com/opensearch-project/OpenSearch/blob/main/modules/analysis-common/src/main/java/org/opensearch/analysis/common/EnglishPluralStemFilter.java\n\tThe algorithm itself was created by Mark Harwood\n\thttps://github.com/markharwood\n*/\n\n/*\n * SPDX-License-Identifier: Apache-2.0\n *\n * The OpenSearch Contributors require contributions made to\n * this file be licensed under the Apache-2.0 license or a\n * compatible open source license.\n */\n\n/*\n * Licensed to Elasticsearch under one or more contributor\n * license agreements. See the NOTICE file distributed with\n * this work for additional information regarding copyright\n * ownership. Elasticsearch licenses this file to you under\n * the Apache License, Version 2.0 (the \"License\"); you may\n * not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\npackage en\n\nimport (\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst PluralStemmerName = \"stemmer_en_plural\"\n\ntype EnglishPluralStemmerFilter struct {\n}\n\nfunc NewEnglishPluralStemmerFilter() *EnglishPluralStemmerFilter {\n\treturn &EnglishPluralStemmerFilter{}\n}\n\nfunc (s *EnglishPluralStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\ttoken.Term = []byte(stem(string(token.Term)))\n\t}\n\n\treturn input\n}\n\nfunc EnglishPluralStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewEnglishPluralStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(PluralStemmerName, EnglishPluralStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\n// ----------------------------------------------------------------------------\n\n// Words ending in oes that retain the e when stemmed\nvar oesExceptions = []string{\"shoes\", \"canoes\", \"oboes\"}\n\n// Words ending in ches that retain the e when stemmed\nvar chesExceptions = []string{\n\t\"cliches\",\n\t\"avalanches\",\n\t\"mustaches\",\n\t\"moustaches\",\n\t\"quiches\",\n\t\"headaches\",\n\t\"heartaches\",\n\t\"porsches\",\n\t\"tranches\",\n\t\"caches\",\n}\n\nfunc stem(word string) string {\n\trunes := []rune(strings.ToLower(word))\n\n\tif len(runes) < 3 || runes[len(runes)-1] != 's' {\n\t\treturn string(runes)\n\t}\n\n\tswitch runes[len(runes)-2] {\n\tcase 'u':\n\t\tfallthrough\n\tcase 's':\n\t\treturn string(runes)\n\tcase 'e':\n\t\t// Modified ies->y logic from original s-stemmer - only work on strings > 4\n\t\t// so spies -> spy still but pies->pie.\n\t\t// The original code also special-cased aies and eies for no good reason as far as I can tell.\n\t\t// ( no words of consequence - eg http://www.thefreedictionary.com/words-that-end-in-aies )\n\t\tif len(runes) > 4 && runes[len(runes)-3] == 'i' {\n\t\t\trunes[len(runes)-3] = 'y'\n\t\t\treturn string(runes[0 : len(runes)-2])\n\t\t}\n\n\t\t// Suffix rules to remove any dangling \"e\"\n\t\tif len(runes) > 3 {\n\t\t\t// xes (but >1 prefix so we can stem \"boxes->box\" but keep \"axes->axe\")\n\t\t\tif len(runes) > 4 && runes[len(runes)-3] == 'x' {\n\t\t\t\treturn string(runes[0 : len(runes)-2])\n\t\t\t}\n\n\t\t\t// oes\n\t\t\tif len(runes) > 3 && runes[len(runes)-3] == 'o' {\n\t\t\t\tif isException(runes, oesExceptions) {\n\t\t\t\t\t// Only remove the S\n\t\t\t\t\treturn string(runes[0 : len(runes)-1])\n\t\t\t\t}\n\t\t\t\t// Remove the es\n\t\t\t\treturn string(runes[0 : len(runes)-2])\n\t\t\t}\n\n\t\t\tif len(runes) > 4 {\n\t\t\t\t// shes/sses\n\t\t\t\tif runes[len(runes)-4] == 's' && (runes[len(runes)-3] == 'h' || runes[len(runes)-3] == 's') {\n\t\t\t\t\treturn string(runes[0 : len(runes)-2])\n\t\t\t\t}\n\n\t\t\t\t// ches\n\t\t\t\tif len(runes) > 4 {\n\t\t\t\t\tif runes[len(runes)-4] == 'c' && runes[len(runes)-3] == 'h' {\n\t\t\t\t\t\tif isException(runes, chesExceptions) {\n\t\t\t\t\t\t\t// Only remove the S\n\t\t\t\t\t\t\treturn string(runes[0 : len(runes)-1])\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// Remove the es\n\t\t\t\t\t\treturn string(runes[0 : len(runes)-2])\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfallthrough\n\tdefault:\n\t\treturn string(runes[0 : len(runes)-1])\n\t}\n}\n\nfunc isException(word []rune, exceptions []string) bool {\n\tfor _, exception := range exceptions {\n\n\t\texceptionRunes := []rune(exception)\n\n\t\texceptionPos := len(exceptionRunes) - 1\n\t\twordPos := len(word) - 1\n\n\t\tmatched := true\n\t\tfor exceptionPos >= 0 && wordPos >= 0 {\n\t\t\tif exceptionRunes[exceptionPos] != word[wordPos] {\n\t\t\t\tmatched = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t\texceptionPos--\n\t\t\twordPos--\n\t\t}\n\t\tif matched {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n"
  },
  {
    "path": "analysis/lang/en/plural_stemmer_test.go",
    "content": "package en\n\nimport \"testing\"\n\nfunc TestEnglishPluralStemmer(t *testing.T) {\n\tdata := []struct {\n\t\tIn, Out string\n\t}{\n\t\t{\"dresses\", \"dress\"},\n\t\t{\"dress\", \"dress\"},\n\t\t{\"axes\", \"axe\"},\n\t\t{\"ad\", \"ad\"},\n\t\t{\"ads\", \"ad\"},\n\t\t{\"gas\", \"ga\"},\n\t\t{\"sass\", \"sass\"},\n\t\t{\"berries\", \"berry\"},\n\t\t{\"dresses\", \"dress\"},\n\t\t{\"spies\", \"spy\"},\n\t\t{\"shoes\", \"shoe\"},\n\t\t{\"headaches\", \"headache\"},\n\t\t{\"computer\", \"computer\"},\n\t\t{\"dressing\", \"dressing\"},\n\t\t{\"clothes\", \"clothe\"},\n\t\t{\"DRESSES\", \"dress\"},\n\t\t{\"frog\", \"frog\"},\n\t\t{\"dress\", \"dress\"},\n\t\t{\"runs\", \"run\"},\n\t\t{\"pies\", \"pie\"},\n\t\t{\"foxes\", \"fox\"},\n\t\t{\"axes\", \"axe\"},\n\t\t{\"foes\", \"fo\"},\n\t\t{\"dishes\", \"dish\"},\n\t\t{\"snitches\", \"snitch\"},\n\t\t{\"cliches\", \"cliche\"},\n\t\t{\"forests\", \"forest\"},\n\t\t{\"yes\", \"ye\"},\n\t}\n\n\tfor _, datum := range data {\n\t\tstemmed := stem(datum.In)\n\n\t\tif stemmed != datum.Out {\n\t\t\tt.Errorf(\"expected %v but got %v\", datum.Out, stemmed)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/en/possessive_filter_en.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage en\n\nimport (\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\n// PossessiveName is the name PossessiveFilter is registered as\n// in the bleve registry.\nconst PossessiveName = \"possessive_en\"\n\nconst rightSingleQuotationMark = '’'\nconst apostrophe = '\\''\nconst fullWidthApostrophe = '＇'\n\nconst apostropheChars = rightSingleQuotationMark + apostrophe + fullWidthApostrophe\n\n// PossessiveFilter implements a TokenFilter which\n// strips the English possessive suffix ('s) from tokens.\n// It handle a variety of apostrophe types, is case-insensitive\n// and doesn't distinguish between possessive and contraction.\n// (ie \"She's So Rad\" becomes \"She So Rad\")\ntype PossessiveFilter struct {\n}\n\nfunc NewPossessiveFilter() *PossessiveFilter {\n\treturn &PossessiveFilter{}\n}\n\nfunc (s *PossessiveFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tlastRune, lastRuneSize := utf8.DecodeLastRune(token.Term)\n\t\tif lastRune == 's' || lastRune == 'S' {\n\t\t\tnextLastRune, nextLastRuneSize := utf8.DecodeLastRune(token.Term[:len(token.Term)-lastRuneSize])\n\t\t\tif nextLastRune == rightSingleQuotationMark ||\n\t\t\t\tnextLastRune == apostrophe ||\n\t\t\t\tnextLastRune == fullWidthApostrophe {\n\t\t\t\ttoken.Term = token.Term[:len(token.Term)-lastRuneSize-nextLastRuneSize]\n\t\t\t}\n\t\t}\n\t}\n\treturn input\n}\n\nfunc PossessiveFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewPossessiveFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(PossessiveName, PossessiveFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/en/possessive_filter_en_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage en\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestEnglishPossessiveFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"marty's\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"MARTY'S\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"marty’s\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"MARTY’S\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"marty＇s\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"MARTY＇S\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"m\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"s\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"'s\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"marty\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"MARTY\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"marty\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"MARTY\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"marty\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"MARTY\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"m\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"s\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tstemmerFilter, err := cache.TokenFilterNamed(PossessiveName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := stemmerFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output, actual)\n\t\t}\n\t}\n}\n\nfunc BenchmarkEnglishPossessiveFilter(b *testing.B) {\n\n\tinput := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"marty's\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"MARTY'S\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"marty’s\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"MARTY’S\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"marty＇s\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"MARTY＇S\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"m\"),\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tstemmerFilter, err := cache.TokenFilterNamed(PossessiveName)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tstemmerFilter.Filter(input)\n\t}\n\n}\n"
  },
  {
    "path": "analysis/lang/en/stemmer_en_snowball.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage en\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/english\"\n)\n\nconst SnowballStemmerName = \"stemmer_en_snowball\"\n\ntype EnglishStemmerFilter struct {\n}\n\nfunc NewEnglishStemmerFilter() *EnglishStemmerFilter {\n\treturn &EnglishStemmerFilter{}\n}\n\nfunc (s *EnglishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tenglish.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc EnglishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewEnglishStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, EnglishStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/en/stemmer_en_test.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage en\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSnowballEnglishStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"enjoy\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"enjoy\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"enjoyed\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"enjoy\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"enjoyable\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"enjoy\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/en/stop_filter_en.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage en\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/en/stop_words_en.go",
    "content": "package en\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_en\"\n\n// EnglishStopWords is the built-in list of stopwords used by the \"stop_en\" TokenFilter.\n//\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\nvar EnglishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/english/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n \n | An English stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\n | Many of the forms below are quite rare (e.g. \"yourselves\") but included for\n |  completeness.\n\n           | PRONOUNS FORMS\n             | 1st person sing\n\ni              | subject, always in upper case of course\n\nme             | object\nmy             | possessive adjective\n               | the possessive pronoun 'mine' is best suppressed, because of the\n               | sense of coal-mine etc.\nmyself         | reflexive\n             | 1st person plural\nwe             | subject\n\n| us           | object\n               | care is required here because US = United States. It is usually\n               | safe to remove it if it is in lower case.\nour            | possessive adjective\nours           | possessive pronoun\nourselves      | reflexive\n             | second person (archaic 'thou' forms not included)\nyou            | subject and object\nyour           | possessive adjective\nyours          | possessive pronoun\nyourself       | reflexive (singular)\nyourselves     | reflexive (plural)\n             | third person singular\nhe             | subject\nhim            | object\nhis            | possessive adjective and pronoun\nhimself        | reflexive\n\nshe            | subject\nher            | object and possessive adjective\nhers           | possessive pronoun\nherself        | reflexive\n\nit             | subject and object\nits            | possessive adjective\nitself         | reflexive\n             | third person plural\nthey           | subject\nthem           | object\ntheir          | possessive adjective\ntheirs         | possessive pronoun\nthemselves     | reflexive\n             | other forms (demonstratives, interrogatives)\nwhat\nwhich\nwho\nwhom\nthis\nthat\nthese\nthose\n\n           | VERB FORMS (using F.R. Palmer's nomenclature)\n             | BE\nam             | 1st person, present\nis             | -s form (3rd person, present)\nare            | present\nwas            | 1st person, past\nwere           | past\nbe             | infinitive\nbeen           | past participle\nbeing          | -ing form\n             | HAVE\nhave           | simple\nhas            | -s form\nhad            | past\nhaving         | -ing form\n             | DO\ndo             | simple\ndoes           | -s form\ndid            | past\ndoing          | -ing form\n\n | The forms below are, I believe, best omitted, because of the significant\n | homonym forms:\n\n |  He made a WILL\n |  old tin CAN\n |  merry month of MAY\n |  a smell of MUST\n |  fight the good fight with all thy MIGHT\n\n | would, could, should, ought might however be included\n\n |          | AUXILIARIES\n |            | WILL\n |will\n\nwould\n\n |            | SHALL\n |shall\n\nshould\n\n |            | CAN\n |can\n\ncould\n\n |            | MAY\n |may\n |might\n |            | MUST\n |must\n |            | OUGHT\n\nought\n\n           | COMPOUND FORMS, increasingly encountered nowadays in 'formal' writing\n              | pronoun + verb\n\ni'm\nyou're\nhe's\nshe's\nit's\nwe're\nthey're\ni've\nyou've\nwe've\nthey've\ni'd\nyou'd\nhe'd\nshe'd\nwe'd\nthey'd\ni'll\nyou'll\nhe'll\nshe'll\nwe'll\nthey'll\n\n              | verb + negation\n\nisn't\naren't\nwasn't\nweren't\nhasn't\nhaven't\nhadn't\ndoesn't\ndon't\ndidn't\n\n              | auxiliary + negation\n\nwon't\nwouldn't\nshan't\nshouldn't\ncan't\ncannot\ncouldn't\nmustn't\n\n             | miscellaneous forms\n\nlet's\nthat's\nwho's\nwhat's\nhere's\nthere's\nwhen's\nwhere's\nwhy's\nhow's\n\n              | rarer forms\n\n | daren't needn't\n\n              | doubtful forms\n\n | oughtn't mightn't\n\n           | ARTICLES\na\nan\nthe\n\n           | THE REST (Overlap among prepositions, conjunctions, adverbs etc is so\n           | high, that classification is pointless.)\nand\nbut\nif\nor\nbecause\nas\nuntil\nwhile\n\nof\nat\nby\nfor\nwith\nabout\nagainst\nbetween\ninto\nthrough\nduring\nbefore\nafter\nabove\nbelow\nto\nfrom\nup\ndown\nin\nout\non\noff\nover\nunder\n\nagain\nfurther\nthen\nonce\n\nhere\nthere\nwhen\nwhere\nwhy\nhow\n\nall\nany\nboth\neach\nfew\nmore\nmost\nother\nsome\nsuch\n\nno\nnor\nnot\nonly\nown\nsame\nso\nthan\ntoo\nvery\n\n | Just for the record, the following words are among the commonest in English\n\n    | one\n    | every\n    | least\n    | less\n    | many\n    | now\n    | ever\n    | never\n    | say\n    | says\n    | said\n    | also\n    | get\n    | go\n    | goes\n    | just\n    | made\n    | make\n    | put\n    | see\n    | seen\n    | whether\n    | like\n    | well\n    | back\n    | even\n    | still\n    | way\n    | take\n    | since\n    | another\n    | however\n    | two\n    | three\n    | four\n    | five\n    | first\n    | second\n    | new\n    | old\n    | high\n    | long\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(EnglishStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/analyzer_es.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage es\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"es\"\n\nfunc AnalyzerConstructor(config map[string]interface{},\n\tcache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnormalizeEsFilter, err := cache.TokenFilterNamed(NormalizeName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopEsFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlightStemmerEsFilter, err := cache.TokenFilterNamed(LightStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopEsFilter,\n\t\t\tnormalizeEsFilter,\n\t\t\tlightStemmerEsFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/analyzer_es_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage es\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSpanishAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"chicana\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"chican\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      7,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"chicano\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"chican\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      7,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// added by marty for better coverage\n\t\t{\n\t\t\tinput: []byte(\"yeses\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"yes\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"jaeces\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"jaez\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"arcos\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"arc\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"caos\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"caos\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"parecer\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"parecer\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      7,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/light_stemmer_es.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage es\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst LightStemmerName = \"stemmer_es_light\"\n\ntype SpanishLightStemmerFilter struct {\n}\n\nfunc NewSpanishLightStemmerFilter() *SpanishLightStemmerFilter {\n\treturn &SpanishLightStemmerFilter{}\n}\n\nfunc (s *SpanishLightStemmerFilter) Filter(\n\tinput analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\trunes := bytes.Runes(token.Term)\n\t\trunes = stem(runes)\n\t\ttoken.Term = analysis.BuildTermFromRunes(runes)\n\t}\n\treturn input\n}\n\nfunc stem(input []rune) []rune {\n\tl := len(input)\n\tif l < 5 {\n\t\treturn input\n\t}\n\n\tswitch input[l-1] {\n\tcase 'o', 'a', 'e':\n\t\treturn input[:l-1]\n\tcase 's':\n\t\tif input[l-2] == 'e' && input[l-3] == 's' && input[l-4] == 'e' {\n\t\t\treturn input[:l-2]\n\t\t}\n\t\tif input[l-2] == 'e' && input[l-3] == 'c' {\n\t\t\tinput[l-3] = 'z'\n\t\t\treturn input[:l-2]\n\t\t}\n\t\tif input[l-2] == 'o' || input[l-2] == 'a' || input[l-2] == 'e' {\n\t\t\treturn input[:l-2]\n\t\t}\n\t}\n\n\treturn input\n}\n\nfunc SpanishLightStemmerFilterConstructor(config map[string]interface{},\n\tcache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewSpanishLightStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(LightStemmerName, SpanishLightStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/spanish_normalize.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage es\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst NormalizeName = \"normalize_es\"\n\ntype SpanishNormalizeFilter struct {\n}\n\nfunc NewSpanishNormalizeFilter() *SpanishNormalizeFilter {\n\treturn &SpanishNormalizeFilter{}\n}\n\nfunc (s *SpanishNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := normalize(token.Term)\n\t\ttoken.Term = term\n\t}\n\treturn input\n}\n\nfunc normalize(input []byte) []byte {\n\trunes := bytes.Runes(input)\n\tfor i := 0; i < len(runes); i++ {\n\t\tswitch runes[i] {\n\t\tcase 'à', 'á', 'â', 'ä':\n\t\t\trunes[i] = 'a'\n\t\tcase 'ò', 'ó', 'ô', 'ö':\n\t\t\trunes[i] = 'o'\n\t\tcase 'è', 'é', 'ê', 'ë':\n\t\t\trunes[i] = 'e'\n\t\tcase 'ù', 'ú', 'û', 'ü':\n\t\t\trunes[i] = 'u'\n\t\tcase 'ì', 'í', 'î', 'ï':\n\t\t\trunes[i] = 'i'\n\t\t}\n\t}\n\n\treturn analysis.BuildTermFromRunes(runes)\n}\n\nfunc NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewSpanishNormalizeFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/spanish_normalize_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage es\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestSpanishNormalizeFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Guía\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Guia\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Belcebú\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Belcebu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Limón\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Limon\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"agüero\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aguero\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"laúd\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"laud\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// empty\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tspanishNormalizeFilter := NewSpanishNormalizeFilter()\n\tfor _, test := range tests {\n\t\tactual := spanishNormalizeFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected %s(% x), got %s(% x)\", test.output[0].Term, test.output[0].Term, actual[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/stemmer_es_snowball.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage es\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/spanish\"\n)\n\nconst SnowballStemmerName = \"stemmer_es_snowball\"\n\ntype SpanishStemmerFilter struct {\n}\n\nfunc NewSpanishStemmerFilter() *SpanishStemmerFilter {\n\treturn &SpanishStemmerFilter{}\n}\n\nfunc (s *SpanishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tspanish.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc SpanishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewSpanishStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, SpanishStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/stemmer_es_snowball_test.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage es\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSnowballSpanishStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"agresivos\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"agres\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"agresivamente\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"agres\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"agresividad\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"agres\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/stop_filter_es.go",
    "content": "// Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\npackage es\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{},\n\tcache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/es/stop_words_es.go",
    "content": "package es\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_es\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar SpanishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | A Spanish stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\n\n | The following is a ranked list (commonest to rarest) of stopwords\n | deriving from a large sample of text.\n\n | Extra words have been added at the end.\n\nde             |  from, of\nla             |  the, her\nque            |  who, that\nel             |  the\nen             |  in\ny              |  and\na              |  to\nlos            |  the, them\ndel            |  de + el\nse             |  himself, from him etc\nlas            |  the, them\npor            |  for, by, etc\nun             |  a\npara           |  for\ncon            |  with\nno             |  no\nuna            |  a\nsu             |  his, her\nal             |  a + el\n  | es         from SER\nlo             |  him\ncomo           |  how\nmás            |  more\npero           |  pero\nsus            |  su plural\nle             |  to him, her\nya             |  already\no              |  or\n  | fue        from SER\neste           |  this\n  | ha         from HABER\nsí             |  himself etc\nporque         |  because\nesta           |  this\n  | son        from SER\nentre          |  between\n  | está     from ESTAR\ncuando         |  when\nmuy            |  very\nsin            |  without\nsobre          |  on\n  | ser        from SER\n  | tiene      from TENER\ntambién        |  also\nme             |  me\nhasta          |  until\nhay            |  there is/are\ndonde          |  where\n  | han        from HABER\nquien          |  whom, that\n  | están      from ESTAR\n  | estado     from ESTAR\ndesde          |  from\ntodo           |  all\nnos            |  us\ndurante        |  during\n  | estados    from ESTAR\ntodos          |  all\nuno            |  a\nles            |  to them\nni             |  nor\ncontra         |  against\notros          |  other\n  | fueron     from SER\nese            |  that\neso            |  that\n  | había      from HABER\nante           |  before\nellos          |  they\ne              |  and (variant of y)\nesto           |  this\nmí             |  me\nantes          |  before\nalgunos        |  some\nqué            |  what?\nunos           |  a\nyo             |  I\notro           |  other\notras          |  other\notra           |  other\nél             |  he\ntanto          |  so much, many\nesa            |  that\nestos          |  these\nmucho          |  much, many\nquienes        |  who\nnada           |  nothing\nmuchos         |  many\ncual           |  who\n  | sea        from SER\npoco           |  few\nella           |  she\nestar          |  to be\n  | haber      from HABER\nestas          |  these\n  | estaba     from ESTAR\n  | estamos    from ESTAR\nalgunas        |  some\nalgo           |  something\nnosotros       |  we\n\n      | other forms\n\nmi             |  me\nmis            |  mi plural\ntú             |  thou\nte             |  thee\nti             |  thee\ntu             |  thy\ntus            |  tu plural\nellas          |  they\nnosotras       |  we\nvosotros       |  you\nvosotras       |  you\nos             |  you\nmío            |  mine\nmía            |\nmíos           |\nmías           |\ntuyo           |  thine\ntuya           |\ntuyos          |\ntuyas          |\nsuyo           |  his, hers, theirs\nsuya           |\nsuyos          |\nsuyas          |\nnuestro        |  ours\nnuestra        |\nnuestros       |\nnuestras       |\nvuestro        |  yours\nvuestra        |\nvuestros       |\nvuestras       |\nesos           |  those\nesas           |  those\n\n               | forms of estar, to be (not including the infinitive):\nestoy\nestás\nestá\nestamos\nestáis\nestán\nesté\nestés\nestemos\nestéis\nestén\nestaré\nestarás\nestará\nestaremos\nestaréis\nestarán\nestaría\nestarías\nestaríamos\nestaríais\nestarían\nestaba\nestabas\nestábamos\nestabais\nestaban\nestuve\nestuviste\nestuvo\nestuvimos\nestuvisteis\nestuvieron\nestuviera\nestuvieras\nestuviéramos\nestuvierais\nestuvieran\nestuviese\nestuvieses\nestuviésemos\nestuvieseis\nestuviesen\nestando\nestado\nestada\nestados\nestadas\nestad\n\n               | forms of haber, to have (not including the infinitive):\nhe\nhas\nha\nhemos\nhabéis\nhan\nhaya\nhayas\nhayamos\nhayáis\nhayan\nhabré\nhabrás\nhabrá\nhabremos\nhabréis\nhabrán\nhabría\nhabrías\nhabríamos\nhabríais\nhabrían\nhabía\nhabías\nhabíamos\nhabíais\nhabían\nhube\nhubiste\nhubo\nhubimos\nhubisteis\nhubieron\nhubiera\nhubieras\nhubiéramos\nhubierais\nhubieran\nhubiese\nhubieses\nhubiésemos\nhubieseis\nhubiesen\nhabiendo\nhabido\nhabida\nhabidos\nhabidas\n\n               | forms of ser, to be (not including the infinitive):\nsoy\neres\nes\nsomos\nsois\nson\nsea\nseas\nseamos\nseáis\nsean\nseré\nserás\nserá\nseremos\nseréis\nserán\nsería\nserías\nseríamos\nseríais\nserían\nera\neras\néramos\nerais\neran\nfui\nfuiste\nfue\nfuimos\nfuisteis\nfueron\nfuera\nfueras\nfuéramos\nfuerais\nfueran\nfuese\nfueses\nfuésemos\nfueseis\nfuesen\nsiendo\nsido\n  |  sed also means 'thirst'\n\n               | forms of tener, to have (not including the infinitive):\ntengo\ntienes\ntiene\ntenemos\ntenéis\ntienen\ntenga\ntengas\ntengamos\ntengáis\ntengan\ntendré\ntendrás\ntendrá\ntendremos\ntendréis\ntendrán\ntendría\ntendrías\ntendríamos\ntendríais\ntendrían\ntenía\ntenías\nteníamos\nteníais\ntenían\ntuve\ntuviste\ntuvo\ntuvimos\ntuvisteis\ntuvieron\ntuviera\ntuvieras\ntuviéramos\ntuvierais\ntuvieran\ntuviese\ntuvieses\ntuviésemos\ntuvieseis\ntuviesen\nteniendo\ntenido\ntenida\ntenidos\ntenidas\ntened\n\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(SpanishStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/eu/stop_filter_eu.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage eu\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/eu/stop_words_eu.go",
    "content": "package eu\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_eu\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar BasqueStopWords = []byte(`# example set of basque stopwords\nal\nanitz\narabera\nasko\nbaina\nbat\nbatean\nbatek\nbati\nbatzuei\nbatzuek\nbatzuetan\nbatzuk\nbera\nberaiek\nberau\nberauek\nbere\nberori\nberoriek\nbeste\nbezala\nda\ndago\ndira\nditu\ndu\ndute\nedo\negin\nere\neta\neurak\nez\ngainera\ngu\ngutxi\nguzti\nhaiei\nhaiek\nhaietan\nhainbeste\nhala\nhan\nhandik\nhango\nhara\nhari\nhark\nhartan\nhau\nhauei\nhauek\nhauetan\nhemen\nhemendik\nhemengo\nhi\nhona\nhonek\nhonela\nhonetan\nhoni\nhor\nhori\nhoriei\nhoriek\nhorietan\nhorko\nhorra\nhorrek\nhorrela\nhorretan\nhorri\nhortik\nhura\nizan\nni\nnoiz\nnola\nnon\nnondik\nnongo\nnor\nnora\nze\nzein\nzen\nzenbait\nzenbat\nzer\nzergatik\nziren\nzituen\nzu\nzuek\nzuen\nzuten\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(BasqueStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fa/analyzer_fa.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fa\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/char/zerowidthnonjoiner\"\n\t\"github.com/blevesearch/bleve/v2/analysis/lang/ar\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"fa\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tzFilter, err := cache.CharFilterNamed(zerowidthnonjoiner.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnormArFilter, err := cache.TokenFilterNamed(ar.NormalizeName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnormFaFilter, err := cache.TokenFilterNamed(NormalizeName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopFaFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tCharFilters: []analysis.CharFilter{\n\t\t\tzFilter,\n\t\t},\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tnormArFilter,\n\t\t\tnormFaFilter,\n\t\t\tstopFaFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fa/analyzer_fa_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fa\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestPersianAnalyzerVerbs(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// active present indicative\n\t\t{\n\t\t\tinput: []byte(\"می‌خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active preterite indicative\n\t\t{\n\t\t\tinput: []byte(\"خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective preterite indicative\n\t\t{\n\t\t\tinput: []byte(\"می‌خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active future indicative\n\t\t{\n\t\t\tinput: []byte(\"خواهد خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active present progressive indicative\n\t\t{\n\t\t\tinput: []byte(\"دارد می‌خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active preterite progressive indicative\n\t\t{\n\t\t\tinput: []byte(\"داشت می‌خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active perfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده‌است\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective perfect indicative\n\t\t{\n\t\t\tinput: []byte(\"می‌خورده‌است\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active pluperfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده بود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective pluperfect indicative\n\t\t{\n\t\t\tinput: []byte(\"می‌خورده بود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active preterite subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective preterite subjunctive\n\t\t{\n\t\t\tinput: []byte(\"می‌خورده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active pluperfect subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective pluperfect subjunctive\n\t\t{\n\t\t\tinput: []byte(\"می‌خورده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive present indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده می‌شود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive preterite indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده شد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective preterite indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده می‌شد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive perfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده شده‌است\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective perfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده می‌شده‌است\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive pluperfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده شده بود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective pluperfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده می‌شده بود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive future indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده خواهد شد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive present progressive indicative\n\t\t{\n\t\t\tinput: []byte(\"دارد خورده می‌شود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive preterite progressive indicative\n\t\t{\n\t\t\tinput: []byte(\"داشت خورده می‌شد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive present subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده شود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive preterite subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده شده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective preterite subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده می‌شده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive pluperfect subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده شده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective pluperfect subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده می‌شده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active present subjunctive\n\t\t{\n\t\t\tinput: []byte(\"بخورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"بخورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestPersianAnalyzerVerbsDefective(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// active present indicative\n\t\t{\n\t\t\tinput: []byte(\"مي خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active preterite indicative\n\t\t{\n\t\t\tinput: []byte(\"خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective preterite indicative\n\t\t{\n\t\t\tinput: []byte(\"مي خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active future indicative\n\t\t{\n\t\t\tinput: []byte(\"خواهد خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active present progressive indicative\n\t\t{\n\t\t\tinput: []byte(\"دارد مي خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active preterite progressive indicative\n\t\t{\n\t\t\tinput: []byte(\"داشت مي خورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active perfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده است\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective perfect indicative\n\t\t{\n\t\t\tinput: []byte(\"مي خورده است\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active pluperfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده بود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective pluperfect indicative\n\t\t{\n\t\t\tinput: []byte(\"مي خورده بود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active preterite subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective preterite subjunctive\n\t\t{\n\t\t\tinput: []byte(\"مي خورده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active pluperfect subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active imperfective pluperfect subjunctive\n\t\t{\n\t\t\tinput: []byte(\"مي خورده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive present indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده مي شود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive preterite indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده شد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective preterite indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده مي شد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive perfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده شده است\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective perfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده مي شده است\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive pluperfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده شده بود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective pluperfect indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده مي شده بود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive future indicative\n\t\t{\n\t\t\tinput: []byte(\"خورده خواهد شد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive present progressive indicative\n\t\t{\n\t\t\tinput: []byte(\"دارد خورده مي شود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive preterite progressive indicative\n\t\t{\n\t\t\tinput: []byte(\"داشت خورده مي شد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive present subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده شود\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive preterite subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده شده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective preterite subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده مي شده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive pluperfect subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده شده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// passive imperfective pluperfect subjunctive\n\t\t{\n\t\t\tinput: []byte(\"خورده مي شده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// active present subjunctive\n\t\t{\n\t\t\tinput: []byte(\"بخورد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"بخورد\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestPersianAnalyzerOthers(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// nouns\n\t\t{\n\t\t\tinput: []byte(\"برگ ها\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"برگ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"برگ‌ها\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"برگ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// non persian\n\t\t{\n\t\t\tinput: []byte(\"English test.\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"english\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"test\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// others\n\t\t{\n\t\t\tinput: []byte(\"خورده مي شده بوده باشد\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"خورده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"برگ‌ها\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"برگ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fa/persian_normalize.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fa\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst NormalizeName = \"normalize_fa\"\n\nconst (\n\tYeh        = '\\u064A'\n\tFarsiYeh   = '\\u06CC'\n\tYehBarree  = '\\u06D2'\n\tKeheh      = '\\u06A9'\n\tKaf        = '\\u0643'\n\tHamzaAbove = '\\u0654'\n\tHehYeh     = '\\u06C0'\n\tHehGoal    = '\\u06C1'\n\tHeh        = '\\u0647'\n)\n\ntype PersianNormalizeFilter struct {\n}\n\nfunc NewPersianNormalizeFilter() *PersianNormalizeFilter {\n\treturn &PersianNormalizeFilter{}\n}\n\nfunc (s *PersianNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := normalize(token.Term)\n\t\ttoken.Term = term\n\t}\n\treturn input\n}\n\nfunc normalize(input []byte) []byte {\n\trunes := bytes.Runes(input)\n\tfor i := 0; i < len(runes); i++ {\n\t\tswitch runes[i] {\n\t\tcase FarsiYeh, YehBarree:\n\t\t\trunes[i] = Yeh\n\t\tcase Keheh:\n\t\t\trunes[i] = Kaf\n\t\tcase HehYeh, HehGoal:\n\t\t\trunes[i] = Heh\n\t\tcase HamzaAbove: // necessary for HEH + HAMZA\n\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\ti--\n\t\t}\n\t}\n\treturn analysis.BuildTermFromRunes(runes)\n}\n\nfunc NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewPersianNormalizeFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fa/persian_normalize_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fa\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestPersianNormalizeFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// FarsiYeh\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"های\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"هاي\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// YehBarree\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"هاے\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"هاي\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Keheh\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"کشاندن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"كشاندن\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// HehYeh\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"كتابۀ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"كتابه\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// HehHamzaAbove\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"كتابهٔ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"كتابه\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// HehGoal\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"زادہ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"زاده\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// empty\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tpersianNormalizeFilter := NewPersianNormalizeFilter()\n\tfor _, test := range tests {\n\t\tactual := persianNormalizeFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected % x, got % x\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fa/stop_filter_fa.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fa\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fa/stop_words_fa.go",
    "content": "package fa\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_fa\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar PersianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license.\n# See http://members.unine.ch/jacques.savoy/clef/index.html.\n# Also see http://www.opensource.org/licenses/bsd-license.html\n# Note: by default this file is used after normalization, so when adding entries\n# to this file, use the arabic 'ي' instead of 'ی'\nانان\nنداشته\nسراسر\nخياه\nايشان\nوي\nتاكنون\nبيشتري\nدوم\nپس\nناشي\nوگو\nيا\nداشتند\nسپس\nهنگام\nهرگز\nپنج\nنشان\nامسال\nديگر\nگروهي\nشدند\nچطور\nده\nو\nدو\nنخستين\nولي\nچرا\nچه\nوسط\nه\nكدام\nقابل\nيك\nرفت\nهفت\nهمچنين\nدر\nهزار\nبله\nبلي\nشايد\nاما\nشناسي\nگرفته\nدهد\nداشته\nدانست\nداشتن\nخواهيم\nميليارد\nوقتيكه\nامد\nخواهد\nجز\nاورده\nشده\nبلكه\nخدمات\nشدن\nبرخي\nنبود\nبسياري\nجلوگيري\nحق\nكردند\nنوعي\nبعري\nنكرده\nنظير\nنبايد\nبوده\nبودن\nداد\nاورد\nهست\nجايي\nشود\nدنبال\nداده\nبايد\nسابق\nهيچ\nهمان\nانجا\nكمتر\nكجاست\nگردد\nكسي\nتر\nمردم\nتان\nدادن\nبودند\nسري\nجدا\nندارند\nمگر\nيكديگر\nدارد\nدهند\nبنابراين\nهنگامي\nسمت\nجا\nانچه\nخود\nدادند\nزياد\nدارند\nاثر\nبدون\nبهترين\nبيشتر\nالبته\nبه\nبراساس\nبيرون\nكرد\nبعضي\nگرفت\nتوي\nاي\nميليون\nاو\nجريان\nتول\nبر\nمانند\nبرابر\nباشيم\nمدتي\nگويند\nاكنون\nتا\nتنها\nجديد\nچند\nبي\nنشده\nكردن\nكردم\nگويد\nكرده\nكنيم\nنمي\nنزد\nروي\nقصد\nفقط\nبالاي\nديگران\nاين\nديروز\nتوسط\nسوم\nايم\nدانند\nسوي\nاستفاده\nشما\nكنار\nداريم\nساخته\nطور\nامده\nرفته\nنخست\nبيست\nنزديك\nطي\nكنيد\nاز\nانها\nتمامي\nداشت\nيكي\nطريق\nاش\nچيست\nروب\nنمايد\nگفت\nچندين\nچيزي\nتواند\nام\nايا\nبا\nان\nايد\nترين\nاينكه\nديگري\nراه\nهايي\nبروز\nهمچنان\nپاعين\nكس\nحدود\nمختلف\nمقابل\nچيز\nگيرد\nندارد\nضد\nهمچون\nسازي\nشان\nمورد\nباره\nمرسي\nخويش\nبرخوردار\nچون\nخارج\nشش\nهنوز\nتحت\nضمن\nهستيم\nگفته\nفكر\nبسيار\nپيش\nبراي\nروزهاي\nانكه\nنخواهد\nبالا\nكل\nوقتي\nكي\nچنين\nكه\nگيري\nنيست\nاست\nكجا\nكند\nنيز\nيابد\nبندي\nحتي\nتوانند\nعقب\nخواست\nكنند\nبين\nتمام\nهمه\nما\nباشند\nمثل\nشد\nاري\nباشد\nاره\nطبق\nبعد\nاگر\nصورت\nغير\nجاي\nبيش\nريزي\nاند\nزيرا\nچگونه\nبار\nلطفا\nمي\nدرباره\nمن\nديده\nهمين\nگذاري\nبرداري\nعلت\nگذاشته\nهم\nفوق\nنه\nها\nشوند\nاباد\nهمواره\nهر\nاول\nخواهند\nچهار\nنام\nامروز\nمان\nهاي\nقبل\nكنم\nسعي\nتازه\nرا\nهستند\nزير\nجلوي\nعنوان\nبود\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(PersianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fi/analyzer_fi.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fi\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"fi\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopFiFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerFiFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopFiFilter,\n\t\t\tstemmerFiFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fi/analyzer_fi_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fi\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestFinishAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"edeltäjiinsä\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"edeltäj\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"edeltäjistään\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"edeltäj\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"olla\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fi/stemmer_fi.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fi\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/finnish\"\n)\n\nconst SnowballStemmerName = \"stemmer_fi_snowball\"\n\ntype FinnishStemmerFilter struct {\n}\n\nfunc NewFinnishStemmerFilter() *FinnishStemmerFilter {\n\treturn &FinnishStemmerFilter{}\n}\n\nfunc (s *FinnishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tfinnish.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc FinnishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewFinnishStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, FinnishStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fi/stop_filter_fi.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fi\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fi/stop_words_fi.go",
    "content": "package fi\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_fi\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar FinnishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n \n| forms of BE\n\nolla\nolen\nolet\non\nolemme\nolette\novat\nole        | negative form\n\noli\nolisi\nolisit\nolisin\nolisimme\nolisitte\nolisivat\nolit\nolin\nolimme\nolitte\nolivat\nollut\nolleet\n\nen         | negation\net\nei\nemme\nette\neivät\n\n|Nom   Gen    Acc    Part   Iness   Elat    Illat  Adess   Ablat   Allat   Ess    Trans\nminä   minun  minut  minua  minussa minusta minuun minulla minulta minulle               | I\nsinä   sinun  sinut  sinua  sinussa sinusta sinuun sinulla sinulta sinulle               | you\nhän    hänen  hänet  häntä  hänessä hänestä häneen hänellä häneltä hänelle               | he she\nme     meidän meidät meitä  meissä  meistä  meihin meillä  meiltä  meille                | we\nte     teidän teidät teitä  teissä  teistä  teihin teillä  teiltä  teille                | you\nhe     heidän heidät heitä  heissä  heistä  heihin heillä  heiltä  heille                | they\n\ntämä   tämän         tätä   tässä   tästä   tähän  tallä   tältä   tälle   tänä   täksi  | this\ntuo    tuon          tuotä  tuossa  tuosta  tuohon tuolla  tuolta  tuolle  tuona  tuoksi | that\nse     sen           sitä   siinä   siitä   siihen sillä   siltä   sille   sinä   siksi  | it\nnämä   näiden        näitä  näissä  näistä  näihin näillä  näiltä  näille  näinä  näiksi | these\nnuo    noiden        noita  noissa  noista  noihin noilla  noilta  noille  noina  noiksi | those\nne     niiden        niitä  niissä  niistä  niihin niillä  niiltä  niille  niinä  niiksi | they\n\nkuka   kenen kenet   ketä   kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who\nketkä  keiden ketkä  keitä  keissä  keistä  keihin keillä  keiltä  keille  keinä  keiksi | (pl)\nmikä   minkä minkä   mitä   missä   mistä   mihin  millä   miltä   mille   minä   miksi  | which what\nmitkä                                                                                    | (pl)\n\njoka   jonka         jota   jossa   josta   johon  jolla   jolta   jolle   jona   joksi  | who which\njotka  joiden        joita  joissa  joista  joihin joilla  joilta  joille  joina  joiksi | (pl)\n\n| conjunctions\n\nettä   | that\nja     | and\njos    | if\nkoska  | because\nkuin   | than\nmutta  | but\nniin   | so\nsekä   | and\nsillä  | for\ntai    | or\nvaan   | but\nvai    | or\nvaikka | although\n\n\n| prepositions\n\nkanssa  | with\nmukaan  | according to\nnoin    | about\npoikki  | across\nyli     | over, across\n\n| other\n\nkun    | when\nniin   | so\nnyt    | now\nitse   | self\n\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(FinnishStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/analyzer_fr.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"fr\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\telisionFilter, err := cache.TokenFilterNamed(ElisionName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopFrFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerFrFilter, err := cache.TokenFilterNamed(LightStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\telisionFilter,\n\t\t\tstopFrFilter,\n\t\t\tstemmerFrFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/analyzer_fr_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestFrenchAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput:  []byte(\"\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"chien chat cheval\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chien\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chat\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cheval\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"chien CHAT CHEVAL\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chien\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chat\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cheval\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"  chien  ,? + = -  CHAT /: > CHEVAL\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chien\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chat\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cheval\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"chien++\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chien\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"mot \\\"entreguillemet\\\"\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"mot\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"entreguilemet\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Jean-François\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"jean\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"francoi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop words\n\t\t{\n\t\t\tinput: []byte(\"le la chien les aux chat du des à cheval\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chien\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chat\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cheval\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// nouns and adjectives\n\t\t{\n\t\t\tinput: []byte(\"lances chismes habitable chiste éléments captifs\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"lanc\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chism\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"habitabl\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chist\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"element\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"captif\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// verbs\n\t\t{\n\t\t\tinput: []byte(\"finissions souffrirent rugissante\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"finision\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"soufrirent\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"rugisant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"C3PO aujourd'hui oeuf ïâöûàä anticonstitutionnellement Java++ \"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"c3po\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aujourd'hui\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"oeuf\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ïaöuaä\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"anticonstitutionel\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"java\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"propriétaire\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"proprietair\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/articles_fr.go",
    "content": "package fr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst ArticlesName = \"articles_fr\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis\n\nvar FrenchArticles = []byte(`\nl\nm\nt\nqu\nn\ns\nj\nd\nc\njusqu\nquoiqu\nlorsqu\npuisqu\n`)\n\nfunc ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(FrenchArticles)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/elision_fr.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/elision\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst ElisionName = \"elision_fr\"\n\nfunc ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tarticlesTokenMap, err := cache.TokenMapNamed(ArticlesName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building elision filter: %v\", err)\n\t}\n\treturn elision.NewElisionFilter(articlesTokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/elision_fr_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestFrenchElision(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"l'avion\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"avion\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\telisionFilter, err := cache.TokenFilterNamed(ElisionName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := elisionFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/light_stemmer_fr.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"bytes\"\n\t\"unicode\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst LightStemmerName = \"stemmer_fr_light\"\n\ntype FrenchLightStemmerFilter struct {\n}\n\nfunc NewFrenchLightStemmerFilter() *FrenchLightStemmerFilter {\n\treturn &FrenchLightStemmerFilter{}\n}\n\nfunc (s *FrenchLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\trunes := bytes.Runes(token.Term)\n\t\trunes = stem(runes)\n\t\ttoken.Term = analysis.BuildTermFromRunes(runes)\n\t}\n\treturn input\n}\n\nfunc stem(input []rune) []rune {\n\n\tinputLen := len(input)\n\n\tif inputLen > 5 && input[inputLen-1] == 'x' {\n\t\tif input[inputLen-3] == 'a' && input[inputLen-2] == 'u' && input[inputLen-4] != 'e' {\n\t\t\tinput[inputLen-2] = 'l'\n\t\t}\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t}\n\n\tif inputLen > 3 && input[inputLen-1] == 'x' {\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t}\n\n\tif inputLen > 3 && input[inputLen-1] == 's' {\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t}\n\n\tif inputLen > 9 && analysis.RunesEndsWith(input, \"issement\") {\n\t\tinput = input[0 : inputLen-6]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 8 && analysis.RunesEndsWith(input, \"issant\") {\n\t\tinput = input[0 : inputLen-4]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 6 && analysis.RunesEndsWith(input, \"ement\") {\n\t\tinput = input[0 : inputLen-4]\n\t\tinputLen = len(input)\n\t\tif inputLen > 3 && analysis.RunesEndsWith(input, \"ive\") {\n\t\t\tinput = input[0 : inputLen-1]\n\t\t\tinputLen = len(input)\n\t\t\tinput[inputLen-1] = 'f'\n\t\t}\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 11 && analysis.RunesEndsWith(input, \"ficatrice\") {\n\t\tinput = input[0 : inputLen-5]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-2] = 'e'\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 10 && analysis.RunesEndsWith(input, \"ficateur\") {\n\t\tinput = input[0 : inputLen-4]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-2] = 'e'\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 9 && analysis.RunesEndsWith(input, \"catrice\") {\n\t\tinput = input[0 : inputLen-3]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-4] = 'q'\n\t\tinput[inputLen-3] = 'u'\n\t\tinput[inputLen-2] = 'e'\n\t\t//s[len-1] = 'r' <-- unnecessary, already 'r'.\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 8 && analysis.RunesEndsWith(input, \"cateur\") {\n\t\tinput = input[0 : inputLen-2]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-4] = 'q'\n\t\tinput[inputLen-3] = 'u'\n\t\tinput[inputLen-2] = 'e'\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 8 && analysis.RunesEndsWith(input, \"atrice\") {\n\t\tinput = input[0 : inputLen-4]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-2] = 'e'\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 7 && analysis.RunesEndsWith(input, \"ateur\") {\n\t\tinput = input[0 : inputLen-3]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-2] = 'e'\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 6 && analysis.RunesEndsWith(input, \"trice\") {\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-3] = 'e'\n\t\tinput[inputLen-2] = 'u'\n\t\tinput[inputLen-1] = 'r'\n\t}\n\n\tif inputLen > 5 && analysis.RunesEndsWith(input, \"ième\") {\n\t\treturn norm(input[0 : inputLen-4])\n\t}\n\n\tif inputLen > 7 && analysis.RunesEndsWith(input, \"teuse\") {\n\t\tinput = input[0 : inputLen-2]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 6 && analysis.RunesEndsWith(input, \"teur\") {\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-1] = 'r'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 5 && analysis.RunesEndsWith(input, \"euse\") {\n\t\treturn norm(input[0 : inputLen-2])\n\t}\n\n\tif inputLen > 8 && analysis.RunesEndsWith(input, \"ère\") {\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-2] = 'e'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 7 && analysis.RunesEndsWith(input, \"ive\") {\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-1] = 'f'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 4 &&\n\t\t(analysis.RunesEndsWith(input, \"folle\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"molle\")) {\n\t\tinput = input[0 : inputLen-2]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-1] = 'u'\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 9 && analysis.RunesEndsWith(input, \"nnelle\") {\n\t\treturn norm(input[0 : inputLen-5])\n\t}\n\n\tif inputLen > 9 && analysis.RunesEndsWith(input, \"nnel\") {\n\t\treturn norm(input[0 : inputLen-3])\n\t}\n\n\tif inputLen > 4 && analysis.RunesEndsWith(input, \"ète\") {\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-2] = 'e'\n\t}\n\n\tif inputLen > 8 && analysis.RunesEndsWith(input, \"ique\") {\n\t\tinput = input[0 : inputLen-4]\n\t\tinputLen = len(input)\n\t}\n\n\tif inputLen > 8 && analysis.RunesEndsWith(input, \"esse\") {\n\t\treturn norm(input[0 : inputLen-3])\n\t}\n\n\tif inputLen > 7 && analysis.RunesEndsWith(input, \"inage\") {\n\t\treturn norm(input[0 : inputLen-3])\n\t}\n\n\tif inputLen > 9 && analysis.RunesEndsWith(input, \"isation\") {\n\t\tinput = input[0 : inputLen-7]\n\t\tinputLen = len(input)\n\t\tif inputLen > 5 && analysis.RunesEndsWith(input, \"ual\") {\n\t\t\tinput[inputLen-2] = 'e'\n\t\t}\n\t\treturn norm(input)\n\t}\n\n\tif inputLen > 9 && analysis.RunesEndsWith(input, \"isateur\") {\n\t\treturn norm(input[0 : inputLen-7])\n\t}\n\n\tif inputLen > 8 && analysis.RunesEndsWith(input, \"ation\") {\n\t\treturn norm(input[0 : inputLen-5])\n\t}\n\n\tif inputLen > 8 && analysis.RunesEndsWith(input, \"ition\") {\n\t\treturn norm(input[0 : inputLen-5])\n\t}\n\n\treturn norm(input)\n\n}\n\nfunc norm(input []rune) []rune {\n\n\tif len(input) > 4 {\n\t\tfor i := 0; i < len(input); i++ {\n\t\t\tswitch input[i] {\n\t\t\tcase 'à', 'á', 'â':\n\t\t\t\tinput[i] = 'a'\n\t\t\tcase 'ô':\n\t\t\t\tinput[i] = 'o'\n\t\t\tcase 'è', 'é', 'ê':\n\t\t\t\tinput[i] = 'e'\n\t\t\tcase 'ù', 'û':\n\t\t\t\tinput[i] = 'u'\n\t\t\tcase 'î':\n\t\t\t\tinput[i] = 'i'\n\t\t\tcase 'ç':\n\t\t\t\tinput[i] = 'c'\n\t\t\t}\n\n\t\t\tch := input[0]\n\t\t\tfor i := 1; i < len(input); i++ {\n\t\t\t\tif input[i] == ch && unicode.IsLetter(ch) {\n\t\t\t\t\tinput = analysis.DeleteRune(input, i)\n\t\t\t\t\ti -= 1\n\t\t\t\t} else {\n\t\t\t\t\tch = input[i]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(input) > 4 && analysis.RunesEndsWith(input, \"ie\") {\n\t\tinput = input[0 : len(input)-2]\n\t}\n\n\tif len(input) > 4 {\n\t\tif input[len(input)-1] == 'r' {\n\t\t\tinput = input[0 : len(input)-1]\n\t\t}\n\t\tif input[len(input)-1] == 'e' {\n\t\t\tinput = input[0 : len(input)-1]\n\t\t}\n\t\tif input[len(input)-1] == 'e' {\n\t\t\tinput = input[0 : len(input)-1]\n\t\t}\n\t\tif input[len(input)-1] == input[len(input)-2] && unicode.IsLetter(input[len(input)-1]) {\n\t\t\tinput = input[0 : len(input)-1]\n\t\t}\n\t}\n\n\treturn input\n}\n\nfunc FrenchLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewFrenchLightStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(LightStemmerName, FrenchLightStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/light_stemmer_fr_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestFrenchLightStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chevaux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cheval\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cheval\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cheval\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"hiboux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"hibou\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"hibou\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"hibou\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chantés\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chanter\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chante\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baronnes\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baron\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"barons\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baron\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baron\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baron\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"peaux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"peau\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"peau\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"peau\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"anneaux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aneau\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"anneau\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aneau\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"neveux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"neveu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"neveu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"neveu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"affreux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"afreu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"affreuse\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"afreu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"investissement\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"investi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"investir\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"investi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"assourdissant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"asourdi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"assourdir\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"asourdi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"pratiquement\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"pratiqu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"pratique\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"pratiqu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"administrativement\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"administratif\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"administratif\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"administratif\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"justificatrice\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"justifi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"justificateur\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"justifi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"justifier\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"justifi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"educatrice\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"eduqu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"eduquer\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"eduqu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"communicateur\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"comuniqu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"communiquer\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"comuniqu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"accompagnatrice\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"acompagn\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"accompagnateur\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"acompagn\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"administrateur\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"administr\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"administrer\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"administr\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"productrice\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"product\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"producteur\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"product\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"acheteuse\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"achet\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"acheteur\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"achet\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"planteur\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"plant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"plante\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"plant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"poreuse\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"poreu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"poreux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"poreu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"plieuse\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"plieu\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bijoutière\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bijouti\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bijoutier\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bijouti\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"caissière\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"caisi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"caissier\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"caisi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abrasive\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abrasif\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abrasif\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abrasif\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"folle\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fou\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fou\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fou\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"personnelle\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"person\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"personne\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"person\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// algo bug: too short length\n\t\t// {\n\t\t// \tinput: analysis.TokenStream{\n\t\t// \t\t&analysis.Token{\n\t\t// \t\t\tTerm: []byte(\"personnel\"),\n\t\t// \t\t},\n\t\t// \t},\n\t\t// \toutput: analysis.TokenStream{\n\t\t// \t\t&analysis.Token{\n\t\t// \t\t\tTerm: []byte(\"person\"),\n\t\t// \t\t},\n\t\t// \t},\n\t\t// },\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"complète\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"complet\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"complet\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"complet\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aromatique\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aromat\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"faiblesse\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"faibl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"faible\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"faibl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"patinage\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"patin\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"patin\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"patin\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"sonorisation\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"sono\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ritualisation\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"rituel\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"rituel\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"rituel\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// algo bug: masked by rules above\n\t\t// {\n\t\t// \tinput: analysis.TokenStream{\n\t\t// \t\t&analysis.Token{\n\t\t// \t\t\tTerm: []byte(\"colonisateur\"),\n\t\t// \t\t},\n\t\t// \t},\n\t\t// \toutput: analysis.TokenStream{\n\t\t// \t\t&analysis.Token{\n\t\t// \t\t\tTerm: []byte(\"colon\"),\n\t\t// \t\t},\n\t\t// \t},\n\t\t// },\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"nomination\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"nomin\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"disposition\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"dispos\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"dispose\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"dispos\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// SOLR-3463 : abusive compression of repeated characters in numbers\n\t\t// Trailing repeated char elision :\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"1234555\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"1234555\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Repeated char within numbers with more than 4 characters :\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"12333345\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"12333345\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Short numbers weren't affected already:\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"1234\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"1234\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Ensure behaviour is preserved for words!\n\t\t// Trailing repeated char elision :\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcdeff\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcdef\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Repeated char within words with more than 4 characters :\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcccddeef\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcdef\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"créées\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cre\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// Combined letter and digit repetition\n\t\t// 10:00pm\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"22hh00\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"22h00\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// bug #214\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"propriétaire\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"proprietair\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(LightStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/minimal_stemmer_fr.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst MinimalStemmerName = \"stemmer_fr_min\"\n\ntype FrenchMinimalStemmerFilter struct {\n}\n\nfunc NewFrenchMinimalStemmerFilter() *FrenchMinimalStemmerFilter {\n\treturn &FrenchMinimalStemmerFilter{}\n}\n\nfunc (s *FrenchMinimalStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\trunes := bytes.Runes(token.Term)\n\t\trunes = minstem(runes)\n\t\ttoken.Term = analysis.BuildTermFromRunes(runes)\n\t}\n\treturn input\n}\n\nfunc minstem(input []rune) []rune {\n\n\tif len(input) < 6 {\n\t\treturn input\n\t}\n\n\tif input[len(input)-1] == 'x' {\n\t\tif input[len(input)-3] == 'a' && input[len(input)-2] == 'u' {\n\t\t\tinput[len(input)-2] = 'l'\n\t\t}\n\t\treturn input[0 : len(input)-1]\n\t}\n\n\tif input[len(input)-1] == 's' {\n\t\tinput = input[0 : len(input)-1]\n\t}\n\tif input[len(input)-1] == 'r' {\n\t\tinput = input[0 : len(input)-1]\n\t}\n\tif input[len(input)-1] == 'e' {\n\t\tinput = input[0 : len(input)-1]\n\t}\n\tif input[len(input)-1] == 'é' {\n\t\tinput = input[0 : len(input)-1]\n\t}\n\tif input[len(input)-1] == input[len(input)-2] {\n\t\tinput = input[0 : len(input)-1]\n\t}\n\treturn input\n}\n\nfunc FrenchMinimalStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewFrenchMinimalStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(MinimalStemmerName, FrenchMinimalStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/minimal_stemmer_fr_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestFrenchMinimalStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chevaux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cheval\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"hiboux\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"hibou\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chantés\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chanter\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chante\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"chant\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baronnes\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baron\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"barons\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baron\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baron\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"baron\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(MinimalStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/stemmer_fr_snowball.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/french\"\n)\n\nconst SnowballStemmerName = \"stemmer_fr_snowball\"\n\ntype FrenchStemmerFilter struct {\n}\n\nfunc NewFrenchStemmerFilter() *FrenchStemmerFilter {\n\treturn &FrenchStemmerFilter{}\n}\n\nfunc (s *FrenchStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tfrench.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc FrenchStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewFrenchStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, FrenchStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/stemmer_fr_snowball_test.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSnowballFrenchStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"antagoniste\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"antagon\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"barbouillait\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"barbouill\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"calculateur\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"calcul\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/stop_filter_fr.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/fr/stop_words_fr.go",
    "content": "package fr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_fr\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar FrenchStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | A French stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\nau             |  a + le\naux            |  a + les\navec           |  with\nce             |  this\nces            |  these\ndans           |  with\nde             |  of\ndes            |  de + les\ndu             |  de + le\nelle           |  she\nen             |  'of them' etc\net             |  and\neux            |  them\nil             |  he\nje             |  I\nla             |  the\nle             |  the\nleur           |  their\nlui            |  him\nma             |  my (fem)\nmais           |  but\nme             |  me\nmême           |  same; as in moi-même (myself) etc\nmes            |  me (pl)\nmoi            |  me\nmon            |  my (masc)\nne             |  not\nnos            |  our (pl)\nnotre          |  our\nnous           |  we\non             |  one\nou             |  where\npar            |  by\npas            |  not\npour           |  for\nqu             |  que before vowel\nque            |  that\nqui            |  who\nsa             |  his, her (fem)\nse             |  oneself\nses            |  his (pl)\nson            |  his, her (masc)\nsur            |  on\nta             |  thy (fem)\nte             |  thee\ntes            |  thy (pl)\ntoi            |  thee\nton            |  thy (masc)\ntu             |  thou\nun             |  a\nune            |  a\nvos            |  your (pl)\nvotre          |  your\nvous           |  you\n\n               |  single letter forms\n\nc              |  c'\nd              |  d'\nj              |  j'\nl              |  l'\nà              |  to, at\nm              |  m'\nn              |  n'\ns              |  s'\nt              |  t'\ny              |  there\n\n               | forms of être (not including the infinitive):\nété\nétée\nétées\nétés\nétant\nsuis\nes\nest\nsommes\nêtes\nsont\nserai\nseras\nsera\nserons\nserez\nseront\nserais\nserait\nserions\nseriez\nseraient\nétais\nétait\nétions\nétiez\nétaient\nfus\nfut\nfûmes\nfûtes\nfurent\nsois\nsoit\nsoyons\nsoyez\nsoient\nfusse\nfusses\nfût\nfussions\nfussiez\nfussent\n\n               | forms of avoir (not including the infinitive):\nayant\neu\neue\neues\neus\nai\nas\navons\navez\nont\naurai\nauras\naura\naurons\naurez\nauront\naurais\naurait\naurions\nauriez\nauraient\navais\navait\navions\naviez\navaient\neut\neûmes\neûtes\neurent\naie\naies\nait\nayons\nayez\naient\neusse\neusses\neût\neussions\neussiez\neussent\n\n               | Later additions (from Jean-Christophe Deschamps)\nceci           |  this\ncela           |  that\ncelà           |  that\ncet            |  this\ncette          |  this\nici            |  here\nils            |  they\nles            |  the (pl)\nleurs          |  their (pl)\nquel           |  which\nquels          |  which\nquelle         |  which\nquelles        |  which\nsans           |  without\nsoi            |  oneself\n\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(FrenchStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ga/articles_ga.go",
    "content": "package ga\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst ArticlesName = \"articles_ga\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis\n\nvar IrishArticles = []byte(`\nd\nm\nb\n`)\n\nfunc ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(IrishArticles)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ga/elision_ga.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ga\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/elision\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst ElisionName = \"elision_ga\"\n\nfunc ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tarticlesTokenMap, err := cache.TokenMapNamed(ArticlesName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building elision filter: %v\", err)\n\t}\n\treturn elision.NewElisionFilter(articlesTokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ga/elision_ga_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ga\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestFrenchElision(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"b'fhearr\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fhearr\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\telisionFilter, err := cache.TokenFilterNamed(ElisionName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := elisionFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ga/stop_filter_ga.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ga\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ga/stop_words_ga.go",
    "content": "package ga\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_ga\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar IrishStopWords = []byte(`\na\nach\nag\nagus\nan\naon\nar\narna\nas\nb'\nba\nbeirt\nbhúr\ncaoga\nceathair\nceathrar\nchomh\nchtó\nchuig\nchun\ncois\ncéad\ncúig\ncúigear\nd'\ndaichead\ndar\nde\ndeich\ndeichniúr\nden\ndhá\ndo\ndon\ndtí\ndá\ndár\ndó\nfaoi\nfaoin\nfaoina\nfaoinár\nfara\nfiche\ngach\ngan\ngo\ngur\nhaon\nhocht\ni\niad\nidir\nin\nina\nins\ninár\nis\nle\nleis\nlena\nlenár\nm'\nmar\nmo\nmé\nna\nnach\nnaoi\nnaonúr\nná\nní\nníor\nnó\nnócha\nocht\nochtar\nos\nroimh\nsa\nseacht\nseachtar\nseachtó\nseasca\nseisear\nsiad\nsibh\nsinn\nsna\nsé\nsí\ntar\nthar\nthú\ntriúr\ntrí\ntrína\ntrínár\ntríocha\ntú\num\nár\né\néis\ní\nó\nón\nóna\nónár\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(IrishStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/gl/stop_filter_gl.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage gl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/gl/stop_words_gl.go",
    "content": "package gl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_gl\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar GalicianStopWords = []byte(`# galican stopwords\na\naínda\nalí\naquel\naquela\naquelas\naqueles\naquilo\naquí\nao\naos\nas\nasí\ná\nben\ncando\nche\nco\ncoa\ncomigo\ncon\nconnosco\ncontigo\nconvosco\ncoas\ncos\ncun\ncuns\ncunha\ncunhas\nda\ndalgunha\ndalgunhas\ndalgún\ndalgúns\ndas\nde\ndel\ndela\ndelas\ndeles\ndesde\ndeste\ndo\ndos\ndun\nduns\ndunha\ndunhas\ne\nel\nela\nelas\neles\nen\nera\neran\nesa\nesas\nese\neses\nesta\nestar\nestaba\nestá\nestán\neste\nestes\nestiven\nestou\neu\né\nfacer\nfoi\nforon\nfun\nhabía\nhai\niso\nisto\nla\nlas\nlle\nlles\nlo\nlos\nmais\nme\nmeu\nmeus\nmin\nmiña\nmiñas\nmoi\nna\nnas\nneste\nnin\nno\nnon\nnos\nnosa\nnosas\nnoso\nnosos\nnós\nnun\nnunha\nnuns\nnunhas\no\nos\nou\nó\nós\npara\npero\npode\npois\npola\npolas\npolo\npolos\npor\nque\nse\nsenón\nser\nseu\nseus\nsexa\nsido\nsobre\nsúa\nsúas\ntamén\ntan\nte\nten\nteñen\nteño\nter\nteu\nteus\nti\ntido\ntiña\ntiven\ntúa\ntúas\nun\nunha\nunhas\nuns\nvos\nvosa\nvosas\nvoso\nvosos\nvós\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(GalicianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hi/analyzer_hi.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hi\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/lang/in\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"hi\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tindicNormalizeFilter, err := cache.TokenFilterNamed(in.NormalizeName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\thindiNormalizeFilter, err := cache.TokenFilterNamed(NormalizeName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopHiFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerHiFilter, err := cache.TokenFilterNamed(StemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tindicNormalizeFilter,\n\t\t\thindiNormalizeFilter,\n\t\t\tstopHiFilter,\n\t\t\tstemmerHiFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hi/analyzer_hi_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hi\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestHindiAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// two ways to write 'hindi' itself\n\t\t{\n\t\t\tinput: []byte(\"हिन्दी\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"हिंद\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"हिंदी\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"हिंद\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hi/hindi_normalize.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hi\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst NormalizeName = \"normalize_hi\"\n\ntype HindiNormalizeFilter struct {\n}\n\nfunc NewHindiNormalizeFilter() *HindiNormalizeFilter {\n\treturn &HindiNormalizeFilter{}\n}\n\nfunc (s *HindiNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := normalize(token.Term)\n\t\ttoken.Term = term\n\t}\n\treturn input\n}\n\nfunc normalize(input []byte) []byte {\n\trunes := bytes.Runes(input)\n\tfor i := 0; i < len(runes); i++ {\n\t\tswitch runes[i] {\n\t\t// dead n -> bindu\n\t\tcase '\\u0928':\n\t\t\tif i+1 < len(runes) && runes[i+1] == '\\u094D' {\n\t\t\t\trunes[i] = '\\u0902'\n\t\t\t\trunes = analysis.DeleteRune(runes, i+1)\n\t\t\t}\n\t\t// candrabindu -> bindu\n\t\tcase '\\u0901':\n\t\t\trunes[i] = '\\u0902'\n\t\t// nukta deletions\n\t\tcase '\\u093C':\n\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\ti--\n\t\tcase '\\u0929':\n\t\t\trunes[i] = '\\u0928'\n\t\tcase '\\u0931':\n\t\t\trunes[i] = '\\u0930'\n\t\tcase '\\u0934':\n\t\t\trunes[i] = '\\u0933'\n\t\tcase '\\u0958':\n\t\t\trunes[i] = '\\u0915'\n\t\tcase '\\u0959':\n\t\t\trunes[i] = '\\u0916'\n\t\tcase '\\u095A':\n\t\t\trunes[i] = '\\u0917'\n\t\tcase '\\u095B':\n\t\t\trunes[i] = '\\u091C'\n\t\tcase '\\u095C':\n\t\t\trunes[i] = '\\u0921'\n\t\tcase '\\u095D':\n\t\t\trunes[i] = '\\u0922'\n\t\tcase '\\u095E':\n\t\t\trunes[i] = '\\u092B'\n\t\tcase '\\u095F':\n\t\t\trunes[i] = '\\u092F'\n\t\t\t// zwj/zwnj -> delete\n\t\tcase '\\u200D', '\\u200C':\n\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\ti--\n\t\t\t// virama -> delete\n\t\tcase '\\u094D':\n\t\t\trunes = analysis.DeleteRune(runes, i)\n\t\t\ti--\n\t\t\t// chandra/short -> replace\n\t\tcase '\\u0945', '\\u0946':\n\t\t\trunes[i] = '\\u0947'\n\t\tcase '\\u0949', '\\u094A':\n\t\t\trunes[i] = '\\u094B'\n\t\tcase '\\u090D', '\\u090E':\n\t\t\trunes[i] = '\\u090F'\n\t\tcase '\\u0911', '\\u0912':\n\t\t\trunes[i] = '\\u0913'\n\t\tcase '\\u0972':\n\t\t\trunes[i] = '\\u0905'\n\t\t\t// long -> short ind. vowels\n\t\tcase '\\u0906':\n\t\t\trunes[i] = '\\u0905'\n\t\tcase '\\u0908':\n\t\t\trunes[i] = '\\u0907'\n\t\tcase '\\u090A':\n\t\t\trunes[i] = '\\u0909'\n\t\tcase '\\u0960':\n\t\t\trunes[i] = '\\u090B'\n\t\tcase '\\u0961':\n\t\t\trunes[i] = '\\u090C'\n\t\tcase '\\u0910':\n\t\t\trunes[i] = '\\u090F'\n\t\tcase '\\u0914':\n\t\t\trunes[i] = '\\u0913'\n\t\t\t// long -> short dep. vowels\n\t\tcase '\\u0940':\n\t\t\trunes[i] = '\\u093F'\n\t\tcase '\\u0942':\n\t\t\trunes[i] = '\\u0941'\n\t\tcase '\\u0944':\n\t\t\trunes[i] = '\\u0943'\n\t\tcase '\\u0963':\n\t\t\trunes[i] = '\\u0962'\n\t\tcase '\\u0948':\n\t\t\trunes[i] = '\\u0947'\n\t\tcase '\\u094C':\n\t\t\trunes[i] = '\\u094B'\n\t\t}\n\t}\n\treturn analysis.BuildTermFromRunes(runes)\n}\n\nfunc NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewHindiNormalizeFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hi/hindi_normalize_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hi\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestHindiNormalizeFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// basics\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अँगरेज़ी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजि\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अँगरेजी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजि\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अँग्रेज़ी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजि\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अँग्रेजी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजि\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेज़ी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजि\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजि\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंग्रेज़ी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजि\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंग्रेजी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अंगरेजि\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test decompositions\n\t\t// removing nukta dot\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"क़िताब\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"किताब\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"फ़र्ज़\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"फरज\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"क़र्ज़\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"करज\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// some other composed nukta forms\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ऱऴख़ग़ड़ढ़य़\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"रळखगडढय\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// removal of format (ZWJ/ZWNJ)\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"शार्‍मा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"शारमा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"शार्‌मा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"शारमा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// removal of chandra\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ॅॆॉॊऍऎऑऒ\\u0972\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ेेोोएएओओअ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// vowel shortening\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"आईऊॠॡऐऔीूॄॣैौ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अइउऋऌएओिुृॢेो\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// empty\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\thindiNormalizeFilter := NewHindiNormalizeFilter()\n\tfor _, test := range tests {\n\t\tactual := hindiNormalizeFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected % x, got % x\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hi/hindi_stemmer_filter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hi\n\nimport (\n\t\"bytes\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StemmerName = \"stemmer_hi\"\n\ntype HindiStemmerFilter struct {\n}\n\nfunc NewHindiStemmerFilter() *HindiStemmerFilter {\n\treturn &HindiStemmerFilter{}\n}\n\nfunc (s *HindiStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\t// if not protected keyword, stem it\n\t\tif !token.KeyWord {\n\t\t\tstemmed := stem(token.Term)\n\t\t\ttoken.Term = stemmed\n\t\t}\n\t}\n\treturn input\n}\n\nfunc stem(input []byte) []byte {\n\tinputLen := utf8.RuneCount(input)\n\n\t// 5\n\tif inputLen > 6 &&\n\t\t(bytes.HasSuffix(input, []byte(\"ाएंगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाएंगे\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाऊंगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाऊंगा\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाइयाँ\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाइयों\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाइयां\"))) {\n\t\treturn analysis.TruncateRunes(input, 5)\n\t}\n\n\t// 4\n\tif inputLen > 5 &&\n\t\t(bytes.HasSuffix(input, []byte(\"ाएगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाएगा\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाओगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाओगे\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"एंगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ेंगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"एंगे\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ेंगे\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ूंगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ूंगा\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ातीं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"नाओं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"नाएं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ताओं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ताएं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ियाँ\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ियों\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ियां\"))) {\n\t\treturn analysis.TruncateRunes(input, 4)\n\t}\n\n\t// 3\n\tif inputLen > 4 &&\n\t\t(bytes.HasSuffix(input, []byte(\"ाकर\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाइए\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाईं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाया\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ेगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ेगा\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ोगी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ोगे\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाने\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाना\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाते\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाती\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाता\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"तीं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाओं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाएं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ुओं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ुएं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ुआं\"))) {\n\t\treturn analysis.TruncateRunes(input, 3)\n\t}\n\n\t// 2\n\tif inputLen > 3 &&\n\t\t(bytes.HasSuffix(input, []byte(\"कर\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाओ\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"िए\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाई\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाए\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ने\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"नी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ना\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ते\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ीं\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ती\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ता\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ाँ\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ां\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ों\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ें\"))) {\n\t\treturn analysis.TruncateRunes(input, 2)\n\t}\n\n\t// 1\n\tif inputLen > 2 &&\n\t\t(bytes.HasSuffix(input, []byte(\"ो\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"े\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ू\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ु\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ी\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ि\")) ||\n\t\t\tbytes.HasSuffix(input, []byte(\"ा\"))) {\n\t\treturn analysis.TruncateRunes(input, 1)\n\t}\n\n\treturn input\n}\n\nfunc StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewHindiStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hi/hindi_stemmer_filter_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hi\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestHindiStemmerFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// masc noun inflections\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडका\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडक\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडके\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडक\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडकों\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडक\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"गुरु\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"गुर\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"गुरुओं\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"गुर\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"दोस्त\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"दोस्त\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"दोस्तों\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"दोस्त\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// feminine noun inflections\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडकी\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडक\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडकियों\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"लडक\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"किताब\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"किताब\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"किताबें\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"किताब\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"किताबों\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"किताब\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"आध्यापीका\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"आध्यापीक\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"आध्यापीकाएं\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"आध्यापीक\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"आध्यापीकाओं\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"आध्यापीक\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// some verb forms\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"खाना\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"खा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"खाता\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"खा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"खाती\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"खा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"खा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"खा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// exceptions\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"कठिनाइयां\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"कठिन\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"कठिन\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"कठिन\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// empty\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\thindiStemmerFilter := NewHindiStemmerFilter()\n\tfor _, test := range tests {\n\t\tactual := hindiStemmerFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected % x, got % x\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hi/stop_filter_hi.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hi\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hi/stop_words_hi.go",
    "content": "package hi\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_hi\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar HindiStopWords = []byte(`# Also see http://www.opensource.org/licenses/bsd-license.html\n# See http://members.unine.ch/jacques.savoy/clef/index.html.\n# This file was created by Jacques Savoy and is distributed under the BSD license.\n# Note: by default this file also contains forms normalized by HindiNormalizer \n# for spelling variation (see section below), such that it can be used whether or \n# not you enable that feature. When adding additional entries to this list,\n# please add the normalized form as well. \nअंदर\nअत\nअपना\nअपनी\nअपने\nअभी\nआदि\nआप\nइत्यादि\nइन \nइनका\nइन्हीं\nइन्हें\nइन्हों\nइस\nइसका\nइसकी\nइसके\nइसमें\nइसी\nइसे\nउन\nउनका\nउनकी\nउनके\nउनको\nउन्हीं\nउन्हें\nउन्हों\nउस\nउसके\nउसी\nउसे\nएक\nएवं\nएस\nऐसे\nऔर\nकई\nकर\nकरता\nकरते\nकरना\nकरने\nकरें\nकहते\nकहा\nका\nकाफ़ी\nकि\nकितना\nकिन्हें\nकिन्हों\nकिया\nकिर\nकिस\nकिसी\nकिसे\nकी\nकुछ\nकुल\nके\nको\nकोई\nकौन\nकौनसा\nगया\nघर\nजब\nजहाँ\nजा\nजितना\nजिन\nजिन्हें\nजिन्हों\nजिस\nजिसे\nजीधर\nजैसा\nजैसे\nजो\nतक\nतब\nतरह\nतिन\nतिन्हें\nतिन्हों\nतिस\nतिसे\nतो\nथा\nथी\nथे\nदबारा\nदिया\nदुसरा\nदूसरे\nदो\nद्वारा\nन\nनहीं\nना\nनिहायत\nनीचे\nने\nपर\nपर  \nपहले\nपूरा\nपे\nफिर\nबनी\nबही\nबहुत\nबाद\nबाला\nबिलकुल\nभी\nभीतर\nमगर\nमानो\nमे\nमें\nयदि\nयह\nयहाँ\nयही\nया\nयिह \nये\nरखें\nरहा\nरहे\nऱ्वासा\nलिए\nलिये\nलेकिन\nव\nवर्ग\nवह\nवह \nवहाँ\nवहीं\nवाले\nवुह \nवे\nवग़ैरह\nसंग\nसकता\nसकते\nसबसे\nसभी\nसाथ\nसाबुत\nसाभ\nसारा\nसे\nसो\nही\nहुआ\nहुई\nहुए\nहै\nहैं\nहो\nहोता\nहोती\nहोते\nहोना\nहोने\n# additional normalized forms of the above\nअपनि\nजेसे\nहोति\nसभि\nतिंहों\nइंहों\nदवारा\nइसि\nकिंहें\nथि\nउंहों\nओर\nजिंहें\nवहिं\nअभि\nबनि\nहि\nउंहिं\nउंहें\nहें\nवगेरह\nएसे\nरवासा\nकोन\nनिचे\nकाफि\nउसि\nपुरा\nभितर\nहे\nबहि\nवहां\nकोइ\nयहां\nजिंहों\nतिंहें\nकिसि\nकइ\nयहि\nइंहिं\nजिधर\nइंहें\nअदि\nइतयादि\nहुइ\nकोनसा\nइसकि\nदुसरे\nजहां\nअप\nकिंहों\nउनकि\nभि\nवरग\nहुअ\nजेसा\nनहिं\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(HindiStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hr/analyzer_hr.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\n// Originated from: http://nlp.ffzg.hr/resources/tools/stemmer-for-croatian/\n\nconst AnalyzerName = \"hr\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsuffixFilter, err := cache.TokenFilterNamed(SuffixTransformationFilterName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerFilter, err := cache.TokenFilterNamed(StemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopFilter,\n\t\t\tsuffixFilter,\n\t\t\tstemmerFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hr/analyzer_hr_test.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hr\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestCroatianAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"Hrvatska\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"hrvatsk\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Hrvatski\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"hrvatsk\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// uppercase letters\n\t\t{\n\t\t\tinput: []byte(\"KOMARAC\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"komarc\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// vowelR\n\t\t{\n\t\t\tinput: []byte(\"crvi\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"crv\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"biti\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t\t// suffix transformation\n\t\t{\n\t\t\tinput: []byte(\"zaključcima\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"zaključk\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hr/stemmer_hr.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hr\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StemmerName = \"stemmer_hr\"\n\n// These regular expressions rules originated from:\n// http://nlp.ffzg.hr/resources/tools/stemmer-for-croatian/\n\nvar stemmingRules = []*regexp.Regexp{\n\tregexp.MustCompile(`^(.+(s|š)k)(ijima|ijega|ijemu|ijem|ijim|ijih|ijoj|ijeg|iji|ije|ija|oga|ome|omu|ima|og|om|im|ih|oj|i|e|o|a|u)$`),\n\tregexp.MustCompile(`^(.+(s|š)tv)(ima|om|o|a|u)$`),\n\tregexp.MustCompile(`^(.+(t|m|p|r|g)anij)(ama|ima|om|a|u|e|i|)$`),\n\tregexp.MustCompile(`^(.+an)(inom|ina|inu|ine|ima|in|om|u|i|a|e|)$`),\n\tregexp.MustCompile(`^(.+in)(ima|ama|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+on)(ovima|ova|ove|ovi|ima|om|a|e|i|u|)$`),\n\tregexp.MustCompile(`^(.+n)(ijima|ijega|ijemu|ijeg|ijem|ijim|ijih|ijoj|iji|ije|ija|iju|ima|ome|omu|oga|oj|om|ih|im|og|o|e|a|u|i|)$`),\n\tregexp.MustCompile(`^(.+(a|e|u)ć)(oga|ome|omu|ega|emu|ima|oj|ih|om|eg|em|og|uh|im|e|a)$`),\n\tregexp.MustCompile(`^(.+ugov)(ima|i|e|a)$`),\n\tregexp.MustCompile(`^(.+ug)(ama|om|a|e|i|u|o)$`),\n\tregexp.MustCompile(`^(.+log)(ama|om|a|u|e|)$`),\n\tregexp.MustCompile(`^(.+[^eo]g)(ovima|ama|ovi|ove|ova|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+(rrar|ott|ss|ll)i)(jem|ja|ju|o|)$`),\n\tregexp.MustCompile(`^(.+uj)(ući|emo|ete|mo|em|eš|e|u|)$`),\n\tregexp.MustCompile(`^(.+(c|č|ć|đ|l|r)aj)(evima|evi|eva|eve|ama|ima|em|a|e|i|u|)$`),\n\tregexp.MustCompile(`^(.+(b|c|d|l|n|m|ž|g|f|p|r|s|t|z)ij)(ima|ama|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+[^z]nal)(ima|ama|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+ijal)(ima|ama|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+ozil)(ima|om|a|e|u|i|)$`),\n\tregexp.MustCompile(`^(.+olov)(ima|i|a|e)$`),\n\tregexp.MustCompile(`^(.+ol)(ima|om|a|u|e|i|)$`),\n\tregexp.MustCompile(`^(.+lem)(ama|ima|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+ram)(ama|om|a|e|i|u|o)$`),\n\tregexp.MustCompile(`^(.+(a|d|e|o)r)(ama|ima|om|u|a|e|i|)$`),\n\tregexp.MustCompile(`^(.+(e|i)s)(ima|om|e|a|u)$`),\n\tregexp.MustCompile(`^(.+(t|n|j|k|j|t|b|g|v)aš)(ama|ima|om|em|a|u|i|e|)$`),\n\tregexp.MustCompile(`^(.+(e|i)š)(ima|ama|om|em|i|e|a|u|)$`),\n\tregexp.MustCompile(`^(.+ikat)(ima|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+lat)(ima|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+et)(ama|ima|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+(e|i|k|o)st)(ima|ama|om|a|e|i|u|o|)$`),\n\tregexp.MustCompile(`^(.+išt)(ima|em|a|e|u)$`),\n\tregexp.MustCompile(`^(.+ova)(smo|ste|hu|ti|še|li|la|le|lo|t|h|o)$`),\n\tregexp.MustCompile(`^(.+(a|e|i)v)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|ama|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|o|)$`),\n\tregexp.MustCompile(`^(.+[^dkml]ov)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|o|)$`),\n\tregexp.MustCompile(`^(.+(m|l)ov)(ima|om|a|u|e|i|)$`),\n\tregexp.MustCompile(`^(.+el)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|o|)$`),\n\tregexp.MustCompile(`^(.+(a|e|š)nj)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|ega|emu|eg|em|im|ih|oj|om|og|a|e|i|o|u)$`),\n\tregexp.MustCompile(`^(.+čin)(ama|ome|omu|oga|ima|og|om|im|ih|oj|a|u|i|o|e|)$`),\n\tregexp.MustCompile(`^(.+roši)(vši|smo|ste|še|mo|te|ti|li|la|lo|le|m|š|t|h|o)$`),\n\tregexp.MustCompile(`^(.+oš)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|)$`),\n\tregexp.MustCompile(`^(.+(e|o)vit)(ijima|ijega|ijemu|ijem|ijim|ijih|ijoj|ijeg|iji|ije|ija|oga|ome|omu|ima|og|om|im|ih|oj|i|e|o|a|u|)$`),\n\tregexp.MustCompile(`^(.+ast)(ijima|ijega|ijemu|ijem|ijim|ijih|ijoj|ijeg|iji|ije|ija|oga|ome|omu|ima|og|om|im|ih|oj|i|e|o|a|u|)$`),\n\tregexp.MustCompile(`^(.+k)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|o|)$`),\n\tregexp.MustCompile(`^(.+(e|a|i|u)va)(jući|smo|ste|jmo|jte|ju|la|le|li|lo|mo|na|ne|ni|no|te|ti|še|hu|h|j|m|n|o|t|v|š|)$`),\n\tregexp.MustCompile(`^(.+ir)(ujemo|ujete|ujući|ajući|ivat|ujem|uješ|ujmo|ujte|avši|asmo|aste|ati|amo|ate|aju|aše|ahu|ala|alo|ali|ale|uje|uju|uj|al|an|am|aš|at|ah|ao)$`),\n\tregexp.MustCompile(`^(.+ač)(ismo|iste|iti|imo|ite|iše|eći|ila|ilo|ili|ile|ena|eno|eni|ene|io|im|iš|it|ih|en|i|e)$`),\n\tregexp.MustCompile(`^(.+ača)(vši|smo|ste|smo|ste|hu|ti|mo|te|še|la|lo|li|le|ju|na|no|ni|ne|o|m|š|t|h|n)$`),\n\tregexp.MustCompile(`^(.+n)(uvši|usmo|uste|ući|imo|ite|emo|ete|ula|ulo|ule|uli|uto|uti|uta|em|eš|uo|ut|e|u|i)$`),\n\tregexp.MustCompile(`^(.+ni)(vši|smo|ste|ti|mo|te|mo|te|la|lo|le|li|m|š|o)$`),\n\tregexp.MustCompile(`^(.+((a|r|i|p|e|u)st|[^o]g|ik|uc|oj|aj|lj|ak|ck|čk|šk|uk|nj|im|ar|at|et|št|it|ot|ut|zn|zv)a)(jući|vši|smo|ste|jmo|jte|jem|mo|te|je|ju|ti|še|hu|la|li|le|lo|na|no|ni|ne|t|h|o|j|n|m|š)$`),\n\tregexp.MustCompile(`^(.+ur)(ajući|asmo|aste|ajmo|ajte|amo|ate|aju|ati|aše|ahu|ala|ali|ale|alo|ana|ano|ani|ane|al|at|ah|ao|aj|an|am|aš)$`),\n\tregexp.MustCompile(`^(.+(a|i|o)staj)(asmo|aste|ahu|ati|emo|ete|aše|ali|ući|ala|alo|ale|mo|ao|em|eš|at|ah|te|e|u|)$`),\n\tregexp.MustCompile(`^(.+(b|c|č|ć|d|e|f|g|j|k|n|r|t|u|v)a)(lama|lima|lom|lu|li|la|le|lo|l)$`),\n\tregexp.MustCompile(`^(.+(t|č|j|ž|š)aj)(evima|evi|eva|eve|ama|ima|em|a|e|i|u|)$`),\n\tregexp.MustCompile(`^(.+([^o]m|ič|nč|uč|b|c|ć|d|đ|h|j|k|l|n|p|r|s|š|v|z|ž)a)(jući|vši|smo|ste|jmo|jte|mo|te|ju|ti|še|hu|la|li|le|lo|na|no|ni|ne|t|h|o|j|n|m|š)$`),\n\tregexp.MustCompile(`^(.+(a|i|o)sta)(dosmo|doste|doše|nemo|demo|nete|dete|nimo|nite|nila|vši|nem|dem|neš|deš|doh|de|ti|ne|nu|du|la|li|lo|le|t|o)$`),\n\tregexp.MustCompile(`^(.+ta)(smo|ste|jmo|jte|vši|ti|mo|te|ju|še|la|lo|le|li|na|no|ni|ne|n|j|o|m|š|t|h)$`),\n\tregexp.MustCompile(`^(.+inj)(asmo|aste|ati|emo|ete|ali|ala|alo|ale|aše|ahu|em|eš|at|ah|ao)$`),\n\tregexp.MustCompile(`^(.+as)(temo|tete|timo|tite|tući|tem|teš|tao|te|li|ti|la|lo|le)$`),\n\tregexp.MustCompile(`^(.+(elj|ulj|tit|ac|ič|od|oj|et|av|ov)i)(vši|eći|smo|ste|še|mo|te|ti|li|la|lo|le|m|š|t|h|o)$`),\n\tregexp.MustCompile(`^(.+(tit|jeb|ar|ed|uš|ič)i)(jemo|jete|jem|ješ|smo|ste|jmo|jte|vši|mo|še|te|ti|ju|je|la|lo|li|le|t|m|š|h|j|o)$`),\n\tregexp.MustCompile(`^(.+(b|č|d|l|m|p|r|s|š|ž)i)(jemo|jete|jem|ješ|smo|ste|jmo|jte|vši|mo|lu|še|te|ti|ju|je|la|lo|li|le|t|m|š|h|j|o)$`),\n\tregexp.MustCompile(`^(.+luč)(ujete|ujući|ujemo|ujem|uješ|ismo|iste|ujmo|ujte|uje|uju|iše|iti|imo|ite|ila|ilo|ili|ile|ena|eno|eni|ene|uj|io|en|im|iš|it|ih|e|i)$`),\n\tregexp.MustCompile(`^(.+jeti)(smo|ste|še|mo|te|ti|li|la|lo|le|m|š|t|h|o)$`),\n\tregexp.MustCompile(`^(.+e)(lama|lima|lom|lu|li|la|le|lo|l)$`),\n\tregexp.MustCompile(`^(.+i)(lama|lima|lom|lu|li|la|le|lo|l)$`),\n\tregexp.MustCompile(`^(.+at)(ijega|ijemu|ijima|ijeg|ijem|ijih|ijim|ima|oga|ome|omu|iji|ije|ija|iju|oj|og|om|im|ih|a|u|i|e|o|)$`),\n\tregexp.MustCompile(`^(.+et)(avši|ući|emo|imo|em|eš|e|u|i)$`),\n\tregexp.MustCompile(`^(.+)(ajući|alima|alom|avši|asmo|aste|ajmo|ajte|ivši|amo|ate|aju|ati|aše|ahu|ali|ala|ale|alo|ana|ano|ani|ane|am|aš|at|ah|ao|aj|an)$`),\n\tregexp.MustCompile(`^(.+)(anje|enje|anja|enja|enom|enoj|enog|enim|enih|anom|anoj|anog|anim|anih|eno|ovi|ova|oga|ima|ove|enu|anu|ena|ama)$`),\n\tregexp.MustCompile(`^(.+)(nijega|nijemu|nijima|nijeg|nijem|nijim|nijih|nima|niji|nije|nija|niju|noj|nom|nog|nim|nih|an|na|nu|ni|ne|no)$`),\n\tregexp.MustCompile(`^(.+)(om|og|im|ih|em|oj|an|u|o|i|e|a)$`),\n}\n\nvar highlightVowelRRegex = regexp.MustCompile(`(^|[^aeiou])r($|[^aeiou])`)\n\nfunc highlightVowelR(term string) string {\n\treturn highlightVowelRRegex.ReplaceAllString(term, `${1}R${2}`)\n}\n\nfunc hasVowel(term string) bool {\n\tterm = highlightVowelR(term)\n\treturn strings.ContainsAny(term, \"aeiouR\")\n}\n\nfunc stem(term string) string {\n\tfor _, rule := range stemmingRules {\n\t\tresults := rule.FindStringSubmatch(term)\n\t\tif len(results) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\troot := results[1]\n\t\tif hasVowel(root) && root != \"\" {\n\t\t\treturn root\n\t\t}\n\t}\n\n\treturn term\n}\n\ntype CroatianStemmerFilter struct{}\n\nfunc NewCroatianStemmerFilter() *CroatianStemmerFilter {\n\treturn &CroatianStemmerFilter{}\n}\n\nfunc (s *CroatianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\ttoken.Term = []byte(stem(string(token.Term)))\n\t}\n\n\treturn input\n}\n\nfunc CroatianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewCroatianStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StemmerName, CroatianStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hr/stop_filter_hr.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hr/stop_words_hr.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_hr\"\n\nvar CroatianStopWords = []byte(`biti\njesam\nbudem\nsam\njesi\nbudeš\nsi\njesmo\nbudemo\nsmo\njeste\nbudete\nste\njesu\nbudu\nsu\nbih\nbijah\nbjeh\nbijaše\nbi\nbje\nbješe\nbijasmo\nbismo\nbjesmo\nbijaste\nbiste\nbjeste\nbijahu\nbiste\nbjeste\nbijahu\nbi\nbiše\nbjehu\nbješe\nbio\nbili\nbudimo\nbudite\nbila\nbilo\nbile\nću\nćeš\nće\nćemo\nćete\nželim\nželiš\nželi\nželimo\nželite\nžele\nmoram\nmoraš\nmora\nmoramo\nmorate\nmoraju\ntrebam\ntrebaš\ntreba\ntrebamo\ntrebate\ntrebaju\nmogu\nmožeš\nmože\nmožemo\nmožete\nza\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(CroatianStopWords)\n\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hr/suffix_transformation_hr.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hr\n\nimport (\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst SuffixTransformationFilterName = \"hr_suffix_transformation_filter\"\n\nvar SuffixTransformations = map[string]string{\n\t\"lozi\":     \"loga\",\n\t\"lozima\":   \"loga\",\n\t\"pjesi\":    \"pjeh\",\n\t\"pjesima\":  \"pjeh\",\n\t\"vojci\":    \"vojka\",\n\t\"bojci\":    \"bojka\",\n\t\"jaci\":     \"jak\",\n\t\"jacima\":   \"jak\",\n\t\"čajan\":    \"čajni\",\n\t\"ijeran\":   \"ijerni\",\n\t\"laran\":    \"larni\",\n\t\"ijesan\":   \"ijesni\",\n\t\"anjac\":    \"anjca\",\n\t\"ajac\":     \"ajca\",\n\t\"ajaca\":    \"ajca\",\n\t\"ljaca\":    \"ljca\",\n\t\"ljac\":     \"ljca\",\n\t\"ejac\":     \"ejca\",\n\t\"ejaca\":    \"ejca\",\n\t\"ojac\":     \"ojca\",\n\t\"ojaca\":    \"ojca\",\n\t\"ajaka\":    \"ajka\",\n\t\"ojaka\":    \"ojka\",\n\t\"šaca\":     \"šca\",\n\t\"šac\":      \"šca\",\n\t\"inzima\":   \"ing\",\n\t\"inzi\":     \"ing\",\n\t\"tvenici\":  \"tvenik\",\n\t\"tetici\":   \"tetika\",\n\t\"teticima\": \"tetika\",\n\t\"nstava\":   \"nstva\",\n\t\"nicima\":   \"nik\",\n\t\"ticima\":   \"tik\",\n\t\"zicima\":   \"zik\",\n\t\"snici\":    \"snik\",\n\t\"kuse\":     \"kusi\",\n\t\"kusan\":    \"kusni\",\n\t\"kustava\":  \"kustva\",\n\t\"dušan\":    \"dušni\",\n\t\"antan\":    \"antni\",\n\t\"bilan\":    \"bilni\",\n\t\"tilan\":    \"tilni\",\n\t\"avilan\":   \"avilni\",\n\t\"silan\":    \"silni\",\n\t\"gilan\":    \"gilni\",\n\t\"rilan\":    \"rilni\",\n\t\"nilan\":    \"nilni\",\n\t\"alan\":     \"alni\",\n\t\"ozan\":     \"ozni\",\n\t\"rave\":     \"ravi\",\n\t\"stavan\":   \"stavni\",\n\t\"pravan\":   \"pravni\",\n\t\"tivan\":    \"tivni\",\n\t\"sivan\":    \"sivni\",\n\t\"atan\":     \"atni\",\n\t\"cenata\":   \"centa\",\n\t\"denata\":   \"denta\",\n\t\"genata\":   \"genta\",\n\t\"lenata\":   \"lenta\",\n\t\"menata\":   \"menta\",\n\t\"jenata\":   \"jenta\",\n\t\"venata\":   \"venta\",\n\t\"tetan\":    \"tetni\",\n\t\"pletan\":   \"pletni\",\n\t\"šave\":     \"šavi\",\n\t\"manata\":   \"manta\",\n\t\"tanata\":   \"tanta\",\n\t\"lanata\":   \"lanta\",\n\t\"sanata\":   \"santa\",\n\t\"ačak\":     \"ačka\",\n\t\"ačaka\":    \"ačka\",\n\t\"ušak\":     \"uška\",\n\t\"atak\":     \"atka\",\n\t\"ataka\":    \"atka\",\n\t\"atci\":     \"atka\",\n\t\"atcima\":   \"atka\",\n\t\"etak\":     \"etka\",\n\t\"etaka\":    \"etka\",\n\t\"itak\":     \"itka\",\n\t\"itaka\":    \"itka\",\n\t\"itci\":     \"itka\",\n\t\"otak\":     \"otka\",\n\t\"otaka\":    \"otka\",\n\t\"utak\":     \"utka\",\n\t\"utaka\":    \"utka\",\n\t\"utci\":     \"utka\",\n\t\"utcima\":   \"utka\",\n\t\"eskan\":    \"eskna\",\n\t\"tičan\":    \"tični\",\n\t\"ojsci\":    \"ojska\",\n\t\"esama\":    \"esma\",\n\t\"metara\":   \"metra\",\n\t\"centar\":   \"centra\",\n\t\"centara\":  \"centra\",\n\t\"istara\":   \"istra\",\n\t\"istar\":    \"istra\",\n\t\"ošću\":     \"osti\",\n\t\"daba\":     \"dba\",\n\t\"čcima\":    \"čka\",\n\t\"čci\":      \"čka\",\n\t\"mac\":      \"mca\",\n\t\"maca\":     \"mca\",\n\t\"voljan\":   \"voljni\",\n\t\"anaka\":    \"anki\",\n\t\"vac\":      \"vca\",\n\t\"vaca\":     \"vca\",\n\t\"saca\":     \"sca\",\n\t\"sac\":      \"sca\",\n\t\"naca\":     \"nca\",\n\t\"nac\":      \"nca\",\n\t\"raca\":     \"rca\",\n\t\"rac\":      \"rca\",\n\t\"aoca\":     \"alca\",\n\t\"alaca\":    \"alca\",\n\t\"alac\":     \"alca\",\n\t\"elaca\":    \"elca\",\n\t\"elac\":     \"elca\",\n\t\"olaca\":    \"olca\",\n\t\"olac\":     \"olca\",\n\t\"olce\":     \"olca\",\n\t\"njac\":     \"njca\",\n\t\"njaca\":    \"njca\",\n\t\"ekata\":    \"ekta\",\n\t\"ekat\":     \"ekta\",\n\t\"izam\":     \"izma\",\n\t\"izama\":    \"izma\",\n\t\"jebe\":     \"jebi\",\n\t\"ašan\":     \"ašni\",\n}\n\ntype SuffixTransformationFilter struct{}\n\nfunc NewSuffixTransformationFilter() *SuffixTransformationFilter {\n\treturn &SuffixTransformationFilter{}\n}\n\nfunc (s *SuffixTransformationFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := string(token.Term)\n\n\t\tfor suffix, newSuffix := range SuffixTransformations {\n\t\t\tif strings.HasSuffix(term, suffix) {\n\t\t\t\tterm = term[:len(term)-len(suffix)] + newSuffix\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\ttoken.Term = []byte(term)\n\t}\n\n\treturn input\n}\n\nfunc SuffixTransformationFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewSuffixTransformationFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SuffixTransformationFilterName, SuffixTransformationFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hu/analyzer_hu.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hu\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"hu\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopHuFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerHuFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopHuFilter,\n\t\t\tstemmerHuFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hu/analyzer_hu_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hu\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestHungarianAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"babakocsi\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"babakocs\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"babakocsijáért\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"babakocs\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"által\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hu/stemmer_hu.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hu\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/hungarian\"\n)\n\nconst SnowballStemmerName = \"stemmer_hu_snowball\"\n\ntype HungarianStemmerFilter struct {\n}\n\nfunc NewHungarianStemmerFilter() *HungarianStemmerFilter {\n\treturn &HungarianStemmerFilter{}\n}\n\nfunc (s *HungarianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\thungarian.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc HungarianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewHungarianStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, HungarianStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hu/stop_filter_hu.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hu\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hu/stop_words_hu.go",
    "content": "package hu\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_hu\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar HungarianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/hungarian/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n \n| Hungarian stop word list\n| prepared by Anna Tordai\n\na\nahogy\nahol\naki\nakik\nakkor\nalatt\náltal\náltalában\namely\namelyek\namelyekben\namelyeket\namelyet\namelynek\nami\namit\namolyan\namíg\namikor\nát\nabban\nahhoz\nannak\narra\narról\naz\nazok\nazon\nazt\nazzal\nazért\naztán\nazután\nazonban\nbár\nbe\nbelül\nbenne\ncikk\ncikkek\ncikkeket\ncsak\nde\ne\neddig\negész\negy\negyes\negyetlen\negyéb\negyik\negyre\nekkor\nel\nelég\nellen\nelő\nelőször\nelőtt\nelső\nén\néppen\nebben\nehhez\nemilyen\nennek\nerre\nez\nezt\nezek\nezen\nezzel\nezért\nés\nfel\nfelé\nhanem\nhiszen\nhogy\nhogyan\nigen\nígy\nilletve\nill.\nill\nilyen\nilyenkor\nison\nismét\nitt\njó\njól\njobban\nkell\nkellett\nkeresztül\nkeressünk\nki\nkívül\nközött\nközül\nlegalább\nlehet\nlehetett\nlegyen\nlenne\nlenni\nlesz\nlett\nmaga\nmagát\nmajd\nmajd\nmár\nmás\nmásik\nmeg\nmég\nmellett\nmert\nmely\nmelyek\nmi\nmit\nmíg\nmiért\nmilyen\nmikor\nminden\nmindent\nmindenki\nmindig\nmint\nmintha\nmivel\nmost\nnagy\nnagyobb\nnagyon\nne\nnéha\nnekem\nneki\nnem\nnéhány\nnélkül\nnincs\nolyan\nott\nössze\nő\nők\nőket\npedig\npersze\nrá\ns\nsaját\nsem\nsemmi\nsok\nsokat\nsokkal\nszámára\nszemben\nszerint\nszinte\ntalán\ntehát\nteljes\ntovább\ntovábbá\ntöbb\núgy\nugyanis\núj\nújabb\nújra\nután\nutána\nutolsó\nvagy\nvagyis\nvalaki\nvalami\nvalamint\nvaló\nvagyok\nvan\nvannak\nvolt\nvoltam\nvoltak\nvoltunk\nvissza\nvele\nviszont\nvolna\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(HungarianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hy/stop_filter_hy.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hy\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/hy/stop_words_hy.go",
    "content": "package hy\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_hy\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar ArmenianStopWords = []byte(`# example set of Armenian stopwords.\nայդ\nայլ\nայն\nայս\nդու\nդուք\nեմ\nեն\nենք\nես\nեք\nէ\nէի\nէին\nէինք\nէիր\nէիք\nէր\nըստ\nթ\nի\nին\nիսկ\nիր\nկամ\nհամար\nհետ\nհետո\nմենք\nմեջ\nմի\nն\nնա\nնաև\nնրա\nնրանք\nոր\nորը\nորոնք\nորպես\nու\nում\nպիտի\nվրա\nև\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(ArmenianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/id/stop_filter_id.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage id\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/id/stop_words_id.go",
    "content": "package id\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_id\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar IndonesianStopWords = []byte(`# from appendix D of: A Study of Stemming Effects on Information\n# Retrieval in Bahasa Indonesia\nada\nadanya\nadalah\nadapun\nagak\nagaknya\nagar\nakan\nakankah\nakhirnya\naku\nakulah\namat\namatlah\nanda\nandalah\nantar\ndiantaranya\nantara\nantaranya\ndiantara\napa\napaan\nmengapa\napabila\napakah\napalagi\napatah\natau\nataukah\nataupun\nbagai\nbagaikan\nsebagai\nsebagainya\nbagaimana\nbagaimanapun\nsebagaimana\nbagaimanakah\nbagi\nbahkan\nbahwa\nbahwasanya\nsebaliknya\nbanyak\nsebanyak\nbeberapa\nseberapa\nbegini\nbeginian\nbeginikah\nbeginilah\nsebegini\nbegitu\nbegitukah\nbegitulah\nbegitupun\nsebegitu\nbelum\nbelumlah\nsebelum\nsebelumnya\nsebenarnya\nberapa\nberapakah\nberapalah\nberapapun\nbetulkah\nsebetulnya\nbiasa\nbiasanya\nbila\nbilakah\nbisa\nbisakah\nsebisanya\nboleh\nbolehkah\nbolehlah\nbuat\nbukan\nbukankah\nbukanlah\nbukannya\ncuma\npercuma\ndahulu\ndalam\ndan\ndapat\ndari\ndaripada\ndekat\ndemi\ndemikian\ndemikianlah\nsedemikian\ndengan\ndepan\ndi\ndia\ndialah\ndini\ndiri\ndirinya\nterdiri\ndong\ndulu\nenggak\nenggaknya\nentah\nentahlah\nterhadap\nterhadapnya\nhal\nhampir\nhanya\nhanyalah\nharus\nharuslah\nharusnya\nseharusnya\nhendak\nhendaklah\nhendaknya\nhingga\nsehingga\nia\nialah\nibarat\ningin\ninginkah\ninginkan\nini\ninikah\ninilah\nitu\nitukah\nitulah\njangan\njangankan\njanganlah\njika\njikalau\njuga\njustru\nkala\nkalau\nkalaulah\nkalaupun\nkalian\nkami\nkamilah\nkamu\nkamulah\nkan\nkapan\nkapankah\nkapanpun\ndikarenakan\nkarena\nkarenanya\nke\nkecil\nkemudian\nkenapa\nkepada\nkepadanya\nketika\nseketika\nkhususnya\nkini\nkinilah\nkiranya\nsekiranya\nkita\nkitalah\nkok\nlagi\nlagian\nselagi\nlah\nlain\nlainnya\nmelainkan\nselaku\nlalu\nmelalui\nterlalu\nlama\nlamanya\nselama\nselama\nselamanya\nlebih\nterlebih\nbermacam\nmacam\nsemacam\nmaka\nmakanya\nmakin\nmalah\nmalahan\nmampu\nmampukah\nmana\nmanakala\nmanalagi\nmasih\nmasihkah\nsemasih\nmasing\nmau\nmaupun\nsemaunya\nmemang\nmereka\nmerekalah\nmeski\nmeskipun\nsemula\nmungkin\nmungkinkah\nnah\nnamun\nnanti\nnantinya\nnyaris\noleh\nolehnya\nseorang\nseseorang\npada\npadanya\npadahal\npaling\nsepanjang\npantas\nsepantasnya\nsepantasnyalah\npara\npasti\npastilah\nper\npernah\npula\npun\nmerupakan\nrupanya\nserupa\nsaat\nsaatnya\nsesaat\nsaja\nsajalah\nsaling\nbersama\nsama\nsesama\nsambil\nsampai\nsana\nsangat\nsangatlah\nsaya\nsayalah\nse\nsebab\nsebabnya\nsebuah\ntersebut\ntersebutlah\nsedang\nsedangkan\nsedikit\nsedikitnya\nsegala\nsegalanya\nsegera\nsesegera\nsejak\nsejenak\nsekali\nsekalian\nsekalipun\nsesekali\nsekaligus\nsekarang\nsekarang\nsekitar\nsekitarnya\nsela\nselain\nselalu\nseluruh\nseluruhnya\nsemakin\nsementara\nsempat\nsemua\nsemuanya\nsendiri\nsendirinya\nseolah\nseperti\nsepertinya\nsering\nseringnya\nserta\nsiapa\nsiapakah\nsiapapun\ndisini\ndisinilah\nsini\nsinilah\nsesuatu\nsesuatunya\nsuatu\nsesudah\nsesudahnya\nsudah\nsudahkah\nsudahlah\nsupaya\ntadi\ntadinya\ntak\ntanpa\nsetelah\ntelah\ntentang\ntentu\ntentulah\ntentunya\ntertentu\nseterusnya\ntapi\ntetapi\nsetiap\ntiap\nsetidaknya\ntidak\ntidakkah\ntidaklah\ntoh\nwaduh\nwah\nwahai\nsewaktu\nwalau\nwalaupun\nwong\nyaitu\nyakni\nyang\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(IndonesianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/in/indic_normalize.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage in\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst NormalizeName = \"normalize_in\"\n\ntype IndicNormalizeFilter struct {\n}\n\nfunc NewIndicNormalizeFilter() *IndicNormalizeFilter {\n\treturn &IndicNormalizeFilter{}\n}\n\nfunc (s *IndicNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\trunes := bytes.Runes(token.Term)\n\t\trunes = normalize(runes)\n\t\ttoken.Term = analysis.BuildTermFromRunes(runes)\n\t}\n\treturn input\n}\n\nfunc NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewIndicNormalizeFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/in/indic_normalize_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage in\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestIndicNormalizeFilter(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// basics\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अाॅअाॅ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ऑऑ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अाॆअाॆ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ऒऒ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अाेअाे\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ओओ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अाैअाै\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"औऔ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अाअा\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"आआ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"अाैर\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"और\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ত্‍\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ৎ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// empty term\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tindicNormalizeFilter := NewIndicNormalizeFilter()\n\tfor _, test := range tests {\n\t\tactual := indicNormalizeFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\tt.Errorf(\"expected % x, got % x for % x\", test.output[0].Term, actual[0].Term, test.input[0].Term)\n\t\t\tt.Errorf(\"expected %s, got %s for %s\", test.output[0].Term, actual[0].Term, test.input[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/in/scripts.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage in\n\nimport (\n\t\"unicode\"\n\n\t\"github.com/bits-and-blooms/bitset\"\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\ntype ScriptData struct {\n\tflag       rune\n\tbase       rune\n\tdecompMask *bitset.BitSet\n}\n\nvar scripts = map[*unicode.RangeTable]*ScriptData{\n\tunicode.Devanagari: {\n\t\tflag: 1,\n\t\tbase: 0x0900,\n\t},\n\tunicode.Bengali: {\n\t\tflag: 2,\n\t\tbase: 0x0980,\n\t},\n\tunicode.Gurmukhi: {\n\t\tflag: 4,\n\t\tbase: 0x0A00,\n\t},\n\tunicode.Gujarati: {\n\t\tflag: 8,\n\t\tbase: 0x0A80,\n\t},\n\tunicode.Oriya: {\n\t\tflag: 16,\n\t\tbase: 0x0B00,\n\t},\n\tunicode.Tamil: {\n\t\tflag: 32,\n\t\tbase: 0x0B80,\n\t},\n\tunicode.Telugu: {\n\t\tflag: 64,\n\t\tbase: 0x0C00,\n\t},\n\tunicode.Kannada: {\n\t\tflag: 128,\n\t\tbase: 0x0C80,\n\t},\n\tunicode.Malayalam: {\n\t\tflag: 256,\n\t\tbase: 0x0D00,\n\t},\n}\n\nfunc flag(ub *unicode.RangeTable) rune {\n\treturn scripts[ub].flag\n}\n\nvar decompositions = [][]rune{\n\t/* devanagari, gujarati vowel candra O */\n\t{0x05, 0x3E, 0x45, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari short O */\n\t{0x05, 0x3E, 0x46, 0x12, flag(unicode.Devanagari)},\n\t/* devanagari, gujarati letter O */\n\t{0x05, 0x3E, 0x47, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari letter AI, gujarati letter AU */\n\t{0x05, 0x3E, 0x48, 0x14, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari, bengali, gurmukhi, gujarati, oriya AA */\n\t{0x05, 0x3E, -1, 0x06, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Gurmukhi) | flag(unicode.Gujarati) | flag(unicode.Oriya)},\n\t/* devanagari letter candra A */\n\t{0x05, 0x45, -1, 0x72, flag(unicode.Devanagari)},\n\t/* gujarati vowel candra E */\n\t{0x05, 0x45, -1, 0x0D, flag(unicode.Gujarati)},\n\t/* devanagari letter short A */\n\t{0x05, 0x46, -1, 0x04, flag(unicode.Devanagari)},\n\t/* gujarati letter E */\n\t{0x05, 0x47, -1, 0x0F, flag(unicode.Gujarati)},\n\t/* gurmukhi, gujarati letter AI */\n\t{0x05, 0x48, -1, 0x10, flag(unicode.Gurmukhi) | flag(unicode.Gujarati)},\n\t/* devanagari, gujarati vowel candra O */\n\t{0x05, 0x49, -1, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari short O */\n\t{0x05, 0x4A, -1, 0x12, flag(unicode.Devanagari)},\n\t/* devanagari, gujarati letter O */\n\t{0x05, 0x4B, -1, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari letter AI, gurmukhi letter AU, gujarati letter AU */\n\t{0x05, 0x4C, -1, 0x14, flag(unicode.Devanagari) | flag(unicode.Gurmukhi) | flag(unicode.Gujarati)},\n\t/* devanagari, gujarati vowel candra O */\n\t{0x06, 0x45, -1, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari short O */\n\t{0x06, 0x46, -1, 0x12, flag(unicode.Devanagari)},\n\t/* devanagari, gujarati letter O */\n\t{0x06, 0x47, -1, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari letter AI, gujarati letter AU */\n\t{0x06, 0x48, -1, 0x14, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* malayalam letter II */\n\t{0x07, 0x57, -1, 0x08, flag(unicode.Malayalam)},\n\t/* devanagari letter UU */\n\t{0x09, 0x41, -1, 0x0A, flag(unicode.Devanagari)},\n\t/* tamil, malayalam letter UU (some styles) */\n\t{0x09, 0x57, -1, 0x0A, flag(unicode.Tamil) | flag(unicode.Malayalam)},\n\t/* malayalam letter AI */\n\t{0x0E, 0x46, -1, 0x10, flag(unicode.Malayalam)},\n\t/* devanagari candra E */\n\t{0x0F, 0x45, -1, 0x0D, flag(unicode.Devanagari)},\n\t/* devanagari short E */\n\t{0x0F, 0x46, -1, 0x0E, flag(unicode.Devanagari)},\n\t/* devanagari AI */\n\t{0x0F, 0x47, -1, 0x10, flag(unicode.Devanagari)},\n\t/* oriya AI */\n\t{0x0F, 0x57, -1, 0x10, flag(unicode.Oriya)},\n\t/* malayalam letter OO */\n\t{0x12, 0x3E, -1, 0x13, flag(unicode.Malayalam)},\n\t/* telugu, kannada letter AU */\n\t{0x12, 0x4C, -1, 0x14, flag(unicode.Telugu) | flag(unicode.Kannada)},\n\t/* telugu letter OO */\n\t{0x12, 0x55, -1, 0x13, flag(unicode.Telugu)},\n\t/* tamil, malayalam letter AU */\n\t{0x12, 0x57, -1, 0x14, flag(unicode.Tamil) | flag(unicode.Malayalam)},\n\t/* oriya letter AU */\n\t{0x13, 0x57, -1, 0x14, flag(unicode.Oriya)},\n\t/* devanagari qa */\n\t{0x15, 0x3C, -1, 0x58, flag(unicode.Devanagari)},\n\t/* devanagari, gurmukhi khha */\n\t{0x16, 0x3C, -1, 0x59, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)},\n\t/* devanagari, gurmukhi ghha */\n\t{0x17, 0x3C, -1, 0x5A, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)},\n\t/* devanagari, gurmukhi za */\n\t{0x1C, 0x3C, -1, 0x5B, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)},\n\t/* devanagari dddha, bengali, oriya rra */\n\t{0x21, 0x3C, -1, 0x5C, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Oriya)},\n\t/* devanagari, bengali, oriya rha */\n\t{0x22, 0x3C, -1, 0x5D, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Oriya)},\n\t/* malayalam chillu nn */\n\t{0x23, 0x4D, 0xFF, 0x7A, flag(unicode.Malayalam)},\n\t/* bengali khanda ta */\n\t{0x24, 0x4D, 0xFF, 0x4E, flag(unicode.Bengali)},\n\t/* devanagari nnna */\n\t{0x28, 0x3C, -1, 0x29, flag(unicode.Devanagari)},\n\t/* malayalam chillu n */\n\t{0x28, 0x4D, 0xFF, 0x7B, flag(unicode.Malayalam)},\n\t/* devanagari, gurmukhi fa */\n\t{0x2B, 0x3C, -1, 0x5E, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)},\n\t/* devanagari, bengali yya */\n\t{0x2F, 0x3C, -1, 0x5F, flag(unicode.Devanagari) | flag(unicode.Bengali)},\n\t/* telugu letter vocalic R */\n\t{0x2C, 0x41, 0x41, 0x0B, flag(unicode.Telugu)},\n\t/* devanagari rra */\n\t{0x30, 0x3C, -1, 0x31, flag(unicode.Devanagari)},\n\t/* malayalam chillu rr */\n\t{0x30, 0x4D, 0xFF, 0x7C, flag(unicode.Malayalam)},\n\t/* malayalam chillu l */\n\t{0x32, 0x4D, 0xFF, 0x7D, flag(unicode.Malayalam)},\n\t/* devanagari llla */\n\t{0x33, 0x3C, -1, 0x34, flag(unicode.Devanagari)},\n\t/* malayalam chillu ll */\n\t{0x33, 0x4D, 0xFF, 0x7E, flag(unicode.Malayalam)},\n\t/* telugu letter MA */\n\t{0x35, 0x41, -1, 0x2E, flag(unicode.Telugu)},\n\t/* devanagari, gujarati vowel sign candra O */\n\t{0x3E, 0x45, -1, 0x49, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari vowel sign short O */\n\t{0x3E, 0x46, -1, 0x4A, flag(unicode.Devanagari)},\n\t/* devanagari, gujarati vowel sign O */\n\t{0x3E, 0x47, -1, 0x4B, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* devanagari, gujarati vowel sign AU */\n\t{0x3E, 0x48, -1, 0x4C, flag(unicode.Devanagari) | flag(unicode.Gujarati)},\n\t/* kannada vowel sign II */\n\t{0x3F, 0x55, -1, 0x40, flag(unicode.Kannada)},\n\t/* gurmukhi vowel sign UU (when stacking) */\n\t{0x41, 0x41, -1, 0x42, flag(unicode.Gurmukhi)},\n\t/* tamil, malayalam vowel sign O */\n\t{0x46, 0x3E, -1, 0x4A, flag(unicode.Tamil) | flag(unicode.Malayalam)},\n\t/* kannada vowel sign OO */\n\t{0x46, 0x42, 0x55, 0x4B, flag(unicode.Kannada)},\n\t/* kannada vowel sign O */\n\t{0x46, 0x42, -1, 0x4A, flag(unicode.Kannada)},\n\t/* malayalam vowel sign AI (if reordered twice) */\n\t{0x46, 0x46, -1, 0x48, flag(unicode.Malayalam)},\n\t/* telugu, kannada vowel sign EE */\n\t{0x46, 0x55, -1, 0x47, flag(unicode.Telugu) | flag(unicode.Kannada)},\n\t/* telugu, kannada vowel sign AI */\n\t{0x46, 0x56, -1, 0x48, flag(unicode.Telugu) | flag(unicode.Kannada)},\n\t/* tamil, malayalam vowel sign AU */\n\t{0x46, 0x57, -1, 0x4C, flag(unicode.Tamil) | flag(unicode.Malayalam)},\n\t/* bengali, oriya vowel sign O, tamil, malayalam vowel sign OO */\n\t{0x47, 0x3E, -1, 0x4B, flag(unicode.Bengali) | flag(unicode.Oriya) | flag(unicode.Tamil) | flag(unicode.Malayalam)},\n\t/* bengali, oriya vowel sign AU */\n\t{0x47, 0x57, -1, 0x4C, flag(unicode.Bengali) | flag(unicode.Oriya)},\n\t/* kannada vowel sign OO */\n\t{0x4A, 0x55, -1, 0x4B, flag(unicode.Kannada)},\n\t/* gurmukhi letter I */\n\t{0x72, 0x3F, -1, 0x07, flag(unicode.Gurmukhi)},\n\t/* gurmukhi letter II */\n\t{0x72, 0x40, -1, 0x08, flag(unicode.Gurmukhi)},\n\t/* gurmukhi letter EE */\n\t{0x72, 0x47, -1, 0x0F, flag(unicode.Gurmukhi)},\n\t/* gurmukhi letter U */\n\t{0x73, 0x41, -1, 0x09, flag(unicode.Gurmukhi)},\n\t/* gurmukhi letter UU */\n\t{0x73, 0x42, -1, 0x0A, flag(unicode.Gurmukhi)},\n\t/* gurmukhi letter OO */\n\t{0x73, 0x4B, -1, 0x13, flag(unicode.Gurmukhi)},\n}\n\nfunc init() {\n\tfor _, scriptData := range scripts {\n\t\tscriptData.decompMask = bitset.New(0x7d)\n\t\tfor _, decomposition := range decompositions {\n\t\t\tch := decomposition[0]\n\t\t\tflags := decomposition[4]\n\t\t\tif (flags & scriptData.flag) != 0 {\n\t\t\t\tscriptData.decompMask.Set(uint(ch))\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc lookupScript(r rune) *unicode.RangeTable {\n\tfor script := range scripts {\n\t\tif unicode.Is(script, r) {\n\t\t\treturn script\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc normalize(input []rune) []rune {\n\tinputLen := len(input)\n\tfor i := 0; i < inputLen; i++ {\n\t\tr := input[i]\n\t\tscript := lookupScript(r)\n\t\tif script != nil {\n\t\t\tscriptData := scripts[script]\n\t\t\tch := r - scriptData.base\n\t\t\tif scriptData.decompMask.Test(uint(ch)) {\n\t\t\t\tinput = compose(ch, script, scriptData, input, i, inputLen)\n\t\t\t\tinputLen = len(input)\n\t\t\t}\n\t\t}\n\t}\n\treturn input[0:inputLen]\n}\n\nfunc compose(ch0 rune, script0 *unicode.RangeTable, scriptData *ScriptData, input []rune, pos int, inputLen int) []rune {\n\tif pos+1 >= inputLen {\n\t\treturn input // need at least 2 characters\n\t}\n\n\tch1 := input[pos+1] - scriptData.base\n\tscript1 := lookupScript(input[pos+1])\n\tif script0 != script1 {\n\t\treturn input // need to be same script\n\t}\n\n\tch2 := rune(-1)\n\tif pos+2 < inputLen {\n\t\tch2 = input[pos+2] - scriptData.base\n\t\tscript2 := lookupScript(input[pos+2])\n\t\tif input[pos+2] == '\\u200D' {\n\t\t\tch2 = 0xff // zero width joiner\n\t\t} else if script2 != script1 {\n\t\t\tch2 = -1 // still allow 2 character match\n\t\t}\n\t}\n\n\tfor _, decomposition := range decompositions {\n\t\tif decomposition[0] == ch0 &&\n\t\t\t(decomposition[4]&scriptData.flag) != 0 {\n\t\t\tif decomposition[1] == ch1 &&\n\t\t\t\t(decomposition[2] < 0 || decomposition[2] == ch2) {\n\t\t\t\tinput[pos] = scriptData.base + decomposition[3]\n\t\t\t\tinput = analysis.DeleteRune(input, pos+1)\n\t\t\t\tif decomposition[2] >= 0 {\n\t\t\t\t\tinput = analysis.DeleteRune(input, pos+1)\n\t\t\t\t}\n\t\t\t\treturn input\n\t\t\t}\n\t\t}\n\t}\n\treturn input\n}\n"
  },
  {
    "path": "analysis/lang/it/analyzer_it.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"it\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\telisionFilter, err := cache.TokenFilterNamed(ElisionName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopItFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerItFilter, err := cache.TokenFilterNamed(LightStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\telisionFilter,\n\t\t\tstopItFilter,\n\t\t\tstemmerItFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/analyzer_it_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestItalianAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"abbandonata\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abbandonat\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"abbandonati\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abbandonat\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"dallo\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t\t// contractions\n\t\t{\n\t\t\tinput: []byte(\"dell'Italia\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ital\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"l'Italiano\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"italian\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test for bug #218\n\t\t{\n\t\t\tinput: []byte(\"Nell'anfora\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"anfor\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/articles_it.go",
    "content": "package it\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst ArticlesName = \"articles_it\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis\n\nvar ItalianArticles = []byte(`\nc\nl\nall\ndall\ndell\nnell\nsull\ncoll\npell\ngl\nagl\ndagl\ndegl\nnegl\nsugl\nun\nm\nt\ns\nv\nd\n`)\n\nfunc ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(ItalianArticles)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/elision_it.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/elision\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst ElisionName = \"elision_it\"\n\nfunc ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tarticlesTokenMap, err := cache.TokenMapNamed(ArticlesName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building elision filter: %v\", err)\n\t}\n\treturn elision.NewElisionFilter(articlesTokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/elision_it_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestItalianElision(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"dell'Italia\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Italia\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\telisionFilter, err := cache.TokenFilterNamed(ElisionName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := elisionFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/light_stemmer_it.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst LightStemmerName = \"stemmer_it_light\"\n\ntype ItalianLightStemmerFilter struct {\n}\n\nfunc NewItalianLightStemmerFilterFilter() *ItalianLightStemmerFilter {\n\treturn &ItalianLightStemmerFilter{}\n}\n\nfunc (s *ItalianLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\trunes := bytes.Runes(token.Term)\n\t\trunes = stem(runes)\n\t\ttoken.Term = analysis.BuildTermFromRunes(runes)\n\t}\n\treturn input\n}\n\nfunc stem(input []rune) []rune {\n\n\tinputLen := len(input)\n\n\tif inputLen < 6 {\n\t\treturn input\n\t}\n\n\tfor i := 0; i < inputLen; i++ {\n\t\tswitch input[i] {\n\t\tcase 'à', 'á', 'â', 'ä':\n\t\t\tinput[i] = 'a'\n\t\tcase 'ò', 'ó', 'ô', 'ö':\n\t\t\tinput[i] = 'o'\n\t\tcase 'è', 'é', 'ê', 'ë':\n\t\t\tinput[i] = 'e'\n\t\tcase 'ù', 'ú', 'û', 'ü':\n\t\t\tinput[i] = 'u'\n\t\tcase 'ì', 'í', 'î', 'ï':\n\t\t\tinput[i] = 'i'\n\t\t}\n\t}\n\n\tswitch input[inputLen-1] {\n\tcase 'e':\n\t\tif input[inputLen-2] == 'i' || input[inputLen-2] == 'h' {\n\t\t\treturn input[0 : inputLen-2]\n\t\t} else {\n\t\t\treturn input[0 : inputLen-1]\n\t\t}\n\tcase 'i':\n\t\tif input[inputLen-2] == 'h' || input[inputLen-2] == 'i' {\n\t\t\treturn input[0 : inputLen-2]\n\t\t} else {\n\t\t\treturn input[0 : inputLen-1]\n\t\t}\n\tcase 'a':\n\t\tif input[inputLen-2] == 'i' {\n\t\t\treturn input[0 : inputLen-2]\n\t\t} else {\n\t\t\treturn input[0 : inputLen-1]\n\t\t}\n\tcase 'o':\n\t\tif input[inputLen-2] == 'i' {\n\t\t\treturn input[0 : inputLen-2]\n\t\t} else {\n\t\t\treturn input[0 : inputLen-1]\n\t\t}\n\t}\n\n\treturn input\n}\n\nfunc ItalianLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewItalianLightStemmerFilterFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(LightStemmerName, ItalianLightStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/light_stemmer_it_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestItalianLightStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ragazzo\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ragazz\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ragazzi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ragazz\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(LightStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/stemmer_it_snowball.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/italian\"\n)\n\nconst SnowballStemmerName = \"stemmer_it_snowball\"\n\ntype ItalianStemmerFilter struct {\n}\n\nfunc NewItalianStemmerFilter() *ItalianStemmerFilter {\n\treturn &ItalianStemmerFilter{}\n}\n\nfunc (s *ItalianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\titalian.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc ItalianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewItalianStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, ItalianStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/stemmer_it_snowball_test.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSnowballItalianStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aizzata\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aizz\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aizzargli\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aizz\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aizzasse\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aizz\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/stop_filter_it.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage it\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/it/stop_words_it.go",
    "content": "package it\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_it\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar ItalianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | An Italian stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\nad             |  a (to) before vowel\nal             |  a + il\nallo           |  a + lo\nai             |  a + i\nagli           |  a + gli\nall            |  a + l'\nagl            |  a + gl'\nalla           |  a + la\nalle           |  a + le\ncon            |  with\ncol            |  con + il\ncoi            |  con + i (forms collo, cogli etc are now very rare)\nda             |  from\ndal            |  da + il\ndallo          |  da + lo\ndai            |  da + i\ndagli          |  da + gli\ndall           |  da + l'\ndagl           |  da + gll'\ndalla          |  da + la\ndalle          |  da + le\ndi             |  of\ndel            |  di + il\ndello          |  di + lo\ndei            |  di + i\ndegli          |  di + gli\ndell           |  di + l'\ndegl           |  di + gl'\ndella          |  di + la\ndelle          |  di + le\nin             |  in\nnel            |  in + el\nnello          |  in + lo\nnei            |  in + i\nnegli          |  in + gli\nnell           |  in + l'\nnegl           |  in + gl'\nnella          |  in + la\nnelle          |  in + le\nsu             |  on\nsul            |  su + il\nsullo          |  su + lo\nsui            |  su + i\nsugli          |  su + gli\nsull           |  su + l'\nsugl           |  su + gl'\nsulla          |  su + la\nsulle          |  su + le\nper            |  through, by\ntra            |  among\ncontro         |  against\nio             |  I\ntu             |  thou\nlui            |  he\nlei            |  she\nnoi            |  we\nvoi            |  you\nloro           |  they\nmio            |  my\nmia            |\nmiei           |\nmie            |\ntuo            |\ntua            |\ntuoi           |  thy\ntue            |\nsuo            |\nsua            |\nsuoi           |  his, her\nsue            |\nnostro         |  our\nnostra         |\nnostri         |\nnostre         |\nvostro         |  your\nvostra         |\nvostri         |\nvostre         |\nmi             |  me\nti             |  thee\nci             |  us, there\nvi             |  you, there\nlo             |  him, the\nla             |  her, the\nli             |  them\nle             |  them, the\ngli            |  to him, the\nne             |  from there etc\nil             |  the\nun             |  a\nuno            |  a\nuna            |  a\nma             |  but\ned             |  and\nse             |  if\nperché         |  why, because\nanche          |  also\ncome           |  how\ndov            |  where (as dov')\ndove           |  where\nche            |  who, that\nchi            |  who\ncui            |  whom\nnon            |  not\npiù            |  more\nquale          |  who, that\nquanto         |  how much\nquanti         |\nquanta         |\nquante         |\nquello         |  that\nquelli         |\nquella         |\nquelle         |\nquesto         |  this\nquesti         |\nquesta         |\nqueste         |\nsi             |  yes\ntutto          |  all\ntutti          |  all\n\n               |  single letter forms:\n\na              |  at\nc              |  as c' for ce or ci\ne              |  and\ni              |  the\nl              |  as l'\no              |  or\n\n               | forms of avere, to have (not including the infinitive):\n\nho\nhai\nha\nabbiamo\navete\nhanno\nabbia\nabbiate\nabbiano\navrò\navrai\navrà\navremo\navrete\navranno\navrei\navresti\navrebbe\navremmo\navreste\navrebbero\navevo\navevi\naveva\navevamo\navevate\navevano\nebbi\navesti\nebbe\navemmo\naveste\nebbero\navessi\navesse\navessimo\navessero\navendo\navuto\navuta\navuti\navute\n\n               | forms of essere, to be (not including the infinitive):\nsono\nsei\nè\nsiamo\nsiete\nsia\nsiate\nsiano\nsarò\nsarai\nsarà\nsaremo\nsarete\nsaranno\nsarei\nsaresti\nsarebbe\nsaremmo\nsareste\nsarebbero\nero\neri\nera\neravamo\neravate\nerano\nfui\nfosti\nfu\nfummo\nfoste\nfurono\nfossi\nfosse\nfossimo\nfossero\nessendo\n\n               | forms of fare, to do (not including the infinitive, fa, fat-):\nfaccio\nfai\nfacciamo\nfanno\nfaccia\nfacciate\nfacciano\nfarò\nfarai\nfarà\nfaremo\nfarete\nfaranno\nfarei\nfaresti\nfarebbe\nfaremmo\nfareste\nfarebbero\nfacevo\nfacevi\nfaceva\nfacevamo\nfacevate\nfacevano\nfeci\nfacesti\nfece\nfacemmo\nfaceste\nfecero\nfacessi\nfacesse\nfacessimo\nfacessero\nfacendo\n\n               | forms of stare, to be (not including the infinitive):\nsto\nstai\nsta\nstiamo\nstanno\nstia\nstiate\nstiano\nstarò\nstarai\nstarà\nstaremo\nstarete\nstaranno\nstarei\nstaresti\nstarebbe\nstaremmo\nstareste\nstarebbero\nstavo\nstavi\nstava\nstavamo\nstavate\nstavano\nstetti\nstesti\nstette\nstemmo\nsteste\nstettero\nstessi\nstesse\nstessimo\nstessero\nstando\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(ItalianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/nl/analyzer_nl.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage nl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"nl\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopNlFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerNlFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopNlFilter,\n\t\t\tstemmerNlFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/nl/analyzer_nl_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage nl\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestDutchAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"lichamelijk\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"licham\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"lichamelijke\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"licham\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"van\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/nl/stemmer_nl.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage nl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/dutch\"\n)\n\nconst SnowballStemmerName = \"stemmer_nl_snowball\"\n\ntype DutchStemmerFilter struct {\n}\n\nfunc NewDutchStemmerFilter() *DutchStemmerFilter {\n\treturn &DutchStemmerFilter{}\n}\n\nfunc (s *DutchStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tdutch.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc DutchStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewDutchStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, DutchStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/nl/stop_filter_nl.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage nl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/nl/stop_words_nl.go",
    "content": "package nl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_nl\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar DutchStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | A Dutch stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\n | This is a ranked list (commonest to rarest) of stopwords derived from\n | a large sample of Dutch text.\n\n | Dutch stop words frequently exhibit homonym clashes. These are indicated\n | clearly below.\n\nde             |  the\nen             |  and\nvan            |  of, from\nik             |  I, the ego\nte             |  (1) chez, at etc, (2) to, (3) too\ndat            |  that, which\ndie            |  that, those, who, which\nin             |  in, inside\neen            |  a, an, one\nhij            |  he\nhet            |  the, it\nniet           |  not, nothing, naught\nzijn           |  (1) to be, being, (2) his, one's, its\nis             |  is\nwas            |  (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river\nop             |  on, upon, at, in, up, used up\naan            |  on, upon, to (as dative)\nmet            |  with, by\nals            |  like, such as, when\nvoor           |  (1) before, in front of, (2) furrow\nhad            |  had, past tense all persons sing. of 'hebben' (have)\ner             |  there\nmaar           |  but, only\nom             |  round, about, for etc\nhem            |  him\ndan            |  then\nzou            |  should/would, past tense all persons sing. of 'zullen'\nof             |  or, whether, if\nwat            |  what, something, anything\nmijn           |  possessive and noun 'mine'\nmen            |  people, 'one'\ndit            |  this\nzo             |  so, thus, in this way\ndoor           |  through by\nover           |  over, across\nze             |  she, her, they, them\nzich           |  oneself\nbij            |  (1) a bee, (2) by, near, at\nook            |  also, too\ntot            |  till, until\nje             |  you\nmij            |  me\nuit            |  out of, from\nder            |  Old Dutch form of 'van der' still found in surnames\ndaar           |  (1) there, (2) because\nhaar           |  (1) her, their, them, (2) hair\nnaar           |  (1) unpleasant, unwell etc, (2) towards, (3) as\nheb            |  present first person sing. of 'to have'\nhoe            |  how, why\nheeft          |  present third person sing. of 'to have'\nhebben         |  'to have' and various parts thereof\ndeze           |  this\nu              |  you\nwant           |  (1) for, (2) mitten, (3) rigging\nnog            |  yet, still\nzal            |  'shall', first and third person sing. of verb 'zullen' (will)\nme             |  me\nzij            |  she, they\nnu             |  now\nge             |  'thou', still used in Belgium and south Netherlands\ngeen           |  none\nomdat          |  because\niets           |  something, somewhat\nworden         |  to become, grow, get\ntoch           |  yet, still\nal             |  all, every, each\nwaren          |  (1) 'were' (2) to wander, (3) wares, (3)\nveel           |  much, many\nmeer           |  (1) more, (2) lake\ndoen           |  to do, to make\ntoen           |  then, when\nmoet           |  noun 'spot/mote' and present form of 'to must'\nben            |  (1) am, (2) 'are' in interrogative second person singular of 'to be'\nzonder         |  without\nkan            |  noun 'can' and present form of 'to be able'\nhun            |  their, them\ndus            |  so, consequently\nalles          |  all, everything, anything\nonder          |  under, beneath\nja             |  yes, of course\neens           |  once, one day\nhier           |  here\nwie            |  who\nwerd           |  imperfect third person sing. of 'become'\naltijd         |  always\ndoch           |  yet, but etc\nwordt          |  present third person sing. of 'become'\nwezen          |  (1) to be, (2) 'been' as in 'been fishing', (3) orphans\nkunnen         |  to be able\nons            |  us/our\nzelf           |  self\ntegen          |  against, towards, at\nna             |  after, near\nreeds          |  already\nwil            |  (1) present tense of 'want', (2) 'will', noun, (3) fender\nkon            |  could; past tense of 'to be able'\nniets          |  nothing\nuw             |  your\niemand         |  somebody\ngeweest        |  been; past participle of 'be'\nandere         |  other\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(DutchStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/no/analyzer_no.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage no\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"no\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopNoFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerNoFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopNoFilter,\n\t\t\tstemmerNoFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/no/analyzer_no_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage no\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestNorwegianAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"havnedistriktene\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"havnedistrikt\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"havnedistrikter\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"havnedistrikt\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"det\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/no/stemmer_no.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage no\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/norwegian\"\n)\n\nconst SnowballStemmerName = \"stemmer_no_snowball\"\n\ntype NorwegianStemmerFilter struct {\n}\n\nfunc NewNorwegianStemmerFilter() *NorwegianStemmerFilter {\n\treturn &NorwegianStemmerFilter{}\n}\n\nfunc (s *NorwegianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tnorwegian.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc NorwegianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewNorwegianStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, NorwegianStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/no/stop_filter_no.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage no\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/no/stop_words_no.go",
    "content": "package no\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_no\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar NorwegianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | A Norwegian stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\n | This stop word list is for the dominant bokmål dialect. Words unique\n | to nynorsk are marked *.\n\n | Revised by Jan Bruusgaard <Jan.Bruusgaard@ssb.no>, Jan 2005\n\nog             | and\ni              | in\njeg            | I\ndet            | it/this/that\nat             | to (w. inf.)\nen             | a/an\net             | a/an\nden            | it/this/that\ntil            | to\ner             | is/am/are\nsom            | who/that\npå             | on\nde             | they / you(formal)\nmed            | with\nhan            | he\nav             | of\nikke           | not\nikkje          | not *\nder            | there\nså             | so\nvar            | was/were\nmeg            | me\nseg            | you\nmen            | but\nett            | one\nhar            | have\nom             | about\nvi             | we\nmin            | my\nmitt           | my\nha             | have\nhadde          | had\nhun            | she\nnå             | now\nover           | over\nda             | when/as\nved            | by/know\nfra            | from\ndu             | you\nut             | out\nsin            | your\ndem            | them\noss            | us\nopp            | up\nman            | you/one\nkan            | can\nhans           | his\nhvor           | where\neller          | or\nhva            | what\nskal           | shall/must\nselv           | self (reflective)\nsjøl           | self (reflective)\nher            | here\nalle           | all\nvil            | will\nbli            | become\nble            | became\nblei           | became *\nblitt          | have become\nkunne          | could\ninn            | in\nnår            | when\nvære           | be\nkom            | come\nnoen           | some\nnoe            | some\nville          | would\ndere           | you\nsom            | who/which/that\nderes          | their/theirs\nkun            | only/just\nja             | yes\netter          | after\nned            | down\nskulle         | should\ndenne          | this\nfor            | for/because\ndeg            | you\nsi             | hers/his\nsine           | hers/his\nsitt           | hers/his\nmot            | against\nå              | to\nmeget          | much\nhvorfor        | why\ndette          | this\ndisse          | these/those\nuten           | without\nhvordan        | how\ningen          | none\ndin            | your\nditt           | your\nblir           | become\nsamme          | same\nhvilken        | which\nhvilke         | which (plural)\nsånn           | such a\ninni           | inside/within\nmellom         | between\nvår            | our\nhver           | each\nhvem           | who\nvors           | us/ours\nhvis           | whose\nbåde           | both\nbare           | only/just\nenn            | than\nfordi          | as/because\nfør            | before\nmange          | many\nogså           | also\nslik           | just\nvært           | been\nvære           | to be\nbåe            | both *\nbegge          | both\nsiden          | since\ndykk           | your *\ndykkar         | yours *\ndei            | they *\ndeira          | them *\ndeires         | theirs *\ndeim           | them *\ndi             | your (fem.) *\ndå             | as/when *\neg             | I *\nein            | a/an *\neit            | a/an *\neitt           | a/an *\nelles          | or *\nhonom          | he *\nhjå            | at *\nho             | she *\nhoe            | she *\nhenne          | her\nhennar         | her/hers\nhennes         | hers\nhoss           | how *\nhossen         | how *\nikkje          | not *\ningi           | noone *\ninkje          | noone *\nkorleis        | how *\nkorso          | how *\nkva            | what/which *\nkvar           | where *\nkvarhelst      | where *\nkven           | who/whom *\nkvi            | why *\nkvifor         | why *\nme             | we *\nmedan          | while *\nmi             | my *\nmine           | my *\nmykje          | much *\nno             | now *\nnokon          | some (masc./neut.) *\nnoka           | some (fem.) *\nnokor          | some *\nnoko           | some *\nnokre          | some *\nsi             | his/hers *\nsia            | since *\nsidan          | since *\nso             | so *\nsomt           | some *\nsomme          | some *\num             | about*\nupp            | up *\nvere           | be *\nvore           | was *\nverte          | become *\nvort           | become *\nvarte          | became *\nvart           | became *\n\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(NorwegianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pl/analyzer_pl.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"pl\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopPlFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerPlFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopPlFilter,\n\t\t\tstemmerPlFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pl/analyzer_pl_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pl\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestPolishAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"śmiało\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"śmieć\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"przypadku\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"przypadek\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"według\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t\t// digits safe\n\t\t{\n\t\t\tinput: []byte(\"text 1000\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"text\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"1000\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"badawczego było opracowanie kompendium które przystępny sposób prezentowało niespecjalistom zakresu kryptografii kwantowej wykorzystanie technik kwantowych do bezpiecznego przesyłu przetwarzania informacji\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"badawczy\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"opracować\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kompendium\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"przystyć\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"prezentować\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"niespecjalista\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"zakres\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kryptografia\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kwantowy\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"wykorzyseć\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"technika\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kwantowy\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bezpieczny\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"przesył\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"przetwarzać\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"informacja\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Ale ta wiedza była utrzymywana w tajemnicy\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"wiedza\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"utrzymywać\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"tajemnik\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pl/stemmer_pl.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/lang/pl/stempel\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst SnowballStemmerName = \"stemmer_pl\"\n\ntype PolishStemmerFilter struct {\n\ttrie stempel.Trie\n}\n\nfunc NewPolishStemmerFilter() (*PolishStemmerFilter, error) {\n\ttrie, err := stempel.LoadTrie()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &PolishStemmerFilter{\n\t\ttrie: trie,\n\t}, nil\n}\n\nfunc (s *PolishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tbuff := []rune(string(token.Term))\n\t\tdiff := s.trie.GetLastOnPath(buff)\n\t\tbuff = stempel.Diff(buff, diff)\n\t\ttoken.Term = []byte(string(buff))\n\t}\n\treturn input\n}\n\nfunc PolishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewPolishStemmerFilter()\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, PolishStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pl/stemmer_pl_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pl\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestPolishStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"utrzymywana\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"utrzymywać\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"tajemnicy\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"tajemnik\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/LICENSE",
    "content": "\n                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License."
  },
  {
    "path": "analysis/lang/pl/stempel/cell.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/stempel/javadata\"\n)\n\ntype cell struct {\n\tref int32\n\tcmd int32\n}\n\nfunc (c *cell) String() string {\n\treturn fmt.Sprintf(\"ref(%d) cmd(%d)\", c.ref, c.cmd)\n}\n\nfunc newCell(r *javadata.Reader) (*cell, error) {\n\tcmd, err := r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading cell cmd: %v\", err)\n\t}\n\t_, err = r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading cell cnt: %v\", err)\n\t}\n\tref, err := r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading cell ref: %v\", err)\n\t}\n\t_, err = r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading cell skip: %v\", err)\n\t}\n\treturn &cell{\n\t\tcmd: cmd,\n\t\tref: ref,\n\t}, nil\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/diff.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\n// Diff transforms the dest rune slice following the rules described\n// in the diff command rune slice.\nfunc Diff(dest, diff []rune) []rune {\n\tif len(diff) == 0 {\n\t\treturn dest\n\t}\n\n\tpos := len(dest) - 1\n\tif pos < 0 {\n\t\treturn dest\n\t}\n\n\tfor i := 0; i < len(diff)/2; i++ {\n\t\tcmd := diff[2*i]\n\t\tparam := diff[2*i+1]\n\t\tparNum := int(param - 'a' + 1)\n\t\tswitch cmd {\n\t\tcase '-':\n\t\t\tpos = pos - parNum + 1\n\t\tcase 'R':\n\t\t\tif pos < 0 || pos >= len(dest) {\n\t\t\t\t// out of bounds, just return\n\t\t\t\treturn dest\n\t\t\t}\n\t\t\tdest[pos] = param\n\t\tcase 'D':\n\t\t\to := pos\n\t\t\tpos -= parNum - 1\n\t\t\tif pos < 0 || pos >= len(dest) {\n\t\t\t\t// out of bounds, just return\n\t\t\t\treturn dest\n\t\t\t}\n\t\t\tdest = append(dest[:pos], dest[o+1:]...)\n\t\tcase 'I':\n\t\t\tpos++\n\t\t\tif pos < 0 || pos > len(dest) {\n\t\t\t\t// out of bounds, just return\n\t\t\t\treturn dest\n\t\t\t}\n\n\t\t\tdest = append(dest, 0)\n\t\t\tcopy(dest[pos+1:], dest[pos:])\n\t\t\tdest[pos] = param\n\t\t}\n\t\tpos--\n\t}\n\treturn dest\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/diff_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestDiff(t *testing.T) {\n\ttests := []struct {\n\t\tin  []rune\n\t\tcmd []rune\n\t\tout []rune\n\t}{\n\t\t// test delete, this command deletes N chars backwards from the current pos\n\t\t// the current pos starts at the end of the string\n\t\t// if you try to delete a negative number of chars or more chars than there\n\t\t// are, you will get the buffer at that time\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  delete 1\n\t\t\tcmd: []rune{'D', 'a'},\n\t\t\tout: []rune{'h', 'e', 'l', 'l'},\n\t\t},\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  delete 2\n\t\t\tcmd: []rune{'D', 'a' + 1},\n\t\t\tout: []rune{'h', 'e', 'l'},\n\t\t},\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  delete 3\n\t\t\tcmd: []rune{'D', 'a' + 2},\n\t\t\tout: []rune{'h', 'e'},\n\t\t},\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  delete 4\n\t\t\tcmd: []rune{'D', 'a' + 3},\n\t\t\tout: []rune{'h'},\n\t\t},\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  delete 5\n\t\t\tcmd: []rune{'D', 'a' + 4},\n\t\t\tout: []rune{},\n\t\t},\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  delete 6 (invalid, return buffer at that point)\n\t\t\tcmd: []rune{'D', 'a' + 5},\n\t\t\tout: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t},\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  delete -1\n\t\t\tcmd: []rune{'D', 'a' - 1},\n\t\t\tout: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t},\n\t\t// delete one char twice\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  delete 1, delete 1\n\t\t\tcmd: []rune{'D', 'a', 'D', 'a'},\n\t\t\tout: []rune{'h', 'e', 'l'},\n\t\t},\n\t\t// test insert\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  insert 'p'\n\t\t\tcmd: []rune{'I', 'p'},\n\t\t\tout: []rune{'h', 'e', 'l', 'l', 'o', 'p'},\n\t\t},\n\t\t// insert twice\n\t\t{\n\t\t\tin: []rune{'h'},\n\t\t\t//  insert 'l', insert 'e'\n\t\t\t// NOTE how the cursor moves backwards, so we have to insert in reverse\n\t\t\tcmd: []rune{'I', 'l', 'I', 'e'},\n\t\t\tout: []rune{'h', 'e', 'l'},\n\t\t},\n\t\t// test replace\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  replace with 'y'\n\t\t\tcmd: []rune{'R', 'y'},\n\t\t\tout: []rune{'h', 'e', 'l', 'l', 'y'},\n\t\t},\n\t\t// test replace again\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  replace with 'y', then replace with 'x'\n\t\t\t// NOTE how the cursor moves backwards as we replace\n\t\t\tcmd: []rune{'R', 'y', 'R', 'x'},\n\t\t\tout: []rune{'h', 'e', 'l', 'x', 'y'},\n\t\t},\n\t\t// test skip, then replace\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  skip 1, then replace with 'y'\n\t\t\tcmd: []rune{'-', 'a', 'R', 'y'},\n\t\t\tout: []rune{'h', 'e', 'l', 'y', 'o'},\n\t\t},\n\t\t// test skip 2, then replace\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  skip 1, then replace with 'y'\n\t\t\tcmd: []rune{'-', 'a' + 1, 'R', 'y'},\n\t\t\tout: []rune{'h', 'e', 'y', 'l', 'o'},\n\t\t},\n\t\t// test skip 2, then replace\n\t\t{\n\t\t\tin: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\t//  skip 5 (too far), then replace with 'y'\n\t\t\t//  get original\n\t\t\tcmd: []rune{'-', 'a' + 4, 'R', 'y'},\n\t\t\tout: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"%s-'%s'\", string(test.in), string(test.cmd)), func(t *testing.T) {\n\t\t\tgot := Diff(test.in, test.cmd)\n\t\t\tif !reflect.DeepEqual(test.out, got) {\n\t\t\t\tt.Errorf(\"expected %v, got %v\", test.out, got)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/file.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"bytes\"\n\t_ \"embed\"\n\t\"github.com/blevesearch/stempel/javadata\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n)\n\n//go:embed pl/stemmer_20000.tbl\nvar stempelFile []byte\n\n// Trie is the external interface to work with the stempel trie\ntype Trie interface {\n\tGetLastOnPath([]rune) []rune\n}\n\n// Open attempts to open a file at the specified path, and use it to\n// build a Trie\nfunc Open(path string) (Trie, error) {\n\tf, err := os.Open(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn buildTrieFromReader(f)\n}\n\n// LoadTrie load trie from embed file\nfunc LoadTrie() (Trie, error) {\n\treturn buildTrieFromReader(bytes.NewReader(stempelFile))\n}\n\n// buildTrieFromReader build trie from io.Reader\nfunc buildTrieFromReader(f io.Reader) (Trie, error) {\n\tr := javadata.NewReader(f)\n\tmethod, err := r.ReadUTF()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar rv Trie\n\tif strings.Contains(method, \"M\") {\n\t\trv, err = newMultiTrie(r)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\trv, err = newTrie(r)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn rv, nil\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/file_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"bufio\"\n\t\"compress/gzip\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"golang.org/x/text/encoding/charmap\"\n)\n\nfunc TestEmpty(t *testing.T) {\n\ttrie, err := Open(\"pl/stemmer_20000.tbl\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tbuff := []rune(\"\")\n\tdiff := trie.GetLastOnPath(buff)\n\tif len(diff) > 0 {\n\t\tt.Fatalf(\"expected empty diff, got %v\", diff)\n\t}\n\tbuff = Diff(buff, diff)\n\tif len(buff) > 0 {\n\t\tt.Fatalf(\"expected empty buff, got %v\", buff)\n\t}\n}\n\n// TestStem only tests that we can successfully stem everything in the\n// dictionary without crashing.  It does not attempt to assert correct output.\nfunc TestStem(t *testing.T) {\n\ttrie, err := Open(\"pl/stemmer_20000.tbl\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\twordFileGz, err := os.Open(\"pl/pl_PL.dic.gz\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\tcerr := wordFileGz.Close()\n\t\tif cerr != nil {\n\t\t\tt.Fatal(cerr)\n\t\t}\n\t}()\n\n\twordFile, err := gzip.NewReader(wordFileGz)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\tcerr := wordFile.Close()\n\t\tif cerr != nil {\n\t\t\tt.Fatal(cerr)\n\t\t}\n\t}()\n\n\tcr := charmap.ISO8859_2.NewDecoder().Reader(wordFile)\n\n\tscanner := bufio.NewScanner(cr)\n\tfor scanner.Scan() {\n\t\tbefore := scanner.Text()\n\t\thasSlash := strings.Index(before, \"/\")\n\t\tif hasSlash > 0 {\n\t\t\tbefore = before[0:hasSlash]\n\t\t}\n\t\tbuff := []rune(before)\n\t\tdiff := trie.GetLastOnPath(buff)\n\t\t_ = Diff(buff, diff)\n\t}\n\n\tif err := scanner.Err(); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/fuzz.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build gofuzz\n// +build gofuzz\n\npackage stempel\n\nvar fuzzTrie Trie\n\nfunc init() {\n\tvar err error\n\tfuzzTrie, err = Open(\"pl/stemmer_20000.tbl\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc Fuzz(data []byte) int {\n\tinRunes := []rune(string(data))\n\tdiff := fuzzTrie.GetLastOnPath(inRunes)\n\t_ = Diff(inRunes, diff)\n\treturn 1\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/javadata/README.md",
    "content": "# javadata\n\nGo library to read data written with java.io.DataOutput\n"
  },
  {
    "path": "analysis/lang/pl/stempel/javadata/fuzz.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build gofuzz\n// +build gofuzz\n\npackage javadata\n\nimport \"bytes\"\n\nfunc Fuzz(data []byte) int {\n\tbr := bytes.NewReader(data)\n\tjdr := NewReader(br)\n\n\tvar err error\n\tfor err == nil {\n\t\t_, err = jdr.ReadUTF()\n\t}\n\tif err != nil {\n\t\treturn 0\n\t}\n\treturn 1\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/javadata/input.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage javadata\n\nimport (\n\t\"bufio\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n)\n\n// ErrMalformedInput returned when malformed input is encountered\nvar ErrMalformedInput = fmt.Errorf(\"malformed input\")\n\n// Reader knows how to read java serialized data\ntype Reader struct {\n\tr *bufio.Reader\n}\n\n// NewReader creates a new java data input reader\nfunc NewReader(r io.Reader) *Reader {\n\treturn &Reader{r: bufio.NewReader(r)}\n}\n\n// ReadBool attempts to reads a bool from the stream\nfunc (r *Reader) ReadBool() (bool, error) {\n\tb, err := r.r.ReadByte()\n\tif err != nil {\n\t\treturn false, err\n\t}\n\treturn b != 0, nil\n}\n\n// ReadInt32 attempts to reads a signed 32-bit integer from the stream\nfunc (r *Reader) ReadInt32() (rv int32, err error) {\n\terr = binary.Read(r.r, binary.BigEndian, &rv)\n\treturn\n}\n\n// ReadUint16 attempts to reads a unsigned 16-bit integer from the stream\nfunc (r *Reader) ReadUint16() (rv uint16, err error) {\n\terr = binary.Read(r.r, binary.BigEndian, &rv)\n\treturn\n}\n\n// ReadCharAsRune attempts to read a java two byte char and return it as a rune\nfunc (r *Reader) ReadCharAsRune() (rv rune, err error) {\n\tvar char uint16\n\terr = binary.Read(r.r, binary.BigEndian, &char)\n\trv = rune(char)\n\treturn\n}\n\n// ReadUTF attempts to reads a UTF-encoded string from the stream\n// this method follows the specific alternate encoding desribed here:\n// https://docs.oracle.com/javase/7/docs/api/java/io/DataInput.html\nfunc (r *Reader) ReadUTF() (string, error) {\n\tutfLen, err := r.ReadUint16()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tbytes := make([]byte, utfLen)\n\trunes := make([]rune, utfLen)\n\t_, err = io.ReadFull(r.r, bytes)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tvar count uint16\n\tvar runeCount uint16\n\n\t// handle simple case of all ascii\n\tfor count < utfLen {\n\t\tc := bytes[count]\n\t\tif bytes[count] > 127 {\n\t\t\tbreak\n\t\t}\n\t\tcount++\n\t\trunes[runeCount] = rune(c)\n\t\truneCount++\n\t}\n\n\t// handle rest\n\tfor count < utfLen {\n\t\tc := bytes[count]\n\t\tswitch bytes[count] >> 4 {\n\t\tcase 0, 1, 2, 3, 4, 5, 6, 7, 8:\n\t\t\t/* 0xxxxxxx*/\n\t\t\tcount++\n\t\t\trunes[runeCount] = rune(c)\n\t\t\truneCount++\n\t\tcase 12, 13:\n\t\t\t/* 110x xxxx   10xx xxxx*/\n\t\t\tcount += 2\n\t\t\tif count > utfLen {\n\t\t\t\treturn \"\", ErrMalformedInput\n\t\t\t}\n\t\t\tchar2 := rune(bytes[count-1])\n\t\t\tif (char2 & 0xC0) != 0x80 {\n\t\t\t\treturn \"\", ErrMalformedInput\n\t\t\t}\n\t\t\trunes[runeCount] = (rune(c)&0x1F)<<6 | char2&0x3F\n\t\t\truneCount++\n\t\tcase 14:\n\t\t\t/* 1110 xxxx  10xx xxxx  10xx xxxx */\n\t\t\tcount += 3\n\t\t\tif count > utfLen {\n\t\t\t\treturn \"\", ErrMalformedInput\n\t\t\t}\n\t\t\tchar2 := rune(bytes[count-2])\n\t\t\tchar3 := rune(bytes[count-1])\n\t\t\tif ((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80) {\n\t\t\t\treturn \"\", ErrMalformedInput\n\t\t\t}\n\t\t\trunes[runeCount] = ((rune(c)&0x0F)<<12 | (char2&0x3F)<<6 | (char3&0x3F)<<0)\n\t\t\truneCount++\n\t\tdefault:\n\t\t\t/* 10xx xxxx,  1111 xxxx */\n\t\t\treturn \"\", ErrMalformedInput\n\t\t}\n\t}\n\treturn string(runes[0:runeCount]), nil\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/javadata/input_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage javadata\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"testing\"\n)\n\nfunc TestReadBool(t *testing.T) {\n\n\ttests := []struct {\n\t\tin  []byte\n\t\tout bool\n\t\terr error\n\t}{\n\t\t{\n\t\t\tin:  []byte{0},\n\t\t\tout: false,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{1},\n\t\t\tout: true,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{27},\n\t\t\tout: true,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{},\n\t\t\terr: io.EOF,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(string(test.in), func(t *testing.T) {\n\t\t\tsr := bytes.NewReader(test.in)\n\t\t\tdr := NewReader(sr)\n\t\t\tactual, err := dr.ReadBool()\n\t\t\tif err != test.err {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif actual != test.out {\n\t\t\t\tt.Errorf(\"expected %t, got %t\", test.out, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestReadUint16(t *testing.T) {\n\n\ttests := []struct {\n\t\tin  []byte\n\t\tout uint16\n\t\terr error\n\t}{\n\t\t{\n\t\t\tin:  []byte{0, 0},\n\t\t\tout: 0,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 1},\n\t\t\tout: 1,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{1, 0},\n\t\t\tout: 256,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{},\n\t\t\terr: io.EOF,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(string(test.in), func(t *testing.T) {\n\t\t\tsr := bytes.NewReader(test.in)\n\t\t\tdr := NewReader(sr)\n\t\t\tactual, err := dr.ReadUint16()\n\t\t\tif err != test.err {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif actual != test.out {\n\t\t\t\tt.Errorf(\"expected %d, got %d\", test.out, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestReadInt32(t *testing.T) {\n\n\ttests := []struct {\n\t\tin  []byte\n\t\tout int32\n\t\terr error\n\t}{\n\t\t{\n\t\t\tin:  []byte{0, 0, 0, 0},\n\t\t\tout: 0,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 0, 0, 1},\n\t\t\tout: 1,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 0, 1, 0},\n\t\t\tout: 256,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 1, 0, 0},\n\t\t\tout: 65536,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{},\n\t\t\terr: io.EOF,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(string(test.in), func(t *testing.T) {\n\t\t\tsr := bytes.NewReader(test.in)\n\t\t\tdr := NewReader(sr)\n\t\t\tactual, err := dr.ReadInt32()\n\t\t\tif err != test.err {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif actual != test.out {\n\t\t\t\tt.Errorf(\"expected %d, got %d\", test.out, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestReadUTF(t *testing.T) {\n\n\ttests := []struct {\n\t\tin  []byte\n\t\tout string\n\t\terr error\n\t}{\n\t\t{\n\t\t\tin:  []byte{0, 3, 'c', 'a', 't'},\n\t\t\tout: \"cat\",\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 2, 0xc2, 0xa3},\n\t\t\tout: \"£\",\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 3, 0xe3, 0x85, 0x85},\n\t\t\tout: \"ㅅ\",\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 6, 0xe3, 0x85, 0x85, 'c', 'a', 't'},\n\t\t\tout: \"ㅅcat\",\n\t\t},\n\t\t{\n\t\t\tin:  []byte{},\n\t\t\terr: io.EOF,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 3},\n\t\t\terr: io.EOF,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 1, 0xc2},\n\t\t\terr: ErrMalformedInput,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 2, 0xc2, 0xc3},\n\t\t\terr: ErrMalformedInput,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 2, 0xe3, 0x85},\n\t\t\terr: ErrMalformedInput,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 3, 0xe3, 0xc5, 0x85},\n\t\t\terr: ErrMalformedInput,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 1, 0xff},\n\t\t\terr: ErrMalformedInput,\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0x0, 0x05, 0x44, 0x61, 0x52, 0xc4, 0x87},\n\t\t\tout: \"DaRć\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(string(test.in), func(t *testing.T) {\n\t\t\tsr := bytes.NewReader(test.in)\n\t\t\tdr := NewReader(sr)\n\t\t\tactual, err := dr.ReadUTF()\n\t\t\tif err != test.err {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif actual != test.out {\n\t\t\t\tt.Errorf(\"expected %s, got %s\", test.out, actual)\n\t\t\t}\n\t\t})\n\t}\n\n}\n\n// func TestFile(t *testing.T) {\n// \tf, err := os.Open(\"stemmer_20000.tbl\")\n// \tif err != nil {\n// \t\tt.Fatal(err)\n// \t}\n// \tr := NewReader(f)\n// \treversed, err := r.ReadBool()\n// \tif err != nil {\n// \t\tt.Fatal(err)\n// \t}\n// \tlog.Printf(\"reversed: %t\", reversed)\n// \troot, err := r.ReadInt32()\n// \tif err != nil {\n// \t\tt.Fatal(err)\n// \t}\n// \tlog.Printf(\"root: %d\", root)\n// \tn, err := r.ReadInt32()\n// \tif err != nil {\n// \t\tt.Fatal(err)\n// \t}\n// \tlog.Printf(\"n is %d\", n)\n// \t// for n > 0 {\n// \t// \tutf, err := r.ReadUTF()\n// \t// \tif err != nil {\n// \t// \t\tt.Error(err)\n// \t// \t}\n// \t// \tlog.Printf(\"read: %s\", utf)\n// \t// \tn--\n// \t// }\n// }\n"
  },
  {
    "path": "analysis/lang/pl/stempel/multi_trie.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/stempel/javadata\"\n)\n\n// multiTrie represents a trie of tries.  When using the multiTrie, each trie\n// is consulted consecutively to find commands to perform on the input.  Thus\n// a multiTrie with seven tries might have up to seven groups of commands to\n// perform on the input.\ntype multiTrie struct {\n\ttries   []*trie\n\tby      int32\n\tforward bool\n}\n\nfunc newMultiTrie(r *javadata.Reader) (rv *multiTrie, err error) {\n\trv = &multiTrie{}\n\trv.forward, err = r.ReadBool()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv.by, err = r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnTries, err := r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor nTries > 0 {\n\t\ttrie, err := newTrie(r)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trv.tries = append(rv.tries, trie)\n\t\tnTries--\n\t}\n\treturn rv, nil\n}\n\nconst eom = rune('*')\n\nfunc (t *multiTrie) GetLastOnPath(key []rune) []rune {\n\tvar rv []rune\n\tlastKey := key\n\tp := make([][]rune, len(t.tries))\n\tlastR := ' '\n\tfor i := 0; i < len(t.tries); i++ {\n\t\tr := t.tries[i].GetLastOnPath(lastKey)\n\t\tif len(r) == 0 || len(r) == 1 && r[0] == eom {\n\t\t\treturn rv\n\t\t}\n\t\tif cannotFollow(lastR, r[0]) {\n\t\t\treturn rv\n\t\t}\n\t\tlastR = r[len(r)-2]\n\t\tp[i] = r\n\t\tif p[i][0] == '-' {\n\t\t\tif i > 0 {\n\t\t\t\tvar err error\n\t\t\t\tkey, err = t.skip(key, lengthPP(p[i-1]))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn rv\n\t\t\t\t}\n\t\t\t}\n\t\t\tvar err error\n\t\t\tkey, err = t.skip(key, lengthPP(p[i]))\n\t\t\tif err != nil {\n\t\t\t\treturn rv\n\t\t\t}\n\t\t}\n\t\trv = append(rv, r...)\n\t\tif len(key) != 0 {\n\t\t\tlastKey = key\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc cannotFollow(after, goes rune) bool {\n\tswitch after {\n\tcase '-', 'D':\n\t\treturn after == goes\n\t}\n\treturn false\n}\n\nvar errIndexOutOfBounds = fmt.Errorf(\"index out of bounds\")\n\nfunc (t *multiTrie) skip(in []rune, count int) ([]rune, error) {\n\tif count > len(in) {\n\t\treturn nil, errIndexOutOfBounds\n\t}\n\tif t.forward {\n\t\treturn in[count:], nil\n\t}\n\treturn in[0 : len(in)-count], nil\n}\n\nfunc lengthPP(cmd []rune) int {\n\trv := 0\n\tfor i := 0; i < len(cmd); i++ {\n\t\tswitch cmd[i] {\n\t\tcase '-', 'D':\n\t\t\ti++\n\t\t\trv += int(cmd[i] - rune('a') + 1)\n\t\tcase 'R':\n\t\t\ti++\n\t\t\trv++\n\t\t\tfallthrough\n\t\tcase 'I':\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (t *multiTrie) String() string {\n\trv := \"\"\n\tfor i, trie := range t.tries {\n\t\trv += fmt.Sprintf(\"trie %d\\n\\n %v\\n--------\\n\", i, trie)\n\t}\n\treturn rv\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/row.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/stempel/javadata\"\n)\n\ntype row struct {\n\tcells map[rune]*cell\n}\n\nfunc (r *row) String() string {\n\trv := \"\"\n\tfor k, v := range r.cells {\n\t\trv += fmt.Sprintf(\"[%s:%v]\\n\", string(k), v)\n\t}\n\treturn rv\n}\n\nfunc newRow(r *javadata.Reader) (*row, error) {\n\trv := &row{\n\t\tcells: make(map[rune]*cell),\n\t}\n\n\tnCells, err := r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading num cells: %v\", err)\n\t}\n\n\tfor nCells > 0 {\n\n\t\tc, err := r.ReadCharAsRune()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error reading cell char: %v\", err)\n\t\t}\n\t\tcell, err := newCell(r)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error reading cell: %v\", err)\n\t\t}\n\n\t\trv.cells[c] = cell\n\t\tnCells--\n\t}\n\treturn rv, nil\n}\n\nfunc (r *row) getCmd(way rune) int32 {\n\tc := r.at(way)\n\tif c != nil {\n\t\treturn c.cmd\n\t}\n\treturn -1\n}\n\nfunc (r *row) getRef(way rune) int32 {\n\tc := r.at(way)\n\tif c != nil {\n\t\treturn c.ref\n\t}\n\treturn -1\n}\n\nfunc (r *row) at(c rune) *cell {\n\treturn r.cells[c]\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/strenum.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"io\"\n)\n\ntype strEnum struct {\n\tr    []rune\n\tfrom int\n\tby   int\n}\n\nfunc newStrEnum(s []rune, up bool) *strEnum {\n\trv := &strEnum{\n\t\tr: s,\n\t}\n\tif up {\n\t\trv.from = 0\n\t\trv.by = 1\n\t} else {\n\t\trv.from = len(s) - 1\n\t\trv.by = -1\n\t}\n\treturn rv\n}\n\nfunc (s *strEnum) next() (rune, error) {\n\tif s.from < 0 || s.from >= len(s.r) {\n\t\treturn 0, io.EOF\n\t}\n\trv := s.r[s.from]\n\ts.from += s.by\n\treturn rv, nil\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/strenum_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestStrenumNext(t *testing.T) {\n\n\ttests := []struct {\n\t\tin     []rune\n\t\tup     bool\n\t\texpect []rune\n\t}{\n\t\t{\n\t\t\tin:     []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\tup:     true,\n\t\t\texpect: []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t},\n\t\t{\n\t\t\tin:     []rune{'h', 'e', 'l', 'l', 'o'},\n\t\t\tup:     false,\n\t\t\texpect: []rune{'o', 'l', 'l', 'e', 'h'},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"%s-up-%t\", string(test.in), test.up), func(t *testing.T) {\n\t\t\tstrenum := newStrEnum(test.in, test.up)\n\t\t\tvar got []rune\n\t\t\tnext, err := strenum.next()\n\t\t\tfor err == nil {\n\t\t\t\tgot = append(got, next)\n\t\t\t\tnext, err = strenum.next()\n\t\t\t}\n\t\t\tif err != io.EOF {\n\t\t\t\tt.Errorf(\"next got err: %v\", err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.expect) {\n\t\t\t\tt.Errorf(\"expected %v, got %v\", test.expect, got)\n\t\t\t}\n\t\t})\n\t}\n\n}\n"
  },
  {
    "path": "analysis/lang/pl/stempel/trie.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stempel\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/stempel/javadata\"\n)\n\n// trie represents the internal trie structure\ntype trie struct {\n\trows    []*row\n\tcmds    []string\n\troot    int32\n\tforward bool\n}\n\nfunc newTrie(r *javadata.Reader) (rv *trie, err error) {\n\trv = &trie{}\n\trv.forward, err = r.ReadBool()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading trie forward: %v\", err)\n\t}\n\trv.root, err = r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading trie root: %v\", err)\n\t}\n\n\t// commands\n\tnCommands, err := r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading trie num commands: %v\", err)\n\t}\n\tfor nCommands > 0 {\n\t\tutfCommand, nerr := r.ReadUTF()\n\t\tif nerr != nil {\n\t\t\treturn nil, fmt.Errorf(\"error reading trie command utf: %v\", nerr)\n\t\t}\n\t\trv.cmds = append(rv.cmds, utfCommand)\n\t\tnCommands--\n\t}\n\n\t// rows\n\tnRows, err := r.ReadInt32()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error reading trie num rows: %v\", err)\n\t}\n\tfor nRows > 0 {\n\t\trow, err := newRow(r)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error reading trie row: %v\", err)\n\t\t}\n\t\trv.rows = append(rv.rows, row)\n\t\tnRows--\n\t}\n\n\treturn rv, nil\n}\n\nfunc (t *trie) getRow(i int) *row {\n\tif i < 0 || i >= len(t.rows) {\n\t\treturn nil\n\t}\n\treturn t.rows[i]\n}\n\nfunc (t *trie) GetLastOnPath(key []rune) []rune {\n\tnow := t.getRow(int(t.root))\n\tvar last []rune\n\tvar w int32\n\te := newStrEnum(key, t.forward)\n\n\t// walk over each rune\n\t// if rune has row in the table, note the cmd (as last)\n\t// if rune has row in table, see if it transitions to another row\n\t// if it does, move to that row and next char on next loop itr\n\t// if it does not, return the last cmd\n\t// if you get to end of string and there is command in row use it\n\t// or return last\n\tfor i := 0; i < len(key)-1; i++ {\n\t\tr, err := e.next()\n\t\tif err != nil {\n\t\t\treturn last\n\t\t}\n\t\tw = now.getCmd(r)\n\t\tif w >= 0 {\n\t\t\tlast = []rune(t.cmds[w])\n\t\t}\n\t\tw = now.getRef(r)\n\t\tif w >= 0 {\n\t\t\tnow = t.getRow(int(w))\n\t\t} else {\n\t\t\treturn last\n\t\t}\n\t}\n\tr, err := e.next()\n\tif err != nil {\n\t\treturn last\n\t}\n\tw = now.getCmd(r)\n\tif err != nil {\n\t\treturn last\n\t}\n\tif w >= 0 {\n\t\treturn []rune(t.cmds[w])\n\t}\n\treturn last\n}\n\nfunc (t *trie) String() string {\n\trv := \"\"\n\tfor _, cmd := range t.cmds {\n\t\trv += fmt.Sprintf(\"cmd: %s\\n\", string(cmd))\n\t}\n\tfor _, row := range t.rows {\n\t\trv += fmt.Sprintf(\"row: %v\\n\", row)\n\t}\n\treturn rv\n}\n"
  },
  {
    "path": "analysis/lang/pl/stop_filter_pl.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pl/stop_words_pl.go",
    "content": "package pl\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_pl\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar PolishStopWords = []byte(` | From https://github.com/stopwords-iso/stopwords-pl/tree/master\n | The MIT License (MIT)\n | See https://github.com/stopwords-iso/stopwords-pl/blob/master/LICENSE\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |  - english text is auto-translate\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n | a polish stop word list. comments begin with vertical bar. each stop\n | word is at the start of a line.\n\na\t\t\t\t| and\naby\t\t\t\t| to\nach\t\t\t\t| ah\nacz\t\t\t\t| although\naczkolwiek\t\t| although\naj\t\t\t\t| ay\nalbo\t\t\t| or\nale\t\t\t\t| but\należ\t\t\t| but\nani\t\t\t\t| or\naż\t\t\t\t| until\nbardziej\t\t| more\nbardzo\t\t\t| very\nbez\t\t\t\t| without\nbo\t\t\t\t| because\nbowiem\t\t\t| because\nby\t\t\t\t| by\nbyli\t\t\t| were\nbym\t\t\t\t| i would\nbynajmniej\t\t| not at all\nbyć\t\t\t\t| to be\nbył\t\t\t\t| was\nbyła\t\t\t| was\nbyło\t\t\t| was\nbyły\t\t\t| were\nbędzie\t\t\t| will be\nbędą\t\t\t| they will\ncali\t\t\t| inches\ncała\t\t\t| whole\ncały\t\t\t| whole\nchce\t\t\t| i want\nchoć\t\t\t| though\nci\t\t\t\t| you\nciebie\t\t\t| you\ncię\t\t\t\t| you\nco\t\t\t\t| what\ncokolwiek\t\t| whatever\ncoraz\t\t\t| getting\ncoś\t\t\t\t| something\nczasami\t\t\t| sometimes\nczasem\t\t\t| sometimes\nczemu\t\t\t| why\nczy\t\t\t\t| whether\nczyli\t\t\t| that is\nczęsto\t\t\t| often\ndaleko\t\t\t| far\ndla\t\t\t\t| for\ndlaczego\t\t| why\ndlatego\t\t\t| which is why\ndo\t\t\t\t| down\ndobrze\t\t\t| all right\ndokąd\t\t\t| where\ndość\t\t\t| enough\ndr\t\t\t\t| dr\ndużo\t\t\t| a lot\ndwa\t\t\t\t| two\ndwaj\t\t\t| two\ndwie\t\t\t| two\ndwoje\t\t\t| two\ndzisiaj\t\t\t| today\ndziś\t\t\t| today\ngdy\t\t\t\t| when\ngdyby\t\t\t| if\ngdyż\t\t\t| because\ngdzie\t\t\t| where\ngdziekolwiek\t| wherever\ngdzieś\t\t\t| somewhere\ngo\t\t\t\t| him\ngodz\t\t\t| time\nhab\t\t\t\t| hab\ni\t\t\t\t| and\nich\t\t\t\t| their\nii\t\t\t\t| ii\niii\t\t\t\t| iii\nile\t\t\t\t| how much\nim\t\t\t\t| them\ninna\t\t\t| different\ninne\t\t\t| other\ninny\t\t\t| other\ninnych\t\t\t| other\ninż\t\t\t\t| eng\niv\t\t\t\t| iv\nix\t\t\t\t| ix\niż\t\t\t\t| that\nja\t\t\t\t| i\njak\t\t\t\t| how\njakaś\t\t\t| some\njakby\t\t\t| as if\njaki\t\t\t| what\njakichś\t\t\t| some\njakie\t\t\t| what\njakiś\t\t\t| some\njakiż\t\t\t| what\njakkolwiek\t\t| however\njako\t\t\t| as\njakoś\t\t\t| somehow\nje\t\t\t\t| them\njeden\t\t\t| one\njedna\t\t\t| one\njednak\t\t\t| but\njednakże\t\t| however\njedno\t\t\t| one\njednym\t\t\t| one\njedynie\t\t\t| only\njego\t\t\t| his\njej\t\t\t\t| her\njemu\t\t\t| him\njest\t\t\t| is\njestem\t\t\t| i am\njeszcze\t\t\t| still\njeśli\t\t\t| if\njeżeli\t\t\t| if\njuż\t\t\t\t| already\nją\t\t\t\t| i\nkażdy\t\t\t| everyone\nkiedy\t\t\t| when\nkierunku\t\t| direction\nkilka\t\t\t| several\nkilku\t\t\t| several\nkimś\t\t\t| someone\nkto\t\t\t\t| who\nktokolwiek\t\t| anyone\nktoś\t\t\t| someone\nktóra\t\t\t| which\nktóre\t\t\t| which\nktórego\t\t\t| whose\nktórej\t\t\t| which\nktóry\t\t\t| which\nktórych\t\t\t| which\nktórym\t\t\t| which\nktórzy\t\t\t| who\nku\t\t\t\t| to\nlat\t\t\t\t| years\nlecz\t\t\t| but\nlub\t\t\t\t| or\nma\t\t\t\t| has\nmają\t\t\t| may\nmam\t\t\t\t| i have\nmamy\t\t\t| we have\nmało\t\t\t| little\nmgr\t\t\t\t| msc\nmi\t\t\t\t| to me\nmiał\t\t\t| had\nmimo\t\t\t| despite\nmiędzy\t\t\t| between\nmnie\t\t\t| me\nmną\t\t\t\t| me\nmogą\t\t\t| they can\nmoi\t\t\t\t| my\nmoim\t\t\t| my\nmoja\t\t\t| my\nmoje\t\t\t| my\nmoże\t\t\t| maybe\nmożliwe\t\t\t| that's possible\nmożna\t\t\t| you can\nmu\t\t\t\t| him\nmusi\t\t\t| has to\nmy\t\t\t\t| we\nmój\t\t\t\t| my\nna\t\t\t\t| on\nnad\t\t\t\t| above\nnam\t\t\t\t| u.s\nnami\t\t\t| us\nnas\t\t\t\t| us\nnasi\t\t\t| our\nnasz\t\t\t| our\nnasza\t\t\t| our\nnasze\t\t\t| our\nnaszego\t\t\t| our\nnaszych\t\t\t| ours\nnatomiast\t\t| whereas\nnatychmiast\t\t| immediately\nnawet\t\t\t| even\nnic\t\t\t\t| nothing\nnich\t\t\t| them\nnie\t\t\t\t| no\nniech\t\t\t| let\nniego\t\t\t| him\nniej\t\t\t| her\nniemu\t\t\t| not him\nnigdy\t\t\t| never\nnim\t\t\t\t| him\nnimi\t\t\t| them\nnią\t\t\t\t| her\nniż\t\t\t\t| than\nno\t\t\t\t| yeah\nnowe\t\t\t| new\nnp\t\t\t\t| e.g.\nnr\t\t\t\t| no\no\t\t\t\t| about\no.o.\t\t\t| o.o.\nobok\t\t\t| near\nod\t\t\t\t| from\nok\t\t\t\t| approx\nokoło\t\t\t| about\non\t\t\t\t| he\nona\t\t\t\t| she\none\t\t\t\t| they\noni\t\t\t\t| they\nono\t\t\t\t| it\noraz\t\t\t| and\noto\t\t\t\t| here\nowszem\t\t\t| yes\npan\t\t\t\t| mr\npana\t\t\t| mr\npani\t\t\t| you\npl\t\t\t\t| pl\npo\t\t\t\t| after\npod\t\t\t\t| under\npodczas\t\t\t| while\npomimo\t\t\t| despite\nponad\t\t\t| above\nponieważ\t\t| because\npowinien\t\t| should\npowinna\t\t\t| she should\npowinni\t\t\t| they should\npowinno\t\t\t| should\npoza\t\t\t| apart from\nprawie\t\t\t| almost\nprof\t\t\t| prof\nprzecież\t\t| yet\nprzed\t\t\t| before\nprzede\t\t\t| above\nprzedtem\t\t| before\nprzez\t\t\t| by\nprzy\t\t\t| by\nraz\t\t\t\t| once\nrazie\t\t\t| case\nroku\t\t\t| year\nrównież\t\t\t| also\nsam\t\t\t\t| alone\nsama\t\t\t| alone\nsię\t\t\t\t| myself\nskąd\t\t\t| from where\nsobie\t\t\t| myself\nsobą\t\t\t| myself\nsposób\t\t\t| way\nswoje\t\t\t| own\nsą\t\t\t\t| are\nta\t\t\t\t| this\ntak\t\t\t\t| yes\ntaka\t\t\t| such\ntaki\t\t\t| such\ntakich\t\t\t| such\ntakie\t\t\t| such\ntakże\t\t\t| too\ntam\t\t\t\t| over there\nte\t\t\t\t| these\ntego\t\t\t| this\ntej\t\t\t\t| this one\ntel\t\t\t\t| phone\ntemu\t\t\t| ago\nten\t\t\t\t| this\nteraz\t\t\t| now\nteż\t\t\t\t| too\nto\t\t\t\t| this\ntobie\t\t\t| you\ntobą\t\t\t| you\ntoteż\t\t\t| this as well\ntotobą\t\t\t| you\ntrzeba\t\t\t| it's necessary to\ntu\t\t\t\t| here\ntutaj\t\t\t| here\ntwoi\t\t\t| yours\ntwoim\t\t\t| yours\ntwoja\t\t\t| your\ntwoje\t\t\t| your\ntwym\t\t\t| your\ntwój\t\t\t| your\nty\t\t\t\t| you\ntych\t\t\t| these\ntylko\t\t\t| just\ntym\t\t\t\t| this\ntys\t\t\t\t| thousand\ntzw\t\t\t\t| so-called\ntę\t\t\t\t| these\nu\t\t\t\t| at\nul\t\t\t\t| st\nvi\t\t\t\t| vi\nvii\t\t\t\t| vii\nviii\t\t\t| viii\nvol\t\t\t\t| vol\nw\t\t\t\t| in\nwam\t\t\t\t| you\nwami\t\t\t| you\nwas\t\t\t\t| mustache\nwasi\t\t\t| yours\nwasz\t\t\t| yours\nwasza\t\t\t| yours\nwasze\t\t\t| yours\nwe\t\t\t\t| in\nwedług\t\t\t| according to\nwie\t\t\t\t| knows\nwiele\t\t\t| many\nwielu\t\t\t| many\nwięc\t\t\t| so\nwięcej\t\t\t| more\nwszyscy\t\t\t| all\nwszystkich\t\t| everyone\nwszystkie\t\t| all\nwszystkim\t\t| everyone\nwszystko\t\t| all\nwtedy\t\t\t| then\nwww\t\t\t\t| www\nwy\t\t\t\t| you\nwłaśnie\t\t\t| exactly\nwśród\t\t\t| among\nxi\t\t\t\t| x.x\nxii\t\t\t\t| xii\nxiii\t\t\t| xii\nxiv\t\t\t\t| xiv\nxv\t\t\t\t| xv\nz\t\t\t\t| with\nza\t\t\t\t| behind\nzapewne\t\t\t| probably\nzawsze\t\t\t| always\nzaś\t\t\t\t| and\nze\t\t\t\t| that\nzeznowu\t\t\t| testify\nznowu\t\t\t| again\nznów\t\t\t| again\nzostał\t\t\t| left\nzł\t\t\t\t| zloty\nżaden\t\t\t| no\nżadna\t\t\t| none\nżadne\t\t\t| none\nżadnych\t\t\t| none\nże\t\t\t\t| that\nżeby\t\t\t| to\n\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(PolishStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pt/analyzer_pt.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pt\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"pt\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopPtFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerPtFilter, err := cache.TokenFilterNamed(LightStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopPtFilter,\n\t\t\tstemmerPtFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pt/analyzer_pt_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pt\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestPortugueseAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"quilométricas\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quilometric\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"quilométricos\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quilometric\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"não\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pt/light_stemmer_pt.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pt\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst LightStemmerName = \"stemmer_pt_light\"\n\ntype PortugueseLightStemmerFilter struct {\n}\n\nfunc NewPortugueseLightStemmerFilter() *PortugueseLightStemmerFilter {\n\treturn &PortugueseLightStemmerFilter{}\n}\n\nfunc (s *PortugueseLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\trunes := bytes.Runes(token.Term)\n\t\trunes = stem(runes)\n\t\ttoken.Term = analysis.BuildTermFromRunes(runes)\n\t}\n\treturn input\n}\n\nfunc stem(input []rune) []rune {\n\n\tinputLen := len(input)\n\n\tif inputLen < 4 {\n\t\treturn input\n\t}\n\n\tinput = removeSuffix(input)\n\tinputLen = len(input)\n\n\tif inputLen > 3 && input[inputLen-1] == 'a' {\n\t\tinput = normFeminine(input)\n\t\tinputLen = len(input)\n\t}\n\n\tif inputLen > 4 {\n\t\tswitch input[inputLen-1] {\n\t\tcase 'e', 'a', 'o':\n\t\t\tinput = input[0 : inputLen-1]\n\t\t\tinputLen = len(input)\n\t\t}\n\t}\n\n\tfor i := 0; i < inputLen; i++ {\n\t\tswitch input[i] {\n\t\tcase 'à', 'á', 'â', 'ä', 'ã':\n\t\t\tinput[i] = 'a'\n\t\tcase 'ò', 'ó', 'ô', 'ö', 'õ':\n\t\t\tinput[i] = 'o'\n\t\tcase 'è', 'é', 'ê', 'ë':\n\t\t\tinput[i] = 'e'\n\t\tcase 'ù', 'ú', 'û', 'ü':\n\t\t\tinput[i] = 'u'\n\t\tcase 'ì', 'í', 'î', 'ï':\n\t\t\tinput[i] = 'i'\n\t\tcase 'ç':\n\t\t\tinput[i] = 'c'\n\t\t}\n\t}\n\n\treturn input\n}\n\nfunc removeSuffix(input []rune) []rune {\n\n\tinputLen := len(input)\n\n\tif inputLen > 4 && analysis.RunesEndsWith(input, \"es\") {\n\t\tswitch input[inputLen-3] {\n\t\tcase 'r', 's', 'l', 'z':\n\t\t\treturn input[0 : inputLen-2]\n\t\t}\n\t}\n\n\tif inputLen > 3 && analysis.RunesEndsWith(input, \"ns\") {\n\t\tinput[inputLen-2] = 'm'\n\t\treturn input[0 : inputLen-1]\n\t}\n\n\tif inputLen > 4 && (analysis.RunesEndsWith(input, \"eis\") || analysis.RunesEndsWith(input, \"éis\")) {\n\t\tinput[inputLen-3] = 'e'\n\t\tinput[inputLen-2] = 'l'\n\t\treturn input[0 : inputLen-1]\n\t}\n\n\tif inputLen > 4 && analysis.RunesEndsWith(input, \"ais\") {\n\t\tinput[inputLen-2] = 'l'\n\t\treturn input[0 : inputLen-1]\n\t}\n\n\tif inputLen > 4 && analysis.RunesEndsWith(input, \"óis\") {\n\t\tinput[inputLen-3] = 'o'\n\t\tinput[inputLen-2] = 'l'\n\t\treturn input[0 : inputLen-1]\n\t}\n\n\tif inputLen > 4 && analysis.RunesEndsWith(input, \"is\") {\n\t\tinput[inputLen-1] = 'l'\n\t\treturn input\n\t}\n\n\tif inputLen > 3 &&\n\t\t(analysis.RunesEndsWith(input, \"ões\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"ães\")) {\n\t\tinput = input[0 : inputLen-1]\n\t\tinputLen = len(input)\n\t\tinput[inputLen-2] = 'ã'\n\t\tinput[inputLen-1] = 'o'\n\t\treturn input\n\t}\n\n\tif inputLen > 6 && analysis.RunesEndsWith(input, \"mente\") {\n\t\treturn input[0 : inputLen-5]\n\t}\n\n\tif inputLen > 3 && input[inputLen-1] == 's' {\n\t\treturn input[0 : inputLen-1]\n\t}\n\treturn input\n}\n\nfunc normFeminine(input []rune) []rune {\n\tinputLen := len(input)\n\n\tif inputLen > 7 &&\n\t\t(analysis.RunesEndsWith(input, \"inha\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"iaca\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"eira\")) {\n\t\tinput[inputLen-1] = 'o'\n\t\treturn input\n\t}\n\n\tif inputLen > 6 {\n\t\tif analysis.RunesEndsWith(input, \"osa\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"ica\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"ida\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"ada\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"iva\") ||\n\t\t\tanalysis.RunesEndsWith(input, \"ama\") {\n\t\t\tinput[inputLen-1] = 'o'\n\t\t\treturn input\n\t\t}\n\n\t\tif analysis.RunesEndsWith(input, \"ona\") {\n\t\t\tinput[inputLen-3] = 'ã'\n\t\t\tinput[inputLen-2] = 'o'\n\t\t\treturn input[0 : inputLen-1]\n\t\t}\n\n\t\tif analysis.RunesEndsWith(input, \"ora\") {\n\t\t\treturn input[0 : inputLen-1]\n\t\t}\n\n\t\tif analysis.RunesEndsWith(input, \"esa\") {\n\t\t\tinput[inputLen-3] = 'ê'\n\t\t\treturn input[0 : inputLen-1]\n\t\t}\n\n\t\tif analysis.RunesEndsWith(input, \"na\") {\n\t\t\tinput[inputLen-1] = 'o'\n\t\t\treturn input\n\t\t}\n\t}\n\treturn input\n}\n\nfunc PortugueseLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewPortugueseLightStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(LightStemmerName, PortugueseLightStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pt/light_stemmer_pt_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pt\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestPortugueseLightStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"doutores\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"doutor\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"doutor\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"doutor\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"homens\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"homem\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"homem\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"homem\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"papéis\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"papel\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"papel\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"papel\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"normais\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"normal\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"normal\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"normal\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"lencóis\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"lencol\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"lencol\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"lencol\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"barris\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"barril\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"barril\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"barril\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"botões\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bota\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"botão\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bota\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// longer\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"o\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"debate\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"político\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"pelo\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"menos\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"o\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"que\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"vem\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"público\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"parece\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"de\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"modo\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"nada\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"surpreendente\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"restrito\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"temas\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"menores\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"mas\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"há\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"evidentemente\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"grandes\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"questões\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"em\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"jogo\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"nas\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"eleições\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"que\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"se\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aproximam\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"o\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"debat\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"politic\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"pelo\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"meno\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"o\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"que\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"vem\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"public\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"parec\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"de\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"modo\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"nada\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"surpreendent\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"restrit\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"tema\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"menor\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"mas\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"há\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"evident\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"grand\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"questa\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"em\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"jogo\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"nas\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"eleica\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"que\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"se\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"aproximam\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(LightStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pt/stop_filter_pt.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pt\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/pt/stop_words_pt.go",
    "content": "package pt\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_pt\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar PortugueseStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | A Portuguese stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\n\n | The following is a ranked list (commonest to rarest) of stopwords\n | deriving from a large sample of text.\n\n | Extra words have been added at the end.\n\nde             |  of, from\na              |  the; to, at; her\no              |  the; him\nque            |  who, that\ne              |  and\ndo             |  de + o\nda             |  de + a\nem             |  in\num             |  a\npara           |  for\n  | é          from SER\ncom            |  with\nnão            |  not, no\numa            |  a\nos             |  the; them\nno             |  em + o\nse             |  himself etc\nna             |  em + a\npor            |  for\nmais           |  more\nas             |  the; them\ndos            |  de + os\ncomo           |  as, like\nmas            |  but\n  | foi        from SER\nao             |  a + o\nele            |  he\ndas            |  de + as\n  | tem        from TER\nà              |  a + a\nseu            |  his\nsua            |  her\nou             |  or\n  | ser        from SER\nquando         |  when\nmuito          |  much\n  | há         from HAV\nnos            |  em + os; us\njá             |  already, now\n  | está       from EST\neu             |  I\ntambém         |  also\nsó             |  only, just\npelo           |  per + o\npela           |  per + a\naté            |  up to\nisso           |  that\nela            |  he\nentre          |  between\n  | era        from SER\ndepois         |  after\nsem            |  without\nmesmo          |  same\naos            |  a + os\n  | ter        from TER\nseus           |  his\nquem           |  whom\nnas            |  em + as\nme             |  me\nesse           |  that\neles           |  they\n  | estão      from EST\nvocê           |  you\n  | tinha      from TER\n  | foram      from SER\nessa           |  that\nnum            |  em + um\nnem            |  nor\nsuas           |  her\nmeu            |  my\nàs             |  a + as\nminha          |  my\n  | têm        from TER\nnuma           |  em + uma\npelos          |  per + os\nelas           |  they\n  | havia      from HAV\n  | seja       from SER\nqual           |  which\n  | será       from SER\nnós            |  we\n  | tenho      from TER\nlhe            |  to him, her\ndeles          |  of them\nessas          |  those\nesses          |  those\npelas          |  per + as\neste           |  this\n  | fosse      from SER\ndele           |  of him\n\n | other words. There are many contractions such as naquele = em+aquele,\n | mo = me+o, but they are rare.\n | Indefinite article plural forms are also rare.\n\ntu             |  thou\nte             |  thee\nvocês          |  you (plural)\nvos            |  you\nlhes           |  to them\nmeus           |  my\nminhas\nteu            |  thy\ntua\nteus\ntuas\nnosso          | our\nnossa\nnossos\nnossas\n\ndela           |  of her\ndelas          |  of them\n\nesta           |  this\nestes          |  these\nestas          |  these\naquele         |  that\naquela         |  that\naqueles        |  those\naquelas        |  those\nisto           |  this\naquilo         |  that\n\n               | forms of estar, to be (not including the infinitive):\nestou\nestá\nestamos\nestão\nestive\nesteve\nestivemos\nestiveram\nestava\nestávamos\nestavam\nestivera\nestivéramos\nesteja\nestejamos\nestejam\nestivesse\nestivéssemos\nestivessem\nestiver\nestivermos\nestiverem\n\n               | forms of haver, to have (not including the infinitive):\nhei\nhá\nhavemos\nhão\nhouve\nhouvemos\nhouveram\nhouvera\nhouvéramos\nhaja\nhajamos\nhajam\nhouvesse\nhouvéssemos\nhouvessem\nhouver\nhouvermos\nhouverem\nhouverei\nhouverá\nhouveremos\nhouverão\nhouveria\nhouveríamos\nhouveriam\n\n               | forms of ser, to be (not including the infinitive):\nsou\nsomos\nsão\nera\néramos\neram\nfui\nfoi\nfomos\nforam\nfora\nfôramos\nseja\nsejamos\nsejam\nfosse\nfôssemos\nfossem\nfor\nformos\nforem\nserei\nserá\nseremos\nserão\nseria\nseríamos\nseriam\n\n               | forms of ter, to have (not including the infinitive):\ntenho\ntem\ntemos\ntém\ntinha\ntínhamos\ntinham\ntive\nteve\ntivemos\ntiveram\ntivera\ntivéramos\ntenha\ntenhamos\ntenham\ntivesse\ntivéssemos\ntivessem\ntiver\ntivermos\ntiverem\nterei\nterá\nteremos\nterão\nteria\nteríamos\nteriam\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(PortugueseStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ro/analyzer_ro.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ro\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"ro\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopRoFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerRoFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopRoFilter,\n\t\t\tstemmerRoFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ro/analyzer_ro_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ro\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestRomanianAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"absenţa\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"absenţ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"absenţi\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"absenţ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"îl\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ro/stemmer_ro.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ro\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/romanian\"\n)\n\nconst SnowballStemmerName = \"stemmer_ro_snowball\"\n\ntype RomanianStemmerFilter struct {\n}\n\nfunc NewRomanianStemmerFilter() *RomanianStemmerFilter {\n\treturn &RomanianStemmerFilter{}\n}\n\nfunc (s *RomanianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tromanian.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc RomanianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewRomanianStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, RomanianStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ro/stop_filter_ro.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ro\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ro/stop_words_ro.go",
    "content": "package ro\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_ro\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/\n// ` was changed to ' to allow for literal string\n\nvar RomanianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license.\n# See http://members.unine.ch/jacques.savoy/clef/index.html.\n# Also see http://www.opensource.org/licenses/bsd-license.html\nacea\naceasta\naceastă\naceea\nacei\naceia\nacel\nacela\nacele\nacelea\nacest\nacesta\naceste\nacestea\naceşti\naceştia\nacolo\nacum\nai\naia\naibă\naici\nal\năla\nale\nalea\nălea\naltceva\naltcineva\nam\nar\nare\naş\naşadar\nasemenea\nasta\năsta\nastăzi\nastea\năstea\năştia\nasupra\naţi\nau\navea\navem\naveţi\nazi\nbine\nbucur\nbună\nca\ncă\ncăci\ncând\ncare\ncărei\ncăror\ncărui\ncât\ncâte\ncâţi\ncătre\ncâtva\nce\ncel\nceva\nchiar\ncînd\ncine\ncineva\ncît\ncîte\ncîţi\ncîtva\ncontra\ncu\ncum\ncumva\ncurând\ncurînd\nda\ndă\ndacă\ndar\ndatorită\nde\ndeci\ndeja\ndeoarece\ndeparte\ndeşi\ndin\ndinaintea\ndintr\ndintre\ndrept\ndupă\nea\nei\nel\nele\neram\neste\neşti\neu\nface\nfără\nfi\nfie\nfiecare\nfii\nfim\nfiţi\niar\nieri\nîi\nîl\nîmi\nîmpotriva\nîn \nînainte\nînaintea\nîncât\nîncît\nîncotro\nîntre\nîntrucât\nîntrucît\nîţi\nla\nlângă\nle\nli\nlîngă\nlor\nlui\nmă\nmâine\nmea\nmei\nmele\nmereu\nmeu\nmi\nmine\nmult\nmultă\nmulţi\nne\nnicăieri\nnici\nnimeni\nnişte\nnoastră\nnoastre\nnoi\nnoştri\nnostru\nnu\nori\noricând\noricare\noricât\norice\noricînd\noricine\noricît\noricum\noriunde\npână\npe\npentru\npeste\npînă\npoate\npot\nprea\nprima\nprimul\nprin\nprintr\nsa\nsă\nsăi\nsale\nsau\nsău\nse\nşi\nsînt\nsîntem\nsînteţi\nspre\nsub\nsunt\nsuntem\nsunteţi\nta\ntăi\ntale\ntău\nte\nţi\nţie\ntine\ntoată\ntoate\ntot\ntoţi\ntotuşi\ntu\nun\nuna\nunde\nundeva\nunei\nunele\nuneori\nunor\nvă\nvi\nvoastră\nvoastre\nvoi\nvoştri\nvostru\nvouă\nvreo\nvreun\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(RomanianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ru/analyzer_ru.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ru\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"ru\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\ttokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopRuFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerRuFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: tokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopRuFilter,\n\t\t\tstemmerRuFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ru/analyzer_ru_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ru\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestRussianAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"километрах\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"километр\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"актеров\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"актер\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"как\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t\t// digits safe\n\t\t{\n\t\t\tinput: []byte(\"text 1000\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"text\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"1000\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Вместе с тем о силе электромагнитной энергии имели представление еще\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"вмест\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"сил\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"электромагнитн\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"энерг\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"имел\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"представлен\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Но знание это хранилось в тайне\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"знан\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"эт\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"хран\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"тайн\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ru/stemmer_ru.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ru\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/russian\"\n)\n\nconst SnowballStemmerName = \"stemmer_ru_snowball\"\n\ntype RussianStemmerFilter struct {\n}\n\nfunc NewRussianStemmerFilter() *RussianStemmerFilter {\n\treturn &RussianStemmerFilter{}\n}\n\nfunc (s *RussianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\trussian.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc RussianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewRussianStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, RussianStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ru/stemmer_ru_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ru\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSnowballRussianStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"актеров\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"актер\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"километров\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"километр\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ru/stop_filter_ru.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ru\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/ru/stop_words_ru.go",
    "content": "package ru\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_ru\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar RussianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | a russian stop word list. comments begin with vertical bar. each stop\n | word is at the start of a line.\n\n | this is a ranked list (commonest to rarest) of stopwords derived from\n | a large text sample.\n\n | letter 'ё' is translated to 'е'.\n\nи              | and\nв              | in/into\nво             | alternative form\nне             | not\nчто            | what/that\nон             | he\nна             | on/onto\nя              | i\nс              | from\nсо             | alternative form\nкак            | how\nа              | milder form of 'no' (but)\nто             | conjunction and form of 'that'\nвсе            | all\nона            | she\nтак            | so, thus\nего            | him\nно             | but\nда             | yes/and\nты             | thou\nк              | towards, by\nу              | around, chez\nже             | intensifier particle\nвы             | you\nза             | beyond, behind\nбы             | conditional/subj. particle\nпо             | up to, along\nтолько         | only\nее             | her\nмне            | to me\nбыло           | it was\nвот            | here is/are, particle\nот             | away from\nменя           | me\nеще            | still, yet, more\nнет            | no, there isnt/arent\nо              | about\nиз             | out of\nему            | to him\nтеперь         | now\nкогда          | when\nдаже           | even\nну             | so, well\nвдруг          | suddenly\nли             | interrogative particle\nесли           | if\nуже            | already, but homonym of 'narrower'\nили            | or\nни             | neither\nбыть           | to be\nбыл            | he was\nнего           | prepositional form of его\nдо             | up to\nвас            | you accusative\nнибудь         | indef. suffix preceded by hyphen\nопять          | again\nуж             | already, but homonym of 'adder'\nвам            | to you\nсказал         | he said\nведь           | particle 'after all'\nтам            | there\nпотом          | then\nсебя           | oneself\nничего         | nothing\nей             | to her\nможет          | usually with 'быть' as 'maybe'\nони            | they\nтут            | here\nгде            | where\nесть           | there is/are\nнадо           | got to, must\nней            | prepositional form of  ей\nдля            | for\nмы             | we\nтебя           | thee\nих             | them, their\nчем            | than\nбыла           | she was\nсам            | self\nчтоб           | in order to\nбез            | without\nбудто          | as if\nчеловек        | man, person, one\nчего           | genitive form of 'what'\nраз            | once\nтоже           | also\nсебе           | to oneself\nпод            | beneath\nжизнь          | life\nбудет          | will be\nж              | short form of intensifer particle 'же'\nтогда          | then\nкто            | who\nэтот           | this\nговорил        | was saying\nтого           | genitive form of 'that'\nпотому         | for that reason\nэтого          | genitive form of 'this'\nкакой          | which\nсовсем         | altogether\nним            | prepositional form of 'его', 'они'\nздесь          | here\nэтом           | prepositional form of 'этот'\nодин           | one\nпочти          | almost\nмой            | my\nтем            | instrumental/dative plural of 'тот', 'то'\nчтобы          | full form of 'in order that'\nнее            | her (acc.)\nкажется        | it seems\nсейчас         | now\nбыли           | they were\nкуда           | where to\nзачем          | why\nсказать        | to say\nвсех           | all (acc., gen. preposn. plural)\nникогда        | never\nсегодня        | today\nможно          | possible, one can\nпри            | by\nнаконец        | finally\nдва            | two\nоб             | alternative form of 'о', about\nдругой         | another\nхоть           | even\nпосле          | after\nнад            | above\nбольше         | more\nтот            | that one (masc.)\nчерез          | across, in\nэти            | these\nнас            | us\nпро            | about\nвсего          | in all, only, of all\nних            | prepositional form of 'они' (they)\nкакая          | which, feminine\nмного          | lots\nразве          | interrogative particle\nсказала        | she said\nтри            | three\nэту            | this, acc. fem. sing.\nмоя            | my, feminine\nвпрочем        | moreover, besides\nхорошо         | good\nсвою           | ones own, acc. fem. sing.\nэтой           | oblique form of 'эта', fem. 'this'\nперед          | in front of\nиногда         | sometimes\nлучше          | better\nчуть           | a little\nтом            | preposn. form of 'that one'\nнельзя         | one must not\nтакой          | such a one\nим             | to them\nболее          | more\nвсегда         | always\nконечно        | of course\nвсю            | acc. fem. sing of 'all'\nмежду          | between\n\n\n  | b: some paradigms\n  |\n  | personal pronouns\n  |\n  | я  меня  мне  мной  [мною]\n  | ты  тебя  тебе  тобой  [тобою]\n  | он  его  ему  им  [него, нему, ним]\n  | она  ее  эи  ею  [нее, нэи, нею]\n  | оно  его  ему  им  [него, нему, ним]\n  |\n  | мы  нас  нам  нами\n  | вы  вас  вам  вами\n  | они  их  им  ими  [них, ним, ними]\n  |\n  |   себя  себе  собой   [собою]\n  |\n  | demonstrative pronouns: этот (this), тот (that)\n  |\n  | этот  эта  это  эти\n  | этого  эты  это  эти\n  | этого  этой  этого  этих\n  | этому  этой  этому  этим\n  | этим  этой  этим  [этою]  этими\n  | этом  этой  этом  этих\n  |\n  | тот  та  то  те\n  | того  ту  то  те\n  | того  той  того  тех\n  | тому  той  тому  тем\n  | тем  той  тем  [тою]  теми\n  | том  той  том  тех\n  |\n  | determinative pronouns\n  |\n  | (a) весь (all)\n  |\n  | весь  вся  все  все\n  | всего  всю  все  все\n  | всего  всей  всего  всех\n  | всему  всей  всему  всем\n  | всем  всей  всем  [всею]  всеми\n  | всем  всей  всем  всех\n  |\n  | (b) сам (himself etc)\n  |\n  | сам  сама  само  сами\n  | самого саму  само  самих\n  | самого самой самого  самих\n  | самому самой самому  самим\n  | самим  самой  самим  [самою]  самими\n  | самом самой самом  самих\n  |\n  | stems of verbs 'to be', 'to have', 'to do' and modal\n  |\n  | быть  бы  буд  быв  есть  суть\n  | име\n  | дел\n  | мог   мож  мочь\n  | уме\n  | хоч  хот\n  | долж\n  | можн\n  | нужн\n  | нельзя\n\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(RussianStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/sv/analyzer_sv.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sv\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"sv\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopSvFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerSvFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\ttoLowerFilter,\n\t\t\tstopSvFilter,\n\t\t\tstemmerSvFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/sv/analyzer_sv_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sv\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSwedishAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"jaktkarlarne\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"jaktkarl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"jaktkarlens\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"jaktkarl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"och\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/sv/stemmer_sv.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sv\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/swedish\"\n)\n\nconst SnowballStemmerName = \"stemmer_sv_snowball\"\n\ntype SwedishStemmerFilter struct {\n}\n\nfunc NewSwedishStemmerFilter() *SwedishStemmerFilter {\n\treturn &SwedishStemmerFilter{}\n}\n\nfunc (s *SwedishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tswedish.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc SwedishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewSwedishStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, SwedishStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/sv/stop_filter_sv.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sv\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/sv/stop_words_sv.go",
    "content": "package sv\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_sv\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar SwedishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt\n | This file is distributed under the BSD License.\n | See http://snowball.tartarus.org/license.php\n | Also see http://www.opensource.org/licenses/bsd-license.html\n |  - Encoding was converted to UTF-8.\n |  - This notice was added.\n |\n | NOTE: To use this file with StopFilterFactory, you must specify format=\"snowball\"\n\n | A Swedish stop word list. Comments begin with vertical bar. Each stop\n | word is at the start of a line.\n\n | This is a ranked list (commonest to rarest) of stopwords derived from\n | a large text sample.\n\n | Swedish stop words occasionally exhibit homonym clashes. For example\n |  så = so, but also seed. These are indicated clearly below.\n\noch            | and\ndet            | it, this/that\natt            | to (with infinitive)\ni              | in, at\nen             | a\njag            | I\nhon            | she\nsom            | who, that\nhan            | he\npå             | on\nden            | it, this/that\nmed            | with\nvar            | where, each\nsig            | him(self) etc\nför            | for\nså             | so (also: seed)\ntill           | to\när             | is\nmen            | but\nett            | a\nom             | if; around, about\nhade           | had\nde             | they, these/those\nav             | of\nicke           | not, no\nmig            | me\ndu             | you\nhenne          | her\ndå             | then, when\nsin            | his\nnu             | now\nhar            | have\ninte           | inte någon = no one\nhans           | his\nhonom          | him\nskulle         | 'sake'\nhennes         | her\ndär            | there\nmin            | my\nman            | one (pronoun)\nej             | nor\nvid            | at, by, on (also: vast)\nkunde          | could\nnågot          | some etc\nfrån           | from, off\nut             | out\nnär            | when\nefter          | after, behind\nupp            | up\nvi             | we\ndem            | them\nvara           | be\nvad            | what\növer           | over\nän             | than\ndig            | you\nkan            | can\nsina           | his\nhär            | here\nha             | have\nmot            | towards\nalla           | all\nunder          | under (also: wonder)\nnågon          | some etc\neller          | or (else)\nallt           | all\nmycket         | much\nsedan          | since\nju             | why\ndenna          | this/that\nsjälv          | myself, yourself etc\ndetta          | this/that\nåt             | to\nutan           | without\nvarit          | was\nhur            | how\ningen          | no\nmitt           | my\nni             | you\nbli            | to be, become\nblev           | from bli\noss            | us\ndin            | thy\ndessa          | these/those\nnågra          | some etc\nderas          | their\nblir           | from bli\nmina           | my\nsamma          | (the) same\nvilken         | who, that\ner             | you, your\nsådan          | such a\nvår            | our\nblivit         | from bli\ndess           | its\ninom           | within\nmellan         | between\nsådant         | such a\nvarför         | why\nvarje          | each\nvilka          | who, that\nditt           | thy\nvem            | who\nvilket         | who, that\nsitta          | his\nsådana         | such a\nvart           | each\ndina           | thy\nvars           | whose\nvårt           | our\nvåra           | our\nert            | your\nera            | your\nvilkas         | whose\n\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(SwedishStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/tr/analyzer_tr.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage tr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/token/apostrophe\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n)\n\nconst AnalyzerName = \"tr\"\n\nfunc AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {\n\tunicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\taposFilter, err := cache.TokenFilterNamed(apostrophe.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttoLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstopTrFilter, err := cache.TokenFilterNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tstemmerTrFilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := analysis.DefaultAnalyzer{\n\t\tTokenizer: unicodeTokenizer,\n\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\taposFilter,\n\t\t\ttoLowerFilter,\n\t\t\tstopTrFilter,\n\t\t\tstemmerTrFilter,\n\t\t},\n\t}\n\treturn &rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/tr/analyzer_tr_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage tr\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestTurkishAnalyzer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t// stemming\n\t\t{\n\t\t\tinput: []byte(\"ağacı\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ağaç\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"ağaç\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ağaç\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// stop word\n\t\t{\n\t\t\tinput:  []byte(\"dolayı\"),\n\t\t\toutput: analysis.TokenStream{},\n\t\t},\n\t\t// apostrophes\n\t\t{\n\t\t\tinput: []byte(\"Kıbrıs'ta\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kıbrıs\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"Van Gölü'ne\"),\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"van\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"göl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(AnalyzerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := analyzer.Analyze(test.input)\n\t\tif len(actual) != len(test.output) {\n\t\t\tt.Fatalf(\"expected length: %d, got %d\", len(test.output), len(actual))\n\t\t}\n\t\tfor i, tok := range actual {\n\t\t\tif !reflect.DeepEqual(tok.Term, test.output[i].Term) {\n\t\t\t\tt.Errorf(\"expected term %s (% x) got %s (% x)\", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/tr/stemmer_tr.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage tr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowballstem\"\n\t\"github.com/blevesearch/snowballstem/turkish\"\n)\n\nconst SnowballStemmerName = \"stemmer_tr_snowball\"\n\ntype TurkishStemmerFilter struct {\n}\n\nfunc NewTurkishStemmerFilter() *TurkishStemmerFilter {\n\treturn &TurkishStemmerFilter{}\n}\n\nfunc (s *TurkishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tenv := snowballstem.NewEnv(string(token.Term))\n\t\tturkish.Stem(env)\n\t\ttoken.Term = []byte(env.Current())\n\t}\n\treturn input\n}\n\nfunc TurkishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewTurkishStemmerFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(SnowballStemmerName, TurkishStemmerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/tr/stemmer_tr_test.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage tr\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestSnowballTurkishStemmer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kimsesizler\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kimsesiz\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kitaplar\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kitap\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"arabanın\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"araba\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bardaklar\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bardak\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kediye\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"kedi\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"yazdım\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"yaz\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tfilter, err := cache.TokenFilterNamed(SnowballStemmerName)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, test := range tests {\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/tr/stop_filter_tr.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage tr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\ttokenMap, err := cache.TokenMapNamed(StopName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn stop.NewStopTokensFilter(tokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/lang/tr/stop_words_tr.go",
    "content": "package tr\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst StopName = \"stop_tr\"\n\n// this content was obtained from:\n// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/\n// ` was changed to ' to allow for literal string\n\nvar TurkishStopWords = []byte(`# Turkish stopwords from LUCENE-559\n# merged with the list from \"Information Retrieval on Turkish Texts\"\n#   (http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf)\nacaba\naltmış\naltı\nama\nancak\narada\naslında\nayrıca\nbana\nbazı\nbelki\nben\nbenden\nbeni\nbenim\nberi\nbeş\nbile\nbin\nbir\nbirçok\nbiri\nbirkaç\nbirkez\nbirşey\nbirşeyi\nbiz\nbize\nbizden\nbizi\nbizim\nböyle\nböylece\nbu\nbuna\nbunda\nbundan\nbunlar\nbunları\nbunların\nbunu\nbunun\nburada\nçok\nçünkü\nda\ndaha\ndahi\nde\ndefa\ndeğil\ndiğer\ndiye\ndoksan\ndokuz\ndolayı\ndolayısıyla\ndört\nedecek\neden\nederek\nedilecek\nediliyor\nedilmesi\nediyor\neğer\nelli\nen\netmesi\netti\nettiği\nettiğini\ngibi\ngöre\nhalen\nhangi\nhatta\nhem\nhenüz\nhep\nhepsi\nher\nherhangi\nherkesin\nhiç\nhiçbir\niçin\niki\nile\nilgili\nise\nişte\nitibaren\nitibariyle\nkadar\nkarşın\nkatrilyon\nkendi\nkendilerine\nkendini\nkendisi\nkendisine\nkendisini\nkez\nki\nkim\nkimden\nkime\nkimi\nkimse\nkırk\nmilyar\nmilyon\nmu\nmü\nmı\nnasıl\nne\nneden\nnedenle\nnerde\nnerede\nnereye\nniye\nniçin\no\nolan\nolarak\noldu\nolduğu\nolduğunu\nolduklarını\nolmadı\nolmadığı\nolmak\nolması\nolmayan\nolmaz\nolsa\nolsun\nolup\nolur\nolursa\noluyor\non\nona\nondan\nonlar\nonlardan\nonları\nonların\nonu\nonun\notuz\noysa\nöyle\npek\nrağmen\nsadece\nsanki\nsekiz\nseksen\nsen\nsenden\nseni\nsenin\nsiz\nsizden\nsizi\nsizin\nşey\nşeyden\nşeyi\nşeyler\nşöyle\nşu\nşuna\nşunda\nşundan\nşunları\nşunu\ntarafından\ntrilyon\ntüm\nüç\nüzere\nvar\nvardı\nve\nveya\nya\nyani\nyapacak\nyapılan\nyapılması\nyapıyor\nyapmak\nyaptı\nyaptığı\nyaptığını\nyaptıkları\nyedi\nyerine\nyetmiş\nyine\nyirmi\nyoksa\nyüz\nzaten\n`)\n\nfunc TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\terr := rv.LoadBytes(TurkishStopWords)\n\treturn rv, err\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(StopName, TokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/test_words.txt",
    "content": "# full line comment\nmarty\nsteve # trailing comment\n| different format of comment\ndustin\nsiri | different style trailing comment\nmultiple words\twith different\twhitespace"
  },
  {
    "path": "analysis/token/apostrophe/apostrophe.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage apostrophe\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"apostrophe\"\n\nconst RightSingleQuotationMark = \"’\"\nconst Apostrophe = \"'\"\nconst Apostrophes = Apostrophe + RightSingleQuotationMark\n\ntype ApostropheFilter struct{}\n\nfunc NewApostropheFilter() *ApostropheFilter {\n\treturn &ApostropheFilter{}\n}\n\nfunc (s *ApostropheFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tfirstApostrophe := bytes.IndexAny(token.Term, Apostrophes)\n\t\tif firstApostrophe >= 0 {\n\t\t\t// found an apostrophe\n\t\t\ttoken.Term = token.Term[0:firstApostrophe]\n\t\t}\n\t}\n\n\treturn input\n}\n\nfunc ApostropheFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewApostropheFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, ApostropheFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/apostrophe/apostrophe_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage apostrophe\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestApostropheFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Türkiye'de\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Türkiye\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"2003'te\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"2003\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Van\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Van\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Gölü'nü\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Gölü\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"gördüm\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"gördüm\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tapostropheFilter := NewApostropheFilter()\n\t\tactual := apostropheFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/token/camelcase/camelcase.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage camelcase\n\nimport (\n\t\"bytes\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"camelCase\"\n\n// CamelCaseFilter splits a given token into a set of tokens where each resulting token\n// falls into one the following classes:\n//  1. Upper case followed by lower case letters.\n//     Terminated by a number, an upper case letter, and a non alpha-numeric symbol.\n//  2. Upper case followed by upper case letters.\n//     Terminated by a number, an upper case followed by a lower case letter, and a non alpha-numeric symbol.\n//  3. Lower case followed by lower case letters.\n//     Terminated by a number, an upper case letter, and a non alpha-numeric symbol.\n//  4. Number followed by numbers.\n//     Terminated by a letter, and a non alpha-numeric symbol.\n//  5. Non alpha-numeric symbol followed by non alpha-numeric symbols.\n//     Terminated by a number, and a letter.\n//\n// It does a one-time sequential pass over an input token, from left to right.\n// The scan is greedy and generates the longest substring that fits into one of the classes.\n//\n// See the test file for examples of classes and their parsings.\ntype CamelCaseFilter struct{}\n\nfunc NewCamelCaseFilter() *CamelCaseFilter {\n\treturn &CamelCaseFilter{}\n}\n\nfunc (f *CamelCaseFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0, len(input))\n\n\tnextPosition := 1\n\tfor _, token := range input {\n\t\truneCount := utf8.RuneCount(token.Term)\n\t\trunes := bytes.Runes(token.Term)\n\n\t\tp := NewParser(runeCount, nextPosition, token.Start)\n\t\tfor i := 0; i < runeCount; i++ {\n\t\t\tif i+1 >= runeCount {\n\t\t\t\tp.Push(runes[i], nil)\n\t\t\t} else {\n\t\t\t\tp.Push(runes[i], &runes[i+1])\n\t\t\t}\n\t\t}\n\t\trv = append(rv, p.FlushTokens()...)\n\t\tnextPosition = p.NextPosition()\n\t}\n\treturn rv\n}\n\nfunc CamelCaseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewCamelCaseFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, CamelCaseFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/camelcase/camelcase_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage camelcase\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestCamelCaseFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput:  tokenStream(\"\"),\n\t\t\toutput: tokenStream(\"\"),\n\t\t},\n\t\t{\n\t\t\tinput:  tokenStream(\"a\"),\n\t\t\toutput: tokenStream(\"a\"),\n\t\t},\n\n\t\t{\n\t\t\tinput:  tokenStream(\"...aMACMac123macILoveGolang\"),\n\t\t\toutput: tokenStream(\"...\", \"a\", \"MAC\", \"Mac\", \"123\", \"mac\", \"I\", \"Love\", \"Golang\"),\n\t\t},\n\t\t{\n\t\t\tinput:  tokenStream(\"Lang\"),\n\t\t\toutput: tokenStream(\"Lang\"),\n\t\t},\n\t\t{\n\t\t\tinput:  tokenStream(\"GLang\"),\n\t\t\toutput: tokenStream(\"G\", \"Lang\"),\n\t\t},\n\t\t{\n\t\t\tinput:  tokenStream(\"GOLang\"),\n\t\t\toutput: tokenStream(\"GO\", \"Lang\"),\n\t\t},\n\t\t{\n\t\t\tinput:  tokenStream(\"GOOLang\"),\n\t\t\toutput: tokenStream(\"GOO\", \"Lang\"),\n\t\t},\n\t\t{\n\t\t\tinput:  tokenStream(\"1234\"),\n\t\t\toutput: tokenStream(\"1234\"),\n\t\t},\n\t\t{\n\t\t\tinput:  tokenStream(\"starbucks\"),\n\t\t\toutput: tokenStream(\"starbucks\"),\n\t\t},\n\t\t{\n\t\t\tinput:  tokenStream(\"Starbucks TVSamsungIsGREAT000\"),\n\t\t\toutput: tokenStream(\"Starbucks\", \" \", \"TV\", \"Samsung\", \"Is\", \"GREAT\", \"000\"),\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tccFilter := NewCamelCaseFilter()\n\t\tactual := ccFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s \\n\\n got %s\", test.output, actual)\n\t\t}\n\t}\n}\n\nfunc tokenStream(termStrs ...string) analysis.TokenStream {\n\ttokenStream := make([]*analysis.Token, len(termStrs))\n\tindex := 0\n\tfor i, termStr := range termStrs {\n\t\ttokenStream[i] = &analysis.Token{\n\t\t\tTerm:     []byte(termStr),\n\t\t\tPosition: i + 1,\n\t\t\tStart:    index,\n\t\t\tEnd:      index + len(termStr),\n\t\t}\n\t\tindex += len(termStr)\n\t}\n\treturn analysis.TokenStream(tokenStream)\n}\n"
  },
  {
    "path": "analysis/token/camelcase/parser.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage camelcase\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc (p *Parser) buildTokenFromTerm(buffer []rune) *analysis.Token {\n\tterm := analysis.BuildTermFromRunes(buffer)\n\ttoken := &analysis.Token{\n\t\tTerm:     term,\n\t\tPosition: p.position,\n\t\tStart:    p.index,\n\t\tEnd:      p.index + len(term),\n\t}\n\tp.position++\n\tp.index += len(term)\n\treturn token\n}\n\n// Parser accepts a symbol and passes it to the current state (representing a class).\n// The state can accept it (and accumulate it). Otherwise, the parser creates a new state that\n// starts with the pushed symbol.\n//\n// Parser accumulates a new resulting token every time it switches state.\n// Use FlushTokens() to get the results after the last symbol was pushed.\ntype Parser struct {\n\tbufferLen int\n\tbuffer    []rune\n\tcurrent   State\n\ttokens    []*analysis.Token\n\tposition  int\n\tindex     int\n}\n\nfunc NewParser(length, position, index int) *Parser {\n\treturn &Parser{\n\t\tbufferLen: length,\n\t\tbuffer:    make([]rune, 0, length),\n\t\ttokens:    make([]*analysis.Token, 0, length),\n\t\tposition:  position,\n\t\tindex:     index,\n\t}\n}\n\nfunc (p *Parser) Push(sym rune, peek *rune) {\n\tif p.current == nil {\n\t\t// the start of parsing\n\t\tp.current = p.NewState(sym)\n\t\tp.buffer = append(p.buffer, sym)\n\n\t} else if p.current.Member(sym, peek) {\n\t\t// same state, just accumulate\n\t\tp.buffer = append(p.buffer, sym)\n\n\t} else {\n\t\t// the old state is no more, thus convert the buffer\n\t\tp.tokens = append(p.tokens, p.buildTokenFromTerm(p.buffer))\n\n\t\t// let the new state begin\n\t\tp.current = p.NewState(sym)\n\t\tp.buffer = make([]rune, 0, p.bufferLen)\n\t\tp.buffer = append(p.buffer, sym)\n\t}\n}\n\n// Note. States have to have different starting symbols.\nfunc (p *Parser) NewState(sym rune) State {\n\tvar found State\n\n\tfound = &LowerCaseState{}\n\tif found.StartSym(sym) {\n\t\treturn found\n\t}\n\n\tfound = &UpperCaseState{}\n\tif found.StartSym(sym) {\n\t\treturn found\n\t}\n\n\tfound = &NumberCaseState{}\n\tif found.StartSym(sym) {\n\t\treturn found\n\t}\n\n\treturn &NonAlphaNumericCaseState{}\n}\n\nfunc (p *Parser) FlushTokens() []*analysis.Token {\n\tp.tokens = append(p.tokens, p.buildTokenFromTerm(p.buffer))\n\treturn p.tokens\n}\n\nfunc (p *Parser) NextPosition() int {\n\treturn p.position\n}\n"
  },
  {
    "path": "analysis/token/camelcase/states.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage camelcase\n\nimport (\n\t\"unicode\"\n)\n\n// States codify the classes that the parser recognizes.\ntype State interface {\n\t// is _sym_ the start character\n\tStartSym(sym rune) bool\n\n\t// is _sym_ a member of a class.\n\t// peek, the next sym on the tape, can also be used to determine a class.\n\tMember(sym rune, peek *rune) bool\n}\n\ntype LowerCaseState struct{}\n\nfunc (s *LowerCaseState) Member(sym rune, peek *rune) bool {\n\treturn unicode.IsLower(sym)\n}\n\nfunc (s *LowerCaseState) StartSym(sym rune) bool {\n\treturn s.Member(sym, nil)\n}\n\ntype UpperCaseState struct {\n\tstartedCollecting bool // denotes that the start character has been read\n\tcollectingUpper   bool // denotes if this is a class of all upper case letters\n}\n\nfunc (s *UpperCaseState) Member(sym rune, peek *rune) bool {\n\tif !(unicode.IsLower(sym) || unicode.IsUpper(sym)) {\n\t\treturn false\n\t}\n\n\tif peek != nil && unicode.IsUpper(sym) && unicode.IsLower(*peek) {\n\t\treturn false\n\t}\n\n\tif !s.startedCollecting {\n\t\t// now we have to determine if upper-case letters are collected.\n\t\ts.startedCollecting = true\n\t\ts.collectingUpper = unicode.IsUpper(sym)\n\t\treturn true\n\t}\n\n\treturn s.collectingUpper == unicode.IsUpper(sym)\n}\n\nfunc (s *UpperCaseState) StartSym(sym rune) bool {\n\treturn unicode.IsUpper(sym)\n}\n\ntype NumberCaseState struct{}\n\nfunc (s *NumberCaseState) Member(sym rune, peek *rune) bool {\n\treturn unicode.IsNumber(sym)\n}\n\nfunc (s *NumberCaseState) StartSym(sym rune) bool {\n\treturn s.Member(sym, nil)\n}\n\ntype NonAlphaNumericCaseState struct{}\n\nfunc (s *NonAlphaNumericCaseState) Member(sym rune, peek *rune) bool {\n\treturn !unicode.IsLower(sym) && !unicode.IsUpper(sym) && !unicode.IsNumber(sym)\n}\n\nfunc (s *NonAlphaNumericCaseState) StartSym(sym rune) bool {\n\treturn s.Member(sym, nil)\n}\n"
  },
  {
    "path": "analysis/token/compound/dict.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage compound\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"dict_compound\"\n\nconst defaultMinWordSize = 5\nconst defaultMinSubWordSize = 2\nconst defaultMaxSubWordSize = 15\nconst defaultOnlyLongestMatch = false\n\ntype DictionaryCompoundFilter struct {\n\tdict             analysis.TokenMap\n\tminWordSize      int\n\tminSubWordSize   int\n\tmaxSubWordSize   int\n\tonlyLongestMatch bool\n}\n\nfunc NewDictionaryCompoundFilter(dict analysis.TokenMap, minWordSize, minSubWordSize, maxSubWordSize int, onlyLongestMatch bool) *DictionaryCompoundFilter {\n\treturn &DictionaryCompoundFilter{\n\t\tdict:             dict,\n\t\tminWordSize:      minWordSize,\n\t\tminSubWordSize:   minSubWordSize,\n\t\tmaxSubWordSize:   maxSubWordSize,\n\t\tonlyLongestMatch: onlyLongestMatch,\n\t}\n}\n\nfunc (f *DictionaryCompoundFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0, len(input))\n\n\tfor _, token := range input {\n\t\trv = append(rv, token)\n\t\ttokenLen := utf8.RuneCount(token.Term)\n\t\tif tokenLen >= f.minWordSize {\n\t\t\tnewtokens := f.decompose(token)\n\t\t\tfor _, newtoken := range newtokens {\n\t\t\t\trv = append(rv, newtoken)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn rv\n}\n\nfunc (f *DictionaryCompoundFilter) decompose(token *analysis.Token) []*analysis.Token {\n\trunes := bytes.Runes(token.Term)\n\trv := make([]*analysis.Token, 0)\n\trlen := len(runes)\n\tfor i := 0; i <= (rlen - f.minSubWordSize); i++ {\n\t\tvar longestMatchToken *analysis.Token\n\t\tfor j := f.minSubWordSize; j <= f.maxSubWordSize; j++ {\n\t\t\tif i+j > rlen {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t_, inDict := f.dict[string(runes[i:i+j])]\n\t\t\tif inDict {\n\t\t\t\tnewtoken := analysis.Token{\n\t\t\t\t\tTerm:     []byte(string(runes[i : i+j])),\n\t\t\t\t\tPosition: token.Position,\n\t\t\t\t\tStart:    token.Start + i,\n\t\t\t\t\tEnd:      token.Start + i + j,\n\t\t\t\t\tType:     token.Type,\n\t\t\t\t\tKeyWord:  token.KeyWord,\n\t\t\t\t}\n\t\t\t\tif f.onlyLongestMatch {\n\t\t\t\t\tif longestMatchToken == nil || utf8.RuneCount(longestMatchToken.Term) < j {\n\t\t\t\t\t\tlongestMatchToken = &newtoken\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\trv = append(rv, &newtoken)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif f.onlyLongestMatch && longestMatchToken != nil {\n\t\t\trv = append(rv, longestMatchToken)\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc DictionaryCompoundFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\n\tminWordSize := defaultMinWordSize\n\tminSubWordSize := defaultMinSubWordSize\n\tmaxSubWordSize := defaultMaxSubWordSize\n\tonlyLongestMatch := defaultOnlyLongestMatch\n\n\tminVal, ok := config[\"min_word_size\"].(float64)\n\tif ok {\n\t\tminWordSize = int(minVal)\n\t}\n\tminSubVal, ok := config[\"min_subword_size\"].(float64)\n\tif ok {\n\t\tminSubWordSize = int(minSubVal)\n\t}\n\tmaxSubVal, ok := config[\"max_subword_size\"].(float64)\n\tif ok {\n\t\tmaxSubWordSize = int(maxSubVal)\n\t}\n\tonlyVal, ok := config[\"only_longest_match\"].(bool)\n\tif ok {\n\t\tonlyLongestMatch = onlyVal\n\t}\n\n\tdictTokenMapName, ok := config[\"dict_token_map\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify dict_token_map\")\n\t}\n\tdictTokenMap, err := cache.TokenMapNamed(dictTokenMapName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building dict compound words filter: %v\", err)\n\t}\n\treturn NewDictionaryCompoundFilter(dictTokenMap, minWordSize, minSubWordSize, maxSubWordSize, onlyLongestMatch), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, DictionaryCompoundFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/compound/dict_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage compound\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenmap\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestStopWordsFilter(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"i\"),\n\t\t\tStart:    0,\n\t\t\tEnd:      1,\n\t\t\tPosition: 1,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"like\"),\n\t\t\tStart:    2,\n\t\t\tEnd:      6,\n\t\t\tPosition: 2,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"to\"),\n\t\t\tStart:    7,\n\t\t\tEnd:      9,\n\t\t\tPosition: 3,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"play\"),\n\t\t\tStart:    10,\n\t\t\tEnd:      14,\n\t\t\tPosition: 4,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"softball\"),\n\t\t\tStart:    15,\n\t\t\tEnd:      23,\n\t\t\tPosition: 5,\n\t\t},\n\t}\n\n\texpectedTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"i\"),\n\t\t\tStart:    0,\n\t\t\tEnd:      1,\n\t\t\tPosition: 1,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"like\"),\n\t\t\tStart:    2,\n\t\t\tEnd:      6,\n\t\t\tPosition: 2,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"to\"),\n\t\t\tStart:    7,\n\t\t\tEnd:      9,\n\t\t\tPosition: 3,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"play\"),\n\t\t\tStart:    10,\n\t\t\tEnd:      14,\n\t\t\tPosition: 4,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"softball\"),\n\t\t\tStart:    15,\n\t\t\tEnd:      23,\n\t\t\tPosition: 5,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"soft\"),\n\t\t\tStart:    15,\n\t\t\tEnd:      19,\n\t\t\tPosition: 5,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"ball\"),\n\t\t\tStart:    19,\n\t\t\tEnd:      23,\n\t\t\tPosition: 5,\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tdictListConfig := map[string]interface{}{\n\t\t\"type\":   tokenmap.Name,\n\t\t\"tokens\": []interface{}{\"factor\", \"soft\", \"ball\", \"team\"},\n\t}\n\t_, err := cache.DefineTokenMap(\"dict_test\", dictListConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdictConfig := map[string]interface{}{\n\t\t\"type\":           \"dict_compound\",\n\t\t\"dict_token_map\": \"dict_test\",\n\t}\n\tdictFilter, err := cache.DefineTokenFilter(\"dict_test\", dictConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\touputTokenStream := dictFilter.Filter(inputTokenStream)\n\tif !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {\n\t\tt.Errorf(\"expected %#v got %#v\", expectedTokenStream, ouputTokenStream)\n\t}\n}\n\nfunc TestStopWordsFilterLongestMatch(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"softestball\"),\n\t\t\tStart:    0,\n\t\t\tEnd:      11,\n\t\t\tPosition: 1,\n\t\t},\n\t}\n\n\texpectedTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"softestball\"),\n\t\t\tStart:    0,\n\t\t\tEnd:      11,\n\t\t\tPosition: 1,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"softest\"),\n\t\t\tStart:    0,\n\t\t\tEnd:      7,\n\t\t\tPosition: 1,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:     []byte(\"ball\"),\n\t\t\tStart:    7,\n\t\t\tEnd:      11,\n\t\t\tPosition: 1,\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tdictListConfig := map[string]interface{}{\n\t\t\"type\":   tokenmap.Name,\n\t\t\"tokens\": []interface{}{\"soft\", \"softest\", \"ball\"},\n\t}\n\t_, err := cache.DefineTokenMap(\"dict_test\", dictListConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdictConfig := map[string]interface{}{\n\t\t\"type\":               \"dict_compound\",\n\t\t\"dict_token_map\":     \"dict_test\",\n\t\t\"only_longest_match\": true,\n\t}\n\tdictFilter, err := cache.DefineTokenFilter(\"dict_test\", dictConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\touputTokenStream := dictFilter.Filter(inputTokenStream)\n\tif !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {\n\t\tt.Errorf(\"expected %#v got %#v\", expectedTokenStream, ouputTokenStream)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/edgengram/edgengram.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage edgengram\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"edge_ngram\"\n\ntype Side bool\n\nconst BACK Side = true\nconst FRONT Side = false\n\ntype EdgeNgramFilter struct {\n\tback      Side\n\tminLength int\n\tmaxLength int\n}\n\nfunc NewEdgeNgramFilter(side Side, minLength, maxLength int) *EdgeNgramFilter {\n\treturn &EdgeNgramFilter{\n\t\tback:      side,\n\t\tminLength: minLength,\n\t\tmaxLength: maxLength,\n\t}\n}\n\nfunc (s *EdgeNgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0, len(input))\n\n\tfor _, token := range input {\n\t\truneCount := utf8.RuneCount(token.Term)\n\t\trunes := bytes.Runes(token.Term)\n\t\tif s.back {\n\t\t\ti := runeCount\n\t\t\t// index of the starting rune for this token\n\t\t\tfor ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ {\n\t\t\t\t// build an ngram of this size starting at i\n\t\t\t\tif i-ngramSize >= 0 {\n\t\t\t\t\tngramTerm := analysis.BuildTermFromRunes(runes[i-ngramSize : i])\n\t\t\t\t\ttoken := analysis.Token{\n\t\t\t\t\t\tPosition: token.Position,\n\t\t\t\t\t\tStart:    token.Start,\n\t\t\t\t\t\tEnd:      token.End,\n\t\t\t\t\t\tType:     token.Type,\n\t\t\t\t\t\tTerm:     ngramTerm,\n\t\t\t\t\t}\n\t\t\t\t\trv = append(rv, &token)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\ti := 0\n\t\t\t// index of the starting rune for this token\n\t\t\tfor ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ {\n\t\t\t\t// build an ngram of this size starting at i\n\t\t\t\tif i+ngramSize <= runeCount {\n\t\t\t\t\tngramTerm := analysis.BuildTermFromRunes(runes[i : i+ngramSize])\n\t\t\t\t\ttoken := analysis.Token{\n\t\t\t\t\t\tPosition: token.Position,\n\t\t\t\t\t\tStart:    token.Start,\n\t\t\t\t\t\tEnd:      token.End,\n\t\t\t\t\t\tType:     token.Type,\n\t\t\t\t\t\tTerm:     ngramTerm,\n\t\t\t\t\t}\n\t\t\t\t\trv = append(rv, &token)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn rv\n}\n\nfunc EdgeNgramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tside := FRONT\n\tback, ok := config[\"back\"].(bool)\n\tif ok && back {\n\t\tside = BACK\n\t}\n\tminVal, ok := config[\"min\"].(float64)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify min\")\n\t}\n\tmin := int(minVal)\n\tmaxVal, ok := config[\"max\"].(float64)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify max\")\n\t}\n\tmax := int(maxVal)\n\n\treturn NewEdgeNgramFilter(side, min, max), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, EdgeNgramFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/edgengram/edgengram_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage edgengram\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestEdgeNgramFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tside   Side\n\t\tmin    int\n\t\tmax    int\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tside: FRONT,\n\t\t\tmin:  1,\n\t\t\tmax:  1,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tside: BACK,\n\t\t\tmin:  1,\n\t\t\tmax:  1,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"e\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tside: FRONT,\n\t\t\tmin:  1,\n\t\t\tmax:  3,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ab\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abc\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tside: BACK,\n\t\t\tmin:  1,\n\t\t\tmax:  3,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"e\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"de\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tside: FRONT,\n\t\t\tmin:  1,\n\t\t\tmax:  3,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"vwxyz\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ab\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abc\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"v\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"vw\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"vwx\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tside: BACK,\n\t\t\tmin:  3,\n\t\t\tmax:  5,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Beryl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ryl\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"eryl\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Beryl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tside: FRONT,\n\t\t\tmin:  3,\n\t\t\tmax:  5,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Beryl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Ber\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Bery\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Beryl\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tedgeNgramFilter := NewEdgeNgramFilter(test.side, test.min, test.max)\n\t\tactual := edgeNgramFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/token/elision/elision.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage elision\n\nimport (\n\t\"fmt\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"elision\"\n\nconst RightSingleQuotationMark = '’'\nconst Apostrophe = '\\''\n\ntype ElisionFilter struct {\n\tarticles analysis.TokenMap\n}\n\nfunc NewElisionFilter(articles analysis.TokenMap) *ElisionFilter {\n\treturn &ElisionFilter{\n\t\tarticles: articles,\n\t}\n}\n\nfunc (s *ElisionFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\tterm := token.Term\n\t\tfor i := 0; i < len(term); {\n\t\t\tr, size := utf8.DecodeRune(term[i:])\n\t\t\tif r == Apostrophe || r == RightSingleQuotationMark {\n\t\t\t\t// see if the prefix matches one of the articles\n\t\t\t\tprefix := term[0:i]\n\t\t\t\t_, articleMatch := s.articles[string(prefix)]\n\t\t\t\tif articleMatch {\n\t\t\t\t\ttoken.Term = term[i+size:]\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\ti += size\n\t\t}\n\t}\n\treturn input\n}\n\nfunc ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tarticlesTokenMapName, ok := config[\"articles_token_map\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify articles_token_map\")\n\t}\n\tarticlesTokenMap, err := cache.TokenMapNamed(articlesTokenMapName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building elision filter: %v\", err)\n\t}\n\treturn NewElisionFilter(articlesTokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, ElisionFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/elision/elision_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage elision\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenmap\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestElisionFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ar\" + string(Apostrophe) + \"word\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"word\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ar\" + string(RightSingleQuotationMark) + \"word\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"word\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\n\tarticleListConfig := map[string]interface{}{\n\t\t\"type\":   tokenmap.Name,\n\t\t\"tokens\": []interface{}{\"ar\"},\n\t}\n\t_, err := cache.DefineTokenMap(\"articles_test\", articleListConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\telisionConfig := map[string]interface{}{\n\t\t\"type\":               \"elision\",\n\t\t\"articles_token_map\": \"articles_test\",\n\t}\n\telisionFilter, err := cache.DefineTokenFilter(\"elision_test\", elisionConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, test := range tests {\n\n\t\tactual := elisionFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/token/hierarchy/hierarchy.go",
    "content": "package hierarchy\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"math\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"hierarchy\"\n\ntype HierarchyFilter struct {\n\tmaxLevels  int\n\tdelimiter  []byte\n\tsplitInput bool\n}\n\nfunc NewHierarchyFilter(delimiter []byte, maxLevels int, splitInput bool) *HierarchyFilter {\n\treturn &HierarchyFilter{\n\t\tmaxLevels:  maxLevels,\n\t\tdelimiter:  delimiter,\n\t\tsplitInput: splitInput,\n\t}\n}\n\nfunc (s *HierarchyFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0, s.maxLevels)\n\n\tvar soFar [][]byte\n\tfor _, token := range input {\n\t\tif s.splitInput {\n\t\t\tparts := bytes.Split(token.Term, s.delimiter)\n\t\t\tfor _, part := range parts {\n\t\t\t\tsoFar, rv = s.buildToken(rv, soFar, part)\n\t\t\t\tif len(soFar) >= s.maxLevels {\n\t\t\t\t\treturn rv\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tsoFar, rv = s.buildToken(rv, soFar, token.Term)\n\t\t\tif len(soFar) >= s.maxLevels {\n\t\t\t\treturn rv\n\t\t\t}\n\t\t}\n\t}\n\n\treturn rv\n}\n\nfunc (s *HierarchyFilter) buildToken(tokenStream analysis.TokenStream, soFar [][]byte, part []byte) (\n\t[][]byte, analysis.TokenStream) {\n\n\tsoFar = append(soFar, part)\n\tterm := bytes.Join(soFar, s.delimiter)\n\n\ttokenStream = append(tokenStream, &analysis.Token{\n\t\tType:     analysis.Shingle,\n\t\tTerm:     term,\n\t\tStart:    0,\n\t\tEnd:      len(term),\n\t\tPosition: 1,\n\t})\n\n\treturn soFar, tokenStream\n}\n\nfunc HierarchyFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tmax := math.MaxInt64\n\tmaxVal, ok := config[\"max\"].(float64)\n\tif ok {\n\t\tmax = int(maxVal)\n\t}\n\n\tsplitInput := true\n\tsplitInputVal, ok := config[\"split_input\"].(bool)\n\tif ok {\n\t\tsplitInput = splitInputVal\n\t}\n\n\tdelimiter, ok := config[\"delimiter\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify delimiter\")\n\t}\n\n\treturn NewHierarchyFilter([]byte(delimiter), max, splitInput), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, HierarchyFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/hierarchy/hierarchy_test.go",
    "content": "package hierarchy\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestHierarchyFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tname       string\n\t\tdelimiter  string\n\t\tmax        int\n\t\tsplitInput bool\n\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tname: \"single token a/b/c, delimiter /\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a/b/c\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      1,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b/c\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t\tdelimiter:  \"/\",\n\t\t\tmax:        10,\n\t\t\tsplitInput: true,\n\t\t},\n\t\t{\n\t\t\tname: \"multiple tokens already split a b c, delimiter /\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"b\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"c\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      1,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b/c\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t\tdelimiter:  \"/\",\n\t\t\tmax:        10,\n\t\t\tsplitInput: true,\n\t\t},\n\t\t{\n\t\t\tname: \"single token a/b/c, delimiter /, limit 2\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a/b/c\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      1,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t\tdelimiter:  \"/\",\n\t\t\tmax:        2,\n\t\t\tsplitInput: true,\n\t\t},\n\t\t{\n\t\t\tname: \"multiple tokens already split a b c, delimiter /, limit 2\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"b\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"c\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      1,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t\tdelimiter:  \"/\",\n\t\t\tmax:        2,\n\t\t\tsplitInput: true,\n\t\t},\n\n\t\t{\n\t\t\tname: \"single token a/b/c, delimiter /, no split\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a/b/c\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b/c\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t\tdelimiter:  \"/\",\n\t\t\tmax:        10,\n\t\t\tsplitInput: false,\n\t\t},\n\t\t{\n\t\t\tname: \"multiple tokens already split a b c, delimiter /, no split\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"b\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"c\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      1,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"a/b/c\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t\tdelimiter:  \"/\",\n\t\t\tmax:        10,\n\t\t\tsplitInput: false,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttest := test\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tfilter := NewHierarchyFilter([]byte(test.delimiter), test.max, test.splitInput)\n\t\t\tactual := filter.Filter(test.input)\n\t\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\t\tt.Errorf(\"expected %s, got %s\", test.output, actual)\n\t\t\t}\n\t\t})\n\t}\n\n}\n"
  },
  {
    "path": "analysis/token/keyword/keyword.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage keyword\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"keyword_marker\"\n\ntype KeyWordMarkerFilter struct {\n\tkeyWords analysis.TokenMap\n}\n\nfunc NewKeyWordMarkerFilter(keyWords analysis.TokenMap) *KeyWordMarkerFilter {\n\treturn &KeyWordMarkerFilter{\n\t\tkeyWords: keyWords,\n\t}\n}\n\nfunc (f *KeyWordMarkerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\t_, isKeyWord := f.keyWords[string(token.Term)]\n\t\tif isKeyWord {\n\t\t\ttoken.KeyWord = true\n\t\t}\n\t}\n\treturn input\n}\n\nfunc KeyWordMarkerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tkeywordsTokenMapName, ok := config[\"keywords_token_map\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify keywords_token_map\")\n\t}\n\tkeywordsTokenMap, err := cache.TokenMapNamed(keywordsTokenMapName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building keyword marker filter: %v\", err)\n\t}\n\treturn NewKeyWordMarkerFilter(keywordsTokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, KeyWordMarkerFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/keyword/keyword_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage keyword\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestKeyWordMarkerFilter(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"a\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walk\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"in\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"the\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"park\"),\n\t\t},\n\t}\n\n\texpectedTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"a\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:    []byte(\"walk\"),\n\t\t\tKeyWord: true,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"in\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"the\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:    []byte(\"park\"),\n\t\t\tKeyWord: true,\n\t\t},\n\t}\n\n\tkeyWordsMap := analysis.NewTokenMap()\n\tkeyWordsMap.AddToken(\"walk\")\n\tkeyWordsMap.AddToken(\"park\")\n\n\tfilter := NewKeyWordMarkerFilter(keyWordsMap)\n\touputTokenStream := filter.Filter(inputTokenStream)\n\tif !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {\n\t\tt.Errorf(\"expected %#v got %#v\", expectedTokenStream[0].KeyWord, ouputTokenStream[0].KeyWord)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/length/length.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage length\n\nimport (\n\t\"fmt\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"length\"\n\ntype LengthFilter struct {\n\tmin int\n\tmax int\n}\n\nfunc NewLengthFilter(min, max int) *LengthFilter {\n\treturn &LengthFilter{\n\t\tmin: min,\n\t\tmax: max,\n\t}\n}\n\nfunc (f *LengthFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0, len(input))\n\n\tfor _, token := range input {\n\t\twordLen := utf8.RuneCount(token.Term)\n\t\tif f.min > 0 && f.min > wordLen {\n\t\t\tcontinue\n\t\t}\n\t\tif f.max > 0 && f.max < wordLen {\n\t\t\tcontinue\n\t\t}\n\t\trv = append(rv, token)\n\t}\n\n\treturn rv\n}\n\nfunc LengthFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tmin := 0\n\tmax := 0\n\n\tminVal, ok := config[\"min\"].(float64)\n\tif ok {\n\t\tmin = int(minVal)\n\t}\n\tmaxVal, ok := config[\"max\"].(float64)\n\tif ok {\n\t\tmax = int(maxVal)\n\t}\n\tif min == max && max == 0 {\n\t\treturn nil, fmt.Errorf(\"either min or max must be non-zero\")\n\t}\n\n\treturn NewLengthFilter(min, max), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, LengthFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/length/length_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage length\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestLengthFilter(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"1\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"two\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"three\"),\n\t\t},\n\t}\n\n\tlengthFilter := NewLengthFilter(3, 4)\n\touputTokenStream := lengthFilter.Filter(inputTokenStream)\n\tif len(ouputTokenStream) != 1 {\n\t\tt.Fatalf(\"expected 1 output token\")\n\t}\n\tif string(ouputTokenStream[0].Term) != \"two\" {\n\t\tt.Errorf(\"expected term `two`, got `%s`\", ouputTokenStream[0].Term)\n\t}\n}\n\nfunc TestLengthFilterNoMax(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"1\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"two\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"three\"),\n\t\t},\n\t}\n\n\tlengthFilter := NewLengthFilter(3, -1)\n\touputTokenStream := lengthFilter.Filter(inputTokenStream)\n\tif len(ouputTokenStream) != 2 {\n\t\tt.Fatalf(\"expected 2 output token\")\n\t}\n\tif string(ouputTokenStream[0].Term) != \"two\" {\n\t\tt.Errorf(\"expected term `two`, got `%s`\", ouputTokenStream[0].Term)\n\t}\n\tif string(ouputTokenStream[1].Term) != \"three\" {\n\t\tt.Errorf(\"expected term `three`, got `%s`\", ouputTokenStream[0].Term)\n\t}\n}\n\nfunc TestLengthFilterNoMin(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"1\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"two\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"three\"),\n\t\t},\n\t}\n\n\tlengthFilter := NewLengthFilter(-1, 4)\n\touputTokenStream := lengthFilter.Filter(inputTokenStream)\n\tif len(ouputTokenStream) != 2 {\n\t\tt.Fatalf(\"expected 2 output token\")\n\t}\n\tif string(ouputTokenStream[0].Term) != \"1\" {\n\t\tt.Errorf(\"expected term `1`, got `%s`\", ouputTokenStream[0].Term)\n\t}\n\tif string(ouputTokenStream[1].Term) != \"two\" {\n\t\tt.Errorf(\"expected term `two`, got `%s`\", ouputTokenStream[0].Term)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/lowercase/lowercase.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package lowercase implements a TokenFilter which converts\n// tokens to lower case according to unicode rules.\npackage lowercase\n\nimport (\n\t\"bytes\"\n\t\"unicode\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\n// Name is the name used to register LowerCaseFilter in the bleve registry\nconst Name = \"to_lower\"\n\ntype LowerCaseFilter struct {\n}\n\nfunc NewLowerCaseFilter() *LowerCaseFilter {\n\treturn &LowerCaseFilter{}\n}\n\nfunc (f *LowerCaseFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\ttoken.Term = toLowerDeferredCopy(token.Term)\n\t}\n\treturn input\n}\n\nfunc LowerCaseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewLowerCaseFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, LowerCaseFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\n// toLowerDeferredCopy will function exactly like\n// bytes.ToLower() only it will reuse (overwrite)\n// the original byte array when possible\n// NOTE: because its possible that the lower-case\n// form of a rune has a different utf-8 encoded\n// length, in these cases a new byte array is allocated\nfunc toLowerDeferredCopy(s []byte) []byte {\n\tj := 0\n\tfor i := 0; i < len(s); {\n\t\twid := 1\n\t\tr := rune(s[i])\n\t\tif r >= utf8.RuneSelf {\n\t\t\tr, wid = utf8.DecodeRune(s[i:])\n\t\t}\n\n\t\tl := unicode.ToLower(r)\n\n\t\t// If the rune is already lowercased, just move to the\n\t\t// next rune.\n\t\tif l == r {\n\t\t\ti += wid\n\t\t\tj += wid\n\t\t\tcontinue\n\t\t}\n\n\t\t// Handles the Unicode edge-case where the last\n\t\t// rune in a word on the greek Σ needs to be converted\n\t\t// differently.\n\t\tif l == 'σ' && i+2 == len(s) {\n\t\t\tl = 'ς'\n\t\t}\n\n\t\tlwid := utf8.RuneLen(l)\n\t\tif lwid > wid {\n\t\t\t// utf-8 encoded replacement is wider\n\t\t\t// for now, punt and defer\n\t\t\t// to bytes.ToLower() for the remainder\n\t\t\t// only known to happen with chars\n\t\t\t//   Rune Ⱥ(570) width 2 - Lower ⱥ(11365) width 3\n\t\t\t//   Rune Ⱦ(574) width 2 - Lower ⱦ(11366) width 3\n\t\t\trest := bytes.ToLower(s[i:])\n\t\t\trv := make([]byte, j+len(rest))\n\t\t\tcopy(rv[:j], s[:j])\n\t\t\tcopy(rv[j:], rest)\n\t\t\treturn rv\n\t\t} else {\n\t\t\tutf8.EncodeRune(s[j:], l)\n\t\t}\n\t\ti += wid\n\t\tj += lwid\n\t}\n\treturn s[:j]\n}\n"
  },
  {
    "path": "analysis/token/lowercase/lowercase_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage lowercase\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestLowerCaseFilter(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"ONE\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"two\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"ThReE\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"steven's\"),\n\t\t},\n\t\t// these characters are chosen in particular\n\t\t// because the utf-8 encoding of the lower-case\n\t\t// version has a different length\n\t\t// Rune İ(304) width 2 - Lower i(105) width 1\n\t\t// Rune Ⱥ(570) width 2 - Lower ⱥ(11365) width 3\n\t\t// Rune Ⱦ(574) width 2 - Lower ⱦ(11366) width 3\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"İȺȾCAT\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"ȺȾCAT\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"ὈΔΥΣΣ\"),\n\t\t},\n\t}\n\n\texpectedTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"one\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"two\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"three\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"steven's\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"iⱥⱦcat\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"ⱥⱦcat\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"ὀδυσς\"),\n\t\t},\n\t}\n\n\tfilter := NewLowerCaseFilter()\n\touputTokenStream := filter.Filter(inputTokenStream)\n\tif !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {\n\t\tt.Errorf(\"expected %#v got %#v\", expectedTokenStream, ouputTokenStream)\n\t\tt.Errorf(\"expected %s got %s\", expectedTokenStream[0].Term, ouputTokenStream[0].Term)\n\t}\n}\n\nfunc BenchmarkLowerCaseFilter(b *testing.B) {\n\tinput := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"A\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"boiling\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"liquid\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"expanding\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"vapor\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"explosion\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"caused\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"by\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"the\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"rupture\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"of\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"a\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"vessel\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"containing\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"a\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"pressurized\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"liquid\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"above\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"its\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"boiling\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"point\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"İȺȾCAT\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"ȺȾCAT\"),\n\t\t},\n\t}\n\tfilter := NewLowerCaseFilter()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfilter.Filter(input)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/ngram/ngram.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ngram\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"ngram\"\n\ntype NgramFilter struct {\n\tminLength int\n\tmaxLength int\n}\n\nfunc NewNgramFilter(minLength, maxLength int) *NgramFilter {\n\treturn &NgramFilter{\n\t\tminLength: minLength,\n\t\tmaxLength: maxLength,\n\t}\n}\n\nfunc (s *NgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0, len(input))\n\n\tfor _, token := range input {\n\t\truneCount := utf8.RuneCount(token.Term)\n\t\trunes := bytes.Runes(token.Term)\n\t\tfor i := 0; i < runeCount; i++ {\n\t\t\t// index of the starting rune for this token\n\t\t\tfor ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ {\n\t\t\t\t// build an ngram of this size starting at i\n\t\t\t\tif i+ngramSize <= runeCount {\n\t\t\t\t\tngramTerm := analysis.BuildTermFromRunes(runes[i : i+ngramSize])\n\t\t\t\t\ttoken := analysis.Token{\n\t\t\t\t\t\tPosition: token.Position,\n\t\t\t\t\t\tStart:    token.Start,\n\t\t\t\t\t\tEnd:      token.End,\n\t\t\t\t\t\tType:     token.Type,\n\t\t\t\t\t\tTerm:     ngramTerm,\n\t\t\t\t\t}\n\t\t\t\t\trv = append(rv, &token)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn rv\n}\n\nfunc NgramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tminVal, ok := config[\"min\"]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify min\")\n\t}\n\n\tmin, err := convertToInt(minVal)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tmaxVal, ok := config[\"max\"]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify max\")\n\t}\n\n\tmax, err := convertToInt(maxVal)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn NewNgramFilter(min, max), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, NgramFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\n// Expects either an int or a flaot64 value\nfunc convertToInt(val interface{}) (int, error) {\n\tvar intVal int\n\tvar floatVal float64\n\tvar ok bool\n\n\tintVal, ok = val.(int)\n\tif ok {\n\t\treturn intVal, nil\n\t}\n\n\tfloatVal, ok = val.(float64)\n\tif ok {\n\t\treturn int(floatVal), nil\n\t}\n\n\treturn 0, fmt.Errorf(\"failed to convert to int value\")\n}\n"
  },
  {
    "path": "analysis/token/ngram/ngram_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ngram\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestNgramFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tmin    int\n\t\tmax    int\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tmin: 1,\n\t\t\tmax: 1,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"b\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"c\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"d\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"e\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmin: 2,\n\t\t\tmax: 2,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ab\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bc\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cd\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"de\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmin: 1,\n\t\t\tmax: 3,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ab\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abc\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"b\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bc\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"bcd\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"c\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cd\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"cde\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"d\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"de\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"e\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tngramFilter := NewNgramFilter(test.min, test.max)\n\t\tactual := ngramFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output, actual)\n\t\t}\n\t}\n}\n\nfunc TestConversionInt(t *testing.T) {\n\tconfig := map[string]interface{}{\n\t\t\"type\": Name,\n\t\t\"min\":  3,\n\t\t\"max\":  8,\n\t}\n\n\tf, err := NgramFilterConstructor(config, nil)\n\n\tif err != nil {\n\t\tt.Errorf(\"Failed to construct the ngram filter: %v\", err)\n\t}\n\n\tngram := f.(*NgramFilter)\n\tif ngram.minLength != 3 && ngram.maxLength != 8 {\n\t\tt.Errorf(\"Failed to construct the bounds. Got %v and %v.\", ngram.minLength, ngram.maxLength)\n\t}\n}\n\nfunc TestConversionFloat(t *testing.T) {\n\tconfig := map[string]interface{}{\n\t\t\"type\": Name,\n\t\t\"min\":  float64(3),\n\t\t\"max\":  float64(8),\n\t}\n\n\tf, err := NgramFilterConstructor(config, nil)\n\n\tif err != nil {\n\t\tt.Errorf(\"Failed to construct the ngram filter: %v\", err)\n\t}\n\n\tngram := f.(*NgramFilter)\n\tif ngram.minLength != 3 && ngram.maxLength != 8 {\n\t\tt.Errorf(\"Failed to construct the bounds. Got %v and %v.\", ngram.minLength, ngram.maxLength)\n\t}\n}\n\nfunc TestBadConversion(t *testing.T) {\n\tconfig := map[string]interface{}{\n\t\t\"type\": Name,\n\t\t\"min\":  \"3\",\n\t}\n\n\t_, err := NgramFilterConstructor(config, nil)\n\n\tif err == nil {\n\t\tt.Errorf(\"Expected conversion error.\")\n\t}\n\n\tif err.Error() != \"failed to convert to int value\" {\n\t\tt.Errorf(\"Wrong error recevied. Got %v.\", err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/porter/porter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage porter\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/go-porterstemmer\"\n)\n\nconst Name = \"stemmer_porter\"\n\ntype PorterStemmer struct {\n}\n\nfunc NewPorterStemmer() *PorterStemmer {\n\treturn &PorterStemmer{}\n}\n\nfunc (s *PorterStemmer) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\t// if it is not a protected keyword, stem it\n\t\tif !token.KeyWord {\n\t\t\ttermRunes := bytes.Runes(token.Term)\n\t\t\tstemmedRunes := porterstemmer.StemWithoutLowerCasing(termRunes)\n\t\t\ttoken.Term = analysis.BuildTermFromRunes(stemmedRunes)\n\t\t}\n\t}\n\treturn input\n}\n\nfunc PorterStemmerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewPorterStemmer(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, PorterStemmerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/porter/porter_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage porter\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestPorterStemmer(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walking\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"talked\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"business\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:    []byte(\"protected\"),\n\t\t\tKeyWord: true,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"cat\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"done\"),\n\t\t},\n\t\t// a term which does stem, but does not change length\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"marty\"),\n\t\t},\n\t}\n\n\texpectedTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walk\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"talk\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"busi\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:    []byte(\"protected\"),\n\t\t\tKeyWord: true,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"cat\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"done\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"marti\"),\n\t\t},\n\t}\n\n\tfilter := NewPorterStemmer()\n\touputTokenStream := filter.Filter(inputTokenStream)\n\tif !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {\n\t\tt.Errorf(\"expected %#v got %#v\", expectedTokenStream[3], ouputTokenStream[3])\n\t}\n}\n\nfunc BenchmarkPorterStemmer(b *testing.B) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walking\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"talked\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"business\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:    []byte(\"protected\"),\n\t\t\tKeyWord: true,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"cat\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"done\"),\n\t\t},\n\t}\n\n\tfilter := NewPorterStemmer()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tfilter.Filter(inputTokenStream)\n\t}\n\n}\n"
  },
  {
    "path": "analysis/token/reverse/reverse.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage reverse\n\nimport (\n\t\"unicode\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\n// Name is the name used to register ReverseFilter in the bleve registry\nconst Name = \"reverse\"\n\ntype ReverseFilter struct {\n}\n\nfunc NewReverseFilter() *ReverseFilter {\n\treturn &ReverseFilter{}\n}\n\nfunc (f *ReverseFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\ttoken.Term = reverse(token.Term)\n\t}\n\treturn input\n}\n\nfunc ReverseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewReverseFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, ReverseFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\n// reverse(..) will generate a reversed version of the provided\n// unicode array and return it back to its caller.\nfunc reverse(s []byte) []byte {\n\tcursorIn := 0\n\tinputRunes := []rune(string(s))\n\tcursorOut := len(s)\n\toutput := make([]byte, len(s))\n\tfor i := 0; i < len(inputRunes); {\n\t\twid := utf8.RuneLen(inputRunes[i])\n\t\ti++\n\t\tfor i < len(inputRunes) {\n\t\t\tr := inputRunes[i]\n\t\t\tif unicode.Is(unicode.Mn, r) || unicode.Is(unicode.Me, r) || unicode.Is(unicode.Mc, r) {\n\t\t\t\twid += utf8.RuneLen(r)\n\t\t\t\ti++\n\t\t\t} else {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tcopy(output[cursorOut-wid:cursorOut], s[cursorIn:cursorIn+wid])\n\t\tcursorIn += wid\n\t\tcursorOut -= wid\n\t}\n\n\treturn output\n}\n"
  },
  {
    "path": "analysis/token/reverse/reverse_test.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage reverse\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestReverseFilter(t *testing.T) {\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"one\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"TWo\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"thRee\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"four's\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"what's this in reverse\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"œ∑´®†\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"İȺȾCAT÷≥≤µ123\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"!@#$%^&*()\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"cafés\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"¿Dónde estás?\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"Me gustaría una cerveza.\"),\n\t\t},\n\t}\n\n\texpectedTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"eno\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"oWT\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"eeRht\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"s'ruof\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"esrever ni siht s'tahw\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"†®´∑œ\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"321µ≤≥÷TACȾȺİ\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\")(*&^%$#@!\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"séfac\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"?sátse ednóD¿\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\".azevrec anu aíratsug eM\"),\n\t\t},\n\t}\n\n\tfilter := NewReverseFilter()\n\toutputTokenStream := filter.Filter(inputTokenStream)\n\tfor i := 0; i < len(expectedTokenStream); i++ {\n\t\tif !bytes.Equal(outputTokenStream[i].Term, expectedTokenStream[i].Term) {\n\t\t\tt.Errorf(\"[%d] expected %s got %s\",\n\t\t\t\ti+1, expectedTokenStream[i].Term, outputTokenStream[i].Term)\n\t\t}\n\t}\n}\n\nfunc BenchmarkReverseFilter(b *testing.B) {\n\tinput := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"A\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"boiling\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"liquid\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"expanding\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"vapor\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"explosion\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"caused\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"by\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"the\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"rupture\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"of\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"a\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"vessel\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"containing\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"pressurized\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"liquid\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"above\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"its\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"boiling\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"point\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"İȺȾCAT\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"Me gustaría una cerveza.\"),\n\t\t},\n\t}\n\tfilter := NewReverseFilter()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfilter.Filter(input)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/shingle/shingle.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage shingle\n\nimport (\n\t\"container/ring\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"shingle\"\n\ntype ShingleFilter struct {\n\tmin            int\n\tmax            int\n\toutputOriginal bool\n\ttokenSeparator string\n\tfill           string\n}\n\nfunc NewShingleFilter(min, max int, outputOriginal bool, sep, fill string) *ShingleFilter {\n\treturn &ShingleFilter{\n\t\tmin:            min,\n\t\tmax:            max,\n\t\toutputOriginal: outputOriginal,\n\t\ttokenSeparator: sep,\n\t\tfill:           fill,\n\t}\n}\n\nfunc (s *ShingleFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0, len(input))\n\n\tring := ring.New(s.max)\n\titemsInRing := 0\n\tcurrentPosition := 0\n\tfor _, token := range input {\n\t\tif s.outputOriginal {\n\t\t\trv = append(rv, token)\n\t\t}\n\n\t\t// if there are gaps, insert filler tokens\n\t\toffset := token.Position - currentPosition\n\t\tfor offset > 1 {\n\t\t\tfillerToken := analysis.Token{\n\t\t\t\tPosition: 0,\n\t\t\t\tStart:    -1,\n\t\t\t\tEnd:      -1,\n\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\tTerm:     []byte(s.fill),\n\t\t\t}\n\t\t\tring.Value = &fillerToken\n\t\t\tif itemsInRing < s.max {\n\t\t\t\titemsInRing++\n\t\t\t}\n\t\t\trv = append(rv, s.shingleCurrentRingState(ring, itemsInRing)...)\n\t\t\tring = ring.Next()\n\t\t\toffset--\n\t\t}\n\t\tcurrentPosition = token.Position\n\n\t\tring.Value = token\n\t\tif itemsInRing < s.max {\n\t\t\titemsInRing++\n\t\t}\n\t\trv = append(rv, s.shingleCurrentRingState(ring, itemsInRing)...)\n\t\tring = ring.Next()\n\t}\n\n\treturn rv\n}\n\nfunc (s *ShingleFilter) shingleCurrentRingState(ring *ring.Ring, itemsInRing int) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0)\n\tfor shingleN := s.min; shingleN <= s.max; shingleN++ {\n\t\t// if there are enough items in the ring\n\t\t// to produce a shingle of this size\n\t\tif itemsInRing >= shingleN {\n\t\t\tthisShingleRing := ring.Move(-(shingleN - 1))\n\t\t\tshingledBytes := make([]byte, 0)\n\t\t\tpos := 0\n\t\t\tstart := -1\n\t\t\tend := 0\n\t\t\tfor i := 0; i < shingleN; i++ {\n\t\t\t\tif i != 0 {\n\t\t\t\t\tshingledBytes = append(shingledBytes, []byte(s.tokenSeparator)...)\n\t\t\t\t}\n\t\t\t\tcurr := thisShingleRing.Value.(*analysis.Token)\n\t\t\t\tif pos == 0 && curr.Position != 0 {\n\t\t\t\t\tpos = curr.Position\n\t\t\t\t}\n\t\t\t\tif start == -1 && curr.Start != -1 {\n\t\t\t\t\tstart = curr.Start\n\t\t\t\t}\n\t\t\t\tif curr.End != -1 {\n\t\t\t\t\tend = curr.End\n\t\t\t\t}\n\t\t\t\tshingledBytes = append(shingledBytes, curr.Term...)\n\t\t\t\tthisShingleRing = thisShingleRing.Next()\n\t\t\t}\n\t\t\ttoken := analysis.Token{\n\t\t\t\tType: analysis.Shingle,\n\t\t\t\tTerm: shingledBytes,\n\t\t\t}\n\t\t\tif pos != 0 {\n\t\t\t\ttoken.Position = pos\n\t\t\t}\n\t\t\tif start != -1 {\n\t\t\t\ttoken.Start = start\n\t\t\t}\n\t\t\tif end != -1 {\n\t\t\t\ttoken.End = end\n\t\t\t}\n\t\t\trv = append(rv, &token)\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc ShingleFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tminVal, ok := config[\"min\"].(float64)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify min\")\n\t}\n\tmin := int(minVal)\n\tmaxVal, ok := config[\"max\"].(float64)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify max\")\n\t}\n\tmax := int(maxVal)\n\n\toutputOriginal := false\n\toutVal, ok := config[\"output_original\"].(bool)\n\tif ok {\n\t\toutputOriginal = outVal\n\t}\n\n\tsep := \" \"\n\tsepVal, ok := config[\"separator\"].(string)\n\tif ok {\n\t\tsep = sepVal\n\t}\n\n\tfill := \"_\"\n\tfillVal, ok := config[\"filler\"].(string)\n\tif ok {\n\t\tfill = fillVal\n\t}\n\n\treturn NewShingleFilter(min, max, outputOriginal, sep, fill), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, ShingleFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/shingle/shingle_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage shingle\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestShingleFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tmin            int\n\t\tmax            int\n\t\toutputOriginal bool\n\t\tseparator      string\n\t\tfiller         string\n\t\tinput          analysis.TokenStream\n\t\toutput         analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tmin:            2,\n\t\t\tmax:            2,\n\t\t\toutputOriginal: false,\n\t\t\tseparator:      \" \",\n\t\t\tfiller:         \"_\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fox\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the quick\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick brown\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown fox\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmin:            3,\n\t\t\tmax:            3,\n\t\t\toutputOriginal: false,\n\t\t\tseparator:      \" \",\n\t\t\tfiller:         \"_\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fox\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the quick brown\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick brown fox\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmin:            2,\n\t\t\tmax:            3,\n\t\t\toutputOriginal: false,\n\t\t\tseparator:      \" \",\n\t\t\tfiller:         \"_\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fox\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the quick\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick brown\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the quick brown\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown fox\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick brown fox\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmin:            3,\n\t\t\tmax:            3,\n\t\t\toutputOriginal: false,\n\t\t\tseparator:      \" \",\n\t\t\tfiller:         \"_\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ugly\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"quick\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"brown\"),\n\t\t\t\t\tPosition: 4,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ugly _ quick\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"_ quick brown\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 3,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmin:            1,\n\t\t\tmax:            5,\n\t\t\toutputOriginal: false,\n\t\t\tseparator:      \" \",\n\t\t\tfiller:         \"_\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"text\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t\t// token 3 removed by stop filter\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"see\"),\n\t\t\t\t\tPosition: 4,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"shingles\"),\n\t\t\t\t\tPosition: 5,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"text\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test text\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"_\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"text _\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test text _\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"see\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"_ see\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"text _ see\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test text _ see\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"shingles\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 5,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"see shingles\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"_ see shingles\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 4,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"text _ see shingles\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test text _ see shingles\"),\n\t\t\t\t\tType:     analysis.Shingle,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmin:            2,\n\t\t\tmax:            2,\n\t\t\toutputOriginal: true,\n\t\t\tseparator:      \" \",\n\t\t\tfiller:         \"_\",\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fox\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the quick\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick brown\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fox\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown fox\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tshingleFilter := NewShingleFilter(test.min, test.max, test.outputOriginal, test.separator, test.filler)\n\t\tactual := shingleFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output, actual)\n\t\t}\n\t}\n}\n\n// TestShingleFilterBug431 tests that the shingle filter is in fact stateless\n// by making using the same filter instance twice and ensuring we do not get\n// contaminated output\nfunc TestShingleFilterBug431(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"fox\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"the quick\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"quick brown\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"brown fox\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"sad\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"dirty\"),\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"sock\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"a sad\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"sad dirty\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"dirty sock\"),\n\t\t\t\t\tType: analysis.Shingle,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tshingleFilter := NewShingleFilter(2, 2, false, \" \", \"_\")\n\tfor _, test := range tests {\n\t\tactual := shingleFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output, actual)\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "analysis/token/snowball/snowball.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage snowball\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\n\t\"github.com/blevesearch/snowball\"\n)\n\nconst Name = \"stemmer_snowball\"\n\ntype SnowballStemmer struct {\n\tlanguage string\n}\n\nfunc NewSnowballStemmer(language string) *SnowballStemmer {\n\treturn &SnowballStemmer{\n\t\tlanguage: language,\n\t}\n}\n\nfunc (s *SnowballStemmer) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\t// if it is not a protected keyword, stem it\n\t\tif !token.KeyWord {\n\t\t\tstemmed, _ := snowball.Stem(string(token.Term), s.language, true)\n\t\t\ttoken.Term = []byte(stemmed)\n\t\t}\n\t}\n\treturn input\n}\n\nfunc SnowballStemmerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tlanguage, ok := config[\"language\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify language\")\n\t}\n\treturn NewSnowballStemmer(language), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, SnowballStemmerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/snowball/snowball_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage snowball\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestSnowballStemmer(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walking\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"talked\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"business\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:    []byte(\"protected\"),\n\t\t\tKeyWord: true,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"cat\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"done\"),\n\t\t},\n\t\t// a term which does stem, but does not change length\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"marty\"),\n\t\t},\n\t}\n\n\texpectedTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walk\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"talk\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"busi\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:    []byte(\"protected\"),\n\t\t\tKeyWord: true,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"cat\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"done\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"marti\"),\n\t\t},\n\t}\n\n\tfilter := NewSnowballStemmer(\"english\")\n\touputTokenStream := filter.Filter(inputTokenStream)\n\tif !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {\n\t\tt.Errorf(\"expected %#v got %#v\", expectedTokenStream[3], ouputTokenStream[3])\n\t}\n}\n\nfunc BenchmarkSnowballStemmer(b *testing.B) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walking\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"talked\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"business\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm:    []byte(\"protected\"),\n\t\t\tKeyWord: true,\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"cat\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"done\"),\n\t\t},\n\t}\n\n\tfilter := NewSnowballStemmer(\"english\")\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tfilter.Filter(inputTokenStream)\n\t}\n\n}\n"
  },
  {
    "path": "analysis/token/stop/stop.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package stop implements a TokenFilter removing tokens found in\n// a TokenMap.\n//\n// It constructor takes the following arguments:\n//\n// \"stop_token_map\" (string): the name of the token map identifying tokens to\n// remove.\npackage stop\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"stop_tokens\"\n\ntype StopTokensFilter struct {\n\tstopTokens analysis.TokenMap\n}\n\nfunc NewStopTokensFilter(stopTokens analysis.TokenMap) *StopTokensFilter {\n\treturn &StopTokensFilter{\n\t\tstopTokens: stopTokens,\n\t}\n}\n\nfunc (f *StopTokensFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tj := 0\n\tfor _, token := range input {\n\t\t_, isStopToken := f.stopTokens[string(token.Term)]\n\t\tif !isStopToken {\n\t\t\tinput[j] = token\n\t\t\tj++\n\t\t}\n\t}\n\n\treturn input[:j]\n}\n\nfunc StopTokensFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tstopTokenMapName, ok := config[\"stop_token_map\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify stop_token_map\")\n\t}\n\tstopTokenMap, err := cache.TokenMapNamed(stopTokenMapName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building stop words filter: %v\", err)\n\t}\n\treturn NewStopTokensFilter(stopTokenMap), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, StopTokensFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/stop/stop_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stop\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenmap\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestStopWordsFilter(t *testing.T) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"a\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walk\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"in\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"the\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"park\"),\n\t\t},\n\t}\n\n\texpectedTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walk\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"park\"),\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tstopListConfig := map[string]interface{}{\n\t\t\"type\":   tokenmap.Name,\n\t\t\"tokens\": []interface{}{\"a\", \"in\", \"the\"},\n\t}\n\t_, err := cache.DefineTokenMap(\"stop_test\", stopListConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tstopConfig := map[string]interface{}{\n\t\t\"type\":           \"stop_tokens\",\n\t\t\"stop_token_map\": \"stop_test\",\n\t}\n\tstopFilter, err := cache.DefineTokenFilter(\"stop_test\", stopConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\touputTokenStream := stopFilter.Filter(inputTokenStream)\n\tif !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) {\n\t\tt.Errorf(\"expected %#v got %#v\", expectedTokenStream, ouputTokenStream)\n\t}\n}\n\nfunc BenchmarkStopWordsFilter(b *testing.B) {\n\n\tinputTokenStream := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"a\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"walk\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"in\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"the\"),\n\t\t},\n\t\t&analysis.Token{\n\t\t\tTerm: []byte(\"park\"),\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\tstopListConfig := map[string]interface{}{\n\t\t\"type\":   tokenmap.Name,\n\t\t\"tokens\": []interface{}{\"a\", \"in\", \"the\"},\n\t}\n\t_, err := cache.DefineTokenMap(\"stop_test\", stopListConfig)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tstopConfig := map[string]interface{}{\n\t\t\"type\":           \"stop_tokens\",\n\t\t\"stop_token_map\": \"stop_test\",\n\t}\n\tstopFilter, err := cache.DefineTokenFilter(\"stop_test\", stopConfig)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tstopFilter.Filter(inputTokenStream)\n\t}\n\n}\n"
  },
  {
    "path": "analysis/token/truncate/truncate.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage truncate\n\nimport (\n\t\"fmt\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"truncate_token\"\n\ntype TruncateTokenFilter struct {\n\tlength int\n}\n\nfunc NewTruncateTokenFilter(length int) *TruncateTokenFilter {\n\treturn &TruncateTokenFilter{\n\t\tlength: length,\n\t}\n}\n\nfunc (s *TruncateTokenFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\twordLen := utf8.RuneCount(token.Term)\n\t\tif wordLen > s.length {\n\t\t\ttoken.Term = analysis.TruncateRunes(token.Term, wordLen-s.length)\n\t\t}\n\t}\n\treturn input\n}\n\nfunc TruncateTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tlenVal, ok := config[\"length\"].(float64)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify length\")\n\t}\n\tlength := int(lenVal)\n\n\treturn NewTruncateTokenFilter(length), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, TruncateTokenFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/truncate/truncate_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage truncate\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestTruncateTokenFilter(t *testing.T) {\n\n\ttests := []struct {\n\t\tlength int\n\t\tinput  analysis.TokenStream\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tlength: 5,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcdefgh\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"abcde\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tlength: 3,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"こんにちは世界\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"こんに\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tlength: 10,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"แยกคำภาษาไทยก็ทำได้นะจ้ะ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"แยกคำภาษาไ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttruncateTokenFilter := NewTruncateTokenFilter(test.length)\n\t\tactual := truncateTokenFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/token/unicodenorm/unicodenorm.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage unicodenorm\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"golang.org/x/text/unicode/norm\"\n)\n\nconst Name = \"normalize_unicode\"\n\nconst NFC = \"nfc\"\nconst NFD = \"nfd\"\nconst NFKC = \"nfkc\"\nconst NFKD = \"nfkd\"\n\nvar forms = map[string]norm.Form{\n\tNFC:  norm.NFC,\n\tNFD:  norm.NFD,\n\tNFKC: norm.NFKC,\n\tNFKD: norm.NFKD,\n}\n\ntype UnicodeNormalizeFilter struct {\n\tform norm.Form\n}\n\nfunc NewUnicodeNormalizeFilter(formName string) (*UnicodeNormalizeFilter, error) {\n\tform, ok := forms[formName]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"no form named %s\", formName)\n\t}\n\treturn &UnicodeNormalizeFilter{\n\t\tform: form,\n\t}, nil\n}\n\nfunc MustNewUnicodeNormalizeFilter(formName string) *UnicodeNormalizeFilter {\n\tfilter, err := NewUnicodeNormalizeFilter(formName)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn filter\n}\n\nfunc (s *UnicodeNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tfor _, token := range input {\n\t\ttoken.Term = s.form.Bytes(token.Term)\n\t}\n\treturn input\n}\n\nfunc UnicodeNormalizeFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\tformVal, ok := config[\"form\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify form\")\n\t}\n\tform := formVal\n\treturn NewUnicodeNormalizeFilter(form)\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, UnicodeNormalizeFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/unicodenorm/unicodenorm_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage unicodenorm\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\n// the following tests come from the lucene\n// test cases for CJK width filter\n// which is our basis for using this\n// as a substitute for that\nfunc TestUnicodeNormalization(t *testing.T) {\n\n\ttests := []struct {\n\t\tformName string\n\t\tinput    analysis.TokenStream\n\t\toutput   analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tformName: NFKD,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Ｔｅｓｔ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"Test\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tformName: NFKD,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"１２３４\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"1234\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tformName: NFKD,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ｶﾀｶﾅ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"カタカナ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tformName: NFKC,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ｳﾞｨｯﾂ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ヴィッツ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tformName: NFKC,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"ﾊﾟﾅｿﾆｯｸ\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"パナソニック\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tformName: NFD,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u212B\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0041\\u030A\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tformName: NFC,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u212B\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u00C5\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tformName: NFKD,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\uFB01\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0066\\u0069\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tformName: NFKC,\n\t\t\tinput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\uFB01\"),\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm: []byte(\"\\u0066\\u0069\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tfilter := MustNewUnicodeNormalizeFilter(test.formName)\n\t\tactual := filter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %s, got %s\", test.output[0].Term, actual[0].Term)\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output[0].Term, actual[0].Term)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/token/unique/unique.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage unique\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"unique\"\n\n// UniqueTermFilter retains only the tokens which mark the first occurrence of\n// a term. Tokens whose term appears in a preceding token are dropped.\ntype UniqueTermFilter struct{}\n\nfunc NewUniqueTermFilter() *UniqueTermFilter {\n\treturn &UniqueTermFilter{}\n}\n\nfunc (f *UniqueTermFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n\tencounteredTerms := make(map[string]struct{}, len(input)/4)\n\tj := 0\n\tfor _, token := range input {\n\t\tterm := string(token.Term)\n\t\tif _, ok := encounteredTerms[term]; ok {\n\t\t\tcontinue\n\t\t}\n\t\tencounteredTerms[term] = struct{}{}\n\t\tinput[j] = token\n\t\tj++\n\t}\n\treturn input[:j]\n}\n\nfunc UniqueTermFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {\n\treturn NewUniqueTermFilter(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenFilter(Name, UniqueTermFilterConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/token/unique/unique_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage unique\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestUniqueTermFilter(t *testing.T) {\n\tvar tests = []struct {\n\t\tinput analysis.TokenStream\n\t\t// expected indices of input which should be included in the output. We\n\t\t// use indices instead of another TokenStream, since position/start/end\n\t\t// should be preserved.\n\t\texpectedIndices []int\n\t}{\n\t\t{\n\t\t\tinput:           tokenStream(),\n\t\t\texpectedIndices: []int{},\n\t\t},\n\t\t{\n\t\t\tinput:           tokenStream(\"a\"),\n\t\t\texpectedIndices: []int{0},\n\t\t},\n\t\t{\n\t\t\tinput:           tokenStream(\"each\", \"term\", \"in\", \"this\", \"sentence\", \"is\", \"unique\"),\n\t\t\texpectedIndices: []int{0, 1, 2, 3, 4, 5, 6},\n\t\t},\n\t\t{\n\t\t\tinput:           tokenStream(\"Lui\", \"è\", \"alto\", \"e\", \"lei\", \"è\", \"bassa\"),\n\t\t\texpectedIndices: []int{0, 1, 2, 3, 4, 6},\n\t\t},\n\t\t{\n\t\t\tinput:           tokenStream(\"a\", \"a\", \"A\", \"a\", \"a\", \"A\"),\n\t\t\texpectedIndices: []int{0, 2},\n\t\t},\n\t}\n\tuniqueTermFilter := NewUniqueTermFilter()\n\tfor _, test := range tests {\n\t\texpected := subStream(test.input, test.expectedIndices)\n\t\tactual := uniqueTermFilter.Filter(test.input)\n\t\tif !reflect.DeepEqual(actual, expected) {\n\t\t\tt.Errorf(\"expected %s \\n\\n got %s\", expected, actual)\n\t\t}\n\t}\n}\n\nfunc tokenStream(termStrs ...string) analysis.TokenStream {\n\ttokenStream := make([]*analysis.Token, len(termStrs))\n\tindex := 0\n\tfor i, termStr := range termStrs {\n\t\ttokenStream[i] = &analysis.Token{\n\t\t\tTerm:     []byte(termStr),\n\t\t\tPosition: i + 1,\n\t\t\tStart:    index,\n\t\t\tEnd:      index + len(termStr),\n\t\t}\n\t\tindex += len(termStr)\n\t}\n\treturn analysis.TokenStream(tokenStream)\n}\n\nfunc subStream(stream analysis.TokenStream, indices []int) analysis.TokenStream {\n\tresult := make(analysis.TokenStream, len(indices))\n\tfor i, index := range indices {\n\t\tresult[i] = stream[index]\n\t}\n\treturn result\n}\n"
  },
  {
    "path": "analysis/tokenizer/character/character.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage character\n\nimport (\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\ntype IsTokenRune func(r rune) bool\n\ntype CharacterTokenizer struct {\n\tisTokenRun IsTokenRune\n}\n\nfunc NewCharacterTokenizer(f IsTokenRune) *CharacterTokenizer {\n\treturn &CharacterTokenizer{\n\t\tisTokenRun: f,\n\t}\n}\n\nfunc (c *CharacterTokenizer) Tokenize(input []byte) analysis.TokenStream {\n\n\trv := make(analysis.TokenStream, 0, 1024)\n\n\toffset := 0\n\tstart := 0\n\tend := 0\n\tcount := 0\n\tfor currRune, size := utf8.DecodeRune(input[offset:]); currRune != utf8.RuneError; currRune, size = utf8.DecodeRune(input[offset:]) {\n\t\tisToken := c.isTokenRun(currRune)\n\t\tif isToken {\n\t\t\tend = offset + size\n\t\t} else {\n\t\t\tif end-start > 0 {\n\t\t\t\t// build token\n\t\t\t\trv = append(rv, &analysis.Token{\n\t\t\t\t\tTerm:     input[start:end],\n\t\t\t\t\tStart:    start,\n\t\t\t\t\tEnd:      end,\n\t\t\t\t\tPosition: count + 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t})\n\t\t\t\tcount++\n\t\t\t}\n\t\t\tstart = offset + size\n\t\t\tend = start\n\t\t}\n\t\toffset += size\n\t}\n\t// if we ended in the middle of a token, finish it\n\tif end-start > 0 {\n\t\t// build token\n\t\trv = append(rv, &analysis.Token{\n\t\t\tTerm:     input[start:end],\n\t\t\tStart:    start,\n\t\t\tEnd:      end,\n\t\t\tPosition: count + 1,\n\t\t\tType:     analysis.AlphaNumeric,\n\t\t})\n\t}\n\treturn rv\n}\n"
  },
  {
    "path": "analysis/tokenizer/character/character_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage character\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\t\"unicode\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestCharacterTokenizer(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\t[]byte(\"Hello World.\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tTerm:     []byte(\"Hello\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      11,\n\t\t\t\t\tTerm:     []byte(\"World\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"dominique@mcdiabetes.com\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t\tTerm:     []byte(\"dominique\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    10,\n\t\t\t\t\tEnd:      20,\n\t\t\t\t\tTerm:     []byte(\"mcdiabetes\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    21,\n\t\t\t\t\tEnd:      24,\n\t\t\t\t\tTerm:     []byte(\"com\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\ttokenizer := NewCharacterTokenizer(unicode.IsLetter)\n\tfor _, test := range tests {\n\t\tactual := tokenizer.Tokenize(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"Expected %v, got %v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/exception/exception.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// package exception implements a Tokenizer which extracts pieces matched by a\n// regular expression from the input data, delegates the rest to another\n// tokenizer, then insert back extracted parts in the token stream. Use it to\n// preserve sequences which a regular tokenizer would alter or remove.\n//\n// Its constructor takes the following arguments:\n//\n// \"exceptions\" ([]string): one or more Go regular expressions matching the\n// sequence to preserve. Multiple expressions are combined with \"|\".\n//\n// \"tokenizer\" (string): the name of the tokenizer processing the data not\n// matched by \"exceptions\".\npackage exception\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"exception\"\n\ntype ExceptionsTokenizer struct {\n\texception *regexp.Regexp\n\tremaining analysis.Tokenizer\n}\n\nfunc NewExceptionsTokenizer(exception *regexp.Regexp, remaining analysis.Tokenizer) *ExceptionsTokenizer {\n\treturn &ExceptionsTokenizer{\n\t\texception: exception,\n\t\tremaining: remaining,\n\t}\n}\n\nfunc (t *ExceptionsTokenizer) Tokenize(input []byte) analysis.TokenStream {\n\trv := make(analysis.TokenStream, 0)\n\tmatches := t.exception.FindAllIndex(input, -1)\n\tcurrInput := 0\n\tlastPos := 0\n\tfor _, match := range matches {\n\t\tstart := match[0]\n\t\tend := match[1]\n\t\tif start > currInput {\n\t\t\t// need to defer to remaining for unprocessed section\n\t\t\tintermediate := t.remaining.Tokenize(input[currInput:start])\n\t\t\t// add intermediate tokens to our result stream\n\t\t\tfor _, token := range intermediate {\n\t\t\t\t// adjust token offsets\n\t\t\t\ttoken.Position += lastPos\n\t\t\t\ttoken.Start += currInput\n\t\t\t\ttoken.End += currInput\n\t\t\t\trv = append(rv, token)\n\t\t\t}\n\t\t\tlastPos += len(intermediate)\n\t\t\tcurrInput = start\n\t\t}\n\n\t\t// create single token with this regexp match\n\t\ttoken := &analysis.Token{\n\t\t\tTerm:     input[start:end],\n\t\t\tStart:    start,\n\t\t\tEnd:      end,\n\t\t\tPosition: lastPos + 1,\n\t\t}\n\t\trv = append(rv, token)\n\t\tlastPos++\n\t\tcurrInput = end\n\n\t}\n\n\tif currInput < len(input) {\n\t\t// need to defer to remaining for unprocessed section\n\t\tintermediate := t.remaining.Tokenize(input[currInput:])\n\t\t// add intermediate tokens to our result stream\n\t\tfor _, token := range intermediate {\n\t\t\t// adjust token offsets\n\t\t\ttoken.Position += lastPos\n\t\t\ttoken.Start += currInput\n\t\t\ttoken.End += currInput\n\t\t\trv = append(rv, token)\n\t\t}\n\t}\n\n\treturn rv\n}\n\nfunc ExceptionsTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {\n\texceptions := []string{}\n\tiexceptions, ok := config[\"exceptions\"].([]interface{})\n\tif ok {\n\t\tfor _, exception := range iexceptions {\n\t\t\texception, ok := exception.(string)\n\t\t\tif ok {\n\t\t\t\texceptions = append(exceptions, exception)\n\t\t\t}\n\t\t}\n\t}\n\taexceptions, ok := config[\"exceptions\"].([]string)\n\tif ok {\n\t\texceptions = append(exceptions, aexceptions...)\n\t}\n\tif len(exceptions) == 0 {\n\t\treturn nil, fmt.Errorf(\"no pattern found in 'exception' property\")\n\t}\n\texceptionPattern := strings.Join(exceptions, \"|\")\n\tr, err := regexp.Compile(exceptionPattern)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unable to build regexp tokenizer: %v\", err)\n\t}\n\n\tremainingName, ok := config[\"tokenizer\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify tokenizer for remaining input\")\n\t}\n\tremaining, err := cache.TokenizerNamed(remainingName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn NewExceptionsTokenizer(r, remaining), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenizer(Name, ExceptionsTokenizerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/exception/exception_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage exception\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestExceptionsTokenizer(t *testing.T) {\n\ttests := []struct {\n\t\tconfig   map[string]interface{}\n\t\tinput    []byte\n\t\tpatterns []string\n\t\tresult   analysis.TokenStream\n\t}{\n\t\t{\n\t\t\tinput: []byte(\"test http://blevesearch.com/ words\"),\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"type\":      \"exception\",\n\t\t\t\t\"tokenizer\": \"unicode\",\n\t\t\t\t\"exceptions\": []interface{}{\n\t\t\t\t\t`[hH][tT][tT][pP][sS]?://(\\S)*`,\n\t\t\t\t\t`[fF][iI][lL][eE]://(\\S)*`,\n\t\t\t\t\t`[fF][tT][pP]://(\\S)*`,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"test\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"http://blevesearch.com/\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    5,\n\t\t\t\t\tEnd:      28,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"words\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    29,\n\t\t\t\t\tEnd:      34,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"what ftp://blevesearch.com/ songs\"),\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"type\":      \"exception\",\n\t\t\t\t\"tokenizer\": \"unicode\",\n\t\t\t\t\"exceptions\": []interface{}{\n\t\t\t\t\t`[hH][tT][tT][pP][sS]?://(\\S)*`,\n\t\t\t\t\t`[fF][iI][lL][eE]://(\\S)*`,\n\t\t\t\t\t`[fF][tT][pP]://(\\S)*`,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"what\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ftp://blevesearch.com/\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    5,\n\t\t\t\t\tEnd:      27,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"songs\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    28,\n\t\t\t\t\tEnd:      33,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []byte(\"please email marty@couchbase.com the URL https://blevesearch.com/\"),\n\t\t\tconfig: map[string]interface{}{\n\t\t\t\t\"type\":      \"exception\",\n\t\t\t\t\"tokenizer\": \"unicode\",\n\t\t\t\t\"exceptions\": []interface{}{\n\t\t\t\t\t`[hH][tT][tT][pP][sS]?://(\\S)*`,\n\t\t\t\t\t`[fF][iI][lL][eE]://(\\S)*`,\n\t\t\t\t\t`[fF][tT][pP]://(\\S)*`,\n\t\t\t\t\t`\\S+@\\S+`,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"please\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"email\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tStart:    7,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"marty@couchbase.com\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tStart:    13,\n\t\t\t\t\tEnd:      32,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"the\"),\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tStart:    33,\n\t\t\t\t\tEnd:      36,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"URL\"),\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tStart:    37,\n\t\t\t\t\tEnd:      40,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"https://blevesearch.com/\"),\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tStart:    41,\n\t\t\t\t\tEnd:      65,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\t// remaining := unicode.NewUnicodeTokenizer()\n\tfor _, test := range tests {\n\n\t\t// build the requested exception tokenizer\n\t\tcache := registry.NewCache()\n\t\ttokenizer, err := cache.DefineTokenizer(\"custom\", test.config)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\t// pattern := strings.Join(test.patterns, \"|\")\n\t\t// r, err := regexp.Compile(pattern)\n\t\t// if err != nil {\n\t\t// \tt.Fatal(err)\n\t\t// }\n\t\t// tokenizer := NewExceptionsTokenizer(r, remaining)\n\t\tactual := tokenizer.Tokenize(test.input)\n\t\tif !reflect.DeepEqual(actual, test.result) {\n\t\t\tt.Errorf(\"expected %v, got %v\", test.result, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/letter/letter.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage letter\n\nimport (\n\t\"unicode\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/character\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"letter\"\n\nfunc TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {\n\treturn character.NewCharacterTokenizer(unicode.IsLetter), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenizer(Name, TokenizerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/regexp/regexp.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage regexp\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"regexp\"\n\nvar IdeographRegexp = regexp.MustCompile(`\\p{Han}|\\p{Hangul}|\\p{Hiragana}|\\p{Katakana}`)\n\ntype RegexpTokenizer struct {\n\tr *regexp.Regexp\n}\n\nfunc NewRegexpTokenizer(r *regexp.Regexp) *RegexpTokenizer {\n\treturn &RegexpTokenizer{\n\t\tr: r,\n\t}\n}\n\nfunc (rt *RegexpTokenizer) Tokenize(input []byte) analysis.TokenStream {\n\tmatches := rt.r.FindAllIndex(input, -1)\n\trv := make(analysis.TokenStream, 0, len(matches))\n\tfor i, match := range matches {\n\t\tmatchBytes := input[match[0]:match[1]]\n\t\tif match[1]-match[0] > 0 {\n\t\t\ttoken := analysis.Token{\n\t\t\t\tTerm:     matchBytes,\n\t\t\t\tStart:    match[0],\n\t\t\t\tEnd:      match[1],\n\t\t\t\tPosition: i + 1,\n\t\t\t\tType:     detectTokenType(matchBytes),\n\t\t\t}\n\t\t\trv = append(rv, &token)\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc RegexpTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {\n\trval, ok := config[\"regexp\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify regexp\")\n\t}\n\tr, err := regexp.Compile(rval)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unable to build regexp tokenizer: %v\", err)\n\t}\n\treturn NewRegexpTokenizer(r), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenizer(Name, RegexpTokenizerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc detectTokenType(termBytes []byte) analysis.TokenType {\n\tif IdeographRegexp.Match(termBytes) {\n\t\treturn analysis.Ideographic\n\t}\n\t_, err := strconv.ParseFloat(string(termBytes), 64)\n\tif err == nil {\n\t\treturn analysis.Numeric\n\t}\n\treturn analysis.AlphaNumeric\n}\n"
  },
  {
    "path": "analysis/tokenizer/regexp/regexp_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage regexp\n\nimport (\n\t\"reflect\"\n\t\"regexp\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestBoundary(t *testing.T) {\n\n\twordRegex := regexp.MustCompile(`\\p{Han}|\\p{Hangul}|\\p{Hiragana}|\\p{Katakana}|\\w+`)\n\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\t[]byte(\"Hello World.\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tTerm:     []byte(\"Hello\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      11,\n\t\t\t\t\tTerm:     []byte(\"World\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"こんにちは世界\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t\tTerm:     []byte(\"に\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t\tTerm:     []byte(\"ち\"),\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t\tTerm:     []byte(\"は\"),\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t\tTerm:     []byte(\"世\"),\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t\tTerm:     []byte(\"界\"),\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"\"),\n\t\t\tanalysis.TokenStream{},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttokenizer := NewRegexpTokenizer(wordRegex)\n\t\tactual := tokenizer.Tokenize(test.input)\n\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"Expected %v, got %v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n\nfunc TestBugProducingEmptyTokens(t *testing.T) {\n\n\twordRegex := regexp.MustCompile(`[0-9a-zA-Z_]*`)\n\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\t[]byte(\"Chatha Edwards Sr.\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t\tTerm:     []byte(\"Chatha\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    7,\n\t\t\t\t\tEnd:      14,\n\t\t\t\t\tTerm:     []byte(\"Edwards\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      17,\n\t\t\t\t\tTerm:     []byte(\"Sr\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttokenizer := NewRegexpTokenizer(wordRegex)\n\t\tactual := tokenizer.Tokenize(test.input)\n\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"Expected %v, got %v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/single/single.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage single\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"single\"\n\ntype SingleTokenTokenizer struct {\n}\n\nfunc NewSingleTokenTokenizer() *SingleTokenTokenizer {\n\treturn &SingleTokenTokenizer{}\n}\n\nfunc (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream {\n\treturn analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tTerm:     input,\n\t\t\tPosition: 1,\n\t\t\tStart:    0,\n\t\t\tEnd:      len(input),\n\t\t\tType:     analysis.AlphaNumeric,\n\t\t},\n\t}\n}\n\nfunc SingleTokenTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {\n\treturn NewSingleTokenTokenizer(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenizer(Name, SingleTokenTokenizerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/single/single_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage single\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestSingleTokenTokenizer(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\t[]byte(\"Hello World\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      11,\n\t\t\t\t\tTerm:     []byte(\"Hello World\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"こんにちは世界\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t\tTerm:     []byte(\"こんにちは世界\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"แยกคำภาษาไทยก็ทำได้นะจ้ะ\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      72,\n\t\t\t\t\tTerm:     []byte(\"แยกคำภาษาไทยก็ทำได้นะจ้ะ\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttokenizer := NewSingleTokenTokenizer()\n\t\tactual := tokenizer.Tokenize(test.input)\n\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"Expected %v, got %v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/unicode/unicode.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage unicode\n\nimport (\n\t\"github.com/blevesearch/segment\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"unicode\"\n\ntype UnicodeTokenizer struct {\n}\n\nfunc NewUnicodeTokenizer() *UnicodeTokenizer {\n\treturn &UnicodeTokenizer{}\n}\n\nfunc (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream {\n\trvx := make([]analysis.TokenStream, 0, 10) // When rv gets full, append to rvx.\n\trv := make(analysis.TokenStream, 0, 1)\n\n\tta := []analysis.Token(nil)\n\ttaNext := 0\n\n\tsegmenter := segment.NewWordSegmenterDirect(input)\n\tstart := 0\n\tpos := 1\n\n\tguessRemaining := func(end int) int {\n\t\tavgSegmentLen := end / (len(rv) + 1)\n\t\tif avgSegmentLen < 1 {\n\t\t\tavgSegmentLen = 1\n\t\t}\n\n\t\tremainingLen := len(input) - end\n\n\t\treturn remainingLen / avgSegmentLen\n\t}\n\n\tfor segmenter.Segment() {\n\t\tsegmentBytes := segmenter.Bytes()\n\t\tend := start + len(segmentBytes)\n\t\tif segmenter.Type() != segment.None {\n\t\t\tif taNext >= len(ta) {\n\t\t\t\tremainingSegments := guessRemaining(end)\n\t\t\t\tif remainingSegments > 1000 {\n\t\t\t\t\tremainingSegments = 1000\n\t\t\t\t}\n\t\t\t\tif remainingSegments < 1 {\n\t\t\t\t\tremainingSegments = 1\n\t\t\t\t}\n\n\t\t\t\tta = make([]analysis.Token, remainingSegments)\n\t\t\t\ttaNext = 0\n\t\t\t}\n\n\t\t\ttoken := &ta[taNext]\n\t\t\ttaNext++\n\n\t\t\ttoken.Term = segmentBytes\n\t\t\ttoken.Start = start\n\t\t\ttoken.End = end\n\t\t\ttoken.Position = pos\n\t\t\ttoken.Type = convertType(segmenter.Type())\n\n\t\t\tif len(rv) >= cap(rv) { // When rv is full, save it into rvx.\n\t\t\t\trvx = append(rvx, rv)\n\n\t\t\t\trvCap := cap(rv) * 2\n\t\t\t\tif rvCap > 256 {\n\t\t\t\t\trvCap = 256\n\t\t\t\t}\n\n\t\t\t\trv = make(analysis.TokenStream, 0, rvCap) // Next rv cap is bigger.\n\t\t\t}\n\n\t\t\trv = append(rv, token)\n\t\t\tpos++\n\t\t}\n\t\tstart = end\n\t}\n\n\tif len(rvx) > 0 {\n\t\tn := len(rv)\n\t\tfor _, r := range rvx {\n\t\t\tn += len(r)\n\t\t}\n\t\trall := make(analysis.TokenStream, 0, n)\n\t\tfor _, r := range rvx {\n\t\t\trall = append(rall, r...)\n\t\t}\n\t\treturn append(rall, rv...)\n\t}\n\n\treturn rv\n}\n\nfunc UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {\n\treturn NewUnicodeTokenizer(), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc convertType(segmentWordType int) analysis.TokenType {\n\tswitch segmentWordType {\n\tcase segment.Ideo:\n\t\treturn analysis.Ideographic\n\tcase segment.Kana:\n\t\treturn analysis.Ideographic\n\tcase segment.Number:\n\t\treturn analysis.Numeric\n\t}\n\treturn analysis.AlphaNumeric\n}\n"
  },
  {
    "path": "analysis/tokenizer/unicode/unicode_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage unicode\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/segment\"\n)\n\nfunc TestUnicode(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\t[]byte(\"Hello World\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tTerm:     []byte(\"Hello\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      11,\n\t\t\t\t\tTerm:     []byte(\"World\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"steven's\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      8,\n\t\t\t\t\tTerm:     []byte(\"steven's\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"こんにちは世界\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tTerm:     []byte(\"こ\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    3,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t\tTerm:     []byte(\"ん\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      9,\n\t\t\t\t\tTerm:     []byte(\"に\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    9,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t\tTerm:     []byte(\"ち\"),\n\t\t\t\t\tPosition: 4,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    12,\n\t\t\t\t\tEnd:      15,\n\t\t\t\t\tTerm:     []byte(\"は\"),\n\t\t\t\t\tPosition: 5,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    15,\n\t\t\t\t\tEnd:      18,\n\t\t\t\t\tTerm:     []byte(\"世\"),\n\t\t\t\t\tPosition: 6,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    18,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t\tTerm:     []byte(\"界\"),\n\t\t\t\t\tPosition: 7,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"age 25\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tTerm:     []byte(\"age\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    4,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t\tTerm:     []byte(\"25\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.Numeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"カ\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tTerm:     []byte(\"カ\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.Ideographic,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttokenizer := NewUnicodeTokenizer()\n\t\tactual := tokenizer.Tokenize(test.input)\n\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"Expected %v, got %v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n\nvar sampleLargeInput = []byte(`There are three characteristics of liquids which are relevant to the discussion of a BLEVE:\nIf a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed).\nThe boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example.\nWhen a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did.\nTypically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures.\nIf the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.`)\n\nfunc BenchmarkTokenizeEnglishText(b *testing.B) {\n\n\ttokenizer := NewUnicodeTokenizer()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttokenizer.Tokenize(sampleLargeInput)\n\t}\n\n}\n\nfunc TestConvertType(t *testing.T) {\n\ttests := []struct {\n\t\tin  int\n\t\tout analysis.TokenType\n\t}{\n\t\t{\n\t\t\tsegment.Ideo, analysis.Ideographic,\n\t\t},\n\t\t{\n\t\t\tsegment.Kana, analysis.Ideographic,\n\t\t},\n\t\t{\n\t\t\tsegment.Number, analysis.Numeric,\n\t\t},\n\t\t{\n\t\t\tsegment.Letter, analysis.AlphaNumeric,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := convertType(test.in)\n\t\tif actual != test.out {\n\t\t\tt.Errorf(\"expected %d, got %d for %d\", test.out, actual, test.in)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/web/web.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage web\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/exception\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"web\"\n\nvar email = `(?:[a-z0-9!#$%&'*+/=?^_` + \"`\" + `{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_` + \"`\" + `{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])`\nvar url = `(?i)\\b((?:[a-z][\\w-]+:(?:/{1,3}|[a-z0-9%])|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s` + \"`\" + `!()\\[\\]{};:'\".,<>?«»“”‘’]))`\nvar twitterHandle = `@([a-zA-Z0-9_]){1,15}`\nvar twitterHashtag = `#([a-zA-Z0-9_])+`\nvar exceptions = []string{email, url, twitterHandle, twitterHashtag}\n\nvar exceptionsRegexp = regexp.MustCompile(strings.Join(exceptions, \"|\"))\n\nfunc TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {\n\tremainingTokenizer, err := cache.TokenizerNamed(unicode.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn exception.NewExceptionsTokenizer(exceptionsRegexp, remainingTokenizer), nil\n}\n\nfunc init() {\n\terr := registry.RegisterTokenizer(Name, TokenizerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/web/web_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage web\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nfunc TestWeb(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\t[]byte(\"Hello info@blevesearch.com\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tTerm:     []byte(\"Hello\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      26,\n\t\t\t\t\tTerm:     []byte(\"info@blevesearch.com\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"That http://blevesearch.com\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t\tTerm:     []byte(\"That\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    5,\n\t\t\t\t\tEnd:      27,\n\t\t\t\t\tTerm:     []byte(\"http://blevesearch.com\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"Hey @blevesearch\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      3,\n\t\t\t\t\tTerm:     []byte(\"Hey\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    4,\n\t\t\t\t\tEnd:      16,\n\t\t\t\t\tTerm:     []byte(\"@blevesearch\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"This #bleve\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t\tTerm:     []byte(\"This\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    5,\n\t\t\t\t\tEnd:      11,\n\t\t\t\t\tTerm:     []byte(\"#bleve\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"What about @blevesearch?\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      4,\n\t\t\t\t\tTerm:     []byte(\"What\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    5,\n\t\t\t\t\tEnd:      10,\n\t\t\t\t\tTerm:     []byte(\"about\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    11,\n\t\t\t\t\tEnd:      23,\n\t\t\t\t\tTerm:     []byte(\"@blevesearch\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tcache := registry.NewCache()\n\ttokenizer, err := cache.TokenizerNamed(Name)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, test := range tests {\n\n\t\tactual := tokenizer.Tokenize(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"Expected %v, got %v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/whitespace/whitespace.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage whitespace\n\nimport (\n\t\"unicode\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/character\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"whitespace\"\n\nfunc TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {\n\treturn character.NewCharacterTokenizer(notSpace), nil\n}\n\nfunc notSpace(r rune) bool {\n\treturn !unicode.IsSpace(r)\n}\n\nfunc init() {\n\terr := registry.RegisterTokenizer(Name, TokenizerConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenizer/whitespace/whitespace_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage whitespace\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/character\"\n)\n\nfunc TestBoundary(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput analysis.TokenStream\n\t}{\n\t\t{\n\t\t\t[]byte(\"Hello World.\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      5,\n\t\t\t\t\tTerm:     []byte(\"Hello\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStart:    6,\n\t\t\t\t\tEnd:      12,\n\t\t\t\t\tTerm:     []byte(\"World.\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"こんにちは世界\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      21,\n\t\t\t\t\tTerm:     []byte(\"こんにちは世界\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"\"),\n\t\t\tanalysis.TokenStream{},\n\t\t},\n\t\t{\n\t\t\t[]byte(\"abc界\"),\n\t\t\tanalysis.TokenStream{\n\t\t\t\t{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      6,\n\t\t\t\t\tTerm:     []byte(\"abc界\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttokenizer := character.NewCharacterTokenizer(notSpace)\n\t\tactual := tokenizer.Tokenize(test.input)\n\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"Expected %v, got %v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n\nvar sampleLargeInput = []byte(`There are three characteristics of liquids which are relevant to the discussion of a BLEVE:\nIf a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed).\nThe boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example.\nWhen a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did.\nTypically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures.\nIf the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.`)\n\nfunc BenchmarkTokenizeEnglishText(b *testing.B) {\n\n\ttokenizer := character.NewCharacterTokenizer(notSpace)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttokenizer.Tokenize(sampleLargeInput)\n\t}\n\n}\n"
  },
  {
    "path": "analysis/tokenmap/custom.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// package token_map implements a generic TokenMap, often used in conjunction\n// with filters to remove or process specific tokens.\n//\n// Its constructor takes the following arguments:\n//\n// \"filename\" (string): the path of a file listing the tokens. Each line may\n// contain one or more whitespace separated tokens, followed by an optional\n// comment starting with a \"#\" or \"|\" character.\n//\n// \"tokens\" ([]interface{}): if \"filename\" is not specified, tokens can be\n// passed directly as a sequence of strings wrapped in a []interface{}.\npackage tokenmap\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\nconst Name = \"custom\"\n\nfunc GenericTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) {\n\trv := analysis.NewTokenMap()\n\n\t// first: try to load by filename\n\tfilename, ok := config[\"filename\"].(string)\n\tif ok {\n\t\terr := rv.LoadFile(filename)\n\t\treturn rv, err\n\t}\n\t// next: look for an inline word list\n\ttokens, ok := config[\"tokens\"].([]interface{})\n\tif ok {\n\t\tfor _, token := range tokens {\n\t\t\ttokenStr, ok := token.(string)\n\t\t\tif ok {\n\t\t\t\trv.AddToken(tokenStr)\n\t\t\t}\n\t\t}\n\t\treturn rv, nil\n\t}\n\treturn nil, fmt.Errorf(\"must specify filename or list of tokens for token map\")\n}\n\nfunc init() {\n\terr := registry.RegisterTokenMap(Name, GenericTokenMapConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "analysis/tokenmap.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage analysis\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n)\n\ntype TokenMap map[string]bool\n\nfunc NewTokenMap() TokenMap {\n\treturn make(TokenMap, 0)\n}\n\n// LoadFile reads in a list of tokens from a text file,\n// one per line.\n// Comments are supported using `#` or `|`\nfunc (t TokenMap) LoadFile(filename string) error {\n\tdata, err := os.ReadFile(filename)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn t.LoadBytes(data)\n}\n\n// LoadBytes reads in a list of tokens from memory,\n// one per line.\n// Comments are supported using `#` or `|`\nfunc (t TokenMap) LoadBytes(data []byte) error {\n\tbytesReader := bytes.NewReader(data)\n\tbufioReader := bufio.NewReader(bytesReader)\n\tline, err := bufioReader.ReadString('\\n')\n\tfor err == nil {\n\t\tt.LoadLine(line)\n\t\tline, err = bufioReader.ReadString('\\n')\n\t}\n\t// if the err was EOF we still need to process the last value\n\tif err == io.EOF {\n\t\tt.LoadLine(line)\n\t\treturn nil\n\t}\n\treturn err\n}\n\nfunc (t TokenMap) LoadLine(line string) {\n\t// find the start of a comment, if any\n\tstartComment := strings.IndexAny(line, \"#|\")\n\tif startComment >= 0 {\n\t\tline = line[:startComment]\n\t}\n\n\ttokens := strings.Fields(line)\n\tfor _, token := range tokens {\n\t\tt.AddToken(token)\n\t}\n}\n\nfunc (t TokenMap) AddToken(token string) {\n\tt[token] = true\n}\n"
  },
  {
    "path": "analysis/tokenmap_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage analysis\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestTokenMapLoadFile(t *testing.T) {\n\ttokenMap := NewTokenMap()\n\terr := tokenMap.LoadFile(\"test_words.txt\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedTokens := NewTokenMap()\n\texpectedTokens.AddToken(\"marty\")\n\texpectedTokens.AddToken(\"steve\")\n\texpectedTokens.AddToken(\"dustin\")\n\texpectedTokens.AddToken(\"siri\")\n\texpectedTokens.AddToken(\"multiple\")\n\texpectedTokens.AddToken(\"words\")\n\texpectedTokens.AddToken(\"with\")\n\texpectedTokens.AddToken(\"different\")\n\texpectedTokens.AddToken(\"whitespace\")\n\n\tif !reflect.DeepEqual(tokenMap, expectedTokens) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedTokens, tokenMap)\n\t}\n}\n"
  },
  {
    "path": "analysis/type.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage analysis\n\nimport (\n\t\"fmt\"\n\t\"time\"\n)\n\ntype CharFilter interface {\n\tFilter([]byte) []byte\n}\n\ntype TokenType int\n\nconst (\n\tAlphaNumeric TokenType = iota\n\tIdeographic\n\tNumeric\n\tDateTime\n\tShingle\n\tSingle\n\tDouble\n\tBoolean\n\tIP\n)\n\n// Token represents one occurrence of a term at a particular location in a\n// field.\ntype Token struct {\n\t// Start specifies the byte offset of the beginning of the term in the\n\t// field.\n\tStart int `json:\"start\"`\n\n\t// End specifies the byte offset of the end of the term in the field.\n\tEnd  int    `json:\"end\"`\n\tTerm []byte `json:\"term\"`\n\n\t// Position specifies the 1-based index of the token in the sequence of\n\t// occurrences of its term in the field.\n\tPosition int       `json:\"position\"`\n\tType     TokenType `json:\"type\"`\n\tKeyWord  bool      `json:\"keyword\"`\n}\n\nfunc (t *Token) String() string {\n\treturn fmt.Sprintf(\"Start: %d  End: %d  Position: %d  Token: %s  Type: %d\", t.Start, t.End, t.Position, string(t.Term), t.Type)\n}\n\ntype TokenStream []*Token\n\n// A Tokenizer splits an input string into tokens, the usual behaviour being to\n// map words to tokens.\ntype Tokenizer interface {\n\tTokenize([]byte) TokenStream\n}\n\n// A TokenFilter adds, transforms or removes tokens from a token stream.\ntype TokenFilter interface {\n\tFilter(TokenStream) TokenStream\n}\n\ntype Analyzer interface {\n\tAnalyze([]byte) TokenStream\n}\n\ntype DefaultAnalyzer struct {\n\tCharFilters  []CharFilter\n\tTokenizer    Tokenizer\n\tTokenFilters []TokenFilter\n}\n\nfunc (a *DefaultAnalyzer) Analyze(input []byte) TokenStream {\n\tif a.CharFilters != nil {\n\t\tfor _, cf := range a.CharFilters {\n\t\t\tinput = cf.Filter(input)\n\t\t}\n\t}\n\ttokens := a.Tokenizer.Tokenize(input)\n\tif a.TokenFilters != nil {\n\t\tfor _, tf := range a.TokenFilters {\n\t\t\ttokens = tf.Filter(tokens)\n\t\t}\n\t}\n\treturn tokens\n}\n\nvar ErrInvalidDateTime = fmt.Errorf(\"unable to parse datetime with any of the layouts\")\n\nvar ErrInvalidTimestampString = fmt.Errorf(\"unable to parse timestamp string\")\nvar ErrInvalidTimestampRange = fmt.Errorf(\"timestamp out of range\")\n\ntype DateTimeParser interface {\n\tParseDateTime(string) (time.Time, string, error)\n}\n\nconst SynonymSourceType = \"synonym\"\n\ntype SynonymSourceVisitor func(name string, item SynonymSource) error\n\ntype SynonymSource interface {\n\tAnalyzer() string\n\tCollection() string\n}\n\ntype ByteArrayConverter interface {\n\tConvert([]byte) (interface{}, error)\n}\n"
  },
  {
    "path": "analysis/util.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage analysis\n\nimport (\n\t\"bytes\"\n\t\"unicode/utf8\"\n)\n\nfunc DeleteRune(in []rune, pos int) []rune {\n\tif pos >= len(in) {\n\t\treturn in\n\t}\n\tcopy(in[pos:], in[pos+1:])\n\treturn in[:len(in)-1]\n}\n\nfunc InsertRune(in []rune, pos int, r rune) []rune {\n\t// create a new slice 1 rune larger\n\trv := make([]rune, len(in)+1)\n\t// copy the characters before the insert pos\n\tcopy(rv[0:pos], in[0:pos])\n\t// set the inserted rune\n\trv[pos] = r\n\t// copy the characters after the insert pos\n\tcopy(rv[pos+1:], in[pos:])\n\treturn rv\n}\n\n// BuildTermFromRunesOptimistic will build a term from the provided runes\n// AND optimistically attempt to encode into the provided buffer\n// if at any point it appears the buffer is too small, a new buffer is\n// allocated and that is used instead\n// this should be used in cases where frequently the new term is the same\n// length or shorter than the original term (in number of bytes)\nfunc BuildTermFromRunesOptimistic(buf []byte, runes []rune) []byte {\n\trv := buf\n\tused := 0\n\tfor _, r := range runes {\n\t\tnextLen := utf8.RuneLen(r)\n\t\tif used+nextLen > len(rv) {\n\t\t\t// alloc new buf\n\t\t\tbuf = make([]byte, len(runes)*utf8.UTFMax)\n\t\t\t// copy work we've already done\n\t\t\tcopy(buf, rv[:used])\n\t\t\trv = buf\n\t\t}\n\t\twritten := utf8.EncodeRune(rv[used:], r)\n\t\tused += written\n\t}\n\treturn rv[:used]\n}\n\nfunc BuildTermFromRunes(runes []rune) []byte {\n\treturn BuildTermFromRunesOptimistic(make([]byte, len(runes)*utf8.UTFMax), runes)\n}\n\nfunc TruncateRunes(input []byte, num int) []byte {\n\trunes := bytes.Runes(input)\n\trunes = runes[:len(runes)-num]\n\tout := BuildTermFromRunes(runes)\n\treturn out\n}\n\nfunc RunesEndsWith(input []rune, suffix string) bool {\n\tinputLen := len(input)\n\tsuffixRunes := []rune(suffix)\n\tsuffixLen := len(suffixRunes)\n\tif suffixLen > inputLen {\n\t\treturn false\n\t}\n\n\tfor i := suffixLen - 1; i >= 0; i-- {\n\t\tif input[inputLen-(suffixLen-i)] != suffixRunes[i] {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n"
  },
  {
    "path": "analysis/util_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage analysis\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestDeleteRune(t *testing.T) {\n\ttests := []struct {\n\t\tin     []rune\n\t\tdelPos int\n\t\tout    []rune\n\t}{\n\t\t{\n\t\t\tin:     []rune{'a', 'b', 'c'},\n\t\t\tdelPos: 1,\n\t\t\tout:    []rune{'a', 'c'},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := DeleteRune(test.in, test.delPos)\n\t\tif !reflect.DeepEqual(actual, test.out) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.out, actual)\n\t\t}\n\t}\n}\n\nfunc TestInsertRune(t *testing.T) {\n\ttests := []struct {\n\t\tin      []rune\n\t\tinsPos  int\n\t\tinsRune rune\n\t\tout     []rune\n\t}{\n\t\t{\n\t\t\tin:      []rune{'a', 'b', 'c'},\n\t\t\tinsPos:  1,\n\t\t\tinsRune: 'x',\n\t\t\tout:     []rune{'a', 'x', 'b', 'c'},\n\t\t},\n\t\t{\n\t\t\tin:      []rune{'a', 'b', 'c'},\n\t\t\tinsPos:  0,\n\t\t\tinsRune: 'x',\n\t\t\tout:     []rune{'x', 'a', 'b', 'c'},\n\t\t},\n\t\t{\n\t\t\tin:      []rune{'a', 'b', 'c'},\n\t\t\tinsPos:  3,\n\t\t\tinsRune: 'x',\n\t\t\tout:     []rune{'a', 'b', 'c', 'x'},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := InsertRune(test.in, test.insPos, test.insRune)\n\t\tif !reflect.DeepEqual(actual, test.out) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.out, actual)\n\t\t}\n\t}\n}\n\nfunc TestBuildTermFromRunes(t *testing.T) {\n\ttests := []struct {\n\t\tin []rune\n\t}{\n\t\t{\n\t\t\tin: []rune{'a', 'b', 'c'},\n\t\t},\n\t\t{\n\t\t\tin: []rune{'こ', 'ん', 'に', 'ち', 'は', '世', '界'},\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tout := BuildTermFromRunes(test.in)\n\t\tback := []rune(string(out))\n\t\tif !reflect.DeepEqual(back, test.in) {\n\t\t\tt.Errorf(\"expected %v to convert back to %v\", out, test.in)\n\t\t}\n\t}\n}\n\nfunc TestBuildTermFromRunesOptimistic(t *testing.T) {\n\ttests := []struct {\n\t\tbuf []byte\n\t\tin  []rune\n\t}{\n\t\t{\n\t\t\tbuf: []byte(\"abc\"),\n\t\t\tin:  []rune{'a', 'b', 'c'},\n\t\t},\n\t\t{\n\t\t\tbuf: []byte(\"こんにちは世界\"),\n\t\t\tin:  []rune{'こ', 'ん', 'に', 'ち', 'は', '世', '界'},\n\t\t},\n\t\t// same, but don't give enough buffer\n\t\t{\n\t\t\tbuf: []byte(\"ab\"),\n\t\t\tin:  []rune{'a', 'b', 'c'},\n\t\t},\n\t\t{\n\t\t\tbuf: []byte(\"こ\"),\n\t\t\tin:  []rune{'こ', 'ん', 'に', 'ち', 'は', '世', '界'},\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tout := BuildTermFromRunesOptimistic(test.buf, test.in)\n\t\tback := []rune(string(out))\n\t\tif !reflect.DeepEqual(back, test.in) {\n\t\t\tt.Errorf(\"expected %v to convert back to %v\", out, test.in)\n\t\t}\n\t}\n}\n\nfunc BenchmarkBuildTermFromRunes(b *testing.B) {\n\tinput := [][]rune{\n\t\t{'a', 'b', 'c'},\n\t\t{'こ', 'ん', 'に', 'ち', 'は', '世', '界'},\n\t}\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, i := range input {\n\t\t\tBuildTermFromRunes(i)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "builder.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype builderImpl struct {\n\tb index.IndexBuilder\n\tm mapping.IndexMapping\n}\n\nfunc (b *builderImpl) Index(id string, data interface{}) error {\n\tif id == \"\" {\n\t\treturn ErrorEmptyID\n\t}\n\n\tdoc := document.NewDocument(id)\n\terr := b.m.MapDocument(doc, data)\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = b.b.Index(doc)\n\treturn err\n}\n\nfunc (b *builderImpl) Close() error {\n\treturn b.b.Close()\n}\n\nfunc newBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) {\n\tif path == \"\" {\n\t\treturn nil, fmt.Errorf(\"builder requires path\")\n\t}\n\n\terr := mapping.Validate()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif config == nil {\n\t\tconfig = map[string]interface{}{}\n\t}\n\n\t// the builder does not have an API to interact with internal storage\n\t// however we can pass k/v pairs through the config\n\tmappingBytes, err := util.MarshalJSON(mapping)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tconfig[\"internal\"] = map[string][]byte{\n\t\tstring(util.MappingInternalKey): mappingBytes,\n\t}\n\n\t// do not use real config, as these are options for the builder,\n\t// not the resulting index\n\tmeta := newIndexMeta(scorch.Name, scorch.Name, map[string]interface{}{})\n\terr = meta.Save(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconfig[\"path\"] = indexStorePath(path)\n\n\tb, err := scorch.NewBuilder(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := &builderImpl{\n\t\tb: b,\n\t\tm: mapping,\n\t}\n\n\treturn rv, nil\n}\n"
  },
  {
    "path": "builder_test.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestBuilder(t *testing.T) {\n\ttmpDir, err := os.MkdirTemp(\"\", \"bleve-scorch-builder-test\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = os.RemoveAll(tmpDir)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error cleaning up test index\")\n\t\t}\n\t}()\n\n\tconf := map[string]interface{}{\n\t\t\"batchSize\": 2,\n\t\t\"mergeMax\":  2,\n\t}\n\tb, err := NewBuilder(tmpDir, NewIndexMapping(), conf)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tdoc := map[string]interface{}{\n\t\t\t\"name\": \"hello\",\n\t\t}\n\t\terr = b.Index(fmt.Sprintf(\"%d\", i), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = b.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err := Open(tmpDir)\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"error closing index: %v\", err)\n\t\t}\n\t}()\n\n\tdocCount, err := idx.DocCount()\n\tif err != nil {\n\t\tt.Errorf(\"error checking doc count: %v\", err)\n\t}\n\tif docCount != 10 {\n\t\tt.Errorf(\"expected doc count to be 10, got %d\", docCount)\n\t}\n\n\tq := NewTermQuery(\"hello\")\n\tq.SetField(\"name\")\n\treq := NewSearchRequest(q)\n\tres, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Errorf(\"error searching index: %v\", err)\n\t}\n\tif res.Total != 10 {\n\t\tt.Errorf(\"expected 10 search hits, got %d\", res.Total)\n\t}\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/bulk.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"bufio\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"os\"\n\n\t\"github.com/spf13/cobra\"\n)\n\nvar batchSize int\n\n// bulkCmd represents the bulk command\nvar bulkCmd = &cobra.Command{\n\tUse:   \"bulk [index path] [data paths ...]\",\n\tShort: \"bulk loads from newline delimited JSON files\",\n\tLong:  `The bulk command will perform batch loading of documents in one or more newline delimited JSON files.`,\n\tAnnotations: map[string]string{\n\t\tcanMutateBleveIndex: \"true\",\n\t},\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\tif len(args) < 2 {\n\t\t\treturn fmt.Errorf(\"must specify at least one path\")\n\t\t}\n\n\t\ti := 0\n\t\tbatch := idx.NewBatch()\n\n\t\tfor _, file := range args[1:] {\n\n\t\t\tfile, err := os.Open(file)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tfmt.Printf(\"Indexing: %s\\n\", file.Name())\n\t\t\tr := bufio.NewReader(file)\n\n\t\t\tfor {\n\t\t\t\tif i%batchSize == 0 {\n\t\t\t\t\tfmt.Printf(\"Indexing batch (%d docs)...\\n\", i)\n\t\t\t\t\terr := idx.Batch(batch)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tbatch = idx.NewBatch()\n\t\t\t\t}\n\n\t\t\t\tb, _ := r.ReadBytes('\\n')\n\t\t\t\tif len(b) == 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\tvar doc interface{} = b\n\t\t\t\tvar err error\n\t\t\t\tif parseJSON {\n\t\t\t\t\terr = json.Unmarshal(b, &doc)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn fmt.Errorf(\"error parsing JSON: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tdocID := randomString(5)\n\t\t\t\terr = batch.Index(docID, doc)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\ti++\n\t\t\t}\n\t\t\terr = idx.Batch(batch)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\terr = file.Close()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t}\n\t\treturn nil\n\t},\n}\n\nvar letters = []rune(\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\")\n\nfunc randomString(n int) string {\n\tb := make([]rune, n)\n\tfor i := range b {\n\t\tb[i] = letters[rand.Intn(len(letters))]\n\t}\n\treturn string(b)\n}\n\nfunc init() {\n\tRootCmd.AddCommand(bulkCmd)\n\n\tbulkCmd.Flags().IntVarP(&batchSize, \"batch\", \"b\", 1000, \"Batch size for loading.\")\n\tbulkCmd.Flags().BoolVarP(&parseJSON, \"json\", \"j\", true, \"Parse the contents as JSON.\")\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/check.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\n\t\"github.com/blevesearch/bleve/v2\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar checkFieldName string\nvar checkCount int\n\n// checkCmd represents the check command\nvar checkCmd = &cobra.Command{\n\tUse:   \"check [index path]\",\n\tShort: \"checks the contents of the index\",\n\tLong:  `The check command will perform consistency checks on the index.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\n\t\tvar fieldNames []string\n\t\tvar err error\n\t\tif checkFieldName == \"\" {\n\t\t\tfieldNames, err = idx.Fields()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tfieldNames = []string{checkFieldName}\n\t\t}\n\t\tfmt.Printf(\"checking fields: %v\\n\", fieldNames)\n\n\t\ttotalProblems := 0\n\t\tfor _, fieldName := range fieldNames {\n\t\t\tfmt.Printf(\"checking field: '%s'\\n\", fieldName)\n\t\t\tproblems, err := checkField(idx, fieldName)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatal(err)\n\t\t\t}\n\t\t\ttotalProblems += problems\n\t\t}\n\n\t\tif totalProblems != 0 {\n\t\t\treturn fmt.Errorf(\"found %d total problems\\n\", totalProblems)\n\t\t}\n\n\t\treturn nil\n\t},\n}\n\nfunc checkField(index bleve.Index, fieldName string) (int, error) {\n\ttermDictionary, err := getDictionary(index, fieldName)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tfmt.Printf(\"field contains %d terms\\n\", len(termDictionary))\n\n\tnumTested := 0\n\tnumProblems := 0\n\tfor term, count := range termDictionary {\n\t\tfmt.Printf(\"checked %d terms\\r\", numTested)\n\t\tif checkCount > 0 && numTested >= checkCount {\n\t\t\tbreak\n\t\t}\n\n\t\ttq := bleve.NewTermQuery(term)\n\t\ttq.SetField(fieldName)\n\t\treq := bleve.NewSearchRequest(tq)\n\t\treq.Size = 0\n\t\tres, err := index.Search(req)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\tif res.Total != count {\n\t\t\tfmt.Printf(\"unexpected mismatch for term '%s', dictionary %d, search hits %d\\n\", term, count, res.Total)\n\t\t\tnumProblems++\n\t\t}\n\n\t\tnumTested++\n\t}\n\tfmt.Printf(\"done checking %d terms, found %d problems\\n\", numTested, numProblems)\n\n\treturn numProblems, nil\n}\n\nfunc getDictionary(index bleve.Index, field string) (map[string]uint64, error) {\n\trv := make(map[string]uint64)\n\ti, err := index.Advanced()\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tr, err := i.Reader()\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\td, err := r.FieldDict(field)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tde, err := d.Next()\n\tfor err == nil && de != nil {\n\t\trv[de.Term] = de.Count\n\t\tde, err = d.Next()\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc init() {\n\tRootCmd.AddCommand(checkCmd)\n\tcheckCmd.Flags().StringVarP(&checkFieldName, \"field\", \"f\", \"\", \"Restrict check to the specified field name.\")\n\tcheckCmd.Flags().IntVarP(&checkCount, \"count\", \"c\", 100, \"Check this many terms.\")\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/count.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/spf13/cobra\"\n)\n\n// countCmd represents the count command\nvar countCmd = &cobra.Command{\n\tUse:   \"count [index path]\",\n\tShort: \"counts the number documents in the index\",\n\tLong:  `The count command will count the number of documents in the index.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\tcount, err := idx.DocCount()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error counting docs in index: %v\", err)\n\t\t}\n\t\tfmt.Printf(\"%d\\n\", count)\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(countCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/create.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/blevesearch/bleve/v2\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar mappingPath, indexType, storeType string\n\n// createCmd represents the create command\nvar createCmd = &cobra.Command{\n\tUse:   \"create [index path]\",\n\tShort: \"creates a new index\",\n\tLong:  `The create command will create a new empty index.`,\n\tAnnotations: map[string]string{\n\t\tcanMutateBleveIndex: \"true\",\n\t},\n\tPersistentPreRunE: func(cmd *cobra.Command, args []string) error {\n\t\t// override RootCmd version which opens existing index\n\t\tif len(args) < 1 {\n\t\t\treturn fmt.Errorf(\"must specify path to index\")\n\t\t}\n\t\treturn nil\n\t},\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\tvar mapping mapping.IndexMapping\n\t\tvar err error\n\t\tmapping, err = buildMapping()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error building mapping: %v\", err)\n\t\t}\n\t\tidx, err = bleve.NewUsing(args[0], mapping, indexType, storeType, nil)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error creating index: %v\", err)\n\t\t}\n\t\t// the inherited Post action will close the index\n\t\treturn nil\n\t},\n}\n\nfunc buildMapping() (mapping.IndexMapping, error) {\n\tmapping := mapping.NewIndexMapping()\n\tif mappingPath != \"\" {\n\t\tmappingBytes, err := os.ReadFile(mappingPath)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\terr = json.Unmarshal(mappingBytes, &mapping)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn mapping, nil\n}\n\nfunc init() {\n\tRootCmd.AddCommand(createCmd)\n\n\tcreateCmd.Flags().StringVarP(&mappingPath, \"mapping\", \"m\", \"\", \"Path to a file containing a JSON representation of an index mapping to use.\")\n\tcreateCmd.Flags().StringVarP(&storeType, \"store\", \"s\", bleve.Config.DefaultKVStore, \"The bleve storage type to use.\")\n\tcreateCmd.Flags().StringVarP(&indexType, \"index\", \"i\", bleve.Config.DefaultIndexType, \"The bleve index type to use.\")\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/dictionary.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/spf13/cobra\"\n)\n\n// dictionaryCmd represents the dictionary command\nvar dictionaryCmd = &cobra.Command{\n\tUse:   \"dictionary [index path] [field name]\",\n\tShort: \"prints the term dictionary for the specified field in the index\",\n\tLong:  `The dictionary command will print the term dictionary for the specified field.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\tif len(args) < 2 {\n\t\t\treturn fmt.Errorf(\"must specify field\")\n\t\t}\n\t\ti, err := idx.Advanced()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index: %v\", err)\n\t\t}\n\t\tr, err := i.Reader()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index reader: %v\", err)\n\t\t}\n\t\td, err := r.FieldDict(args[1])\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting field dictionary: %v\", err)\n\t\t}\n\n\t\tde, err := d.Next()\n\t\tfor err == nil && de != nil {\n\t\t\tfmt.Printf(\"%s - %d\\n\", de.Term, de.Count)\n\t\t\tde, err = d.Next()\n\t\t}\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error iterating dictionary: %v\", err)\n\t\t}\n\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(dictionaryCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/dump.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar docID string\n\n// dumpCmd represents the dump command\nvar dumpCmd = &cobra.Command{\n\tUse:   \"dump [index path]\",\n\tShort: \"dumps the contents of the index\",\n\tLong:  `The dump command will dump (possibly a section of) the index.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\ti, err := idx.Advanced()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index: %v\", err)\n\t\t}\n\t\tr, err := i.Reader()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index reader: %v\", err)\n\t\t}\n\t\tupsideDownReader, ok := r.(*upsidedown.IndexReader)\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"dump is only supported by index type upsidedown\")\n\t\t}\n\n\t\tdumpChan := upsideDownReader.DumpAll()\n\t\tfor rowOrErr := range dumpChan {\n\t\t\tswitch rowOrErr := rowOrErr.(type) {\n\t\t\tcase error:\n\t\t\t\treturn fmt.Errorf(\"error dumping: %v\", rowOrErr)\n\t\t\tcase upsidedown.UpsideDownCouchRow:\n\t\t\t\tfmt.Printf(\"%v\\n\", rowOrErr)\n\t\t\t\tfmt.Printf(\"Key:   % -100x\\nValue: % -100x\\n\\n\", rowOrErr.Key(), rowOrErr.Value())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(dumpCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/dumpDoc.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/spf13/cobra\"\n)\n\n// dumpDocCmd represents the dumpDoc command\nvar dumpDocCmd = &cobra.Command{\n\tUse:   \"doc [index path] [doc id]\",\n\tShort: \"dump only the rows relating to this doc ID\",\n\tLong:  `The doc sub-command of dump will only dump the rows relating to this doc ID.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\tif len(args) < 2 {\n\t\t\treturn fmt.Errorf(\"must specify docid\")\n\t\t}\n\n\t\ti, err := idx.Advanced()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index: %v\", err)\n\t\t}\n\t\tr, err := i.Reader()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index reader: %v\", err)\n\t\t}\n\t\tupsideDownReader, ok := r.(*upsidedown.IndexReader)\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"dump doc is only supported by index type upsidedown\")\n\t\t}\n\n\t\tdumpChan := upsideDownReader.DumpDoc(args[1])\n\t\tfor rowOrErr := range dumpChan {\n\t\t\tswitch rowOrErr := rowOrErr.(type) {\n\t\t\tcase error:\n\t\t\t\treturn fmt.Errorf(\"error dumping: %v\", rowOrErr)\n\t\t\tcase upsidedown.UpsideDownCouchRow:\n\t\t\t\tfmt.Printf(\"%v\\n\", rowOrErr)\n\t\t\t\tfmt.Printf(\"Key:   % -100x\\nValue: % -100x\\n\\n\", rowOrErr.Key(), rowOrErr.Value())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tdumpCmd.AddCommand(dumpDocCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/dumpFields.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/spf13/cobra\"\n)\n\n// dumpFieldsCmd represents the dumpFields command\nvar dumpFieldsCmd = &cobra.Command{\n\tUse:   \"fields [index path]\",\n\tShort: \"dump only the field rows\",\n\tLong:  `The fields sub-command of dump will only dump the field rows.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\ti, err := idx.Advanced()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index: %v\", err)\n\t\t}\n\t\tr, err := i.Reader()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index reader: %v\", err)\n\t\t}\n\t\tupsideDownReader, ok := r.(*upsidedown.IndexReader)\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"dump fields is only supported by index type upsidedown\")\n\t\t}\n\n\t\tdumpChan := upsideDownReader.DumpFields()\n\t\tfor rowOrErr := range dumpChan {\n\t\t\tswitch rowOrErr := rowOrErr.(type) {\n\t\t\tcase error:\n\t\t\t\treturn fmt.Errorf(\"error dumping: %v\", rowOrErr)\n\t\t\tcase upsidedown.UpsideDownCouchRow:\n\t\t\t\tfmt.Printf(\"%v\\n\", rowOrErr)\n\t\t\t\tfmt.Printf(\"Key:   % -100x\\nValue: % -100x\\n\\n\", rowOrErr.Key(), rowOrErr.Value())\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tdumpCmd.AddCommand(dumpFieldsCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/fields.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/spf13/cobra\"\n)\n\n// fieldsCmd represents the fields command\nvar fieldsCmd = &cobra.Command{\n\tUse:   \"fields [index path]\",\n\tShort: \"lists the fields in this index\",\n\tLong:  `The fields command will list the fields used in this index.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\ti, err := idx.Advanced()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index: %v\", err)\n\t\t}\n\t\tr, err := i.Reader()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting index reader: %v\", err)\n\t\t}\n\t\tfields, err := r.Fields()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error getting fields: %v\", err)\n\t\t}\n\t\tfor i, field := range fields {\n\t\t\tfmt.Printf(\"%d - %s\\n\", i, field)\n\t\t}\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(fieldsCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/index.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/spf13/cobra\"\n)\n\nvar keepDir, keepExt, parseJSON bool\n\n// indexCmd represents the index command\nvar indexCmd = &cobra.Command{\n\tUse:   \"index [index path] [data paths ...]\",\n\tShort: \"adds the files to the index\",\n\tLong:  `The index command adds the specified files to the index.`,\n\tAnnotations: map[string]string{\n\t\tcanMutateBleveIndex: \"true\",\n\t},\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\tif len(args) < 2 {\n\t\t\treturn fmt.Errorf(\"must specify at least one path\")\n\t\t}\n\t\tfor file := range handleArgs(args[1:]) {\n\t\t\tvar doc interface{}\n\t\t\t// index the files\n\t\t\tdocID := file.filename\n\t\t\tif !keepDir {\n\t\t\t\t_, docID = filepath.Split(docID)\n\t\t\t}\n\t\t\tif !keepExt {\n\t\t\t\text := filepath.Ext(docID)\n\t\t\t\tdocID = docID[0 : len(docID)-len(ext)]\n\t\t\t}\n\t\t\tdoc = file.contents\n\t\t\tvar err error\n\t\t\tif parseJSON {\n\t\t\t\terr = json.Unmarshal(file.contents, &doc)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"error parsing JSON: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfmt.Printf(\"Indexing: %s\\n\", docID)\n\t\t\terr = idx.Index(docID, doc)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"error indexing: %v\", err)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t},\n}\n\ntype file struct {\n\tfilename string\n\tcontents []byte\n}\n\nfunc handleArgs(args []string) chan file {\n\trv := make(chan file)\n\tgo getAllFiles(args, rv)\n\treturn rv\n}\n\nfunc getAllFiles(args []string, rv chan file) {\n\tfor _, arg := range args {\n\t\targ = filepath.Clean(arg)\n\t\terr := filepath.Walk(arg, func(path string, finfo os.FileInfo, err error) error {\n\t\t\tif err != nil {\n\t\t\t\tlog.Print(err)\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif finfo.IsDir() {\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tbytes, err := os.ReadFile(path)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatal(err)\n\t\t\t}\n\t\t\trv <- file{\n\t\t\t\tfilename: filepath.Base(path),\n\t\t\t\tcontents: bytes,\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t}\n\tclose(rv)\n}\n\nfunc init() {\n\tRootCmd.AddCommand(indexCmd)\n\n\tindexCmd.Flags().BoolVarP(&keepDir, \"keepDir\", \"d\", false, \"Keep the directory in the document id.\")\n\tindexCmd.Flags().BoolVarP(&keepExt, \"keepExt\", \"x\", false, \"Keep the extension in the document id.\")\n\tindexCmd.Flags().BoolVarP(&parseJSON, \"json\", \"j\", true, \"Parse the contents as JSON.\")\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/mapping.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"github.com/spf13/cobra\"\n)\n\n// mappingCmd represents the mapping command\nvar mappingCmd = &cobra.Command{\n\tUse:   \"mapping [index path]\",\n\tShort: \"prints the mapping used for this index\",\n\tLong:  `The mapping command prints a JSON representation of the mapping used for this index.`,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tmapping := idx.Mapping()\n\t\tjsonBytes, err := json.MarshalIndent(mapping, \"\", \"  \")\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t\tfmt.Printf(\"%s\\n\", jsonBytes)\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(mappingCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/query.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar limit, skip, repeat int\nvar explain, highlight, fields bool\nvar qtype, qfield, sortby string\n\n// queryCmd represents the query command\nvar queryCmd = &cobra.Command{\n\tUse:   \"query [index path] [query]\",\n\tShort: \"queries the index\",\n\tLong:  `The query command will execute a query against the index.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\tif len(args) < 2 {\n\t\t\treturn fmt.Errorf(\"must specify query\")\n\t\t}\n\n\t\tquery := buildQuery(args)\n\t\tfor i := 0; i < repeat; i++ {\n\t\t\treq := bleve.NewSearchRequestOptions(query, limit, skip, explain)\n\t\t\tif highlight {\n\t\t\t\treq.Highlight = bleve.NewHighlightWithStyle(\"ansi\")\n\t\t\t}\n\t\t\tif fields {\n\t\t\t\treq.Fields = []string{\"*\"}\n\t\t\t}\n\t\t\tif sortby != \"\" {\n\t\t\t\tif strings.Contains(sortby, \",\") {\n\t\t\t\t\treq.SortBy(strings.Split(sortby, \",\"))\n\t\t\t\t} else {\n\t\t\t\t\treq.SortBy([]string{sortby})\n\t\t\t\t}\n\t\t\t}\n\t\t\tres, err := idx.Search(req)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"error running query: %v\", err)\n\t\t\t}\n\t\t\tfmt.Println(res)\n\t\t}\n\t\treturn nil\n\t},\n}\n\nfunc buildQuery(args []string) query.Query {\n\tvar q query.Query\n\tswitch qtype {\n\tcase \"prefix\":\n\t\tpquery := bleve.NewPrefixQuery(strings.Join(args[1:], \" \"))\n\t\tif qfield != \"\" {\n\t\t\tpquery.SetField(qfield)\n\t\t}\n\t\tq = pquery\n\tcase \"term\":\n\t\tpquery := bleve.NewTermQuery(strings.Join(args[1:], \" \"))\n\t\tif qfield != \"\" {\n\t\t\tpquery.SetField(qfield)\n\t\t}\n\t\tq = pquery\n\tdefault:\n\t\t// build a search with the provided parameters\n\t\tqueryString := strings.Join(args[1:], \" \")\n\t\tq = bleve.NewQueryStringQuery(queryString)\n\t}\n\treturn q\n}\n\nfunc init() {\n\tRootCmd.AddCommand(queryCmd)\n\n\tqueryCmd.Flags().IntVarP(&repeat, \"repeat\", \"r\", 1, \"Repeat the query this many times.\")\n\tqueryCmd.Flags().IntVarP(&limit, \"limit\", \"l\", 10, \"Limit number of results returned.\")\n\tqueryCmd.Flags().IntVarP(&skip, \"skip\", \"s\", 0, \"Skip the first N results.\")\n\tqueryCmd.Flags().BoolVarP(&explain, \"explain\", \"x\", false, \"Explain the result scoring.\")\n\tqueryCmd.Flags().BoolVar(&highlight, \"highlight\", true, \"Highlight matching text in results.\")\n\tqueryCmd.Flags().BoolVar(&fields, \"fields\", false, \"Load stored fields.\")\n\tqueryCmd.Flags().StringVarP(&qtype, \"type\", \"t\", \"query_string\", \"Type of query to run.\")\n\tqueryCmd.Flags().StringVarP(&qfield, \"field\", \"f\", \"\", \"Restrict query to field, not applicable to query_string queries.\")\n\tqueryCmd.Flags().StringVarP(&sortby, \"sort-by\", \"b\", \"\", \"Sort by field.\")\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/registry.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/spf13/cobra\"\n)\n\n// registryCmd represents the registry command\nvar registryCmd = &cobra.Command{\n\tUse:   \"registry\",\n\tShort: \"registry lists the bleve components compiled into this executable\",\n\tLong:  `The registry command will list all of the bleve components compiled into this executable.`,\n\tPersistentPreRunE: func(cmd *cobra.Command, args []string) error {\n\t\t// override to do nothing\n\t\treturn nil\n\t},\n\tPersistentPostRunE: func(cmd *cobra.Command, args []string) error {\n\t\t// override to do nothing\n\t\treturn nil\n\t},\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttypes, instances := registry.CharFilterTypesAndInstances()\n\t\tprintType(\"Char Filter\", types, instances)\n\n\t\ttypes, instances = registry.TokenizerTypesAndInstances()\n\t\tprintType(\"Tokenizer\", types, instances)\n\n\t\ttypes, instances = registry.TokenMapTypesAndInstances()\n\t\tprintType(\"Token Map\", types, instances)\n\n\t\ttypes, instances = registry.TokenFilterTypesAndInstances()\n\t\tprintType(\"Token Filter\", types, instances)\n\n\t\ttypes, instances = registry.AnalyzerTypesAndInstances()\n\t\tprintType(\"Analyzer\", types, instances)\n\n\t\ttypes, instances = registry.DateTimeParserTypesAndInstances()\n\t\tprintType(\"Date Time Parser\", types, instances)\n\n\t\ttypes, instances = registry.KVStoreTypesAndInstances()\n\t\tprintType(\"KV Store\", types, instances)\n\n\t\ttypes, instances = registry.FragmentFormatterTypesAndInstances()\n\t\tprintType(\"Fragment Formatter\", types, instances)\n\n\t\ttypes, instances = registry.FragmenterTypesAndInstances()\n\t\tprintType(\"Fragmenter\", types, instances)\n\n\t\ttypes, instances = registry.HighlighterTypesAndInstances()\n\t\tprintType(\"Highlighter\", types, instances)\n\t},\n}\n\nfunc printType(label string, types, instances []string) {\n\tsort.Strings(types)\n\tsort.Strings(instances)\n\tfmt.Printf(\"%s Types:\\n\", label)\n\tfor _, name := range types {\n\t\tfmt.Printf(\"\\t%s\\n\", name)\n\t}\n\tfmt.Println()\n\tfmt.Printf(\"%s Instances:\\n\", label)\n\tfor _, name := range instances {\n\t\tfmt.Printf(\"\\t%s\\n\", name)\n\t}\n\tfmt.Println()\n}\n\nfunc init() {\n\tRootCmd.AddCommand(registryCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/root.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\n\t\"github.com/blevesearch/bleve/v2\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar cfgFile string\n\nvar idx bleve.Index\n\n// DefaultOpenReadOnly allows some distributions of this command to default\n// to always opening the index read-only\nvar DefaultOpenReadOnly = false\n\nconst canMutateBleveIndex = \"canMutateBleveIndex\"\n\n// CanMutateBleveIndex returns true if the command is capable\n// of mutating the bleve index, or false if its operation is\n// read-only\nfunc CanMutateBleveIndex(c *cobra.Command) bool {\n\tfor k, v := range c.Annotations {\n\t\tif k == canMutateBleveIndex {\n\t\t\tif b, err := strconv.ParseBool(v); err == nil && b {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// RootCmd represents the base command when called without any subcommands\nvar RootCmd = &cobra.Command{\n\tUse:   \"bleve\",\n\tShort: \"command-line tool to interact with a bleve index\",\n\tLong:  `Bleve is a command-line tool to interact with a bleve index.`,\n\tPersistentPreRunE: func(cmd *cobra.Command, args []string) error {\n\t\tif cmd.Use == \"bash\" || cmd.Use == \"zsh\" || cmd.Use == \"fish\" || cmd.Use == \"powershell\" {\n\t\t\t// Not applicable to cobra's completion subcommands\n\t\t\treturn nil\n\t\t}\n\t\tif len(args) < 1 {\n\t\t\treturn fmt.Errorf(\"must specify path to index\")\n\t\t}\n\t\truntimeConfig := map[string]interface{}{\n\t\t\t\"read_only\": DefaultOpenReadOnly,\n\t\t}\n\t\tvar err error\n\t\tidx, err = bleve.OpenUsing(args[0], runtimeConfig)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error opening bleve index: %v\", err)\n\t\t}\n\t\treturn nil\n\t},\n\tPersistentPostRunE: func(cmd *cobra.Command, args []string) error {\n\t\tif cmd.Use == \"bash\" || cmd.Use == \"zsh\" || cmd.Use == \"fish\" || cmd.Use == \"powershell\" {\n\t\t\t// Not applicable to cobra's completion subcommands\n\t\t\treturn nil\n\t\t}\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error closing bleve index: %v\", err)\n\t\t}\n\t\treturn nil\n\t},\n}\n\n// Execute adds all child commands to the root command sets flags appropriately.\n// This is called by main.main(). It only needs to happen once to the rootCmd.\nfunc Execute() {\n\tif err := RootCmd.Execute(); err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(-1)\n\t}\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/scorch/ascii.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\n\t\"github.com/blevesearch/bleve/v2/index/scorch/mergeplan\"\n\t\"github.com/spf13/cobra\"\n)\n\n// asciiCmd represents the ascii command\nvar asciiCmd = &cobra.Command{\n\tUse:   \"ascii\",\n\tShort: \"ascii prints an ascii representation of the segments in a snapshot\",\n\tLong:  `The ascii command prints an ascii representation of the segments in a given snapshot.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\n\t\tif len(args) < 2 {\n\t\t\treturn fmt.Errorf(\"snapshot epoch required\")\n\t\t} else if len(args) < 3 {\n\t\t\tsnapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsnapshot, err := index.LoadSnapshot(snapshotEpoch)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsegments := snapshot.Segments()\n\t\t\tvar mergePlanSegments []mergeplan.Segment\n\t\t\tfor _, v := range segments {\n\t\t\t\tmergePlanSegments = append(mergePlanSegments, v)\n\t\t\t}\n\n\t\t\tstr := mergeplan.ToBarChart(args[1], 25, mergePlanSegments, nil)\n\t\t\tfmt.Printf(\"%s\\n\", str)\n\t\t}\n\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(asciiCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/scorch/deleted.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\n\t\"github.com/spf13/cobra\"\n)\n\n// deletedCmd represents the deleted command\nvar deletedCmd = &cobra.Command{\n\tUse:   \"deleted\",\n\tShort: \"deleted prints the deleted bitmap for segments in the index snapshot\",\n\tLong:  `The delete command prints the deleted bitmap for segments in the index snapshot.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\n\t\tif len(args) < 2 {\n\t\t\treturn fmt.Errorf(\"snapshot epoch required\")\n\t\t} else if len(args) < 3 {\n\t\t\tsnapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsnapshot, err := index.LoadSnapshot(snapshotEpoch)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsegments := snapshot.Segments()\n\t\t\tfor i, segmentSnap := range segments {\n\t\t\t\tdeleted := segmentSnap.Deleted()\n\t\t\t\tfmt.Printf(\"%d %v\\n\", i, deleted)\n\t\t\t}\n\t\t}\n\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(deletedCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/scorch/info.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/spf13/cobra\"\n)\n\n// dictCmd represents the dict command\nvar infoCmd = &cobra.Command{\n\tUse:   \"info\",\n\tShort: \"info prints basic info about the index\",\n\tLong:  `The info command prints basic info about the index.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\n\t\treader, err := index.Reader()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tcount, err := reader.DocCount()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfmt.Printf(\"doc count: %d\\n\", count)\n\n\t\t// var numSnapshots int\n\t\t// var rootSnapshot uint64\n\t\t// index.VisitBoltSnapshots(func(snapshotEpoch uint64) error {\n\t\t// \tif rootSnapshot == 0 {\n\t\t// \t\trootSnapshot = snapshotEpoch\n\t\t// \t}\n\t\t// \tnumSnapshots++\n\t\t// \treturn nil\n\t\t// })\n\t\t// fmt.Printf(\"has %d snapshot(s), root: %d\\n\", numSnapshots, rootSnapshot)\n\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(infoCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/scorch/internal.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\n\t\"github.com/spf13/cobra\"\n)\n\nvar ascii bool\n\n// internalCmd represents the internal command\nvar internalCmd = &cobra.Command{\n\tUse:   \"internal\",\n\tShort: \"internal prints the internal k/v pairs in a snapshot\",\n\tLong:  `The internal command prints the internal k/v pairs in a snapshot.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\n\t\tif len(args) < 2 {\n\t\t\treturn fmt.Errorf(\"snapshot epoch required\")\n\t\t} else if len(args) < 3 {\n\t\t\tsnapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsnapshot, err := index.LoadSnapshot(snapshotEpoch)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tinternal := snapshot.Internal()\n\t\t\tfor k, v := range internal {\n\t\t\t\tif ascii {\n\t\t\t\t\tfmt.Printf(\"%s %s\\n\", k, string(v))\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Printf(\"%x %x\\n\", k, v)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(internalCmd)\n\tinternalCmd.Flags().BoolVarP(&ascii, \"ascii\", \"a\", false, \"print key/value in ascii\")\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/scorch/root.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar index *scorch.Scorch\n\n// RootCmd represents the base command when called without any subcommands\nvar RootCmd = &cobra.Command{\n\tUse:   \"scorch\",\n\tShort: \"command-line tool to interact with a scorch index\",\n\tLong:  `Scorch is a command-line tool to interact with a scorch index.`,\n\tPersistentPreRunE: func(cmd *cobra.Command, args []string) error {\n\n\t\tif len(args) < 1 {\n\t\t\treturn fmt.Errorf(\"must specify path to scorch index\")\n\t\t}\n\n\t\treadOnly := true\n\t\tconfig := map[string]interface{}{\n\t\t\t\"read_only\": readOnly,\n\t\t\t\"path\":      args[0],\n\t\t}\n\n\t\tidx, err := scorch.NewScorch(scorch.Name, config, nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\terr = idx.Open()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error opening: %v\", err)\n\t\t}\n\n\t\tindex = idx.(*scorch.Scorch)\n\n\t\treturn nil\n\t},\n\tPersistentPostRunE: func(cmd *cobra.Command, args []string) error {\n\t\treturn nil\n\t},\n}\n\n// Execute adds all child commands to the root command sets flags appropriately.\n// This is called by main.main(). It only needs to happen once to the rootCmd.\nfunc Execute() {\n\tif err := RootCmd.Execute(); err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(-1)\n\t}\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/scorch/snapshot.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\n\tseg \"github.com/blevesearch/scorch_segment_api/v2\"\n\t\"github.com/spf13/cobra\"\n)\n\n// snapshotCmd represents the snapshot command\nvar snapshotCmd = &cobra.Command{\n\tUse:   \"snapshot\",\n\tShort: \"info prints details about the snapshots in the index\",\n\tLong:  `The snapshot command prints details about the snapshots in the index.`,\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\n\t\tif len(args) < 2 {\n\t\t\tsnapshotEpochs, err := index.RootBoltSnapshotEpochs()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, snapshotEpoch := range snapshotEpochs {\n\t\t\t\tfmt.Printf(\"snapshot epoch: %d\\n\", snapshotEpoch)\n\t\t\t}\n\t\t} else if len(args) < 3 {\n\t\t\tsnapshotEpoch, err := strconv.ParseUint(args[1], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsnapshot, err := index.LoadSnapshot(snapshotEpoch)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsegments := snapshot.Segments()\n\t\t\tfor i, segmentSnap := range segments {\n\t\t\t\tsegment := segmentSnap.Segment()\n\t\t\t\tif segment, ok := segment.(seg.PersistedSegment); ok {\n\t\t\t\t\tfmt.Printf(\"%d %s\\n\", i, segment.Path())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn nil\n\t},\n}\n\nfunc init() {\n\tRootCmd.AddCommand(snapshotCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/cmd/scorch.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cmd\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/cmd/bleve/cmd/scorch\"\n)\n\n// make scorch command-line tool a bleve sub-command\n\nfunc init() {\n\tRootCmd.AddCommand(scorch.RootCmd)\n}\n"
  },
  {
    "path": "cmd/bleve/gendocs.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build ignore\n// +build ignore\n\npackage main\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/cmd/bleve/cmd\"\n\n\t\"github.com/spf13/cobra/doc\"\n)\n\n// you can generate markdown docs by running\n//\n//   $ go run gendocs.go\n//\n// this also requires doc sub-package of cobra\n// which is not kept in this repo\n// you can acquire it by running\n//\n//   $ gvt restore\n\nfunc main() {\n\tcmd.RootCmd.DisableAutoGenTag = true\n\tidentity := func(s string) string {\n\t\treturn fmt.Sprintf(`{{< relref \"docs/%s\" >}}`, s)\n\t}\n\temptyStr := func(s string) string { return \"\" }\n\tdoc.GenMarkdownTreeCustom(cmd.RootCmd, \"./\", emptyStr, identity)\n}\n"
  },
  {
    "path": "cmd/bleve/main.go",
    "content": "// Copyright © 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage main\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/cmd/bleve/cmd\"\n\n\t// to support standard set of build tags\n\t_ \"github.com/blevesearch/bleve/v2/config\"\n)\n\nfunc main() {\n\tcmd.Execute()\n}\n"
  },
  {
    "path": "config/README.md",
    "content": "# Bleve Config\n\n**NOTE** you probably do not need this package.  It is only intended for general purpose applications that want to include large parts of Bleve regardless of whether or not the code is directly using it.\n\n## General Purpose Applications\n\nA general purpose application, that must allow users to express the need for Bleve components at runtime can accomplish this by:\n\n```\nimport _ \"github.com/blevesearch/bleve/config\"\n```"
  },
  {
    "path": "config/config.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t// token maps\n\t_ \"github.com/blevesearch/bleve/v2/analysis/tokenmap\"\n\n\t// fragment formatters\n\t_ \"github.com/blevesearch/bleve/v2/search/highlight/format/ansi\"\n\t_ \"github.com/blevesearch/bleve/v2/search/highlight/format/html\"\n\n\t// fragmenters\n\t_ \"github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple\"\n\n\t// highlighters\n\t_ \"github.com/blevesearch/bleve/v2/search/highlight/highlighter/ansi\"\n\t_ \"github.com/blevesearch/bleve/v2/search/highlight/highlighter/html\"\n\t_ \"github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple\"\n\n\t// char filters\n\t_ \"github.com/blevesearch/bleve/v2/analysis/char/asciifolding\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/char/html\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/char/regexp\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/char/zerowidthnonjoiner\"\n\n\t// analyzers\n\t_ \"github.com/blevesearch/bleve/v2/analysis/analyzer/custom\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/analyzer/simple\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/analyzer/web\"\n\n\t// token filters\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/apostrophe\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/camelcase\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/compound\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/edgengram\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/elision\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/keyword\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/length\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/ngram\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/reverse\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/shingle\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/stop\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/truncate\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/token/unique\"\n\n\t// tokenizers\n\t_ \"github.com/blevesearch/bleve/v2/analysis/tokenizer/exception\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/tokenizer/single\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/tokenizer/web\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace\"\n\n\t// date time parsers\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/flexible\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/iso\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/optional\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/percent\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/sanitized\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds\"\n\n\t// languages\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/ar\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/bg\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/ca\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/cjk\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/ckb\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/cs\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/da\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/de\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/el\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/en\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/es\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/eu\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/fa\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/fi\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/fr\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/ga\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/gl\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/hi\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/hr\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/hu\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/hy\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/id\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/in\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/it\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/nl\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/no\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/pl\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/pt\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/ro\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/ru\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/sv\"\n\t_ \"github.com/blevesearch/bleve/v2/analysis/lang/tr\"\n\n\t// kv stores\n\t_ \"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n\t_ \"github.com/blevesearch/bleve/v2/index/upsidedown/store/goleveldb\"\n\t_ \"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t_ \"github.com/blevesearch/bleve/v2/index/upsidedown/store/moss\"\n\n\t// index types\n\t_ \"github.com/blevesearch/bleve/v2/index/upsidedown\"\n)\n"
  },
  {
    "path": "config.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"expvar\"\n\t\"io\"\n\t\"log\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight/highlighter/html\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar bleveExpVar = expvar.NewMap(\"bleve\")\n\ntype configuration struct {\n\tCache                  *registry.Cache\n\tDefaultHighlighter     string\n\tDefaultKVStore         string\n\tDefaultMemKVStore      string\n\tDefaultIndexType       string\n\tSlowSearchLogThreshold time.Duration\n\tanalysisQueue          *index.AnalysisQueue\n}\n\nfunc (c *configuration) SetAnalysisQueueSize(n int) {\n\tif c.analysisQueue != nil {\n\t\tc.analysisQueue.Close()\n\t}\n\tc.analysisQueue = index.NewAnalysisQueue(n)\n}\n\nfunc (c *configuration) Shutdown() {\n\tc.SetAnalysisQueueSize(0)\n}\n\nfunc newConfiguration() *configuration {\n\treturn &configuration{\n\t\tCache:         registry.NewCache(),\n\t\tanalysisQueue: index.NewAnalysisQueue(4),\n\t}\n}\n\n// Config contains library level configuration\nvar Config *configuration\n\nfunc init() {\n\tbootStart := time.Now()\n\n\t// build the default configuration\n\tConfig = newConfiguration()\n\n\t// set the default highlighter\n\tConfig.DefaultHighlighter = html.Name\n\n\t// default kv store\n\tConfig.DefaultKVStore = \"\"\n\n\t// default mem only kv store\n\tConfig.DefaultMemKVStore = gtreap.Name\n\n\t// default index\n\tConfig.DefaultIndexType = scorch.Name\n\n\tbootDuration := time.Since(bootStart)\n\tbleveExpVar.Add(\"bootDuration\", int64(bootDuration))\n\tindexStats = NewIndexStats()\n\tbleveExpVar.Set(\"indexes\", indexStats)\n\n\tinitDisk()\n}\n\nvar logger = log.New(io.Discard, \"bleve\", log.LstdFlags)\n\n// SetLog sets the logger used for logging\n// by default log messages are sent to io.Discard\nfunc SetLog(l *log.Logger) {\n\tlogger = l\n}\n"
  },
  {
    "path": "config_app.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build appengine || appenginevm\n// +build appengine appenginevm\n\npackage bleve\n\n// in the appengine environment we cannot support disk based indexes\n// so we do no extra configuration in this method\nfunc initDisk() {\n\n}\n"
  },
  {
    "path": "config_disk.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !appengine && !appenginevm\n// +build !appengine,!appenginevm\n\npackage bleve\n\nimport \"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n\n// in normal environments we configure boltdb as the default storage\nfunc initDisk() {\n\t// default kv store\n\tConfig.DefaultKVStore = boltdb.Name\n}\n"
  },
  {
    "path": "data/test/sample-data.json",
    "content": "[{\"title\":\"Edinburgh/Leith\",\"name\":\"Ocean Apartments\",\"address\":\"2 Western Harbour Midway\",\"directions\":null,\"phone\":\"+44 131 553 7394\",\"tollfree\":null,\"email\":null,\"fax\":null,\"url\":\"http://www.oceanservicedapts.com\",\"checkin\":\"15H00\",\"checkout\":\"10H00\",\"price\":\"from £70\",\"geo\":{\"lat\":55.9812,\"lon\":-3.2248,\"accuracy\":\"RANGE_INTERPOLATED\"},\"type\":\"hotel\",\"id\":8576,\"country\":\"United Kingdom\",\"city\":\"Edinburgh\",\"state\":null,\"reviews\":[{\"content\":\"In my personal opinion, this hotel is one of the many hidden gems of Istanbul. Located in an area not very easy to get if you are driving yourself. I would advise taking a taxi to get there. The service from the first \\\"hello\\\" until the last \\\"bye\\\" was impecable. The terrace restaurant wiew is second to none. The food was excellent. The staff was very nice. In short, next time I am going to Isnbul, I dont believe I will stay anywehere else. Highly recomended.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":5.0,\"Value\":5.0,\"Sleep Quality\":5.0,\"Rooms\":5.0,\"Location\":5.0},\"author\":\"Lindsey Wiegand III\",\"date\":\"2013-01-01 16:30:13 +0300\"},{\"content\":\"When you first arrive at TomTom Suites you might wonder where you are coming to as the area looks a little run down! But it is in a great location in a street with no passing traffic. Its so quiet. Access to the main sites is easy. Downhill, the tram is within a 5 minute walk and uphill, Iskatel Cadesi ( a buzzing main street with loads of shops and restaurants) and a 2 minute walk round the corner leads to some narrow streets full of atmosphere and restaurants This boutique hotel itself is a sea of tranquility with beautifully appointed rooms, a great breakfast and very helpful staff. Our only regret was that the Terrace bar wasn't open but as the weather was unseasonably inclement it wasn't a problem. We had a great time and would definitely recommend the TomTom Suites\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":5.0,\"Value\":5.0,\"Sleep Quality\":5.0,\"Rooms\":5.0,\"Location\":4.0},\"author\":\"Bulah Weissnat\",\"date\":\"2013-06-01 17:06:53 +0300\"},{\"content\":\"Staying at Tomtom was the best possible choice we could have made, everything was simply perfect: the location, the facilities, the staff, the quality of the food, the elegance and modern style of the furniture, the superb view of the terrace where we had breakfast as well as a wonderful dinner and a lovely evening. Many details made us feel at home, such as the Ipod dock in the rooms and the Ipads we could use free of charge during breakfast. Most of all, the friendliness of the staff, all of them extremely helpful. I'd highlight Chiara, who gave us many tips, especially telling us to go to Bagdah st., on the Asian side, a place great place that no books mentioned; and Ali, a wonderful guy, much more than a concierge, and a perfect host who did everything he could to make our stay as good as it gets. the proximity to Istklal street, and the tram make the location convenient for both day and night. Considering everything, including the fair rates cherged, I doubt there's a better choice in town.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":5.0,\"Value\":5.0,\"Sleep Quality\":5.0,\"Rooms\":5.0,\"Location\":5.0},\"author\":\"Ottis Pacocha\",\"date\":\"2012-08-07 08:16:49 +0300\"},{\"content\":\"A really wonderful hotel in a superb location on a traffic free road. It was so peaceful it was easy to forget that we were in the middle of one of the world's busiest cities. The staff were deligthful and couldn't do enough to help. The hotel is in a converted monastary which has been sympathetically refurbished, the artwork depicting istanbul brings a touch of colour - especially in the lift shaft. Our room was enormous and well appointed with a luxurious large bathroom - and yes, a jacuzzi bath. The rooftop restaurant and bar was a perfect way to relax at the end of the day and must have some of the best views in Istanbul over the Bosphorous and the Golden Horn. The food was excellent, modern turkish using french cooking techniques and local produce. Overall I can't wait to go back and I can't recommend the place highly enough.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":5.0,\"Value\":5.0,\"Sleep Quality\":5.0,\"Rooms\":5.0,\"Location\":5.0},\"author\":\"Dr. Amira Murazik\",\"date\":\"2013-10-14 03:30:16 +0300\"},{\"content\":\"My wife, mother and I really enjoyed our recent stay at TomTom. We always prefer smaller boutique places that are well designed and in actual neighborhoods vs. larger impersonal luxury hotels. TomTom didn't dissapoint! As previously mentioned, beautifully designed and furnished rooms, the best toiletries, etc. The roof deck is very nice with a great view. The location is amazing, in-between everywhere that you want to go but far enough away from the touristy madness. I'd argue it's one of the best situated hotels in Istanbul. As mentioned, it's a very steep but quick walk up to the main street of Isklal. However, this can be challenging if you have any mobility issues. In general, the area is rather hilly but so is most of Istanbul. In the other direction, you can walk up into a neighborhood renowned for antiques and nice cafes or down to the tram that takes you to the main tourist attractions or the road to get to Ortokoy or Bebek. At no point does it feel like you are in the \\\"tourist district\\\" but you never feel unwelcome or out of place. As for noise, we found the building itself to be quiet and well insulated. You didn't hear noise from the rooms around you and minimal noise from the room above. Nowhere near as bad as what an earlier reviewer described. The only real noise to speak of does come from the kids in the morning and the afternoon who walk on the dead-end road to and from school. Other than this, and some kids playing soccer on occasion, the street outside of the hotel is pretty quiet Breakfast in the morning is great. Freshly baked pastries (done at the hotel) and fruit, cheeses and meats, served by extremely friendly staff. Fuat in particular is a delight to interact with and was extremely helpful. The staff overall is very friendly and accommodating. Everyone at the front desk regardless of time of day did whatever they could to make sure we were receiving the best service possible. There are however a few things that need improvement to make the hotel even better. -While breakfast is excellent, we were less than impressed with dinner. To the point that we ate there once and did not choose to repeat the experience. The service was good but the quality of the food and the cooking was pretty bad esp. considering the prices of the food vs. what can be had in the neighborhood for much less. We are admitted foodies, but we didn't hold the hotel restaurant up to lofty expectations. This needs to be addressed. -While the staff is very friendly and eager to help, the hotel would benefit from a dedicated concierge or someone who really knows about the various restaurants around the city. With a hotel of this caliber, we expected them to be more informed about different places and to let us know how traffic can affect getting there, etc. If you didn't ask them these kinds of questions, it didn't occur to them to tell you. -Lastly, with the windows closed in the room there is zero airflow without having the heat on and it's extremely dry. I understand them not wanting to run the AC unless the temperature warrants, but there should be a way to circulate air without opening the windows in the spring (see my earlier comment about noise from school kids in the morning). In conclusion, we wouldn't hesitate to recommend TomTom to friends and hope to stay there again the next time we are in Istanbul.\",\"ratings\":{\"Service\":4.0,\"Cleanliness\":5.0,\"Overall\":5.0,\"Value\":4.0,\"Sleep Quality\":4.0,\"Rooms\":5.0,\"Location\":5.0},\"author\":\"Marcelle Haley\",\"date\":\"2015-07-17 19:17:23 +0300\"},{\"content\":\"We had been at Istanbul for business purposes and used to stay at the well-known brands at Taxim. This time the trip was for leisure and we were looking for a charming, cozy, friendly, clean, staff friendly boutique hotel. So, THIS IS TomTom suites and we consider ourselves very lucky to stay there. Due to the Italian embassy its socak is not crowded and safe 24/7. Breakfast variety was very satisfactory, view from terrace wonderful, staff very friendly, very quiet and clean. If you are looking for a hotel with ID, we strongly recommend it. Be aware that Istiklal street is only 5 minute walking, but uphill.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":5.0,\"Value\":5.0,\"Location\":4.0,\"Rooms\":5.0},\"author\":\"Peggie Little\",\"date\":\"2014-04-22 04:05:24 +0300\"}],\"public_likes\":[\"Ms. Braulio Kuhic\"],\"vacancy\":true,\"description\":\"Modern, stylish contemporary serviced apartments 4 miles for Edinburgh's city Centre.\",\"alias\":\"Serviced Apartments\",\"pets_ok\":false,\"free_breakfast\":true,\"free_internet\":false,\"free_parking\":true}\n,{\"title\":\"Edinburgh/Old Town\",\"name\":\"Euro Hostel Edinburgh\",\"address\":null,\"directions\":null,\"phone\":\"+44  8454 900 461\",\"tollfree\":null,\"email\":null,\"fax\":null,\"url\":\"http://www.euro-hostels.co.uk/Edinburgh_hostel/\",\"checkin\":null,\"checkout\":null,\"price\":null,\"geo\":{\"lat\":55.94825,\"lon\":-3.18805,\"accuracy\":\"APPROXIMATE\"},\"type\":\"hotel\",\"id\":8661,\"country\":\"United Kingdom\",\"city\":\"Edinburgh\",\"state\":null,\"reviews\":[{\"content\":\"A plain and simple hotel, located on a busy street with many company offices nearby I can't imagine a tourist staying here. It is far away from the city center, and anything that a tourist might be interested in. In general, you will need a car, or a taxi to get to anything unless your business office is within walking distance. About €25 from the airport via taxi, this fairly modern looking hotel is very plain and simple in desgin, layout and comforts. Functional, and at a rate of about €85 per night - it was almost a bargain. Surprisingly quite given the location on the street, my \\\"no smoking\\\" room was clearly smoked in. Upon complaining - it was explained to me that the room was in fact no-smoking. No offer to move me, no thought that there might be an issue .... who knows, perhaps they had heard this before. Breakfast included, a simple breakfast that was OK in general, and nothing special. I would only recommend staying here if you are close enough to what you need to go to that you can walk.\",\"ratings\":{\"Service\":3.0,\"Cleanliness\":2.0,\"Overall\":2.0,\"Value\":3.0,\"Sleep Quality\":2.0,\"Rooms\":2.0,\"Location\":1.0},\"author\":\"Rachel O'Hara\",\"date\":\"2014-03-12 14:29:07 +0300\"},{\"content\":\"We (2 couples) recently stayed at the Hotel for 4 nights in their standard rooms -- 1 room off the garden and 1 in the main building.Both rooms were on the small side but the quality of the rooms more than compensated for their size.Breakfast was excellent.The hotels main asset in our eyes were all their staff who were very professional and looked after all our needs exceptionally well.The location of the hotel is excellent close to shops sights and restaurants. If you are travelling to Paris for a short trip i would recommend staying here.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":4.0,\"Value\":4.0,\"Sleep Quality\":5.0,\"Rooms\":3.0,\"Location\":5.0},\"author\":\"Viola Reinger\",\"date\":\"2012-09-21 02:53:29 +0300\"}],\"public_likes\":[\"Laila Jacobs\",\"Dr. Andreane Berge\",\"Ophelia Walter\",\"Mac Hackett\",\"Belle Bartell\",\"Spencer Erdman\",\"Elna Monahan\",\"Shanelle Hayes\",\"Ms. Wallace Larkin\"],\"vacancy\":false,\"description\":\"Kincaid's Court, Guthrie Street. In Cowgate, open every summer from June 8th until September 2nd. Budget accommodation in 43 apartments used as student residences during term time.\",\"alias\":null,\"pets_ok\":false,\"free_breakfast\":false,\"free_internet\":true,\"free_parking\":true}\n,{\"title\":\"Edinburgh/Old Town\",\"name\":\"The Sheraton Grand Hotel\",\"address\":null,\"directions\":null,\"phone\":\"+44  131 229 9131\",\"tollfree\":null,\"email\":null,\"fax\":null,\"url\":null,\"checkin\":null,\"checkout\":null,\"price\":null,\"geo\":{\"lat\":55.947,\"lon\":-3.2073,\"accuracy\":\"APPROXIMATE\"},\"type\":\"hotel\",\"id\":8662,\"country\":\"United Kingdom\",\"city\":\"Edinburgh\",\"state\":null,\"reviews\":[{\"content\":\"Really loved this hotel, it was beautifully decorated (very much interior designed) and was spotlessly clean. We had a booked a superior double and when we arrived we were told we'd been upgraded. The room was fairly large by Paris standards and had a day bed, which I assume could sleep a 3rd person/child. The bathroom was really modern and had a bath and large separate shower with large overhead shower head, plus hand held shower. All the decor was tasteful and we were at the back of the hotel overlooking the small courtyard garden, so very quiet, although the hotel is on a quiet street anyway. Short walk to the Jardin du Luxembourg and less that 5 mins to a metro, which was on a direct line to the Gare du Nord, so perfect for us as we took the Eurostar from London. Hotel booked us a table at a great restaurant, superb food, which they recommended. Breakfast was served to order and you got croissants, pain au chocolat, bread and could choose omelettes etc. The fruit salad was freshly made, the yoghurts were the posh ones in glass jars. Would really recommend the hotel, but not for small children - there are a lot of carefully placed vases and objets d'art that little fingers will want to touch...\",\"ratings\":{\"Cleanliness\":5.0,\"Sleep Quality\":5.0,\"Overall\":5.0,\"Value\":4.0,\"Service\":5.0},\"author\":\"Brittany Ledner Jr.\",\"date\":\"2012-06-21 07:48:16 +0300\"},{\"content\":\"I spent a wonderful week at the Villa Madame, finding the staff very helpful and gracious. My 5th floor room was very clean and light, quiet, and comfortable. The hotel is extremely well located only a few short blocks to the metro, Jardins du Luxemburg, and shopping. The hotel offered free wifi, and much more surprisingly, free international telephone service from my room via voip. Breakfasts were excellent, and the garden area was quiet and comfortable. Just around the corner is Maison du Jardin, a very excellent small restaurant with prix fixe 31 euro dinners. Staff made other great recommendations as well. I would unhesitatingly recommend and will return.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":5.0,\"Value\":4.0,\"Sleep Quality\":5.0,\"Rooms\":5.0,\"Location\":5.0},\"author\":\"Lauren Ortiz\",\"date\":\"2014-10-01 14:04:03 +0300\"},{\"content\":\"My friend and I were backbacking through London and Paris and decided to splurge a little on this hotel. It turned out amazing. You really don't want to risk a hostel in Paris, they are serious dumps. I did a hostel in Paris in 2007. I picked this hotel due to it's proximity to nightlife in the Latin Quarter and it looked nice. The bathroom had a walk in shower with glass door. Flatscreen TV had an aux speaker in the bathroom to hear the tv while doing your thing. Beds were very comfortable and the room we got was big and facing the street. The street was loud during mid afternoon due to the Catholic school playground down the block. We booked a room with two singles and when we went to check in we were told they had inadvertantly given our room away. I started to freak because I was sure we would get a fast one pulled as usually happens in western europe. We were happy that after my face of death look we were upgraded to a larger room with two double beds. Unfortunately for the poor lady who had booked the room they bumped her and I don't know where she was then moved. Overall the staff was slow on check'in. Breakfast was carbs and coffee. They have a person dedicated only to breakfast and coffee however it seems odd because this is a very small hotel. We were the youngest guests in the hotel by about 50 years. Everyone else looked like they were late fifties to mid seventee's. We are single thirtysomethings. I would recommend this hotel to anyone who wants a peacefull nights sleep in Paris.\",\"ratings\":{\"Service\":4.0,\"Cleanliness\":5.0,\"Overall\":4.0,\"Value\":2.0,\"Location\":5.0,\"Rooms\":4.0},\"author\":\"Turner Ferry\",\"date\":\"2013-11-29 14:38:35 +0300\"},{\"content\":\"The Villa Madame is a lovely, comfortable, well-located boutique hotel with excellent service. My wife and I enjoyed our stay there immensely. This hotel is chic and luxuriously comfortable especially for the price. Admittedly, the room we had (Classic Double Room) was the smallest hotel room my wife and I had ever seen at about 3m x 3.5m (10' x 12'). The large outdoor terrace, fantastic bathroom (with Hermès products and great water pressure), deliciously comfortable bed, iPod docking station, super-fast free wi-fi and excellent service more than made up for the tiny room. Alex at the front desk was a delight as she answered our every question and was always happy to chat and share information. The location is excellent being only a few minutes walk from either the Rennes or St-Sulpice Metro stops, 100m from the beautiful Jardin du Luxembourg and within easy walking distance of the myriad of shops and restaurants on both the Rue de Rennes and Boulevard Saint-Germain. The hotel serves a decent continental breakfast which seems expensive at 18€ but we found a package that was below the normal rate and included breakfast. The breakfast room, as other reviewers have noted, is small and has only four tables for two. All of the tables were occupied each time we went for breakfast but the hotel happily served us as we sat in the little lounge area so it wasn't ever a problem. They don't serve lunch or dinner but there are two brasseries within 50m of the front door and many more restaurants and coffee shops just a few blocks away. There were a few other minor concerns that I had regarding this hotel: - The minibar is stocked only with two bottles of water. It would be nice if there was a limited selection of other beverages and also a kettle for tea/coffee service. - The satellite television has almost 900 channels (yes, I went through them all late one night) but half of them are Arabic and almost all of the rest are in French save for one German language station, a couple of Italian stations and only Bloomberg for the English speakers. Although one doesn't go to Paris to watch TV in the room, it's nice to relax and watch the news or a movie after a long day on the town but unless the above suits you, you won't have much cause to even turn on the set. - The street is very small and it is very difficult to find parking. This is likely not a problem for most visitors but is something to keep in mind especially if you rent a car. Overall, my wife and I loved this hotel. The few cons are heavily outweighed by the comforts we enjoyed. We are very seasoned travelers and despite the tiny room, this hotel experience was one of our best ever. We will definitely stay at the Villa Madame again.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":5.0,\"Value\":5.0,\"Location\":5.0,\"Rooms\":5.0},\"author\":\"Blaze Williamson\",\"date\":\"2014-04-26 13:59:54 +0300\"}],\"public_likes\":[\"Madelynn Littel\",\"Marielle Daugherty\",\"Micah Stiedemann\",\"Sandra Howe\",\"Angela Oga\"],\"vacancy\":true,\"description\":\"21 Festival Square. Against the backdrop of majestic Edinburgh Castle, the Sheraton Grand Hotel and Spa combines city centre convenience with warm Scottish hospitality.\",\"alias\":null,\"pets_ok\":false,\"free_breakfast\":true,\"free_internet\":false,\"free_parking\":false}\n,{\"title\":\"Edinburgh/Old Town\",\"name\":\"Radisson Blu Hotel\",\"address\":\"80 High St\",\"directions\":null,\"phone\":\"+44  131 557 9797\",\"tollfree\":null,\"email\":null,\"fax\":null,\"url\":\"http://www.radissonblu.co.uk/hotel-edinburgh\",\"checkin\":null,\"checkout\":null,\"price\":null,\"geo\":{\"lat\":55.95014,\"lon\":-3.18667,\"accuracy\":\"ROOFTOP\"},\"type\":\"hotel\",\"id\":8663,\"country\":\"United Kingdom\",\"city\":\"Edinburgh\",\"state\":null,\"reviews\":[],\"public_likes\":[],\"vacancy\":false,\"description\":\"The Royal Mile. Less than a five-minute walk from major shopping and business districts, and the Edinburgh International Conference Centre is only a short taxi ride away.\",\"alias\":null,\"pets_ok\":false,\"free_breakfast\":true,\"free_internet\":true,\"free_parking\":false}\n,{\"title\":\"Edinburgh/Old Town\",\"name\":\"Hotel Missoni\",\"address\":null,\"directions\":null,\"phone\":\"+44  131 220 6666\",\"tollfree\":null,\"email\":null,\"fax\":null,\"url\":\"http://www.hotelmissoni.com/hotelmissoni-edinburgh\",\"checkin\":null,\"checkout\":null,\"price\":null,\"geo\":{\"lat\":55.9491,\"lon\":-3.19275,\"accuracy\":\"APPROXIMATE\"},\"type\":\"hotel\",\"id\":8664,\"country\":\"United Kingdom\",\"city\":\"Edinburgh\",\"state\":null,\"reviews\":[{\"content\":\"We stayed for 3 nights in March, 2 consecutive and then one at the end of the trip. The area near the Termini station is not the prettiest, but it is very convenient if you are using public transportation. My suggestion when exiting Termini station is to go RIGHT and walk down about 4 blocks and then right again and over 2 blocks. We were travelling with quite a bit of luggage and we ended up going the wrong way too many times. We had no problems in the area and loved the convenience of the location. We walked to Termini to either pick up the Metro train or a bus to most all of the sites. The room was spotless and the Breakfast was delicious. Assunta - the owner, could not have been more helpful, although her sense of direction is not like ours in America. Just a short walk maybe a lot longer than we Americans are used to. Hoping on the #70 bus gets you to almost any tourist site (or close to) and most buses all head back to Termini.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":5.0,\"Overall\":4.0,\"Value\":5.0,\"Sleep Quality\":5.0,\"Rooms\":3.0,\"Location\":3.0},\"author\":\"Hermina Schinner\",\"date\":\"2015-03-06 22:56:15 +0300\"},{\"content\":\"This motel may not be the Ritz Carlton but if your looking for value, a great location and safe neighborhood then you've chosen the right place! It is right across the street from the CBS studios, the line for the price is right is just around the corner. My son and I stayed there for a week and received great service from the maids right on to the front desk. The front desk was more helpful than any concierge I have ever seen and they are just a wonderful hard working family. If I ever go back to LA I will definitely stay at the Beverly Inn even if it's only for the friendly service.\",\"ratings\":{\"Service\":5.0,\"Cleanliness\":4.0,\"Overall\":4.0,\"Value\":5.0,\"Sleep Quality\":5.0,\"Rooms\":4.0,\"Location\":5.0},\"author\":\"Jose Swaniawski Sr.\",\"date\":\"2014-02-12 20:15:16 +0300\"},{\"content\":\"This place is car motel that has seen much better days; the beds are old and offer no support, the televisions, carpets, and furnishings are likewise well-used, and the overall effect can be somewhat depressing. However, it does offer limited parking and convenient location at a very inexpensive rate and the bathrooms were clean; it attracts tourists who are more interested in the surrounding neighborhoods (which are quite nice with wonderful restaurants) and less interested in where they stay for the night. There was a cafe down the street when I was there in 2002, it's along major bus lines, with a terrific market close by across the street. It is also right across from the CBS studios, and about four blocks walking from an old-style diner and bakery. The comments on other sites focus on the ethnic background of the clerks and manager, but I found them pleasant and accommodating. That is why I actually felt safe there as a single female traveler (although I suspect others would not feel this way). This is just slightly above average for what you would expect for the price, but you do get what you pay for.\",\"ratings\":{\"Cleanliness\":1.0,\"Overall\":2.0,\"Value\":4.0,\"Service\":3.0,\"Rooms\":2.0},\"author\":\"Mr. Ellis Heller\",\"date\":\"2012-03-02 00:20:56 +0300\"},{\"content\":\"Dirty, cock roach infested, unsafe, and very noisy. This motel has not been updated in 25 years or more. It is very noisy (even with ear plugs) because you are a stone throw away from the Dolphin Expressway. The carpet and ceramic floors are filthy along with the furniture and bedding. Someone tried entering my room in the middle of the night, thank goodness for deadbolt and door chain. Photos online are very deceiving. RECOMMEND - DO NOT STAY AT THIS MOTEL. This motel should not be part of the Choice Hotel chain. Should be called \\\"Last Choice\\\".\",\"ratings\":{\"Service\":2.0,\"Cleanliness\":1.0,\"Overall\":1.0,\"Value\":1.0,\"Sleep Quality\":1.0,\"Rooms\":1.0,\"Location\":1.0},\"author\":\"Lottie Gerhold IV\",\"date\":\"2014-06-12 22:56:30 +0300\"},{\"content\":\"Creepy, dirty, dark, depressing. It looks like the stereotypical motel in the movies where a drug deal goes bad and people get murdered. Rooms smell of chemical perfumed disinfectant and it burns your nose and lungs... but you will be too afraid for your safety to open a window or door! We stayed at this hotel from 3pm to 10pm (never actually slept overnight, thank god!) because we had a late night flight and it was pouring rain in Miami. The hotel says it's \\\"newly renovated\\\" and had free WIFI, so we figured it would be a nice place to spend a few hours and relax. HA! NOOOO! We laid on top of the beds, just to watch tv and instantly became itchy. Nothing about this hotel was \\\"newly renovated.\\\" The WIFI was extremely slow. The main picture of this hotel is deceiving. It looks a LOT worse and run-down in real life! Don't bother with this place.\",\"ratings\":{\"Service\":2.0,\"Cleanliness\":1.0,\"Overall\":1.0,\"Value\":1.0,\"Sleep Quality\":1.0,\"Rooms\":1.0},\"author\":\"Niko Keebler\",\"date\":\"2014-12-12 12:32:09 +0300\"},{\"content\":\"We didn't stay at the hotel so I can't comment on the rooms. We did leave our car there while we went on a cruise ($5/night parking.) While we were gone someone siphoned ALL of the gas out of our vehicle. I have called the manager twice to alert her to the problem. I've left messages concerning \\\"a security issue at the hotel\\\" and no one has returned my call. I guess they don't care about security. Next time I will spend the $20/day to park at the port since that seems to be the only secure parking to be had.\",\"ratings\":{\"Overall\":1.0},\"author\":\"Miss Alysha Goldner\",\"date\":\"2012-01-17 00:40:45 +0300\"},{\"content\":\"I stayed at this motel for one night with my partner in August 2010. We had a flight early in the morning from Miami airport so we wanted a hotel close to the airport. I booked this through BOOKING.COM and payed £45. I have never stayed in Miami before so we did not know what areas were good and what was bad. When we checked in I wasa little concerned as there was a security hut and a guard at the door. We checked in and it was very run down and dirty. They asked for a credit card but I insisted on paying cash as there was no way I was going to hand over my card details. We parked our car and tried to find our room. The halls were all outdoors and very run down. There was a sign saying it had just had a refurb. I could not see where from the outside. It seemed that the area in Miami the motel was in was not a very good one, this concerned us a little. We finally found our room and went in. The room was very basic and smelt of damp. The dead lock on the door was broken and there was no safe in the room. We felt that uncomfortable that at night we pushed our suit cases up to the door. The shower had no presure and was only luke warm and the paint was peeling off from the bathroom. What really appauld us was the floor. I took my shoes and socks off and was walking around in bare foot. After two minutes my partner said \\\"LOOK AT YOUR FEET\\\". They were black. The floor was that dirty that in 2 mins my feet were black. I wet a towel and rubbed it along the floor. The towel changed to black and the carpet changed colour. The floor could do with a really good clean. The TV reception was very poor and fuzzy and we gave up in the end. Also the internet/Wifi had little/no signal. It was noisey outside and there were Police sirens sounding all night outside. We found it hard to sleep. We only stayed the night as we had to get up at 4 am to check in for our flight and we checked in at 8pm that evening. Otherwise we would have moved hotel. The only plus side was it was close to the airport. We did not stay for the breakfast but if itn was anything like the room we would have passed anyway. We did not use the pool as it was on the otherside of the motel right next to a main highway. It looked dirty and very uninviting. Please only stay at this motel if you have to or if it is free and you are feeling brave.\",\"ratings\":{\"Service\":1.0,\"Cleanliness\":1.0,\"Overall\":1.0,\"Value\":1.0,\"Sleep Quality\":1.0,\"Rooms\":2.0,\"Location\":3.0},\"author\":\"Miss Weldon Flatley\",\"date\":\"2015-05-21 14:38:02 +0300\"}],\"public_likes\":[\"Ms. Jaleel Bartell\",\"Rodger Jerde\",\"Hanna Simonis\"],\"vacancy\":false,\"description\":\"1 George IV Bridge. Situated on the Royal Mile and designed by Rosita Missoni.\",\"alias\":null,\"pets_ok\":true,\"free_breakfast\":true,\"free_internet\":false,\"free_parking\":false}\n,{\"title\":\"Edinburgh/South\",\"name\":\"Argyle Backpackers\",\"address\":\"14 Argyle Pl\",\"directions\":\"The number 41 bus (catch it outside Waverley railway station) goes right past the front door.\",\"phone\":\"+44 131 667 9991\",\"tollfree\":null,\"email\":null,\"fax\":null,\"url\":\"http://www.argyle-backpackers.co.uk/\",\"checkin\":null,\"checkout\":null,\"price\":\"Dorm from £13\",\"geo\":{\"lat\":55.9385,\"lon\":-3.1912,\"accuracy\":\"ROOFTOP\"},\"type\":\"hotel\",\"id\":8685,\"country\":\"United Kingdom\",\"city\":\"Edinburgh\",\"state\":null,\"reviews\":[{\"content\":\"Made a one night reservation at this \\\"hotel\\\" without checking the reviews, big mistake. The hotel looks OK from the highway and looked conveniently located. Should have known when the very unfriendly girl at the front desk gave me the room key wrapped in a post-it note because they had run out of the little envelopes. The room smelled weird, when we opened the chest drawers we saw what looked like small roaches, we never took anything out of the suitcase. It was very cold that night in Miami and the heating unit blew cold air. The next morning we woke up to find out there was no hot water in the whole hotel, after many calls and trips to the front desk I was informed that there was someone coming to fix the boiler. To make a long story short, the hot water came back on at 12:00pm; checkout time is 11:00am. They wanted us to leave without taking a shower, the maid was also annoyed at us because we didn't leave the room and she had to finish to go home. Later that night while at a restaurant in South Beach, I noticed some itchy bumps on the left side of the back of my neck and scalp, as well as the knuckles of my fingers, they turned out to be BED BUG BITES!!!\",\"ratings\":{\"Service\":1.0,\"Cleanliness\":1.0,\"Overall\":1.0,\"Value\":1.0,\"Sleep Quality\":1.0,\"Rooms\":1.0,\"Location\":4.0},\"author\":\"Consuelo Thiel\",\"date\":\"2013-03-28 07:41:35 +0300\"},{\"content\":\"Restaurant service was o.k. Your continental breakfast is a joke. We were there 2 nights. We had 4 rooms which were reserved in August 2008. The last couple to get there Friday niight got a terrible room. Nothing in it but the bed. They wre given a different room the next morning but the damage was done. No we will not be back.\",\"ratings\":{\"Cleanliness\":1.0,\"Overall\":1.0,\"Value\":1.0,\"Service\":1.0,\"Rooms\":1.0},\"author\":\"Ettie Bartell\",\"date\":\"2012-09-04 06:14:54 +0300\"},{\"content\":\"Disappointing, after all these years... I have been staying at the Capri since 2001, when my family moved from SF to the North Bay. It used to be a great deal for us ex-pats and while not a luxury hotel, it was clean and comfortable (plus in a great location).My son and I stayed at the Capri this summer and we were terribly let down. They no longer offer specials and while there is renovation occurring, our room was dingy and the bed very uncomfortable. My 9 year old son said, Mom, let's not stay at the Capri anymore. Sadly, I had to agree.\",\"ratings\":{\"Service\":3.0,\"Business service\":-1.0,\"Cleanliness\":2.0,\"Check in / front desk\":3.0,\"Overall\":2.0,\"Value\":2.0,\"Rooms\":2.0,\"Location\":4.0},\"author\":\"Harry O'Kon I\",\"date\":\"2014-09-10 11:07:33 +0300\"},{\"content\":\"Best Deal in the Marina If you don't mind 1960's decor this place will fit the bill. It's very reasonable and very clean. Sometimes it books up with the Euro tours. Ask to stay on the 3rd floor, with the high ceilings and roof windows. I have actually heard the fog horns at night. There is also plenty of free parking right on-site.This hotel is right off Union street and within walking distance of the Marina Green. Tons of great restaurants and clubs. One of my favorites is the Brazen Head, which is right across the street. A small English Pub with great food and drinks. It's hard to spot but there's a real small sign out front. Also, El Canasta (sp?) has a great steak burrito.\",\"ratings\":{\"Service\":-1.0,\"Business service\":-1.0,\"Cleanliness\":-1.0,\"Check in / front desk\":-1.0,\"Overall\":4.0,\"Value\":-1.0,\"Rooms\":-1.0,\"Location\":-1.0},\"author\":\"Sibyl Lind\",\"date\":\"2014-09-20 18:19:21 +0300\"},{\"content\":\"Bed Bugs and Ants The ants didn't bother me. It was the bed bugs I detested.Before staying at the Buena Vista I didn't know what a bed bug looked. But the spots on my arms that looked like flea bites kept appearing after I got home. So I googled them.Yep, sure enough! Up popped a photo of the same type of bug that I had killed after I found it crawling on my husband's pillow while we were staying at the Buena Vista this January 2008!I ordered some all natural bed bug powder to dust all over my house. Even my kids are showing up with the spots.When I called the hotel after I got home to tell them about it they said they would block off that room. But when I called a day or two later there was someone in that room. It doesn't matter if they block off and treat one room. They've got to do the entire premises.\",\"ratings\":{\"Service\":5.0,\"Business service\":-1.0,\"Cleanliness\":1.0,\"Check in / front desk\":5.0,\"Overall\":2.0,\"Value\":5.0,\"Rooms\":1.0,\"Location\":5.0},\"author\":\"Deshawn Rippin\",\"date\":\"2014-10-17 15:46:51 +0300\"},{\"content\":\"Not a great place to stay This place is falling apart...was once a nice little place to stay but not now. Lobby was shabby and dirty, room was not better...there was mold in the tub, no movies here to buy on cable, iron in room was broke....etc. For the $160 they charged per night ( with a AAA card), I would not go back...there is a very nice small, totally remodeled motel two blocks down that we should have stayed at for the same price and most definitly will next time...It is called Hotel Del Sol...check it out, thehoteldelsol.com....much, much better for the $$$$.\",\"ratings\":{\"Service\":2.0,\"Business service\":1.0,\"Cleanliness\":1.0,\"Check in / front desk\":3.0,\"Overall\":2.0,\"Value\":1.0,\"Rooms\":2.0,\"Location\":4.0},\"author\":\"Wayne Tremblay III\",\"date\":\"2012-10-27 01:48:16 +0300\"},{\"content\":\"Great Budget Accommodation If you want a small budget sized no frills hotel that offers a resonable level of service then this is for you. The rooms were large and the housekeeping very good. The front desk service was always helpful and friendly with good advice. Traffic noise was not as bad as expected and nor was the beds. Only minus was the continental breakfast American style which differed somewhat from what we experienced in other countries. Too sweet for our taste.Overall a great experience and well located although being closer to eateries would have been appreciated. Can recommend the Liquor Store accross the road. Their staff were great!\",\"ratings\":{\"Service\":3.0,\"Business service\":-1.0,\"Cleanliness\":3.0,\"Check in / front desk\":-1.0,\"Overall\":3.0,\"Value\":3.0,\"Rooms\":4.0,\"Location\":-1.0},\"author\":\"Margaretta Miller\",\"date\":\"2012-04-19 20:46:49 +0300\"}],\"public_likes\":[\"Narciso Wiegand\",\"Graciela Bailey\",\"Kavon Bruen\",\"Aditya Feest\",\"Caleb Medhurst\",\"Ross Rippin\",\"Germaine Kunde\"],\"vacancy\":true,\"description\":\"Two good self-catering kitchens, garden, conservatory/seating area, choice of different sized dorms, and private rooms.Definitely not a party hostel.\",\"alias\":null,\"pets_ok\":true,\"free_breakfast\":true,\"free_internet\":true,\"free_parking\":false}\n,{\"title\":\"Abbeville\",\"name\":\"Chez Mel\",\"alt\":null,\"address\":\"63-65 rue Saint-Vulfran\",\"directions\":null,\"phone\":\"+33 3 22 19 48 64\",\"tollfree\":null,\"email\":null,\"url\":null,\"hours\":null,\"image\":null,\"price\":null,\"content\":\"With an old style setting and musical accompaniment, this is a hearty and family-friendly crêpe restaurant. It is also a tea room in the afternoon.\",\"geo\":{\"lat\":50.104437,\"lon\":1.829432,\"accuracy\":\"RANGE_INTERPOLATED\"},\"activity\":\"eat\",\"type\":\"landmark\",\"id\":33,\"country\":\"France\",\"city\":\"Abbeville\",\"state\":\"Picardie\"}\n,{\"title\":\"Aberdour\",\"name\":\"Aberdour Castle\",\"alt\":null,\"address\":null,\"directions\":null,\"phone\":null,\"tollfree\":null,\"email\":null,\"url\":\"http://www.historic-scotland.gov.uk/propertyresults/propertyoverview.htm?PropID=PL_001\",\"hours\":null,\"image\":null,\"price\":null,\"content\":\"Is a fascinating, 12th-century castle which was granted by Robert the Bruce to his friend and nephew, Thomas Randolph, Earl of Moray. It includes the beautiful and well-maintained castle gardens, as well as a spectacular beehive-shaped dovecot built at the end of the sixteenth century.\",\"geo\":{\"lat\":56.0552,\"lon\":-3.2985,\"accuracy\":\"APPROXIMATE\"},\"activity\":\"see\",\"type\":\"landmark\",\"id\":35,\"country\":\"United Kingdom\",\"city\":\"Aberdour\",\"state\":null}\n,{\"title\":\"Aberdour\",\"name\":\"The Silver Sands Beach\",\"alt\":null,\"address\":null,\"directions\":null,\"phone\":null,\"tollfree\":null,\"email\":null,\"url\":null,\"hours\":null,\"image\":null,\"price\":null,\"content\":\"is one of Scotland's seven Blue Flag awarded beaches, and is incredibly popular in summer time. For those after a bit of peace and quiet, the '''Black Sands Beach''' may be more to your tastes.\",\"geo\":{\"lat\":56.0544,\"lon\":-3.2863,\"accuracy\":\"ROOFTOP\"},\"activity\":\"see\",\"type\":\"landmark\",\"id\":36,\"country\":\"United Kingdom\",\"city\":\"Aberdour\",\"state\":null}\n,{\"title\":\"Aberdour\",\"name\":\"Aberdour Railway Station\",\"alt\":null,\"address\":null,\"directions\":null,\"phone\":null,\"tollfree\":null,\"email\":null,\"url\":null,\"hours\":null,\"image\":null,\"price\":null,\"content\":\"is a beautifully kept and cared for example of a traditional station, and regularly wins the &quot;Best Station and Gardens in Great Britain&quot; award.\",\"geo\":{\"lat\":56.05471,\"lon\":-3.30089,\"accuracy\":\"RANGE_INTERPOLATED\"},\"activity\":\"see\",\"type\":\"landmark\",\"id\":37,\"country\":\"United Kingdom\",\"city\":\"Aberdour\",\"state\":null}]"
  },
  {
    "path": "doc.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n/*\nPackage bleve is a library for indexing and searching text.\n\nExample Opening New Index, Indexing Data\n\n\tmessage := struct{\n\t    Id:   \"example\"\n\t    From: \"xyz@couchbase.com\",\n\t    Body: \"bleve indexing is easy\",\n\t}\n\n\tmapping := bleve.NewIndexMapping()\n\tindex, _ := bleve.New(\"example.bleve\", mapping)\n\tindex.Index(message.Id, message)\n\nExample Opening Existing Index, Searching Data\n\n\tindex, _ := bleve.Open(\"example.bleve\")\n\tquery := bleve.NewQueryStringQuery(\"bleve\")\n\tsearchRequest := bleve.NewSearchRequest(query)\n\tsearchResult, _ := index.Search(searchRequest)\n*/\npackage bleve\n"
  },
  {
    "path": "docs/create_and_search_your_first_index.md",
    "content": "# Creating a Bleve Index\n\nA simple how-to example using Bleve in Go to create an index, add documents, and run search queries with results.\n\n```go\npackage main\n\nimport (\n    \"fmt\"\n    \"log\"\n\n    bleve \"github.com/blevesearch/bleve/v2\"\n)\n\ntype Document struct {\n    ID      string `json:\"id\"`\n    Title   string `json:\"title\"`\n    Content string `json:\"content\"`\n}\n\nfunc main() {\n    indexPath := \"example.bleve\"\n    // Create a new index\n    mapping := bleve.NewIndexMapping()\n    index, err := bleve.New(indexPath, mapping)\n    if err != nil {\n        log.Fatal(err)\n    }\n    defer index.Close()\n\n    // Add documents\n    documents := []Document{\n        {\n            ID:      \"doc\",\n            Title:   \"Bleve documentation\",\n            Content: \"Bleve provides full-text search capabilities.\",\n        },\n        {\n            ID:      \"doc1\",\n            Title:   \"Elasticsearch documentation\",\n            Content: \"Elasticsearch provides full-text search capabilities as well.\",\n        },\n    }\n\n    // Iterate and index the documents\n    batch := index.NewBatch()\n    for _, doc := range documents {\n        batch.Index(doc.ID, doc)\n    }\n    if err := index.Batch(batch); err != nil {\n        log.Fatal(err)\n    }\n\n    // Search the created index\n    query := bleve.NewQueryStringQuery(\"bleve\")\n    searchRequest := bleve.NewSearchRequest(query)\n    searchRequest.Explain = true\n    searchRequest.Fields = []string{\"title\", \"content\"}\n    searchResult, err := index.Search(searchRequest)\n    if err != nil {\n        log.Fatal(err)\n    }\n    fmt.Println(searchResult)\n}\n\n```\n\n## Output\n\n```bash\n$ go run main.go\n\n1 matches, showing 1 through 1, took 262.333µs\n    1. doc (0.471405)\n        title\n                Bleve documentation\n        content\n                Bleve provides full-text search capabilities.\n```\n\n## Step-by-Step Breakdown\n\n### 1. Index Creation\n\n```go\n// Create a new index mapping\nmapping := bleve.NewIndexMapping()\n// Create a new index (this creates a directory on disk)\nindex, err := bleve.New(indexPath, mapping)\n```\n\n**What happens:**\n\n- Creates an index mapping with default settings\n- Creates a new index directory `example.bleve/`\n- Sets up the underlying storage (Scorch engine by default)\n\n### 2. Document Indexing\n\n```go\n// Index a document with a unique ID\nerr := index.Index(\"doc\", map[string]interface{}{\n    \"title\":   \"My Document\",\n    \"content\": \"This is the document content\",\n    \"author\":  \"John Doe\",\n})\n```\n\n**What happens:**\n\n- Document gets a unique ID (`doc`)\n- Fields are automatically mapped based on their Go types\n- Text fields are analyzed (tokenized, lowercased, etc.) based on the mapping chosen (here, the default one)\n- Document is stored in the search index\n\n### 3. Searching\n\n```go\n// Create a query\nquery := bleve.NewQueryStringQuery(\"search terms\")\nrequest := bleve.NewSearchRequest(query)\n\n// Execute search\nresults, err := index.Search(request)\n```\n\n**What happens:**\n\n- Query string is parsed and analyzed\n- Index is searched for matching documents\n- Results are scored and ranked by relevance by the algorithm used\n- Document metadata and highlights are returned\n\n## Working with Existing Indexes\n\nTo open an existing index instead of creating a new one:\n\n```go\n// Open existing index\nindex, err := bleve.Open(\"example.bleve\")\nif err != nil {\n    log.Fatal(err)\n}\ndefer index.Close()\n```\n\n## Different Query Types\n\n### 1. Query String Query (Simple)\n\n```go\nquery := bleve.NewQueryStringQuery(\"golang programming\")\n```\n\n### 2. Match Query (Exact Field)\n\n```go\nquery := bleve.NewMatchQuery(\"bleve\")\nquery.SetField(\"title\")  // Search only in title field\n```\n\n### 3. Boolean Query (Complex)\n\n```go\nmustQuery := bleve.NewMatchQuery(\"golang\")\nshouldQuery := bleve.NewMatchQuery(\"programming\")\n\nboolQuery := bleve.NewBooleanQuery()\nboolQuery.AddMust(mustQuery)\nboolQuery.AddShould(shouldQuery)\n```\n\n### 4. Range Query (Numeric/Date)\n\n```go\nminPrice := 20.50\nmaxPrice := 40.75\nquery := bleve.NewNumericRangeQuery(&minPrice, &maxPrice)\nquery.SetField(\"price\")\n```\n\n## Advanced Index Configuration\n\n### Custom Field Mapping\n\n```go\n// We can create customised mapping as well by specifying about analyzers\nmapping := bleve.NewIndexMapping()\n\n// Text field with custom analyzer\ntitleMapping := bleve.NewTextFieldMapping()\ntitleMapping.Analyzer = \"en\"  // English analyzer\n\n// Numeric field\npriceMapping := bleve.NewNumericFieldMapping()\n\n// Date field\ndateMapping := bleve.NewDateTimeFieldMapping()\n\n// Document mapping\ndocMapping := bleve.NewDocumentMapping()\ndocMapping.AddFieldMappingsAt(\"title\", titleMapping)\ndocMapping.AddFieldMappingsAt(\"price\", priceMapping)\ndocMapping.AddFieldMappingsAt(\"created_at\", dateMapping)\n\n// Add to index mapping\nmapping.AddDocumentMapping(\"product\", docMapping)\n\n// Create index with custom mapping\nindex, err := bleve.New(\"products.bleve\", mapping)\n```\n\n### Batch Operations\n\nFor better performance when indexing many documents, we can do indexing in batches:\n\n```go\nbatch := index.NewBatch()\n\nfor _, doc := range documents {\n    batch.Index(doc.ID, doc)\n}\n\n// Execute batch\nerr := index.Batch(batch)\n```\n"
  },
  {
    "path": "docs/geo.md",
    "content": "# Geo spatial search\n\nRedirect to [geo/README.md](https://github.com/blevesearch/bleve/blob/master/geo/README.md)\n"
  },
  {
    "path": "docs/hierarchy.md",
    "content": "# Hierarchical nested search\n\n* *v2.6.0* (and after) will come with support for **Array indexing and hierarchical nested search**.\n* We've achieved this by embedding nested documents within our bleve (scorch) indexes.\n* Usage of zap file format: [v17](https://github.com/blevesearch/zapx/blob/master/zap.md). Here we preserve hierarchical document relationships within segments, continuing to conform to the segmented architecture of *scorch*.\n\n## Supported\n\n* Indexing `Arrays` allows specifying fields that contain arrays of objects. Each object in the array can have its own set of fields, enabling the representation of hierarchical data structures within a single document.\n\n```json\n{\n    \"id\": \"1\",\n    \"name\": \"John Doe\",\n    \"addresses\": [\n        {\n            \"type\": \"home\",\n            \"street\": \"123 Main St\",\n            \"city\": \"Hometown\",\n            \"zip\": \"12345\"\n        },\n        {\n            \"type\": \"work\",\n            \"street\": \"456 Corporate Blvd\",\n            \"city\": \"Metropolis\",\n            \"zip\": \"67890\"\n        }\n    ]\n}\n```\n\n* Multi-level arrays: Arrays can contain objects that themselves have array fields, allowing for deeply nested structures, such as a list of projects, each with its own list of tasks.\n\n```json\n{\n    \"id\": \"2\",\n    \"name\": \"Jane Smith\",\n    \"projects\": [\n        {\n            \"name\": \"Project Alpha\",\n            \"tasks\": [\n                {\"title\": \"Task 1\", \"status\": \"completed\"},\n                {\"title\": \"Task 2\", \"status\": \"in-progress\"}\n            ]\n        },\n        {\n            \"name\": \"Project Beta\",\n            \"tasks\": [\n                {\"title\": \"Task A\", \"status\": \"not-started\"},\n                {\"title\": \"Task B\", \"status\": \"completed\"}\n            ]\n        }\n    ]\n}\n```\n\n* Multiple arrays: A document can have multiple fields that are arrays, each representing different hierarchical data, such as a list of phone numbers and a list of email addresses.\n\n```json\n{\n    \"id\": \"3\",\n    \"name\": \"Alice Johnson\",\n    \"phones\": [\n        {\"type\": \"mobile\", \"number\": \"555-1234\"},\n        {\"type\": \"home\", \"number\": \"555-5678\"}\n    ],\n    \"emails\": [\n        {\"type\": \"personal\", \"address\": \"alice@example.com\"},\n        {\"type\": \"work\", \"address\": \"alice@work.com\"}\n    ]\n}\n```\n\n* Hybrid arrays: Multi-level and multiple arrays can be combined within the same document to represent complex hierarchical data structures, such as a company with multiple departments, each having its own list of employees and projects.\n\n```json\n{\n    \"id\": \"doc1\",\n    \"company\": {\n        \"id\": \"c1\",\n        \"name\": \"TechCorp\",\n        \"departments\": [\n            {\n                \"name\": \"Engineering\",\n                \"budget\": 2000000,\n                \"employees\": [\n                    {\"name\": \"Alice\", \"role\": \"Engineer\"},\n                    {\"name\": \"Bob\", \"role\": \"Manager\"}\n                ],\n                \"projects\": [\n                    {\"title\": \"Project X\", \"status\": \"ongoing\"},\n                    {\"title\": \"Project Y\", \"status\": \"completed\"}\n                ]\n            },\n            {\n                \"name\": \"Sales\",\n                \"budget\": 300000,\n                \"employees\": [\n                    {\"name\": \"Eve\", \"role\": \"Salesperson\"},\n                    {\"name\": \"Mallory\", \"role\": \"Manager\"}\n                ],\n                \"projects\": [\n                    {\"title\": \"Project A\", \"status\": \"completed\"},\n                    {\"title\": \"Project B\", \"status\": \"ongoing\"}\n                ]\n            }\n        ],\n        \"locations\": [\n            {\"city\": \"Athens\",\"country\": \"Greece\"},\n            {\"city\": \"Berlin\",\"country\": \"USA\"}\n        ]\n    }\n}\n```\n\n* Earlier versions of Bleve only supported flat arrays of primitive types (e.g., strings, numbers), and would flatten nested structures, losing the hierarchical relationships, so the above complex documents could not be accurately represented or queried. For example, the \"employees\" and \"projects\" fields within each department would be flattened, making it impossible to associate employees with their respective departments.\n\n* From v2.6.0 onwards, Bleve allows for accurate representation and querying of complex nested structures, preserving the relationships between different levels of the hierarchy, across multi-level, multiple and hybrid arrays.\n\n* The addition of `nested` document mappings enable defining fields that contain arrays of objects, giving the option to preserve the hierarchical relationships within the array during indexing. Having `nested` as false (default) will flatten the objects within the array, losing the hierarchy, which was the earlier behavior.\n\n```json\n{\n    \"departments\": {  \n        \"dynamic\": false,\n        \"enabled\": true,\n        \"nested\": true,\n        \"properties\": {\n            \"employees\": {\n                \"dynamic\": false,\n                \"enabled\": true,\n                \"nested\": true \n            },\n            \"projects\": {\n                \"dynamic\": false,\n                \"enabled\": true,\n                \"nested\": true\n            }\n        }\n    },\n    \"locations\": {\n        \"dynamic\": false,\n        \"enabled\": true,\n        \"nested\": true\n    }\n}\n```\n\n* Any Bleve query (e.g., `match`, `phrase`, `term`, `fuzzy`, `numeric/date range` etc.) can be executed against fields within nested documents, with no special handling required. The query processor will automatically traverse the nested structures to find matches. Additional search constructs\nlike vector search, synonym search, hybrid and pre-filtered vector search integrate seamlessly with hierarchy search.\n\n* Conjunction Queries (AND queries) and other queries that depend on term co-occurrence within the same hierarchical context will respect the boundaries of nested documents. This means that terms must appear within the same nested object to be considered a match. For example, a conjunction query searching for an employee named \"Alice\" with the role \"Engineer\" within the \"Engineering\" department will only return results where both name and role terms are found within the same employee object, which is itself within a \"Engineering\" department object.\n\n* Some other search constructs will have enhanced precision with hierarchy search.\n  * Field-Level Highlighting: Only fields within the matched nested object are retrieved and highlighted, ensuring highlights appear in the correct hierarchical context. For example, a match in `departments[name=Engineering].employees` highlights only employees in that department.\n\n  * Nested Faceting / Aggregations: Facets are computed within matched nested objects, producing context-aware buckets. E.g., a facet on `departments.projects.status` returns ongoing or completed only for projects in matched departments.\n\n  * Sorting by Nested Fields: Sorting can use fields from the relevant nested object, e.g., ordering companies by `departments.budget` sorts based on the budget of the specific matched department, not unrelated departments.\n\n* Vector Search (KNN / Multi-KNN): When a document contains an array of objects with vector/multi-vector fields, the final document score and ranking are identical whether or not the array is marked as `nested`. In both cases, the highest-scoring vector is selected; either directly from the array (non-nested) or from the best-matching nested object with its score bubbled up to the parent document.\n\n* Pre-Filtered Vector Search: When vector search is combined with filters on fields inside a nested array, the filters are applied first to pick which nested items are eligible. Vector similarity is then computed only on the vector fields of those filtered nested objects. For example, if `departments.employees` is a `nested` array, a pre-filtered KNN query for employees with a `skills_vector` matching `machine learning engineer`, a role of `Manager`, and belonging to the `Sales` department will first narrow the candidate set to only employees who meet the requirement, and then compute vector similarity on the `skills_vector` of that filtered subset. This ensures that vector search results come only from the employees that satisfy the filter, and not from unrelated employees in other departments.\n\n## Indexing\n\nBelow is an example of using the Bleve API to index documents with hierarchical structures, using hybrid arrays and nested mappings.\n\n```go\n// Define a document to be indexed.\ndocJSON :=\n    `{\n        \"company\": {\n            \"id\": \"c3\",\n            \"name\": \"WebSolutions\",\n            \"departments\": [\n                {\n                    \"name\": \"HR\",\n                    \"budget\": 800000,\n                    \"employees\": [\n                        {\"name\": \"Eve\", \"role\": \"Manager\"},\n                        {\"name\": \"Frank\", \"role\": \"HR\"}\n                    ],\n                    \"projects\": [\n                        {\"title\": \"Project Beta\", \"status\": \"completed\"},\n                        {\"title\": \"Project B\", \"status\": \"ongoing\"}\n                    ]\n                },\n                {\n                    \"name\": \"Engineering\",\n                    \"budget\": 200000,\n                    \"employees\": [\n                        {\"name\": \"Heidi\", \"role\": \"Support Engineer\"},\n                        {\"name\": \"Ivan\", \"role\": \"Manager\"}\n                    ],\n                    \"projects\": [\n                        {\"title\": \"Project Helpdesk\", \"status\": \"ongoing\"},\n                        {\"title\": \"Project FAQ\", \"status\": \"completed\"}\n                    ]\n                }\n            ],\n            \"locations\": [\n                {\"city\": \"Edinburgh\", \"country\": \"UK\"},\n                {\"city\": \"London\", \"country\": \"Canada\"}\n            ]\n        }\n    }`\n\n// Define departments as a nested document mapping (since it contains arrays of objects)\n// and index name and budget fields\ndepartmentsMapping := bleve.NewNestedDocumentMapping()\ndepartmentsMapping.AddFieldMappingsAt(\"name\", bleve.NewTextFieldMapping())\ndepartmentsMapping.AddFieldMappingsAt(\"budget\", bleve.NewNumericFieldMapping())\n\n// Define employees as a nested document mapping within departments (since it contains arrays of objects)\n// and index name and role fields\nemployeesMapping := bleve.NewNestedDocumentMapping()\nemployeesMapping.AddFieldMappingsAt(\"name\", bleve.NewTextFieldMapping())\nemployeesMapping.AddFieldMappingsAt(\"role\", bleve.NewTextFieldMapping())\ndepartmentsMapping.AddSubDocumentMapping(\"employees\", employeesMapping)\n\n// Define projects as a nested document mapping within departments (since it contains arrays of objects)\n// and index title and status fields\nprojectsMapping := bleve.NewNestedDocumentMapping()\nprojectsMapping.AddFieldMappingsAt(\"title\", bleve.NewTextFieldMapping())\nprojectsMapping.AddFieldMappingsAt(\"status\", bleve.NewTextFieldMapping())\ndepartmentsMapping.AddSubDocumentMapping(\"projects\", projectsMapping)\n\n// Define locations as a nested document mapping (since it contains arrays of objects) \n// and index city and country fields\nlocationsMapping := bleve.NewNestedDocumentMapping()\nlocationsMapping.AddFieldMappingsAt(\"city\", bleve.NewTextFieldMapping())\nlocationsMapping.AddFieldMappingsAt(\"country\", bleve.NewTextFieldMapping())\n\n// Define company as a document mapping and index its name field and \n// add departments and locations as sub-document mappings\ncompanyMapping := bleve.NewDocumentMapping()\ncompanyMapping.AddFieldMappingsAt(\"name\", bleve.NewTextFieldMapping())\ncompanyMapping.AddSubDocumentMapping(\"departments\", departmentsMapping)\ncompanyMapping.AddSubDocumentMapping(\"locations\", locationsMapping)\n\n// Define the final index mapping and add company as a sub-document mapping in the default mapping\nindexMapping := bleve.NewIndexMapping()\nindexMapping.DefaultMapping.AddSubDocumentMapping(\"company\", companyMapping)\n\n// Create the index with the defined mapping\nindex, err := bleve.New(\"hierarchy_example.bleve\", indexMapping)\nif err != nil {\n    panic(err)\n}\n\n// Unmarshal the document JSON into a map, for indexing\nvar doc map[string]interface{}\nerr = json.Unmarshal([]byte(docJSON), &doc)\nif err != nil {\n    panic(err)\n}\n\n// Index the document\nerr = index.Index(\"doc1\", doc)\nif err != nil {\n    panic(err)\n}\n```\n\n## Querying\n\n```go\n// Open the index\nindex, err := bleve.Open(\"hierarchy_example.bleve\")\nif err != nil {\n    panic(err)\n}\n\nvar (\n    req *bleve.SearchRequest\n    res *bleve.SearchResult\n)\n\n// Example 1: Simple Match Query on a field within a nested document, should work as if it were a flat field\nq1 := bleve.NewMatchQuery(\"Engineer\")\nq1.SetField(\"company.departments.employees.role\")\nreq = bleve.NewSearchRequest(q1)\nres, err = index.Search(req)\nif err != nil {\n    panic(err)\n}\nfmt.Println(\"Match Query Results:\", res)\n\n// Example 2: Conjunction Query (AND) on fields within the same nested document\n// like finding employees with name \"Eve\" and role \"Manager\". This will only match\n// if both terms are in the same employee object.\nq1 = bleve.NewMatchQuery(\"Eve\")\nq1.SetField(\"company.departments.employees.name\")\nq2 := bleve.NewMatchQuery(\"Manager\")\nq2.SetField(\"company.departments.employees.role\")\nconjQuery := bleve.NewConjunctionQuery(\n    q1,\n    q2,\n)\nreq = bleve.NewSearchRequest(conjQuery)\nres, err = index.Search(req)\nif err != nil {\n    panic(err)\n}\nfmt.Println(\"Conjunction Query Results:\", res)\n\n// Example 3: Multi-level Nested Query, finding projects with status \"ongoing\"\n// within the \"Engineering\" department. This ensures both conditions are met\n// within the correct hierarchy, i.e., the ongoing project must belong to the\n// Engineering department.\nq1 = bleve.NewMatchQuery(\"Engineering\")\nq1.SetField(\"company.departments.name\")\nq2 = bleve.NewMatchQuery(\"ongoing\")\nq2.SetField(\"company.departments.projects.status\")\nmultiLevelQuery := bleve.NewConjunctionQuery(\n    q1,\n    q2,\n)\nreq = bleve.NewSearchRequest(multiLevelQuery)\nres, err = index.Search(req)\nif err != nil {\n    panic(err)\n}\nfmt.Println(\"Multi-level Nested Query Results:\", res)\n\n// Example 4: Multiple Arrays Query, finding documents with a location in \"London\"\n// and an employee with the role \"Manager\". This checks conditions across different arrays.\nq1 = bleve.NewMatchQuery(\"London\")\nq1.SetField(\"company.locations.city\")\nq2 = bleve.NewMatchQuery(\"Manager\")\nq2.SetField(\"company.departments.employees.role\")\nmultiArrayQuery := bleve.NewConjunctionQuery(\n    q1,\n    q2,\n)\nreq = bleve.NewSearchRequest(multiArrayQuery)\nres, err = index.Search(req)\nif err != nil {\n    panic(err)\n}\nfmt.Println(\"Multiple Arrays Query Results:\", res)\n\n// Hybrid Arrays Query, combining multi-level and multiple arrays,\n// finding documents with a Manager named Ivan working in Edinburgh, UK\nq1 = bleve.NewMatchQuery(\"Ivan\")\nq1.SetField(\"company.departments.employees.name\")\nq2 = bleve.NewMatchQuery(\"Manager\")\nq2.SetField(\"company.departments.employees.role\")\nq3 := bleve.NewMatchQuery(\"Edinburgh\")\nq3.SetField(\"company.locations.city\")\nq4 := bleve.NewMatchQuery(\"UK\")\nq4.SetField(\"company.locations.country\")\nhybridArrayQuery := bleve.NewConjunctionQuery(\n    bleve.NewConjunctionQuery(\n        q1,\n        q2,\n    ),\n    bleve.NewConjunctionQuery(\n        q3,\n        q4,\n    ),\n)\nreq = bleve.NewSearchRequest(hybridArrayQuery)\nres, err = index.Search(req)\nif err != nil {\n    panic(err)\n}\nfmt.Println(\"Hybrid Arrays Query Results:\", res)\n\n// Close the index when done\nerr = index.Close()\nif err != nil {\n    panic(err)\n}\n```\n"
  },
  {
    "path": "docs/index_update.md",
    "content": "# Ability to reduce downtime during index mapping updates\n\n* *v2.5.4* (and after) will come with support to delete or modify any field mapping in the index mapping without requiring a full rebuild of the index\n* We do this by storing which portions of the field has to be deleted within zap and then lazily executing the deletion during subsequent merging of the segments\n\n## Usage\n\nWhile opening an index, if an updated mapping is provided as a string under the key `updated_mapping` within the `runtimeConfig` parameter of `OpenUsing`, then we open the index and try to update it to use the new mapping provided.\n\nIf the update fails, the index is unchanged and an error is returned explaining why the update was unsuccessful.\n\n## What can be deleted and what can't be deleted?\n\nFields can be partially deleted by changing their Index, Store, and DocValues parameters from true to false, or completely removed by deleting the field itself.\n\nAdditionally, document mappings can be deleted either by fully removing them from the index mapping or by setting the Enabled value to false, which deletes all fields defined within that mapping.\n\nHowever, if any of the following conditions are met, the index is considered non-updatable.\n\n* Any additional fields or enabled document mappings in the new index mapping\n* Any changes to IncludeInAll, type, IncludeTermVectors and SkipFreqNorm\n* Any document mapping having its enabled value changing from false to true\n* Text fields with a different analyser or date time fields with a different date time format\n* Vector and VectorBase64 fields changing dims, similarity or vectorIndexOptimizedFor\n* Any changes when field is part of `_all`\n* Full field deletions when it is covered by any dynamic setting (Index, Store or DocValues Dynamic)\n* Any changes to dynamic settings at the top level or any enabled document mapping\n* If multiple fields sharing the same field name either from different type mappings or aliases are present, then any non compatible changes across all of these fields\n\n## How to enforce immediate deletion?\n\nSince the deletion is only done during merging, a [force merge](https://github.com/blevesearch/bleve/blob/b82baf10b205511cf12da5cb24330abd9f5b1b74/index/scorch/merge.go#L164) may be used to completely remove the stale data.\n\n## Sample code to update an existing index\n\n```go\nnewMapping := `<Updated Index Mapping>`\nconfig := map[string]interface{}{\n    \"updated_mapping\": newMapping,\n}\nindex, err := bleve.OpenUsing(\"<Path to Index>\", config)\nif err != nil {\n    return err\n}\n```\n"
  },
  {
    "path": "docs/pagination.md",
    "content": "# Pagination\n\n## Why pagination matters\n\nSearch queries can match many documents. Pagination lets you fetch and display results in chunks, keeping responses small and fast.\n\nBy default, Bleve returns the first 10 hits sorted by relevance (score), highest first.\n\n## Two pagination modes\n\n- `From`/`Size`: simple and stateless; cost grows with page depth.\n- `SearchAfter`/`SearchBefore`: efficient for deep paging; requires passing sort keys from the previous page.\n\nBoth modes can be used with any valid sort.\n\n## `Size`/`From`\n\nOffset-based pagination uses `Size` (page length) and `From` (number of hits to skip). Bleve collects at least `Size + From` ordered results, then returns the `Size` slice starting at `From`.\n\nJSON example:\n\n```json\n{\n  \"query\": { \"match\": \"California\" },\n  \"sort\": [\"-_score\"],\n  \"size\": 5,\n  \"from\": 10\n}\n```\n\nThe result would be 5 hits starting from the 5th hit.\n\nWhen to use:\n\n- Simple, stateless pagination for shallow pages.\n- Avoid for deep pages, as memory grows with `From` for deeper pages.\n\n## `SearchAfter` and `SearchBefore`\n\nThis returns the next (or previous) page based on a boundary defined by the sort keys of a specific hit. This keeps resource usage proportional to the page size, even for deep pages.\n\nRules:\n\n- Use either `SearchAfter` (forward) or `SearchBefore` (backward), not both at once.\n- The length of `SearchAfter`/`SearchBefore` must match the length of `Sort`.\n- Values are strings representing the sort keys, in the same order as `Sort`.\n- Keep the same `query` and `sort` across pages for consistent navigation.\n\nWhere do sort keys come from?\n\n- Each hit includes `Sort` (and `DecodedSort` from Bleve v2.5.2). Take the last hit's sort keys for `SearchAfter`, or the first hit's sort keys for `SearchBefore`.\n- If the field/fields to be searched over is numeric, datetime or geo, the values in the `Sort` field may have garbled values; this is because of how Bleve represents such data types internally. To use such fields as sort keys, use the `DecodedSort` field, which decodes the internal representations. This feature is available from Bleve v2.5.4.\n\n> When using `DecodedSort`, the `Sort` array in the search request needs to explicitly declare the type of the field for proper decoding. Hence, the `Sort` array must contain either `SortField` objects (for numeric and datetime) or `SortGeoDistance` objects (for geo) rather than just the field names. More info on `SortField` and `SortGeoDistance` can be found in [sort_facet.md](sort_facet.md).\n\nForward pagination over `_id` and `_score`:\n\n```json\n{\n  \"query\": { \"match\": \"California\" },\n  \"sort\": [\"_id\", \"_score\"],\n  \"search_after\": [\"hotel_10180\", \"0.998\"],\n  \"size\": 3\n}\n```\n\nBackward pagination over `_id` and `_score`:\n\n```json\n{\n  \"query\": { \"match\": \"California\" },\n  \"sort\": [\"_id\", \"_score\"],\n  \"search_before\": [\"hotel_17595\", \"0.623\"],\n  \"size\": 4\n}\n```\n\nPagination using numeric, datetime and geo fields. Notice how we specify the sort objects, with the \"type\" field explicitly declared in case of numeric and datetime:\n\n```json\n{\n  \"query\": {\n    \"match_all\": {}\n  },\n  \"size\": 10,\n  \"sort\": [\n    {\"by\": \"field\", \"field\": \"price\", \"type\": \"number\"},\n    {\"by\": \"field\", \"field\": \"created_at\", \"type\": \"date\"},\n    {\"by\": \"geo_distance\", \"field\": \"location\", \"location\": {\"lat\": 40.7128,\"lon\": -74.0060}}\n  ],\n  \"search_after\": [\"99.99\", \"2023-10-15T10:30:00Z\", \"5.2\"]\n}\n```\n\n## Total Sort Order\n\nPagination is deterministic. Ensure your `Sort` defines a total order, so that documents with the same sort keys are not left out:\n\n- Sort strings can be field names (prefix with `-` for descending), `\"_score\"`, or `\"_id\"`.\n- Always include a stable tie-breaker as the last key, typically `\"_id\"`.\n- Examples:\n  - `[\"country\", \"-age\", \"_id\"]`\n  - `[\"-_score\", \"_id\"]` (default score desc with a tie-breaker)\n\n## Performance guidance\n\n- Offset pagination cost grows with `From` (collects at least `Size + From` results before slicing).\n- `SearchAfter`/`SearchBefore` keeps memory and network proportional to `Size`.\n- For large datasets and deep navigation, prefer using `SearchAfter` and `SearchBefore`.\n"
  },
  {
    "path": "docs/persister.md",
    "content": "# Scorch Index Memory and File Management\n\n## Memory Management\n\nWhen data is indexed in Scorch — using either the `index.Index()` or `index.Batch()` API — it is added as part of an in-memory \"segment\". Memory management in Scorch indexing mainly relates to handling these in-memory segments during workloads that involve inserts or updates.\n\nIn scenarios with a continuous stream of incoming data, a large number of in-memory segments can accumulate over time. This is where the persister component comes into play—its job is to flush these in-memory segments to disk.\n\nStarting with v2.5.0, Scorch supports parallel flushing of in-memory segments to disk, where the persister checks the total in-memory data and distributes the flush across multiple workers. This feature is disabled by default and can be enabled using two configuration options:\n\n- `NumPersisterWorkers`: This factor decides how many maximum workers can be spawned to flush out the in-memory segments. Each worker will work on a disjoint subset of segments, merge them, and flush them out to the disk. By default the persister deploys only one worker.\n- `MaxSizeInMemoryMergePerWorker`: This config decides what's the maximum amount of input data in bytes a single worker can work upon. By default this value is equal to 0 which means that this config is disabled and the worker tries to merge all the data in one shot. Also note that it's imperative that the user set this config if `NumPersisterWorkers > 1`.\n\nIf the index is tuned to have a higher `NumPersisterWorkers` value, the memory can potentially drain out faster and ensure stronger consistency behaviour — but there would be a lot of on-disk files, and the background merger would experience the pressure of managing this large number of files, which can be resource-intensive.\n\n- Tuning this config is very dependent on the available CPU resources, and something to keep in mind here is that the process's RSS can increase if the number of workers — and each of them working upon a large amount of data — is high.\n\nIncreasing the `MaxSizeInMemoryMergePerWorker` value would mean that each worker acts upon a larger amount of data and spends more time merging and flushing it out to disk — which can be healthy behaviour in terms of I/O, although it comes at the cost of time.\n\n- Changing this config is usecase dependent, for example in usecases where the payload or per doc size is generally large in size (for eg vector usecases), it would be beneficial to have a larger value for this.\n\nSo, having the ideal values for these two configs is definitely dependent on the use case and can involve a bunch of experiments, keeping the resource usage in mind.\n\n## File Management\n\nThe persister introducing some number of file segments into the system would change the state of the system, and the merger would wake up and try to manage these on-disk files.\n\nManagement of these files is crucial when it comes to query latency because a higher number of files would dictate searching through a larger number of files and also higher read amplification to some extent, because the backing data structures can potentially be compacted in size across files.\n\nThe merger sees the files on disk and plans out which segments to merge so that the final layout of segment tiers (each tier having multiple files), which grow in a logarithmic way (the chances of larger tiers growing in number would decrease), is maintained. This also implies that deciding this first-tier size becomes important in deciding the number of segment files across all tiers.\n\nStarting with v2.5.0, this first-tier size is dependent on the file size using the `FloorSegmentFileSize` config, because that's a better metric to consider (unlike the legacy live doc count metric) in order to ensure that the behaviour is in line with the use case and aware of the payload/doc size.\n\n- This config can also be tuned to dictate how the I/O behaviour should be within an index. While tuning this config, it should be in proportion to the `MaxSizeInMemoryMergePerWorker` since that dictates the amount of data flushed out per flush.\n- The observation here is that `FloorSegmentFileSize` is lesser than `MaxSizeInMemoryMergePerWorker` and for an optimal I/O during indexing, this value can be set close to `MaxSizeInMemoryMergePerWorker/6`.\n\n## Setting a Persister/Merger Config in Index\n\nThe configs are set via the `kvConfig` parameter in the `NewUsing()` or `OpenUsing()` API:\n\n```go\n// setting the persister and merger configs\nkvConfig := map[string]interface{}{\n    \"scorchPersisterOptions\": map[string]interface{}{\n        \"NumPersisterWorkers\":           4,\n        \"MaxSizeInMemoryMergePerWorker\": 20000000,\n    },\n    \"scorchMergePlanOptions\": map[string]interface{}{\n        \"FloorSegmentFileSize\": 10000000,\n    },\n}\n// passing the config to the index\nindex, err := bleve.NewUsing(\"example.bleve\", bleve.NewIndexMapping(), bleve.Config.DefaultIndexType, bleve.Config.DefaultMemKVStore, kvConfig)\nif err != nil {\n    panic(err)\n}\n```\n"
  },
  {
    "path": "docs/query-openapi-spec.yaml",
    "content": "openapi: 3.0.3\ninfo:\n  title: Bleve JSON Query Language\n  description: |\n    A comprehensive specification for the Bleve Query JSON Language.\n\n    Bleve is a text indexing library for Go that provides full-text search capabilities.\n    This specification defines the JSON structure for all supported query types in Bleve.\n\n    ## Query Types Supported:\n\n    ### Basic Queries\n    - **Term Query**: Exact term matching\n    - **Match Query**: Full-text search with analysis\n    - **Match Phrase Query**: Phrase matching with proximity\n    - **Prefix Query**: Prefix matching\n    - **Wildcard Query**: Pattern matching with * and ?\n    - **Regexp Query**: Regular expression matching\n    - **Fuzzy Query**: Fuzzy matching with edit distance\n    - **Query String Query**: Human-readable query syntax\n\n    ### Range Queries\n    - **Numeric Range Query**: Numeric value ranges\n    - **Date Range Query**: Date/time ranges\n    - **Term Range Query**: Lexicographic term ranges\n\n    ### Boolean Queries\n    - **Boolean Query**: Must/should/must_not combinations\n    - **Conjunction Query**: All queries must match\n    - **Disjunction Query**: Any query can match\n\n    ### Special Queries\n    - **Match All Query**: Matches all documents\n    - **Match None Query**: Matches no documents\n    - **Bool Field Query**: Boolean field matching\n    - **Doc ID Query**: Document ID matching\n\n    ### Geographic Queries\n    - **Geo Distance Query**: Distance-based geographic search\n    - **Geo Bounding Box Query**: Bounding box geographic search\n    - **Geo Bounding Polygon Query**: Polygon geographic search\n    - **Geo Shape Query**: Complex geographic shape matching\n\n    ### Vector Queries\n    - **KNN Query**: K-nearest neighbors vector search\n\n    ### Network Queries\n    - **IP Range Query**: IP address range matching\n\n    ## Common Properties\n\n    All queries support:\n      - **boost**: Query boost value (float, default 1.0)\n      - **field**: Field to search (string, optional)\n\n    ## Query Parsing\n\n    Bleve automatically detects query types based on the presence of specific JSON fields.\n    The query parser uses a heuristic approach to determine the query type from the JSON structure.\n\n  version: 2.0.0\n  contact:\n    name: Bleve Search\n    url: https://github.com/blevesearch/bleve\n  license:\n    name: Apache 2.0\n    url: https://www.apache.org/licenses/LICENSE-2.0\n\ntags:\n  - name: Basic Queries\n    description: Basic text search queries\n  - name: Range Queries\n    description: Range-based queries for numeric, date, and term values\n  - name: Boolean Queries\n    description: Compound queries combining multiple sub-queries\n  - name: Special Queries\n    description: Special purpose queries like match all/none\n  - name: Geographic Queries\n    description: Geographic and spatial search queries\n  - name: Vector Queries\n    description: Vector similarity and KNN queries\n  - name: Network Queries\n    description: Network and IP address queries\n\ncomponents:\n  schemas:\n    # Base Query Schema\n    Query:\n      type: object\n      description: Base query object - all queries extend this\n      properties:\n        boost:\n          type: number\n          format: float\n          description: Query boost value\n          default: 1.0\n          minimum: 0\n        field:\n          type: string\n          description: Field to search (optional, uses default field if not specified)\n      additionalProperties: true\n\n    # Basic Queries\n    TermQuery:\n      allOf:\n        - type: object\n          properties:\n            term:\n              type: string\n              description: Exact term to search for\n              example: \"search\"\n          required:\n            - term\n          example:\n            term: \"search\"\n            field: \"title\"\n            boost: 2.0\n\n    MatchQuery:\n      allOf:\n        - type: object\n          properties:\n            match:\n              type: string\n              description: Text to match (will be analyzed)\n              example: \"full text search\"\n            operator:\n              type: string\n              enum: [and, or]\n              description: Boolean operator for multiple terms\n              default: \"or\"\n            fuzziness:\n              oneOf:\n                - type: integer\n                  minimum: 0\n                  maximum: 2\n                  description: Edit distance for fuzzy matching\n                - type: string\n                  enum: [\"auto\"]\n                  description: Automatic fuzziness\n              default: 0\n            prefix_length:\n              type: integer\n              description: Prefix length for fuzzy matching\n              minimum: 0\n              default: 0\n            analyzer:\n              type: string\n              description: Analyzer to use (optional)\n          required:\n            - match\n          example:\n            match: \"full text search\"\n            field: \"content\"\n            operator: \"and\"\n            fuzziness: 1\n\n    MatchPhraseQuery:\n      allOf:\n        - type: object\n          properties:\n            match_phrase:\n              type: string\n              description: Phrase to match exactly\n              example: \"exact phrase match\"\n            fuzziness:\n              oneOf:\n                - type: integer\n                  minimum: 0\n                  maximum: 2\n                - type: string\n                  enum: [\"auto\"]\n              default: 0\n            analyzer:\n              type: string\n              description: Analyzer to use (optional)\n          required:\n            - match_phrase\n          example:\n            match_phrase: \"exact phrase match\"\n            field: \"content\"\n            fuzziness: \"auto\"\n\n    PrefixQuery:\n      allOf:\n        - type: object\n          properties:\n            prefix:\n              type: string\n              description: Prefix to match\n              example: \"pref\"\n          required:\n            - prefix\n          example:\n            prefix: \"pref\"\n            field: \"title\"\n\n    WildcardQuery:\n      allOf:\n        - type: object\n          properties:\n            wildcard:\n              type: string\n              description: Wildcard pattern (* matches any sequence, ? matches single character)\n              example: \"te*t\"\n          required:\n            - wildcard\n          example:\n            wildcard: \"te*t\"\n            field: \"title\"\n\n    RegexpQuery:\n      allOf:\n        - type: object\n          properties:\n            regexp:\n              type: string\n              description: Regular expression pattern (should not include ^ or $)\n              example: \"te[st]\"\n          required:\n            - regexp\n          example:\n            regexp: \"te[st]\"\n            field: \"title\"\n\n    FuzzyQuery:\n      allOf:\n        - type: object\n          properties:\n            term:\n              type: string\n              description: Term for fuzzy matching\n              example: \"search\"\n            fuzziness:\n              oneOf:\n                - type: integer\n                  minimum: 0\n                  maximum: 2\n                - type: string\n                  enum: [\"auto\"]\n              default: 1\n            prefix_length:\n              type: integer\n              description: Prefix length for fuzzy matching\n              minimum: 0\n              default: 0\n          required:\n            - term\n          example:\n            term: \"search\"\n            field: \"title\"\n            fuzziness: \"auto\"\n\n    QueryStringQuery:\n      allOf:\n        - type: object\n          properties:\n            query:\n              type: string\n              description: Query string in human-readable syntax\n              example: \"title:search AND content:full text\"\n          required:\n            - query\n          example:\n            query: \"title:search AND content:full text\"\n\n    # Range Queries\n    NumericRangeQuery:\n      allOf:\n        - type: object\n          properties:\n            min:\n              type: number\n              format: float\n              description: Minimum value (inclusive by default)\n            max:\n              type: number\n              format: float\n              description: Maximum value (exclusive by default)\n            inclusive_min:\n              type: boolean\n              description: Whether minimum is inclusive\n              default: true\n            inclusive_max:\n              type: boolean\n              description: Whether maximum is inclusive\n              default: false\n          example:\n            min: 10.5\n            max: 100.0\n            inclusive_min: true\n            inclusive_max: false\n            field: \"price\"\n\n    DateRangeQuery:\n      allOf:\n        - type: object\n          properties:\n            start:\n              type: string\n              format: date-time\n              description: Start date/time\n            end:\n              type: string\n              format: date-time\n              description: End date/time\n            inclusive_start:\n              type: boolean\n              description: Whether start is inclusive\n              default: true\n            inclusive_end:\n              type: boolean\n              description: Whether end is inclusive\n              default: false\n          example:\n            start: \"2023-01-01T00:00:00Z\"\n            end: \"2023-12-31T23:59:59Z\"\n            inclusive_start: true\n            inclusive_end: true\n            field: \"created_at\"\n\n    TermRangeQuery:\n      allOf:\n        - type: object\n          properties:\n            min:\n              type: string\n              description: Minimum term (lexicographic)\n            max:\n              type: string\n              description: Maximum term (lexicographic)\n            inclusive_min:\n              type: boolean\n              description: Whether minimum is inclusive\n              default: true\n            inclusive_max:\n              type: boolean\n              description: Whether maximum is inclusive\n              default: false\n          example:\n            min: \"a\"\n            max: \"m\"\n            inclusive_min: true\n            inclusive_max: false\n            field: \"name\"\n\n    # Boolean Queries\n    BooleanQuery:\n      allOf:\n        - type: object\n          properties:\n            must:\n              oneOf:\n                - type: array\n                  items:\n                    $ref: '#/components/schemas/Query'\n                - $ref: '#/components/schemas/Query'\n              description: Queries that must match (can be single query or array)\n            should:\n              oneOf:\n                - type: array\n                  items:\n                    $ref: '#/components/schemas/Query'\n                - $ref: '#/components/schemas/Query'\n              description: Queries that should match (boost score, can be single query or array)\n            must_not:\n              oneOf:\n                - type: array\n                  items:\n                    $ref: '#/components/schemas/Query'\n                - $ref: '#/components/schemas/Query'\n              description: Queries that must not match (can be single query or array)\n            filter:\n              oneOf:\n                - type: array\n                  items:\n                    $ref: '#/components/schemas/Query'\n                - $ref: '#/components/schemas/Query'\n              description: Queries that filter results (no score impact, can be single query or array)\n            min_should:\n              type: integer\n              description: Minimum number of should queries that must match\n              minimum: 0\n          example:\n            must:\n              - term: \"important\"\n                field: \"status\"\n              - match: \"urgent\"\n                field: \"title\"\n            should:\n              - match: \"critical\"\n                field: \"content\"\n              - match: \"priority\"\n                field: \"tags\"\n            must_not:\n              - term: \"archived\"\n                field: \"status\"\n            filter:\n              - term: \"published\"\n                field: \"status\"\n            boost: 1.5\n\n    ConjunctionQuery:\n      allOf:\n        - type: object\n          properties:\n            conjuncts:\n              type: array\n              items:\n                $ref: '#/components/schemas/Query'\n              description: Array of queries that must all match\n              minItems: 1\n          required:\n            - conjuncts\n          example:\n            conjuncts:\n              - term: \"search\"\n                field: \"title\"\n              - match: \"full text\"\n                field: \"content\"\n\n    DisjunctionQuery:\n      allOf:\n        - type: object\n          properties:\n            disjuncts:\n              type: array\n              items:\n                $ref: '#/components/schemas/Query'\n              description: Array of queries where at least one must match\n              minItems: 1\n            min:\n              type: number\n              format: float\n              description: Minimum number of disjuncts that must match\n              default: 1\n              minimum: 0\n          required:\n            - disjuncts\n          example:\n            disjuncts:\n              - term: \"urgent\"\n                field: \"title\"\n              - term: \"critical\"\n                field: \"content\"\n            min: 1\n\n    # Special Queries\n    MatchAllQuery:\n      allOf:\n        - type: object\n          properties:\n            match_all:\n              type: object\n              description: Empty object indicating match all\n          example:\n            match_all: {}\n            boost: 1.0\n\n    MatchNoneQuery:\n      allOf:\n        - type: object\n          properties:\n            match_none:\n              type: object\n              description: Empty object indicating match none\n          example:\n            match_none: {}\n\n    BoolFieldQuery:\n      allOf:\n        - type: object\n          properties:\n            bool:\n              type: boolean\n              description: Boolean value to match\n          required:\n            - bool\n          example:\n            bool: true\n            field: \"published\"\n\n    DocIDQuery:\n      allOf:\n        - type: object\n          properties:\n            ids:\n              type: array\n              items:\n                type: string\n              description: Array of document IDs to match\n              minItems: 1\n          required:\n            - ids\n          example:\n            ids: [\"doc1\", \"doc2\", \"doc3\"]\n\n    # Geographic Queries\n    GeoDistanceQuery:\n      allOf:\n        - type: object\n          properties:\n            location:\n              oneOf:\n                - type: array\n                  items:\n                    type: number\n                    format: float\n                  minItems: 2\n                  maxItems: 2\n                  description: [longitude, latitude]\n                - type: object\n                  properties:\n                    lon:\n                      type: number\n                      format: float\n                    lat:\n                      type: number\n                      format: float\n            distance:\n              type: string\n              description: Distance string (e.g., \"10km\", \"5mi\")\n          required:\n            - location\n            - distance\n          example:\n            location: [-122.4194, 37.7749]\n            distance: \"10km\"\n            field: \"location\"\n\n    GeoBoundingBoxQuery:\n      allOf:\n        - type: object\n          properties:\n            top_left:\n              oneOf:\n                - type: array\n                  items:\n                    type: number\n                    format: float\n                  minItems: 2\n                  maxItems: 2\n                - type: object\n                  properties:\n                    lon:\n                      type: number\n                      format: float\n                    lat:\n                      type: number\n                      format: float\n            bottom_right:\n              oneOf:\n                - type: array\n                  items:\n                    type: number\n                    format: float\n                  minItems: 2\n                  maxItems: 2\n                - type: object\n                  properties:\n                    lon:\n                      type: number\n                      format: float\n                    lat:\n                      type: number\n                      format: float\n          required:\n            - top_left\n            - bottom_right\n          example:\n            top_left: [-122.5, 37.8]\n            bottom_right: [-122.3, 37.7]\n            field: \"location\"\n\n    GeoBoundingPolygonQuery:\n      allOf:\n        - type: object\n          properties:\n            polygon_points:\n              type: array\n              items:\n                oneOf:\n                  - type: array\n                    items:\n                      type: number\n                      format: float\n                    minItems: 2\n                    maxItems: 2\n                  - type: object\n                    properties:\n                      lon:\n                        type: number\n                        format: float\n                      lat:\n                        type: number\n                        format: float\n              description: Array of polygon points [lon, lat]\n              minItems: 3\n          required:\n            - polygon_points\n          example:\n            polygon_points:\n              - [-122.5, 37.8]\n              - [-122.3, 37.8]\n              - [-122.3, 37.7]\n              - [-122.5, 37.7]\n            field: \"location\"\n\n    GeoShapeQuery:\n      allOf:\n        - type: object\n          properties:\n            geometry:\n              type: object\n              description: GeoJSON geometry object\n              properties:\n                type:\n                  type: string\n                  enum: [Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, GeometryCollection]\n                coordinates:\n                  description: Geometry coordinates (format depends on geometry type)\n                  oneOf:\n                    - type: array\n                      description: Coordinates for Point, LineString, MultiPoint, etc.\n                    - type: array\n                      items:\n                        type: array\n                      description: Nested coordinates for Polygon, MultiLineString, etc.\n                    - type: array\n                      items:\n                        type: array\n                        items:\n                          type: array\n                      description: Deeply nested coordinates for MultiPolygon, etc.\n              required:\n                - type\n                - coordinates\n          required:\n            - geometry\n          example:\n            geometry:\n              type: \"Polygon\"\n              coordinates:\n                - [[-122.5, 37.8], [-122.3, 37.8], [-122.3, 37.7], [-122.5, 37.7], [-122.5, 37.8]]\n            field: \"location\"\n\n    # Vector Queries\n    KNNQuery:\n      allOf:\n        - type: object\n          properties:\n            field:\n              type: string\n              description: Vector field name\n            vector:\n              type: array\n              items:\n                type: number\n                format: float\n              description: Query vector\n              minItems: 1\n            k:\n              type: integer\n              format: int64\n              description: Number of nearest neighbors to return\n              minimum: 1\n              maximum: 10000\n            params:\n              type: object\n              description: Additional parameters for vector search\n          required:\n            - field\n            - vector\n            - k\n          example:\n            field: \"embedding\"\n            vector: [0.1, 0.2, 0.3, 0.4, 0.5]\n            k: 10\n            boost: 1.0\n\n    # Network Queries\n    IPRangeQuery:\n      allOf:\n        - type: object\n          properties:\n            cidr:\n              type: string\n              description: CIDR notation for IP range (e.g., \"192.168.1.0/24\")\n          required:\n            - cidr\n          example:\n            cidr: \"192.168.1.0/24\"\n            field: \"ip_address\"\n\n    # Phrase Query (internal use)\n    PhraseQuery:\n      allOf:\n        - type: object\n          properties:\n            terms:\n              type: array\n              items:\n                type: string\n              description: Array of terms in phrase order\n          required:\n            - terms\n          example:\n            terms: [\"exact\", \"phrase\", \"match\"]\n            field: \"content\"\n\n    MultiPhraseQuery:\n      allOf:\n        - type: object\n          properties:\n            terms:\n              type: array\n              items:\n                type: array\n                items:\n                  type: string\n              description: Array of term arrays for multi-phrase matching\n          required:\n            - terms\n          example:\n            terms:\n              - [\"exact\", \"precise\"]\n              - [\"phrase\"]\n              - [\"match\", \"search\"]\n\n  examples:\n    # Basic Query Examples\n    simple_term:\n      summary: Simple Term Query\n      value:\n        term: \"search\"\n        field: \"title\"\n        boost: 2.0\n\n    simple_match:\n      summary: Simple Match Query\n      value:\n        match: \"full text search\"\n        field: \"content\"\n        operator: \"and\"\n\n    fuzzy_search:\n      summary: Fuzzy Search\n      value:\n        term: \"search\"\n        field: \"title\"\n        fuzziness: \"auto\"\n        prefix_length: 2\n\n    # Boolean Query Examples\n    complex_boolean:\n      summary: Complex Boolean Query\n      value:\n        must:\n          - term: \"important\"\n            field: \"status\"\n          - match: \"urgent\"\n            field: \"title\"\n        should:\n          - match: \"critical\"\n            field: \"content\"\n          - match: \"priority\"\n            field: \"tags\"\n        must_not:\n          - term: \"archived\"\n            field: \"status\"\n        filter:\n          - term: \"published\"\n            field: \"status\"\n        boost: 1.5\n\n    # Range Query Examples\n    price_range:\n      summary: Price Range Query\n      value:\n        min: 10.0\n        max: 100.0\n        inclusive_min: true\n        inclusive_max: false\n        field: \"price\"\n\n    date_range:\n      summary: Date Range Query\n      value:\n        start: \"2023-01-01T00:00:00Z\"\n        end: \"2023-12-31T23:59:59Z\"\n        inclusive_start: true\n        inclusive_end: true\n        field: \"created_at\"\n\n    # Geographic Query Examples\n    geo_distance:\n      summary: Geographic Distance Query\n      value:\n        location: [-122.4194, 37.7749]\n        distance: \"10km\"\n        field: \"location\"\n\n    geo_bounding_box:\n      summary: Geographic Bounding Box Query\n      value:\n        top_left: [-122.5, 37.8]\n        bottom_right: [-122.3, 37.7]\n        field: \"location\"\n\n    # Vector Query Examples\n    knn_search:\n      summary: K-Nearest Neighbors Query\n      value:\n        field: \"embedding\"\n        vector: [0.1, 0.2, 0.3, 0.4, 0.5]\n        k: 10\n        boost: 1.0\n\n    # Query String Examples\n    query_string:\n      summary: Query String Query\n      value:\n        query: \"title:search AND (content:full text OR content:search) AND -status:archived\"\n\n  responses:\n    ValidationSuccess:\n      description: Query validation successful\n      content:\n        application/json:\n          schema:\n            type: object\n            properties:\n              valid:\n                type: boolean\n                example: true\n              query_type:\n                type: string\n                example: \"term\"\n              message:\n                type: string\n                example: \"Query is valid\"\n\n    ValidationError:\n      description: Query validation failed\n      content:\n        application/json:\n          schema:\n            type: object\n            properties:\n              valid:\n                type: boolean\n                example: false\n              error:\n                type: string\n                example: \"unknown query type\"\n              details:\n                type: string\n                example: \"No recognized query fields found\"\n\nexternalDocs:\n  description: Bleve Documentation\n  url: https://github.com/blevesearch/bleve\n"
  },
  {
    "path": "docs/score_fusion.md",
    "content": "# Score Fusion for Hybrid Search\n\nBleve supports **hybrid search** that combines full-text search (FTS) with vector (kNN) search to leverage the strengths of both approaches:\n\n* **Full-text search** excels at exact keyword matching, filtering, and structured queries\n* **Vector search** captures semantic similarity and handles synonyms and paraphrasing naturally\n\nWith *v2.5.4* onwards - when using hybrid search, you can choose different **score fusion strategies** to combine results from both search methods. This document describes the available fusion strategies and how to use them.\n\n## Fusion Strategies\n\n### Additive Score Fusion (Default)\n\nBy default, Bleve combines FTS and kNN scores using a simple weighted addition. See the [Vector Search documentation](vectors.md#querying) for details on the default hybrid search behavior and examples.\n\nWhile this approach works well with proper boost tuning, it can be sensitive to different score scales and distributions. The fusion strategies below (RRF and RSF) provide more robust alternatives that handle score normalization automatically.\n\n### Reciprocal Rank Fusion (RRF)\n\nReciprocal Rank Fusion is a **rank-based** algorithm that combines results based on their position in each result list, rather than their raw scores. This makes it robust to different score scales and distributions.\n\n**Algorithm:**\n\nFor each document appearing in FTS or kNN results, the RRF score is calculated as:\n\n```math\nRRF\\_score = w_{\\text{fts}} \\cdot \\frac{1}{k + \\text{rank}_{\\text{fts}}} + \\sum_{i=1}^{n} w_{\\text{knn}_i} \\cdot \\frac{1}{k + \\text{rank}_{\\text{knn}_i}}\n```\n\nWhere:\n\n* $\\text{rank}_{\\text{fts}}$: 1-indexed rank of the document in the FTS result list (or 0 if not present)\n* $\\text{rank}_{\\text{knn}_i}$: 1-indexed rank of the document in the i-th kNN result list (or 0 if not present)\n* $k$: rank constant (default: 60) that dampens the impact of rank differences\n* $w_{\\text{fts}}$: weight from the FTS query boost value\n* $w_{\\text{knn}_i}$: weight from the i-th kNN query boost value\n* $\\sum_{i=1}^{n}$: summation over all kNN queries (you can add multiple kNN queries)\n\n**Advantages:**\n\n* Distribution-agnostic - no need for score normalization\n* Works out of the box with minimal tuning\n* Prioritizes documents appearing in both result lists\n* Robust to outliers since only ranks matter\n\n**Disadvantages:**\n\n* Ignores score magnitude (loses some information)\n* May be sensitive to imbalanced result list sizes\n\n**Usage:**\n\n```go\n// Create a hybrid search with RRF fusion\nsearchRequest := bleve.NewSearchRequest(bleve.NewMatchQuery(\"dark chocolate\"))\nsearchRequest.Score = bleve.ScoreRRF  // Alternatively, set to \"rrf\"\n\n// Add first kNN component\nsearchRequest.AddKNN(\n    \"embedding\",                             // Vector field\n    []float32{0.1, 0.2, 0.3, 0.4},          // Query vector\n    30,                                      // k neighbors\n    1.0,                                     // kNN weight (boost)\n)\n\n// Add second kNN component (optional - you can add multiple)\nsearchRequest.AddKNN(\n    \"image_embedding\",                       // Different vector field\n    []float32{0.5, 0.3, 0.1, 0.8},          // Query vector\n    20,                                      // k neighbors\n    0.5,                                     // kNN weight (boost)\n)\n\n// Optional: Configure RRF parameters\nparams := bleve.RequestParams{\n    ScoreRankConstant: 60,                   // Rank constant (default: 60)\n    ScoreWindowSize: 150                     // Window size (default: size)\n}\nsearchRequest.AddParams(params)\n\nsearchResult, err := index.Search(searchRequest)\n```\n\n### Relative Score Fusion (RSF)\n\nRelative Score Fusion is a **score-based** strategy that normalizes scores from both modalities into a common [0, 1] range using min-max normalization before combining them.\n\n**Algorithm:**\n\n1. **Min-max normalize** each result set independently:\n\n    ```math\n    \\text{normalized\\_score} = \\frac{\\text{score} - \\text{min\\_score}}{\\text{max\\_score} - \\text{min\\_score}}\n    ```\n\n2. **Combine** normalized scores using weighted addition:\n\n    ```math\n    RSF\\_score = w_{\\text{fts}} \\cdot \\text{normalized\\_score\\_fts} + \\sum_{i=1}^{n} w_{\\text{knn}_i} \\cdot \\text{normalized\\_score\\_knn}_i\n    ```\n\nWhere:\n\n* $w_{\\text{fts}}$: weight from the FTS query boost value\n* $w_{\\text{knn}_i}$: weight from the i-th kNN query boost value\n* $\\sum_{i=1}^{n}$: summation over all kNN queries (you can add multiple kNN queries)\n\n**Advantages:**\n\n* Score-aware - retains relevance magnitude information\n* Resolves incompatible score ranges\n* Easy to understand\n\n**Disadvantages:**\n\n* Sensitive to outliers - a single extreme score can skew normalization\n* Doesn't account for the shape or distribution of scores\n\n**Usage:**\n\n```go\n// Create a hybrid search with RSF fusion\nsearchRequest := bleve.NewSearchRequest(bleve.NewMatchQuery(\"machine learning\"))\nsearchRequest.Score = bleve.ScoreRSF  // Or set to \"rsf\"\n\n// Add first kNN component\nsearchRequest.AddKNN(\n    \"content_vector\",                        // Vector field\n    []float32{0.5, 0.3, 0.1, 0.8},          // Query vector\n    20,                                      // k neighbors\n    1.0,                                     // kNN weight (boost)\n)\n\n// Add second kNN component (optional - you can add multiple)\nsearchRequest.AddKNN(\n    \"title_vector\",                          // Different vector field\n    []float32{0.2, 0.7, 0.4, 0.1},          // Query vector\n    15,                                      // k neighbors\n    0.8,                                     // kNN weight (boost)\n)\n\n// Optional: Configure RRF parameters\nparams := bleve.RequestParams{\n    ScoreWindowSize: 150                     // Window size (default: size)\n}\nsearchRequest.AddParams(params)\n\nsearchResult, err := index.Search(searchRequest)\n```\n\n## Parameters\n\n### Score\n\nThe `Score` field in your search request specifies which fusion strategy to use:\n\n* **`ScoreRRF (\"rrf\")`**: Reciprocal Rank Fusion\n* **`ScoreRSF (\"rsf\")`**: Relative Score Fusion  \n* **Omitted or empty**: Default additive fusion with scores returned\n\n### Params\n\nThe `Params` object contains additional parameters for score fusion:\n\n#### Score Window Size\n\n`ScoreWindowSize` is the maximum number of results to consider from each result list for fusion.\n\n* **Default**: Same as `Size` parameter\n* **Minimum**: Must be ≥ `Size` and ≥ 1\n* **Purpose**: Controls the tradeoff between relevance and performance\n\nA larger window size increases the chance of finding relevant results but requires more computation. For pagination to work consistently, ensure:\n\n```text\nFrom + Size <= ScoreWindowSize\n```\n\n**Example:**\n\n```json\n{\n  \"score\": \"rrf\",\n  \"params\": {\n    \"score_window_size\": 150\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n```\n\nWith window size set to 150, you can paginate through up to 150 results. If you try to access results beyond this (e.g., `from=160`), you'll get an empty result set.\n\n#### Score Rank Constant\n\n> *Only applicable for RRF*\n\n`ScoreRankConstant` controls how much the rank position affects the reciprocal rank score.\n\n* **Default**: 60\n* **Range**: Any positive integer\n* **Effect**: Higher values dampen the impact of rank differences\n\n**Example:**\n\n```json\n{\n  \"score\": \"rrf\",\n  \"params\": {\n    \"score_rank_constant\": 60\n  }\n}\n```\n\n## Weighting Queries\n\nThe boost value in your query components controls their relative importance in hybrid search:\n\n```go\n// FTS query with boost 2.0\nquery := bleve.NewMatchQuery(\"search term\")\nquery.SetBoost(2.0)\n\nsearchRequest := bleve.NewSearchRequest(query)\n\n// kNN query with boost 1.0\nsearchRequest.AddKNN(\"vec\", queryVector, 10, 1.0)\n```\n\nFor RRF and RSF, weights determine the **relative importance** of each component's contribution, rather than scaling raw scores.\n\n**Example:** If `fts_boost = 2.0` and `knn_boost = 1.0`, the FTS contribution is twice as important as the kNN contribution in the final ranking in RRF or RSF.\n\n## Restrictions\n\nWhen using score fusion (`Score` set to `\"rrf\"` or `\"rsf\"`), certain features are not supported:\n\n* **SearchAfter/SearchBefore**: Not compatible with score fusion. For pagination, use `From` and `Size` only.\n* **Sort**: Only descending score sort (`-_score`) or default sorting is allowed\n* **Faceting**: Only documents included in the FTS result list are considered. Documents that appear exclusively in the KNN result list are ignored during faceting.\n\n## Choosing a Fusion Strategy\n\n| Use Case | Recommended Strategy |\n|----------|---------------------|\n| Different score scales (e.g., TF-IDF + L2 distance) | **RRF/RSF** |\n| Minimal tuning, out-of-the-box performance | **RRF** |\n| Want to preserve score magnitude importance | **RSF** |\n| Have well-tuned boost values already | **Additive (default)** |\n| Score distributions have extreme outliers | **RRF** |\n"
  },
  {
    "path": "docs/scoring.md",
    "content": "# Scoring models for document hits\n\n* Search is performed on a collection fields using compound queries such as conjunction/disjunction/boolean etc. However, the scoring itself is done independently for each field and then aggregated to get the final score for a document hit.\n* Default scoring scheme for document hits involving text hits: `tf-idf`.\n* Nearest-neighbor/vector hits scoring depends on chosen `knn distance` metric, highlighted [here](https://github.com/blevesearch/bleve/blob/master/docs/vectors.md#supported).\n* Hybrid search scoring will combine `tf-idf` scores with `knn distance` numbers.\n* *v2.5.0* (and after) will come with support for `bm25` scoring for exact searches.\n\n## BM25\n\nWhen it comes to scoring a document hit for a specific field, BM25 scoring mechanism requires the following stats:\n\n* fieldLen - The number of analyzed terms in the current document's field.\n* avgFieldLen - The average number of analyzed terms in the field across all the documents.\n* docTotal - The total number of documents in the index.\n* docTerm - The total number of documents containing the query term within the index.\n\nThe scoring formula followed in BM25 is\n\n```math\n\\sum_{i}^n IDF(q_i) {{f(q_i,D) * (k1 + 1)}\\over{f(q_i,D) + k1 * (1-b+b*{{fieldLen}\\over{avgFieldLen}})}}\n```\n\n$IDF(q_i)$ here refers to Inverse Document Frequency talks about how rare (and hence rich in information) is a particular query term $`q_i`$ across all the documents in the index, which is calculated as\n\n```math\n    \\ln(1 + {{docTotal - docTerm + 0.5}\\over{docTerm + 0.5}})\n```\n\nComing back to the BM25 scoring, $f(q_i,D)$ refers to the frequency of the query term in document $D$. The entire equation has certain multipliers\n\n* $k1$ - helps in controlling the saturation of the score with respect to query term in a document. Basically if the query term's frequency is too high, the score value gets saturated and doesn't increase beyond a certain point.\n* $b$ - controls the extent to which the $fieldLen$ normalizes the term's frequency.\n\n### How to enable and use BM25\n\nBleve v2.5.0 updated the `indexMapping` construct with the concept of `scoringModel`. This is a global (meaning applicable to all the fields) setting which drives which scoring algorithm to apply while scoring the document hits. Supported scoring models are defined [here](https://github.com/blevesearch/bleve_index_api/blob/f54d76f0a71a838837159aa44ced0404bb6ec25f/indexing_options.go#L27)\n\nFor instance, while defining the index mapping for the data model that's been decided by the user, following snippet can be referred to enable BM25\n\n```go\nindexMapping := bleve.NewIndexMapping()\nindexMapping.TypeField = \"type\"\nindexMapping.DefaultAnalyzer = \"en\"\nindexMapping.ScoringModel = \"bm25\"\n```\n\nDuring search time there's explicit change involved, unless the user wants to perform a global scoring.\n\n### Global Scoring\n\nLet's say that the user has a dataset which is quite large (let's say 3 million) and to have good throughput, they create 3 shards (with the same index mapping) for the \"index\". Each of these shards can be `bleve.Index` type and while performing a search over the entire dataset, a `bleve.IndexAlias` can be created over which a search can be performed. This parallelizes things pretty good, both on the indexing path and the search path.\n\nThe concept of global scoring is applicable when the index is \"sharded\" (similar to above situation). This is because each index has data which is disjoint, and thereby while performing the scoring on document hits on each of them, the value of stats is not complete at a global level, since we're doing a search over the entire dataset using the `bleve.IndexAlias`. For eg: `docTotal` value while scoring the document hits would be 1 million which is incorrect at a global level.\n\nSo in order to keep the scoring roughly same across varying count of the number of shards involved, we provide a mechanism to enable \"global scoring\". In this type of search, an initial roundtrip is performed to gather and aggregate the stats necessary for the scoring mechanism and in the second phase, the actual search is performed. So naturally this comes at a cost of latency. As a reference here's how the user can go about with it\n\n```go\nmultiPartIndex := bleve.NewIndexAlias(shard1, shard2)\n// set the alias with the same index mapping which both the shards use.\nerr = multiPartIndex.SetIndexMapping(indexMapping)\nif err != nil {\n    return err\n}\n\nctx := context.Background()\nctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring)\n\nres, err := multiPartIndex.SearchInContext(ctx, searchRequest)\n```\n\nA note here is that, this would only matter if the relative order of the document hits vary quite a bit (vs single shard case). This would be possible when the shard count increases quite a bit, in low doc count situations or if there is a heavy skew in the data distribution amongst the shards for some reason. Ideally the shards are created when the data is quite large and each of them index same amount of data - in which case the scores won't fluctuate much to affect the relative hit order and the user can choose to avoid the global scoring mechanism altogether.\n\n## TF-IDF\n\nTF-IDF is the default scoring mechanism involved (for backward compatibility reasons) and requires no change from the user at index or search time to avail it.\n\nThe scoring formula involved is\n\n```math\n    \\sum_{i}^n f(q_i, D) * {{1}\\over{\\sqrt{fieldLen}}} * IDF(q_i)\n```\n\nwhere $IDF(q_i)$ is\n\n```math\n    1 + {{docTotal}\\over{1 + docTerm}}\n```\n\nNote: TF-IDF formula doesn't accommodate logic for score saturation due to term frequency or fieldLen. So, it's recommended to use BM25 scoring by explicitly setting it in the index mapping.\n"
  },
  {
    "path": "docs/search_autocomplete.md",
    "content": "# Edge N-gram Autocomplete in Bleve\n\nSearch autocomplete is a feature which we see in search boxes when suggestions appear while we type.\nSo when we type `jav`, we see suggestions like: `java` `javascript` `javascript programming` etc.\nThis is helpful because it saves users time in finding what they are looking for.\n\n![Alt Text](/docs/images/search_autocomplete.png \"search autocomplete suggestion\")\n\n## 2. How Does It Work?\n\nAutocomplete generally works in three steps:\n\n1. **Index Time**: Breaking text into searchable pieces (tokens)\n2. **Query Time**: Matching user input against indexed tokens  \n3. **Results**: Returning relevant suggestions quickly\n\nBut before we jump into the flow, let's understand different methods to achieve this and why edge n-grams are the most efficient approach.\n\n## 3. Different Tokenization Methods\n\nThere are several tokenization approaches, each with its own strengths and weaknesses:\n\n### 3.1 Single Token Tokenizer\n\n```go\n// analysis/tokenizer/single/single.go\nfunc (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream {\n    return analysis.TokenStream{\n        &analysis.Token{\n            Term:     input,    // Here entire input as one token\n            Position: 1,\n            Start:    0,\n            End:      len(input),\n            Type:     analysis.AlphaNumeric,\n        },\n    }\n}\n```\n\n**How it works**: Treats the entire input as a single token.\n\n**Example**: \"JavaScript Programming\" → [`\"JavaScript Programming\"`]\n\n**Pros**:\n\n- Simple and fast\n- Perfect for exact phrase matching\n- Minimal index size\n\n**Cons**:\n\n- No autocomplete support (can't match partial text)\n- Not flexible for search\n\n**Use case**: Keyword fields, IDs, exact phrase matching\n\n### 3.2 Whitespace Token Tokenizer\n\n```go\n// analysis/tokenizer/whitespace/whitespace.go  \nfunc TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {\n    return character.NewCharacterTokenizer(notSpace), nil\n}\n\nfunc notSpace(r rune) bool {\n    return !unicode.IsSpace(r)  // Split on whitespace\n}\n```\n\n**How it works**: Splits text on whitespace characters.\n\n**Example**: \"JavaScript Programming\" → [`\"JavaScript\"`, `\"Programming\"`]\n\n**Pros**:\n\n- Simple word-based tokenization\n- Works well for basic prefix search\n\n**Cons**:\n\n- Only matches from word beginnings\n- No support for partial word matching\n- Limited autocomplete capabilities\n\n**Use case**: Basic search, word-level indexing\n\n### 3.3 N-gram Method\n\n```go\n// analysis/token/ngram/ngram.go\nfunc (s *NgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n    rv := make(analysis.TokenStream, 0, len(input))\n    \n    for _, token := range input {\n        runeCount := utf8.RuneCount(token.Term)\n        runes := bytes.Runes(token.Term)\n        \n        // ..generate all possible n-grams\n    }\n    return rv\n}\n```\n\n**How it works**: Creates ALL possible substrings of specified lengths.\n\n**Example**: \"java\" with 2-3 grams → [`\"ja\"`, `\"av\"`, `\"va\"`, `\"jav\"`, `\"ava\"`]\n\n**Pros**:\n\n- Supports partial matching anywhere in the text\n- Coverage of all possible substrings\n\n**Cons**:\n\n- **MASSIVE index size** (exponential growth)\n- Brings more noise as irrelevant matches (e.g., \"av\" matching \"java\")\n- Poor performance for autocomplete\n- High memory usage\n\n**Use case**: Full-text substring search (not ideal for autocomplete)\n\n### 3.4 Edge N-gram Method (preferred for autocomplete)\n\n```go\n// analysis/token/edgengram/edgengram.go\nfunc (s *EdgeNgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream {\n    rv := make(analysis.TokenStream, 0, len(input))\n    \n    for _, token := range input {\n        runeCount := utf8.RuneCount(token.Term)\n        runes := bytes.Runes(token.Term)\n        // ..builds tokens based from either end, specified in the input\n    }\n    return rv\n}\n```\n\n**How it works**: Creates substrings only from the beginning (or end) of each word.\n\n**Example**: \"javascript\" with front edge n-grams (1-5) → [`\"j\"`, `\"ja\"`, `\"jav\"`, `\"java\"`, `\"javas\"`]\n\n**Pros**:\n\n- Perfect for autocomplete (matches prefixes naturally)\n- Efficient index size (linear growth vs exponential)  \n- Fast queries (direct term matching, no complex processing)\n- Intuitive results (matches what users expect)\n- Highly scalable for large datasets\n\n**Cons**:\n\n- Only supports prefix matching (but that's preferred for autocomplete!)\n- Slightly larger index than basic tokenization\n\n**Use case**: Search autocomplete, prefix-based search\n\n## 4. Why Edge N-grams Are Most Efficient for Autocomplete\n\nLet's see what happens when a user types \"java\" with edge_ngram tokenizer:\n\n```text\nIndex contains: [\"j\", \"ja\", \"jav\", \"java\", \"javas\", \"javasc\", \"javascr\", ...]\nUser types: \"java\"  \nQuery: ExactTermQuery(\"java\")\nProcess: Direct hash table lookup for term \"java\"\nResult: Instant match, then retrieve documents containing this term\n```\n\n**Key advantages of Edge N-gram approach:**\n\n1. **O(1) lookup**: Direct term matching\n2. **No query-time processing**: Terms are pre-computed at index time  \n3. **Better caching**: Exact term queries cache better than prefix queries\n4. **Consistent performance**: Query time doesn't increase with index size\n\n## 5. Step-by-Step Implementation: Building Search Autocomplete\n\nNow let's see how to implement this in practice using our exact configuration. We'll build a complete autocomplete system step by step.\n\n### Step 1: Create Custom Edge N-gram Token Filter\n\nFirst, we need to create a custom token filter for edge n-grams. Here's how it looks in our configuration:\n\n```go\n// Create a new index mapping\nindexMapping := mapping.NewIndexMapping()\n\n// 1. Define the edgeGram token filter\nedgeGramFilter := map[string]interface{}{\n    \"type\": edgengram.Name,\n    \"min\":  2.0,\n    \"max\":  4.0,\n    \"back\": false,\n}\n\n// Register the token filter\nif err := indexMapping.AddCustomTokenFilter(\"Engram\", edgeGramFilter); err != nil {\n    log.Fatal(err)\n}\n```\n\n**What each setting does:**\n\n- `\"type\": \"edge_ngram\"` - Tells Bleve to use the edge n-gram filter\n- `\"min\": 2` - Start creating tokens from 2 characters (\"ja\", \"sc\", etc.)\n- `\"max\": 4` - Stop at 4 characters (\"java\", \"scri\", etc.)\n- `\"back\": \"false\"` - Create tokens from the front (beginning) of words\n\n### Step 2: Create Custom Analyzer\n\nNext, we create an analyzer that uses our custom token filter along with other helpful filters:\n\n```go\n// 2. Define a custom analyzer that uses it\ncustomAnalyzer := map[string]interface{}{\n    \"type\":      custom.Name,\n    \"tokenizer\": \"unicode\",\n    \"char_filters\": []string{\n         zerowidthnonjoiner.Name,\n    },\n    \"token_filters\": []string{\n        \"Engram\", // our custom edge_ngram filter\n        \"to_lower\",\n        \"stop_en\",\n    },\n}\n\nif err := indexMapping.AddCustomAnalyzer(\"edgeGramAnalyzer\", customAnalyzer); err != nil {\n    log.Fatal(err)\n}\n```\n\n**The pipeline works like this:**\n\n1. **Input Text**: \"Schaumbergfest Event\"\n\n2. **Tokenizer** (`unicode`): Splits into words\n\n   ```text\n   [\"Schaumbergfest\", \"Event\"]\n   ```\n\n3. **Character Filter** (`zero_width_spaces`): Removes invisible characters\n\n   ```text\n   [\"Schaumbergfest\", \"Event\"] (cleaned)\n   ```\n\n4. **Token Filter 1** (`Engram`): Creates edge n-grams (2-4 chars)\n\n   ```text\n   [\"Sc\", \"Sch\", \"Scha\", \"Ev\", \"Eve\", \"Even\"]\n   ```\n\n5. **Token Filter 2** (`to_lower`): Makes everything lowercase\n\n   ```text\n   [\"sc\", \"sch\", \"scha\", \"ev\", \"eve\", \"even\"]\n   ```\n\n6. **Token Filter 3** (`stop_en`): Removes common words (none in this case)\n\n   ```text\n   [\"sc\", \"sch\", \"scha\", \"ev\", \"eve\", \"even\"] (final tokens)\n   ```\n\n### Step 3: Configure Field Mapping\n\nNow we tell Bleve which fields to apply our autocomplete analyzer to:\n\n```go\n    // 3. Assign analyzer to a field mapping\n    fieldMapping := mapping.NewTextFieldMapping()\n    fieldMapping.Analyzer = \"edgeGramAnalyzer\"\n    \n    indexMapping.DefaultMapping.AddFieldMappingsAt(\"title\", fieldMapping)\n    \n    indexPath := \"example.bleve\"\n    index, err := bleve.New(indexPath, indexMapping)\n    if err != nil {\n        log.Fatal(err)\n    }\n```\n\n### Step 4: How It Works in Real Search\n\nWhen someone searches for \"sc\", here's what happens:\n\n**Index contains these tokens:**\n\n```text\n\"sc\" → [document1: \"Schaumbergfest\", document2: \"Script\", ...]\n\"sch\" → [document1: \"Schaumbergfest\", ...]  \n\"scha\" → [document1: \"Schaumbergfest\", ...]\n```\n\n**User types \"sc\":**\n\n1. Query: `name:sc`\n2. Bleve looks up exact term \"sc\" in the index\n3. Finds document with \"Schaumbergfest\" and \"Script\"\n4. Returns suggestion instantly\n\n```go\n\ntype Document struct {\n  ID    string `json:\"id\"`\n  Title string `json:\"title\"`\n}\n// 4. Index Documents\ndocuments := []Document{\n  {\n    ID:    \"doc1\",\n    Title: \"Schaumbergfest\",\n  },\n  {\n    ID:    \"doc2\",\n    Title: \"Script\",\n  },\n}\n\nbatch := index.NewBatch()\nfor _, doc := range documents {\n  batch.Index(doc.ID, doc)\n}\nif err := index.Batch(batch); err != nil {\n  log.Fatal(err)\n}\n\n// 5. Search the created index\nquery := bleve.NewMatchQuery(\"sc\")\nquery.SetField(\"title\")\nsearchRequest := bleve.NewSearchRequest(query)\nsearchRequest.Explain = true\nsearchRequest.Fields = []string{\"title\"}\nsearchResult, err := index.Search(searchRequest)\nif err != nil {\n  log.Fatal(err)\n}\nfmt.Println(searchResult)\n```\n\nOutput:\n\n```bash\n\n$ go run main.go\n\n2 matches, showing 1 through 2, took 189.041µs\n    1. doc2 (0.343255)\n        title\n                Script\n    2. doc1 (0.343255)\n        title\n                Schaumbergfest\n```\n\nNote: To run code, enclose code starting from Step 1 in func main.\n"
  },
  {
    "path": "docs/sort_facet.md",
    "content": "<h2>Purpose of Docvalues</h2>\n\n<h3>Background</h3>\n\n<p align=\"justify\">What are docValues? In the index mapping, there is an option to enable or disable docValues for a specific field mapping. However, what does it actually mean to activate or deactivate docValues, and how does it impact the end user? This document aims to address these questions.</p>\n<pre>\n\t\"default_mapping\": {\n\t\t\"dynamic\": true,\n\t\t\"enabled\": true,\n\t\t\"properties\": {\n\t\t\t\"loremIpsum\": {\n\t\t\t\"enabled\": true,\n\t\t\t\"dynamic\": false,\n\t\t\t\"fields\": [\n\t\t\t\t{\n\t\t\t\t\t\"name\": \"loremIpsum\",\n\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\"store\": false,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"include_term_vectors\": false,\n\t\t\t\t\t\"include_in_all\": false,\n\t\t\t\t\t\"docvalues\": true\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}\n</pre>\n<p align=\"justify\">Enabling docValues will always result in an increase in the size of your Bleve index, leading to a corresponding increase in disk usage. But what advantages can you expect in return? This document also quantitatively assesses this trade-off with a test case.</p>\n\n<p align=\"justify\">In a more general sense, we recommend enabling docValues on a field mapping if you anticipate queries that involve sorting and/or facet operations on that field. It's important to note, though, that sorting and faceting will work irrespective of whether docValues are enabled or not. This may lead you to wonder if there's any real benefit to enabling docValues since you're allocating extra disk space without an apparent return. The real advantage, however, becomes evident in enhanced query response times and reduced memory consumption during active usage. By accepting a minor increase in the disk space used by your Full-Text Search (FTS) index, you can anticipate better performance in handling search requests that involve sorting and faceting.</p>\n\n<h3>Usage</h3>\n\n<p align=\"justify\">The initial use of docValues comes into play when sorting is involved. In the search request JSON, there is a field named \"sort.\" This optional \"sort\" field can have a slice of JSON objects as its value. Each JSON object must belong to one of the following types: \n\t<ul>\n\t\t<li>SortDocID</li>\n\t\t<li>SortScore (which is the default if none is specified)</li>\n\t\t<li>SortGeoDistance</li> \n\t\t<li>SortField</li>\n\t</ul>\n</p>\n<p align=\"justify\">DocValues are relevant only when any of the JSON objects in the \"sort\" field are of type SortGeoDistance or SortField. This means that if you expect queries on a field F, where the queries either do not specify a value for the \"sort\" field or provide a JSON object of type SortDocID or SortScore, enabling docValues will not improve sorting operations, and as a result, query latency will remain unchanged. It's worth noting that the default sorting object, SortScore, does not require docValues to be enabled for any of the field mappings. Therefore, a search request without a sorting operation will not utilize docValues at all.</p>\n<div style=\"overflow-x: auto;\">\n<table>\n\t<tr>\n\t\t<th>No Sort Objects</th>\n\t\t<th>SortDocID</th>\n\t\t<th>SortScore</th>\n\t\t<th>SortField</th>\n\t\t<th>SortGeoDistance</th>\n\t</tr>\n\t<tr>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\":\"dolor\"\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\":\"sit_amet\"\n  },\n  \"sort\":[\n    {\n     \"by\":\"id\",\n     \"desc\":true\n    }\n    ],\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\":\"sit_amet\"\n  },\n  \"sort\":[\n    {\n     \"by\":\"score\",\n    }\n    ],\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\":\"sit_amet\"\n  },\n  \"sort\":[\n    {\n     \"by\":\"field\",\n     \"field\":\"dolor\",\n     \"type\":\"auto\",\n     \"mode\":\"min\",\n     \"missing\":\"last\"\n    }\n    ],\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\": \"dolor\"\n  },\n  \"sort\": [\n    {\n      \"by\": \"geo_distance\",\n      \"field\": \"sit_amet\",\n      \"location\": [\n        123.223,\n        34.33\n      ],\n      \"unit\": \"km\"\n    }\n  ],\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t</tr>\n\t<tr align=\"center\">\n\t\t<td>No DocValues used</td>\n\t\t<td>No DocValues used</td>\n\t\t<td>No DocValues used</td>\n\t\t<td>DocValues used for field \"dolor\". Field Mapping for \"dolor\" may enable docValues.</td>\n\t\t<td>DocValues used, for field \"sit_amet\". \nField Mapping for \"sit_amet\" may enable docValues.</td>\n\t</tr>\n</table>\n</div>\n<p align=\"justify\">Now, let's consider faceting. The search request object also includes another field called \"facets,\" where you can specify a collection of facet requests, with each request being associated with a unique name. Each of these facet requests can fall into one of three types:\n<ul>\n\t<li>Date range</li> \n\t<li>Numeric range</li>\n\t<li>Term facet</li> \n</ul>\nEnabling docValues for the fields associated with such facet requests might provide benefits in this context.</p>\n<div style=\"overflow-x: auto;\">\n<table>\n\t<tr>\n\t\t<th>No Facet Request</th>\n\t\t<th>Date Range Facet</th>\n\t\t<th>Numeric Range Facet</th>\n\t\t<th>Term Facet</th>\n\t</tr>\n\t<tr>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\": \"dolor\"\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\": \"sit_amet\"\n  },\n  \"facet\": {\n    \"facetA\": {\n      \"size\": 1,\n      \"field\": \"dolor\",\n      \"date_ranges\": [\n        {\n          \"name\": \"lorem\",\n          \"start\": \"20/August/2001\",\n          \"end\": \"22/August/2002\",\n          \"datetime_parser\": \"custDT\"\n        }\n      ]\n    }\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\": \"sit_amet\"\n  },\n  \"facet\": {\n    \"facetA\": {\n      \"size\": 1,\n      \"field\": \"dolor\",\n      \"numeric_ranges\":[\n          { \n            \"name\":\"lorem\",\n            \"min\":22,\n            \"max\":34\n          }\n        ]\n    }\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\": \"sit_amet\"\n  },\n  \"facet\": {\n    \"facetA\": {\n      \"size\": 1,\n      \"field\": \"dolor\"\n    }\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t</tr>\n\t<tr align=\"center\">\n\t\t<td>No DocValues used</td>\n\t\t<td colspan=\"3\">DocValues used for field \"dolor\". Field Mapping for \"dolor\" may enable docValues.</td>\t\n  </tr>\n</table>\n</div>\n\n<p align=\"justify\">In summary, when a search request is received by the Bleve index, it extracts all the fields from the sort objects and facet objects. To potentially benefit from docValues, you should consider enabling docValues for the fields mentioned in SortField and SortGeoDistance sort objects, as well as the fields associated with all the facet objects. By doing so, you can optimize sorting and faceting operations in your search queries.</p>\n\n<div style=\"overflow-x: auto;\">\n<table>\n\t<tr>\n\t\t<th>Combo A</th>\n\t\t<th>Combo B</th>\n\t</tr>\n\t<tr>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\": \"sit_amet\"\n  },\n  \"facet\": {\n    \"facetA\": {\n      \"size\": 1,\n      \"field\": \"dolor\",\n      \"date_ranges\": [\n        {\n          \"name\": \"lorem\",\n          \"start\": \"20/August/2001\",\n          \"end\": \"22/August/2002\",\n          \"datetime_parser\": \"custDT\"\n        }\n      ]\n    }\n  },\n  \"sort\":[\n    {\n     \"by\":\"field\",\n     \"field\":\"lorem\",\n     \"type\":\"auto\",\n     \"mode\":\"min\",\n     \"missing\":\"last\"\n    }\n    ],\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"lorem ipsum\",\n    \"field\": \"sit_amet\"\n  },\n  \"facet\": {\n    \"facetA\": {\n      \"size\": 1,\n      \"field\": \"dolor\",\n      \"numeric_ranges\":[\n          { \n            \"name\":\"lorem\",\n            \"min\":22,\n            \"max\":34\n          }\n        ]\n    }\n  },\n  \"sort\": [\n    {\n      \"by\": \"geo_distance\",\n      \"field\": \"ipsum\",\n      \"location\": [\n        123.223,\n        34.33\n      ],\n      \"unit\": \"km\"\n    }\n  ],\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t</tr>\n\t<tr align=\"center\">\n\t\t<td>DocValues used for field \"dolor\" and \"lorem\". Field Mapping for \"dolor\" and \"lorem\" may enable docValues.</td>\n\t\t<td>DocValues used for field \"dolor\" and \"ipsum\". Field Mapping for \"dolor\" and \"ipsum\" may enable docValues.</td>\n\t</tr>\n</table>\n</div>\n\n<h3>Empirical Analysis</h3>\n\n<p align=\"justify\">To evaluate our hypothesis, I've set up a sample dataset on my personal computer and I've created two Bleve indexes: one with docvalues enabled for three fields (<code>dummyDate</code>, <code>dummyNumber</code>, and <code>dummyTerm</code>), and another where I've disabled docValues for the same three fields. These field mappings were incorporated into the Default Mapping. It's important to mention that for both indexes, DocValues for dynamic fields were enabled, as the default mapping is dynamic.</p>\n\n<p align=\"justify\">The values for <code>dummyDate</code> and <code>dummyNumber</code> were configured to increase monotonically, with <code>dummyDate</code> representing a date value and `dummyNumber` representing a numeric value. This setup was intentional to ensure that facet aggregation would consistently result in cache hits and misses, providing a useful testing scenario.</p>\n\n<div style=\"overflow-x: auto;\">\n<table>\n\t<tr>\n\t\t<th>Index A</th>\n\t\t<th>Index B</th>\n\t</tr>\n\t<tr>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n   \"default_mapping\": {\n    \"dynamic\": true,\n    \"enabled\": true,\n    \"properties\": {\n     \"dummyNumber\": {\n      \"enabled\": true,\n      \"dynamic\": false,\n      \"fields\": [\n       {\n        \"name\": \"dummyNumber\",\n        \"type\": \"text\",\n        \"store\": false,\n        \"index\": true,\n        \"include_term_vectors\": false,\n        \"include_in_all\": false,\n        \"docvalues\": true\n       }\n      ]\n     },\n     \"dummyTerm\": {\n      \"enabled\": true,\n      \"dynamic\": false,\n      \"fields\": [\n       {\n        \"name\": \"dummyTerm\",\n        \"type\": \"text\",\n        \"store\": false,\n        \"index\": true,\n        \"include_term_vectors\": false,\n        \"include_in_all\": false,\n        \"docvalues\": true\n       }\n      ]\n     },\n     \"dummyDate\": {\n      \"enabled\": true,\n      \"dynamic\": false,\n      \"fields\": [\n       {\n        \"name\": \"dummyDate\",\n        \"type\": \"text\",\n        \"store\": false,\n        \"index\": true,\n        \"include_term_vectors\": false,\n        \"include_in_all\": false,\n        \"docvalues\": true\n       }\n      ]\n     }\n    }\n   }\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n   \"default_mapping\": {\n    \"dynamic\": true,\n    \"enabled\": true,\n    \"properties\": {\n     \"dummyNumber\": {\n      \"enabled\": true,\n      \"dynamic\": false,\n      \"fields\": [\n       {\n        \"name\": \"dummyNumber\",\n        \"type\": \"text\",\n        \"store\": false,\n        \"index\": true,\n        \"include_term_vectors\": false,\n        \"include_in_all\": false,\n        \"docvalues\": false\n       }\n      ]\n     },\n     \"dummyTerm\": {\n      \"enabled\": true,\n      \"dynamic\": false,\n      \"fields\": [\n       {\n        \"name\": \"dummyTerm\",\n        \"type\": \"text\",\n        \"store\": false,\n        \"index\": true,\n        \"include_term_vectors\": false,\n        \"include_in_all\": false,\n        \"docvalues\": false\n       }\n      ]\n     },\n     \"dummyDate\": {\n      \"enabled\": true,\n      \"dynamic\": false,\n      \"fields\": [\n       {\n        \"name\": \"dummyDate\",\n        \"type\": \"text\",\n        \"store\": false,\n        \"index\": true,\n        \"include_term_vectors\": false,\n        \"include_in_all\": false,\n        \"docvalues\": false\n       }\n      ]\n     }\n    }\n   }\n\t\t\t</pre>\n\t\t</td>\n\t</tr>\n\t<tr align=\"center\">\n\t\t<td>Docvalues enabled across all three field mappings</td>\n\t\t<td>Docvalues disabled across all three field mappings</td>\n\t</tr>\n</table>\n</div>\n\nDocument Format used for the test scenario:\n\n<div style=\"overflow-x: auto;\">\n<table>\n\t<tr>\n\t\t<th>Document 1</th>\n\t\t<th>Document 2</th>\n\t\t<th>... Document i</th>\n\t\t<th>Document 5000</th>\n\t</tr>\n\t<tr>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n\t\"dummyTerm\":\"Term\",\n\t\"dummyDate\":\"2000-01-01T00:00:00,\n\t\"dummyNumber:1\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n\t\"dummyTerm\":\"Term\",\n\t\"dummyDate\":\"2000-01-01T01:00:00,\n\t\"dummyNumber:2\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n\t\"dummyTerm\":\"Term\",\n\t\"dummyDate\":\"2000-01-01T01:00:00\"+(i hours),\n\t\"dummyNumber:i\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n\t\"dummyTerm\":\"Term\",\n\t\"dummyDate\":2000-01-01T01:00:00 + (5000 hours),\n\t\"dummyNumber:5000\n}\n\t\t\t</pre>\n\t\t</td>\n</table>\n</div>\n\n<p align=\"justify\">Now I ran the following set of search requests across both the indexes, while increasing the number of documents indexed from 2000 to 4000.</p>\n\n<div style=\"overflow-x: auto;\">\n<table>\n\t<tr>\n\t\t<th>Request 1</th>\n\t\t<th>Request 2</th>\n\t\t<th>... Request i</th>\n\t\t<th>Request 1000</th>\n\t</tr>\n\t<tr>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"term\",\n    \"field\":\"dummyTerm\"\n  },\n  \"facets\":{\n    \"myDate\":{\n      \"field\":\"dummyDate\",\n      \"size\":100000,\n      \"date_ranges\":[\n        {\n          \"start\":\"2000-01-01T00:00:00\",\n          \"end\":\"2000-01-01T01:00:00\"\n        }\n      ]\n    },\n    \"myNum\":{\n      \"field\":\"dummyNumber\",\n      \"size\":100000,\n      \"numeric_ranges\":[\n        {\n          \"min\": 1000,\n          \"max\": 1001\n        }\n      ]\n    }\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"term\",\n    \"field\":\"dummyTerm\"\n  },\n  \"facets\":{\n    \"myDate\":{\n      \"field\":\"dummyDate\",\n      \"size\":100000,\n      \"date_ranges\":[\n        {\n          \"start\":\"2000-01-01T01:00:00\",\n          \"end\":\"2000-01-01T02:00:00\"\n        }\n      ]\n    },\n    \"myNum\":{\n      \"field\":\"dummyNumber\",\n      \"size\":100000,\n      \"numeric_ranges\":[\n        {\n          \"min\": 999,\n          \"max\": 1000\n        }\n      ]\n    }\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"term\",\n    \"field\":\"dummyTerm\"\n  },\n  \"facets\":{\n    \"myDate\":{\n      \"field\":\"dummyDate\",\n      \"size\":100000,\n      \"date_ranges\":[\n        {\n          \"start\":\"2000-01-01T00:00:00\" + i hour\n          \"end\":\"2000-01-01T00:00:00\" + (i+1) hour\n        }\n      ]\n    },\n    \"myNum\":{\n      \"field\":\"dummyNumber\",\n      \"size\":100000,\n      \"numeric_ranges\":[\n        {\n          \"min\": 1000-i,\n          \"max\": 1000-i+1\n        }\n      ]\n    }\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n\t\t<td style=\"vertical-align: top; width: 20%;\">\n\t\t\t<pre>\n{\n  \"explain\": true,\n  \"fields\": [\n    \"*\"\n  ],\n  \"highlight\": {},\n  \"query\": {\n    \"match\": \"term\",\n    \"field\":\"dummyTerm\"\n  },\n  \"facets\":{\n    \"myDate\":{\n      \"field\":\"dummyDate\",\n      \"size\":100000,\n      \"date_ranges\":[\n        {\n          \"start\":\"2000-01-01T01:00:00\" + 1000 hour,\n          \"end\":\"2000-01-01T02:00:00\" + 1001 hour\n        }\n      ]\n    },\n    \"myNum\":{\n      \"field\":\"dummyNumber\",\n      \"size\":100000,\n      \"numeric_ranges\":[\n        {\n          \"min\": 0,\n          \"max\": 1\n        }\n      ]\n    }\n  },\n  \"size\": 10,\n  \"from\": 0\n}\n\t\t\t</pre>\n\t\t</td>\n</table>\n</div>\n\n\n<div style=\"overflow-x: auto;\">\n<table>\n\t<tr>\n\t\t<th>Bleve index size growth with increase in indexed documents</th>\n\t\t<th>Total query time for 1000 queries with increase in number of indexed documents</th>\n\t</tr>\n\t\t<td><img src = \"images/indexSizeVsNumDocs.png\" alt = \"indexSizeVsNumDocs.png\"/></td>\n\t\t<td><img src = \"images/queryTimevsNumDocs.png\" alt = \"queryTimevsNumDocs.png\"/></td>\n\t</tr>\n</table>\n</div>\n\n<div style=\"overflow-x: auto;\">\n<table>\n\t<tr>\n\t\t<th style=\"width:50%\">Average increase in index size (in bytes) by enabling DocValues</th>\n\t\t<th style=\"width:50%\">Average reduction in time taken to perform 1000 queries (in milliseconds) by enabling DocValues</th>\n\t</tr>\n\t<tr>\n\t\t<td align=\"center\"><code>7762.47</code></td>\n\t\t<td align=\"center\"><code>27.034</code></td>\n\t</tr>\n</table>\nEven at this small scale, with a small document size and a very limited number of indexed documents, we still observe a noticeable tradeoff. With just a slight increase in the index size (an average of 7KB) we obtain a 20ms reduction in the total execution time, on average, for only 1000 queries.\n\n<h3>Technical Information</h3>\n\n<p align=\"justify\">When a search request involves facet or sorting operations on a field F, these operations occur after the main search query is executed. For instance, if the main query yields a result of 200 documents, the sorting and faceting processes will be applied to these 200 documents. However, the main query result only provides a set of document IDs, not the actual document contents.</p>\n\n<p align=\"justify\">Here's where docValues become essential. If the field mapping for F is docValue enabled, the system can directly access the values for the field from the stored docValue part in the index file. This means that for each document ID returned in the search result, the field values are readily available.</p>\n\n<p align=\"justify\">However, if docValues are not enabled for field F, the system must take a different approach. It needs to \"fetch the document\" from the index file, read the value for field F, and cache this field-document pair in memory for further processing. The issue becomes apparent in the latter scenario. By not enabling docValues for field F, you essentially retrieve all the documents in the search result (at the worst case), which can be a substantial amount of data. Moreover, you have to cache this information in memory, leading to increased memory usage. As a result, query latency significantly suffers because you're essentially fetching and processing all documents, which can be both time-consuming and resource-intensive. Enabling docValues for the relevant fields is, therefore, a crucial optimization to enhance query performance and reduce memory overhead in such situations.</p>\n"
  },
  {
    "path": "docs/synonyms.md",
    "content": "# Synonym search\n\n* *v2.5.0* (and after) will come with support for **synonym definition indexing and search**.\n* We've achieved this by embedding synonym indexes within our bleve (scorch) indexes.\n* Usage of zap file format: [v16](https://github.com/blevesearch/zapx/blob/master/zap.md). Here we co-locate text, vector and synonym indexes as neighbors within segments, continuing to conform to the segmented architecture of *scorch*.\n\n## Supported\n\n* Indexing `Synonym Definitions` allows specifying equivalent terms that will be used to construct the synonym index. There are currently two types of `Synonym Definitions` supported:\n\n    1. Equivalent Mapping:\n\n        In this type, all terms in the *synonyms* list are considered equal and can replace one another. Any of these terms can match a query or document containing any other term in the group, ensuring full synonym coverage.\n\n        ```json\n        {\n            \"synonyms\": [\n                \"tranquil\",\n                \"peaceful\",\n                \"calm\",\n                \"relaxed\",\n                \"unruffled\"\n            ]\n        }\n        ```\n\n    2. Explicit Mapping:\n\n        In this mapping, only the terms in the *input* list (\"blazing\") will have the terms in *synonyms* as their synonyms. The input terms are not equivalent to each other, and the synonym relationship is explicitly directional, applying only from the *input* to the *synonyms*.\n\n        ```json\n        {\n            \"input\": [\n                \"blazing\"\n            ],\n            \"synonyms\": [\n                \"intense\",\n                \"radiant\",\n                \"burning\",\n                \"fiery\",\n                \"glowing\"\n            ]\n        }\n        ```\n\n* The addition of `Synonym Sources` in the index mapping enables associating a set of `synonym definitions` (called a `synonym collection`) with a specific analyzer. This allows for preprocessing of terms in both the *input* and *synonyms* lists before the synonym index is created. By using an analyzer, you can normalize or transform terms (e.g., case folding, stemming) to improve synonym matching.\n\n    ```json\n    {\n        \"analysis\": {\n            \"synonym_sources\": {\n                \"english\": {\n                    \"collection\": \"en_thesaurus\",\n                    \"analyzer\": \"en\"\n                },\n                \"german\": {\n                    \"collection\": \"de_thesaurus\",\n                    \"analyzer\": \"de\"\n                }\n            }\n        }\n   }\n   ```\n\n    There are two `synonym sources` named \"english\" and \"german,\" each associated with its respective `synonym collection` and analyzer. In any text field mapping, a `synonym source` can be specified to enable synonym expansion when the field is queried. The analyzer of the synonym source must match the analyzer of the field mapping to which it is applied.\n\n* Any text-based Bleve query (e.g., match, phrase, term, fuzzy, etc.) will use the `synonym source` (if available) for the queried field to expand the search terms using the thesaurus created from user-defined synonym definitions. The behavior for specific query types is as follows:\n\n    1. Queries with `fuzziness` parameter: For queries like match, phrase, and match-phrase that support the `fuzziness` parameter, the queried terms are fuzzily matched with the thesaurus's LHS terms to generate candidate terms. These terms are then combined with the results of fuzzy matching against the field dictionary, which contains the terms present in the queried field.\n\n    2. Wildcard, Regexp, and Prefix queries: These queries follow a similar approach. First, the thesaurus is used to expand terms (e.g., LHS terms that match the prefix or regex). The resulting terms are then combined with candidate terms from dictionary expansion.\n\n## Indexing\n\nBelow is an example of using the Bleve API to define synonym sources, index synonym definitions, and associate them with a text field mapping:\n\n```go\n// Define a document to be indexed.\ndoc := struct {\n    Text string `json:\"text\"`\n}{\n    Text: \"hardworking employee\",\n}\n\n// Define a synonym definition where \"hardworking\" has equivalent terms.\nsynDef := &bleve.SynonymDefinition{\n    Synonyms: []string{\n        \"hardworking\",\n        \"industrious\",\n        \"conscientious\",\n        \"persistent\",\n    },\n}\n\n// Define the name of the `synonym collection`.\n// This collection groups multiple synonym definitions.\nsynonymCollection := \"collection1\"\n\n// Define the name of the `synonym source`.\n// This source will be associated with specific field mappings.\nsynonymSourceName := \"english\"\n\n// Define the analyzer to process terms in the synonym definitions.\n// This analyzer must match the one applied to the field using the synonym source.\nanalyzer := \"en\"\n\n// Configure the synonym source by associating it with the synonym collection and analyzer.\nsynonymSourceConfig := map[string]interface{}{\n    \"collection\": synonymCollection,\n    \"analyzer\":   analyzer,\n}\n\n// Create a new index mapping.\nbleveMapping := bleve.NewIndexMapping()\n\n// Add the synonym source configuration to the index mapping.\nerr := bleveMapping.AddSynonymSource(synonymSourceName, synonymSourceConfig)\nif err != nil {\n    panic(err)\n}\n\n// Create a text field mapping with the specified analyzer and synonym source.\ntextFieldMapping := bleve.NewTextFieldMapping()\ntextFieldMapping.Analyzer = analyzer\ntextFieldMapping.SynonymSource = synonymSourceName\n\n// Associate the text field mapping with the \"text\" field in the default document mapping.\nbleveMapping.DefaultMapping.AddFieldMappingsAt(\"text\", textFieldMapping)\n\n// Create a new index with the specified mapping.\nindex, err := bleve.New(\"example.bleve\", bleveMapping)\nif err != nil {\n    panic(err)\n}\n\n// Index the document into the created index.\nerr = index.Index(\"doc1\", doc)\nif err != nil {\n    panic(err)\n}\n\n// Check if the index supports synonym indexing and add the synonym definition.\nif synIndex, ok := index.(bleve.SynonymIndex); ok {\n    err = synIndex.IndexSynonym(\"synDoc1\", synonymCollection, synDef)\n    if err != nil {\n        panic(err)\n    }\n} else {\n    // If the index does not support synonym indexing, raise an error.\n    panic(\"expected synonym index\")\n}\n```\n\n## Querying\n\n```go\n// Query the index created above.\n// Create a match query for the term \"persistent\".\nquery := bleve.NewMatchQuery(\"persistent\")\n\n// Specify the field to search within, in this case, the \"text\" field.\nquery.SetField(\"text\")\n\n// Create a search request with the query and enable explanation to understand how results are scored.\nsearchRequest := bleve.NewSearchRequest(query)\nsearchRequest.Explain = true\n\n// Execute the search on the index.\nsearchResult, err := index.Search(searchRequest)\nif err != nil {\n    // Handle any errors that occur during the search.\n    panic(err)\n}\n\n// The search result will contain one match: \"doc1\". This document includes the term \"hardworking\", \n// which is a synonym for the queried term \"persistent\". The synonym relationship is based on \n// the user-defined thesaurus associated with the index.\n// Print the search results, which will include the explanation for the match.\nfmt.Println(searchResult)\n```\n"
  },
  {
    "path": "docs/vectors.md",
    "content": "# Nearest neighbor (vector) search\n\n* *v2.4.0* (and after) will come with support for **vectors' indexing and search**.\n* We've achieved this by embedding [FAISS](https://github.com/facebookresearch/faiss) indexes within our bleve (scorch) indexes.\n* Introduction of a new zap file format: [v16](https://github.com/blevesearch/zapx/blob/master/zap.md) - which will be the default going forward. Here we co-locate text and vector indexes as neighbors within segments, continuing to conform to the segmented architecture of *scorch*.\n\n## Pre-requisite(s)\n\n* Induction of [FAISS](https://github.com/blevesearch/faiss) into our eco system, which is a fork of the original [facebookresearch/faiss](https://github.com/facebookresearch/faiss)\n* FAISS is a C++ library that needs to be compiled and it's shared libraries need to be situated at an accessible path for your application.\n* A `vectors` GO TAG needs to be set for bleve to access all the supporting code. This TAG must be set only after the FAISS shared library is made available. Failure to do either will inhibit you from using this feature.\n* Please follow these [instructions](#setup-instructions) below for any assistance in the area.\n* Releases of `blevesearch/bleve` work with select checkpoints of `blevesearch/faiss` owing to API changes and improvements (tracking over the `bleve` branch):\n\n    | bleve version(s) | blevesearch/faiss version |\n    | --- | --- |\n    | `v2.4.0` | [blevesearch/faiss@7b119f4](https://github.com/blevesearch/faiss/tree/7b119f4b9c408989b696b36f8cc53908e53de6db) (modified v1.7.4) |\n    | `v2.4.1`, `v2.4.2` | [blevesearch/faiss@d9db66a](https://github.com/blevesearch/faiss/tree/d9db66a38518d99eb334218697e1df0732f3fdf8) (modified v1.7.4) |\n    | `v2.4.3`, `v2.4.4` | [blevesearch/faiss@b747c55](https://github.com/blevesearch/faiss/tree/b747c55a93a9627039c34d44b081f375dca94e57) (modified v1.8.0) |\n    | `v2.5.0`, `v2.5.1` | [blevesearch/faiss@352484e](https://github.com/blevesearch/faiss/tree/352484e0fc9d1f8f46737841efe5f26e0f383f71) (modified v1.10.0) |\n    | `v2.5.2`, `v2.5.3`, `v2.5.4` | [blevesearch/faiss@b3d4e00](https://github.com/blevesearch/faiss/tree/b3d4e00a69425b95e0b283da7801efc9f66b580d) (modified v1.11.0) |\n    | `v2.5.5`, `v2.5.6`, `v2.5.7` | [blevesearch/faiss@8a59a0c](https://github.com/blevesearch/faiss/tree/8a59a0c552fa2d14fa871f6b6bc793de1d277f5e) (modified v1.12.0) |\n    | `v2.6.0` | [blevesearch/faiss@1f14a3e](https://github.com/blevesearch/faiss/tree/1f14a3e4ed5ec1efcb0a66055516980da5d0a453) (modified v1.13.2) |\n\n## Supported\n\n* The `vector` field type is an array that is to hold float32 values only.\n* The `vector_base64` field type to support base64 encoded strings using little endian byte ordering (v2.4.1+)\n* Supported similarity metrics are: [`\"cosine\"` (v2.4.3+), `\"dot_product\"`, `\"l2_norm\"`].\n  * `cosine` paths will additionally normalize vectors before indexing and search.\n* Supported dimensionality is between 1 and 2048 (v2.4.0), and up to **4096** (v2.4.1+).\n* Supported vector index optimizations: `latency`, `memory_efficient` (v2.4.1+), `recall`.\n* Vectors from documents that do not conform to the index mapping dimensionality are simply discarded at index time.\n* The dimensionality of the query vector must match the dimensionality of the indexed vectors to obtain any results.\n* Pure kNN searches can be performed, but the `query` attribute within the search request must be set - to `{\"match_none\": {}}` in this case. The `query` attribute is made optional when `knn` is available with v2.4.1+.\n* Hybrid searches are supported, where results from `query` are unioned (for now) with results from `knn`. The tf-idf scores from exact searches are simply summed with the similarity distances to determine the aggregate scores.\n\n```text\naggregate_score = (query_boost * query_hit_score) + (knn_boost * knn_hit_distance)\n```\n\n* Advanced score fusion strategies (v2.5.4+) are available if requested for - see [score fusion](score_fusion.md#score-fusion-for-hybrid-search).\n* Multi kNN searches are supported - the `knn` object within the search request accepts an array of requests. These sub objects are unioned by default but this behavior can be overridden by setting `knn_operator` to `\"and\"`.\n* Previously supported pagination settings will work as they were, with size/limit being applied over the top-K hits combined with any exact search hits.\n* Pre-filtered vector and hybrid search (v2.4.3+): Apply any Bleve filter query first to narrow down candidates before running kNN search, making vector and hybrid searches faster and more relevant.\n* Fields containing multiple vectors (v2.5.7+):\n  * A single document may contain multiple vectors within the same field, in the form of either:\n    * an array of vectors (multi-vector field)\n    * an array of objects each containing a vector (nested-vector field)\n  * **All vectors in the field must share the same dimensionality**.\n  * For single-kNN queries, each document is scored using its single best-matching vector.\n  * For multi-kNN queries, the system selects the best-matching vector for each query vector within the document.\n\n## Indexing\n\n```go\n// Example document with single-vector, multi-vector, and nested-vector fields\ndoc := struct {\n    Id         string      `json:\"id\"`\n    Text       string      `json:\"text\"`\n    Vec        []float32   `json:\"vec\"`        // Single-vector field\n    Embeddings [][]float32 `json:\"embeddings\"` // Multi-vector field: array of vectors (v2.5.7+)\n    Sections   []struct {  // Nested-vector field: array of objects with vectors (v2.5.7+)\n        Text string `json:\"text\"`\n        Vec  []float32 `json:\"vec\"`\n    } `json:\"sections\"`\n}{\n    Id:   \"example\",\n    Text: \"hello from united states\",\n    Vec:  []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, // Single-vector field of dimensionality 10\n    Embeddings: [][]float32{ // Multi-vector field containing 2 vectors of dimensionality 10\n        {10, 11, 12, 13, 14, 15, 16, 17, 18, 19}, // First vector\n        {20, 21, 22, 23, 24, 25, 26, 27, 28, 29}, // Second vector\n    },\n    Sections: []struct { // Nested-vector field containing 2 objects each with a vector of dimensionality 10\n        Text string `json:\"text\"`\n        Vec  []float32 `json:\"vec\"`\n    }{\n        {Text: \"first section\", Vec: []float32{30, 31, 32, 33, 34, 35, 36, 37, 38, 39}},\n        {Text: \"second section\", Vec: []float32{40, 41, 42, 43, 44, 45, 46, 47, 48, 49}},\n    },\n}\n\n// Field mappings\ntextFieldMapping := bleve.NewTextFieldMapping()\nvectorFieldMapping := bleve.NewVectorFieldMapping()\nvectorFieldMapping.Dims = 10              // Set vector dimensionality\nvectorFieldMapping.Similarity = \"l2_norm\" // Euclidean distance\n\n// Sub-document mappings\nsectionsMapping := bleve.NewDocumentMapping()\nsectionsMapping.AddFieldMappingsAt(\"text\", textFieldMapping)\nsectionsMapping.AddFieldMappingsAt(\"vec\", vectorFieldMapping)\n\n// Index mapping\nbleveMapping := bleve.NewIndexMapping()\nbleveMapping.DefaultMapping.AddFieldMappingsAt(\"text\", textFieldMapping)\nbleveMapping.DefaultMapping.AddFieldMappingsAt(\"vec\", vectorFieldMapping)        // Single-vector\nbleveMapping.DefaultMapping.AddFieldMappingsAt(\"embeddings\", vectorFieldMapping) // Multi-vector\nbleveMapping.DefaultMapping.AddSubDocumentMapping(\"sections\", sectionsMapping)   // Nested-vector\n\n// Create the index\nindex, err := bleve.New(\"example.bleve\", bleveMapping)\nif err != nil {\n    panic(err)\n}\n\n// Index the document\nerr = index.Index(doc.Id, doc)\nif err != nil {\n    panic(err)\n}\n```\n\n## Querying\n\n```go\n// ------------------------------------\n// Single-vector field search (v2.4.0+)\n// ------------------------------------\nsearchRequest := bleve.NewSearchRequest(bleve.NewMatchNoneQuery())\nsearchRequest.AddKNN(\n    \"vec\",                                   // Vector field\n    []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9}, // Query vector\n    5,                                       // top-k\n    1,                                       // boost\n)\nsearchResult, err := index.Search(searchRequest)\nif err != nil {\n    panic(err)\n}\n// Scores are 1 / squared L2 distance, e.g., score = 0.25 for squared distance of 4\nfmt.Printf(\"Single-vector field kNN result:\\n%s\\n\", searchResult)\n```\n\n```go\n// -----------------------------------\n// Multi-vector field search (v2.5.7+)\n// -----------------------------------\nsearchRequest = bleve.NewSearchRequest(bleve.NewMatchNoneQuery())\nsearchRequest.AddKNN(\n    \"embeddings\",\n    []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9},\n    5,\n    1.0,\n)\nsearchResult, err = index.Search(searchRequest)\nif err != nil {\n    panic(err)\n}\n// Scores are based on the **best-matching vector** from the multi-vector field.\n// Example: distances to doc vectors {10..19} and {20..29} → pick the closer one (smaller squared L2),\n// then score = 1 / squared L2 distance.\nfmt.Printf(\"Multi-vector field kNN result:\\n%s\\n\", searchResult)\n```\n\n```go\n// ------------------------------------\n// Nested-vector field search (v2.5.7+)\n// ------------------------------------\nsearchRequest = bleve.NewSearchRequest(bleve.NewMatchNoneQuery())\nsearchRequest.AddKNN(\n    \"sections.vec\",\n    []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9},\n    5,\n    1.0,\n)\nsearchResult, err = index.Search(searchRequest)\nif err != nil {\n    panic(err)\n}\n// Scores are based on the **best-matching vector** from the nested-vector field.\n// Example: distances to doc vectors {30..39} and {40..49} → pick the closer one (smaller squared L2),\n// then score = 1 / squared L2 distance.\nfmt.Printf(\"Nested-vector field kNN result:\\n%s\\n\", searchResult)\n```\n\n```go\n// -----------------------------------------------------\n// Multi kNN queries on multi-vector documents (v2.5.7+)\n// -----------------------------------------------------\nsearchRequest = bleve.NewSearchRequest(bleve.NewMatchNoneQuery())\nsearchRequest.AddKNN(\n    \"embeddings\",\n    []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9},\n    5,\n    1.0,\n)\nsearchRequest.AddKNN(\n    \"embeddings\",\n    []float32{1, 2, 2, 5, 5, 6, 8, 7, 9, 10},\n    8,\n    1.0,\n)\nsearchResult, err = index.Search(searchRequest)\nif err != nil {\n    panic(err)\n}\n// Document score explanation:\n// - For each query vector, Bleve selects the **closest vector** in the multi-vector field.\n// - Scores from multiple queries are then **normalized and summed** to get the final document score.\n// For example, if the closest vector to the first query has squared L2 distance 4 (score 0.25)\n// and the closest vector to the second query has squared L2 distance 1 (score 1.0),\n// and both queries use equal boost values of 1.0, the normalization factor is 1/√2 (where 2 is the number of kNN queries).\n// Then the total document score = 1/√2 * 0.25 + 1/√2 * 1.0 = 0.1768 + 0.7071 = 0.8839.\n// Note: If the boost values differ, or if more queries are used, the normalization factor and score calculation will change accordingly.\nfmt.Printf(\"Multi kNN queries result:\\n%s\\n\", searchResult)\n```\n\n```go\n// --------------------------------------\n// Hybrid search: text + vector (v2.4.0+)\n// --------------------------------------\nsearchRequest = bleve.NewSearchRequest(bleve.NewMatchQuery(\"united states\"))\nsearchRequest.AddKNN(\n    \"vec\",\n    []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9},\n    5,\n    1,\n)\nsearchResult, err = index.Search(searchRequest)\nif err != nil {\n    panic(err)\n}\n// Score = sum of text relevance score + kNN vector score\n// Example: text score 0.5 + vector score 0.25 = total score 0.75\nfmt.Printf(\"Hybrid search result:\\n%s\\n\", searchResult)\n```\n\n## Querying with filters (v2.4.3+)\n\n```go\n// Pre-filtered vector/hybrid search: filter query narrows candidates before KNN search\nsearchRequest = bleve.NewSearchRequest(bleve.NewMatchNoneQuery()) // replace with any Bleve query for Pre-filtered Hybrid Search\nfilterQuery := bleve.NewTermQuery(\"hello\")                        // Filter query to narrow candidates\nsearchRequest.AddKNNWithFilter(\n    \"vec\",                                   // Vector field name\n    []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9}, // Query vector (must match indexed vector dims)\n    5,                                       // Number of nearest neighbors to return (k)\n    1,                                       // Boost factor for KNN score\n    filterQuery,                             // Filter query applied before KNN search\n)\nsearchResult, err = index.Search(searchRequest)\nif err != nil {\n    panic(err)\n}\n// Scores are computed only among documents matching the filter query\n// Example: if only one document matches the filter and has squared L2 distance 4 to the query vector,\n// its score will be 0.25 (1 / 4) and returned as the top result.\nfmt.Printf(\"Pre-filtered kNN search result:\\n%s\\n\", searchResult)\n```\n\n## Setup Instructions\n\n* Using `cmake` is a recommended approach by FAISS authors.\n* More details here - [faiss/INSTALL](https://github.com/blevesearch/faiss/blob/main/INSTALL.md).\n\n### Linux\n\nAlso documented here - [go-faiss/README](https://github.com/blevesearch/go-faiss/blob/master/README.md).\n\n```shell\ngit clone https://github.com/blevesearch/faiss.git\ncd faiss\ncmake -B build -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON .\nmake -C build\nsudo make -C build install\n```\n\nBuilding will produce the dynamic library `faiss_c`. You will need to install it in a place where your system will find it (e.g. /usr/lib). You can do this with:\n\n```shell\nsudo cp build/c_api/libfaiss_c.so /usr/local/lib\n```\n\n### OSX\n\nWhile you shouldn't need to do any different over osX x86_64, with aarch64 - some instructions need adjusting (see [facebookresearch/faiss#2111](https://github.com/facebookresearch/faiss/issues/2111)) ..\n\n```shell\nLDFLAGS=\"-L/opt/homebrew/opt/llvm/lib\" CPPFLAGS=\"-I/opt/homebrew/opt/llvm/include\" CXX=/opt/homebrew/opt/llvm/bin/clang++ CC=/opt/homebrew/opt/llvm/bin/clang cmake -B build -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON -DFAISS_ENABLE_PYTHON=OFF .\nmake -C build\nsudo make -C build install\nsudo cp build/c_api/libfaiss_c.dylib /usr/local/lib\n```\n\n### Sanity check\n\nOnce the supporting library is built and made available, a sanity run is recommended to make sure all unit tests and especially those accessing the vectors' code pass. Here's how ..\n\n```shell\ngo test -ldflags \"-r /usr/local/lib\" ./... -tags=vectors\n```\n"
  },
  {
    "path": "document/document.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeDocument int\n\nfunc init() {\n\tvar d Document\n\treflectStaticSizeDocument = int(reflect.TypeOf(d).Size())\n}\n\ntype Document struct {\n\tid               string\n\tFields           []Field     `json:\"fields\"`\n\tNestedDocuments  []*Document `json:\"nested_documents\"`\n\tCompositeFields  []*CompositeField\n\tStoredFieldsSize uint64\n\tindexed          bool\n}\n\nfunc (d *Document) StoredFieldsBytes() uint64 {\n\treturn d.StoredFieldsSize\n}\n\nfunc NewDocument(id string) *Document {\n\treturn &Document{\n\t\tid:              id,\n\t\tFields:          make([]Field, 0),\n\t\tCompositeFields: make([]*CompositeField, 0),\n\t}\n}\n\nfunc NewSynonymDocument(id string) *Document {\n\treturn &Document{\n\t\tid:     id,\n\t\tFields: make([]Field, 0),\n\t}\n}\n\nfunc (d *Document) Size() int {\n\tsizeInBytes := reflectStaticSizeDocument + size.SizeOfPtr +\n\t\tlen(d.id)\n\n\tfor _, entry := range d.Fields {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tfor _, entry := range d.CompositeFields {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (d *Document) AddField(f Field) *Document {\n\tswitch f := f.(type) {\n\tcase *CompositeField:\n\t\td.CompositeFields = append(d.CompositeFields, f)\n\tdefault:\n\t\td.Fields = append(d.Fields, f)\n\t}\n\treturn d\n}\n\nfunc (d *Document) GoString() string {\n\tfields := \"\"\n\tfor i, field := range d.Fields {\n\t\tif i != 0 {\n\t\t\tfields += \", \"\n\t\t}\n\t\tfields += fmt.Sprintf(\"%#v\", field)\n\t}\n\tcompositeFields := \"\"\n\tfor i, field := range d.CompositeFields {\n\t\tif i != 0 {\n\t\t\tcompositeFields += \", \"\n\t\t}\n\t\tcompositeFields += fmt.Sprintf(\"%#v\", field)\n\t}\n\treturn fmt.Sprintf(\"&document.Document{ID:%s, Fields: %s, CompositeFields: %s}\", d.ID(), fields, compositeFields)\n}\n\nfunc (d *Document) NumPlainTextBytes() uint64 {\n\trv := uint64(0)\n\tfor _, field := range d.Fields {\n\t\trv += field.NumPlainTextBytes()\n\t}\n\tfor _, compositeField := range d.CompositeFields {\n\t\tfor _, field := range d.Fields {\n\t\t\tif compositeField.includesField(field.Name()) {\n\t\t\t\trv += field.NumPlainTextBytes()\n\t\t\t}\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (d *Document) ID() string {\n\treturn d.id\n}\n\nfunc (d *Document) SetID(id string) {\n\td.id = id\n}\n\nfunc (d *Document) AddIDField() {\n\td.AddField(NewTextFieldCustom(\"_id\", nil, []byte(d.ID()), index.IndexField|index.StoreField, nil))\n}\n\nfunc (d *Document) VisitFields(visitor index.FieldVisitor) {\n\tfor _, f := range d.Fields {\n\t\tvisitor(f)\n\t}\n}\n\nfunc (d *Document) VisitComposite(visitor index.CompositeFieldVisitor) {\n\tfor _, f := range d.CompositeFields {\n\t\tvisitor(f)\n\t}\n}\n\nfunc (d *Document) HasComposite() bool {\n\treturn len(d.CompositeFields) > 0\n}\n\nfunc (d *Document) VisitSynonymFields(visitor index.SynonymFieldVisitor) {\n\tfor _, f := range d.Fields {\n\t\tif sf, ok := f.(index.SynonymField); ok {\n\t\t\tvisitor(sf)\n\t\t}\n\t}\n}\n\nfunc (d *Document) SetIndexed() {\n\td.indexed = true\n}\n\nfunc (d *Document) Indexed() bool {\n\treturn d.indexed\n}\n\nfunc (d *Document) AddNestedDocument(doc *Document) {\n\td.NestedDocuments = append(d.NestedDocuments, doc)\n}\n\nfunc (d *Document) VisitNestedDocuments(visitor func(doc index.Document)) {\n\tfor _, doc := range d.NestedDocuments {\n\t\tvisitor(doc)\n\t}\n}\n"
  },
  {
    "path": "document/document_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"testing\"\n)\n\nfunc TestDocumentNumPlainTextBytes(t *testing.T) {\n\n\ttests := []struct {\n\t\tdoc *Document\n\t\tnum uint64\n\t}{\n\t\t{\n\t\t\tdoc: NewDocument(\"a\"),\n\t\t\tnum: 0,\n\t\t},\n\t\t{\n\t\t\tdoc: NewDocument(\"b\").\n\t\t\t\tAddField(NewTextField(\"name\", nil, []byte(\"hello\"))),\n\t\t\tnum: 5,\n\t\t},\n\t\t{\n\t\t\tdoc: NewDocument(\"c\").\n\t\t\t\tAddField(NewTextField(\"name\", nil, []byte(\"hello\"))).\n\t\t\t\tAddField(NewTextField(\"desc\", nil, []byte(\"x\"))),\n\t\t\tnum: 6,\n\t\t},\n\t\t{\n\t\t\tdoc: NewDocument(\"d\").\n\t\t\t\tAddField(NewTextField(\"name\", nil, []byte(\"hello\"))).\n\t\t\t\tAddField(NewTextField(\"desc\", nil, []byte(\"x\"))).\n\t\t\t\tAddField(NewNumericField(\"age\", nil, 1.0)),\n\t\t\tnum: 14,\n\t\t},\n\t\t{\n\t\t\tdoc: NewDocument(\"e\").\n\t\t\t\tAddField(NewTextField(\"name\", nil, []byte(\"hello\"))).\n\t\t\t\tAddField(NewTextField(\"desc\", nil, []byte(\"x\"))).\n\t\t\t\tAddField(NewNumericField(\"age\", nil, 1.0)).\n\t\t\t\tAddField(NewCompositeField(\"_all\", true, nil, nil)),\n\t\t\tnum: 28,\n\t\t},\n\t\t{\n\t\t\tdoc: NewDocument(\"e\").\n\t\t\t\tAddField(NewTextField(\"name\", nil, []byte(\"hello\"))).\n\t\t\t\tAddField(NewTextField(\"desc\", nil, []byte(\"x\"))).\n\t\t\t\tAddField(NewNumericField(\"age\", nil, 1.0)).\n\t\t\t\tAddField(NewCompositeField(\"_all\", true, nil, []string{\"age\"})),\n\t\t\tnum: 20,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := test.doc.NumPlainTextBytes()\n\t\tif actual != test.num {\n\t\t\tt.Errorf(\"expected doc '%s' to have %d plain text bytes, got %d\", test.doc.ID(), test.num, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "document/field.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype Field interface {\n\t// Name returns the path of the field from the root DocumentMapping.\n\t// A root field path is \"field\", a subdocument field is \"parent.field\".\n\tName() string\n\t// ArrayPositions returns the intermediate document and field indices\n\t// required to resolve the field value in the document. For example, if the\n\t// field path is \"doc1.doc2.field\" where doc1 and doc2 are slices or\n\t// arrays, ArrayPositions returns 2 indices used to resolve \"doc2\" value in\n\t// \"doc1\", then \"field\" in \"doc2\".\n\tArrayPositions() []uint64\n\tOptions() index.FieldIndexingOptions\n\tAnalyze()\n\tValue() []byte\n\n\t// NumPlainTextBytes should return the number of plain text bytes\n\t// that this field represents - this is a common metric for tracking\n\t// the rate of indexing\n\tNumPlainTextBytes() uint64\n\n\tSize() int\n\n\tEncodedFieldType() byte\n\tAnalyzedLength() int\n\tAnalyzedTokenFrequencies() index.TokenFrequencies\n}\n"
  },
  {
    "path": "document/field_boolean.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeBooleanField int\n\nfunc init() {\n\tvar f BooleanField\n\treflectStaticSizeBooleanField = int(reflect.TypeOf(f).Size())\n}\n\nconst DefaultBooleanIndexingOptions = index.StoreField | index.IndexField | index.DocValues\n\ntype BooleanField struct {\n\tname              string\n\tarrayPositions    []uint64\n\toptions           index.FieldIndexingOptions\n\tvalue             []byte\n\tnumPlainTextBytes uint64\n\tlength            int\n\tfrequencies       index.TokenFrequencies\n}\n\nfunc (b *BooleanField) Size() int {\n\tvar freqSize int\n\tif b.frequencies != nil {\n\t\tfreqSize = b.frequencies.Size()\n\t}\n\treturn reflectStaticSizeBooleanField + size.SizeOfPtr +\n\t\tlen(b.name) +\n\t\tlen(b.arrayPositions)*size.SizeOfUint64 +\n\t\tlen(b.value) +\n\t\tfreqSize\n}\n\nfunc (b *BooleanField) Name() string {\n\treturn b.name\n}\n\nfunc (b *BooleanField) ArrayPositions() []uint64 {\n\treturn b.arrayPositions\n}\n\nfunc (b *BooleanField) Options() index.FieldIndexingOptions {\n\treturn b.options\n}\n\nfunc (b *BooleanField) Analyze() {\n\ttokens := make(analysis.TokenStream, 0)\n\ttokens = append(tokens, &analysis.Token{\n\t\tStart:    0,\n\t\tEnd:      len(b.value),\n\t\tTerm:     b.value,\n\t\tPosition: 1,\n\t\tType:     analysis.Boolean,\n\t})\n\n\tb.length = len(tokens)\n\tb.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options)\n}\n\nfunc (b *BooleanField) Value() []byte {\n\treturn b.value\n}\n\nfunc (b *BooleanField) Boolean() (bool, error) {\n\tif len(b.value) == 1 {\n\t\treturn b.value[0] == 'T', nil\n\t}\n\treturn false, fmt.Errorf(\"boolean field has %d bytes\", len(b.value))\n}\n\nfunc (b *BooleanField) GoString() string {\n\treturn fmt.Sprintf(\"&document.BooleanField{Name:%s, Options: %s, Value: %s}\", b.name, b.options, b.value)\n}\n\nfunc (b *BooleanField) NumPlainTextBytes() uint64 {\n\treturn b.numPlainTextBytes\n}\n\nfunc (b *BooleanField) EncodedFieldType() byte {\n\treturn 'b'\n}\n\nfunc (b *BooleanField) AnalyzedLength() int {\n\treturn b.length\n}\n\nfunc (b *BooleanField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn b.frequencies\n}\n\nfunc NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField {\n\treturn &BooleanField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\tvalue:             value,\n\t\toptions:           DefaultBooleanIndexingOptions,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField {\n\treturn NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultBooleanIndexingOptions)\n}\n\nfunc NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options index.FieldIndexingOptions) *BooleanField {\n\tnumPlainTextBytes := 5\n\tv := []byte(\"F\")\n\tif b {\n\t\tnumPlainTextBytes = 4\n\t\tv = []byte(\"T\")\n\t}\n\treturn &BooleanField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\tvalue:             v,\n\t\toptions:           options,\n\t\tnumPlainTextBytes: uint64(numPlainTextBytes),\n\t}\n}\n"
  },
  {
    "path": "document/field_composite.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeCompositeField int\n\nfunc init() {\n\tvar cf CompositeField\n\treflectStaticSizeCompositeField = int(reflect.TypeOf(cf).Size())\n}\n\nconst DefaultCompositeIndexingOptions = index.IndexField\n\ntype CompositeField struct {\n\tname                 string\n\tincludedFields       map[string]bool\n\texcludedFields       map[string]bool\n\tdefaultInclude       bool\n\toptions              index.FieldIndexingOptions\n\ttotalLength          int\n\tcompositeFrequencies index.TokenFrequencies\n}\n\nfunc NewCompositeField(name string, defaultInclude bool, include []string, exclude []string) *CompositeField {\n\treturn NewCompositeFieldWithIndexingOptions(name, defaultInclude, include, exclude, DefaultCompositeIndexingOptions)\n}\n\nfunc NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options index.FieldIndexingOptions) *CompositeField {\n\trv := &CompositeField{\n\t\tname:                 name,\n\t\toptions:              options,\n\t\tdefaultInclude:       defaultInclude,\n\t\tincludedFields:       make(map[string]bool, len(include)),\n\t\texcludedFields:       make(map[string]bool, len(exclude)),\n\t\tcompositeFrequencies: make(index.TokenFrequencies),\n\t}\n\n\tfor _, i := range include {\n\t\trv.includedFields[i] = true\n\t}\n\tfor _, e := range exclude {\n\t\trv.excludedFields[e] = true\n\t}\n\n\treturn rv\n}\n\nfunc (c *CompositeField) Size() int {\n\tsizeInBytes := reflectStaticSizeCompositeField + size.SizeOfPtr +\n\t\tlen(c.name)\n\n\tfor k := range c.includedFields {\n\t\tsizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool\n\t}\n\n\tfor k := range c.excludedFields {\n\t\tsizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool\n\t}\n\tif c.compositeFrequencies != nil {\n\t\tsizeInBytes += c.compositeFrequencies.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (c *CompositeField) Name() string {\n\treturn c.name\n}\n\nfunc (c *CompositeField) ArrayPositions() []uint64 {\n\treturn []uint64{}\n}\n\nfunc (c *CompositeField) Options() index.FieldIndexingOptions {\n\treturn c.options\n}\n\nfunc (c *CompositeField) Analyze() {\n}\n\nfunc (c *CompositeField) Value() []byte {\n\treturn []byte{}\n}\n\nfunc (c *CompositeField) NumPlainTextBytes() uint64 {\n\treturn 0\n}\n\nfunc (c *CompositeField) includesField(field string) bool {\n\tshouldInclude := c.defaultInclude\n\t_, fieldShouldBeIncluded := c.includedFields[field]\n\tif fieldShouldBeIncluded {\n\t\tshouldInclude = true\n\t}\n\t_, fieldShouldBeExcluded := c.excludedFields[field]\n\tif fieldShouldBeExcluded {\n\t\tshouldInclude = false\n\t}\n\treturn shouldInclude\n}\n\nfunc (c *CompositeField) Compose(field string, length int, freq index.TokenFrequencies) {\n\tif c.includesField(field) {\n\t\tc.totalLength += length\n\t\tc.compositeFrequencies.MergeAll(field, freq)\n\t}\n}\n\nfunc (c *CompositeField) EncodedFieldType() byte {\n\treturn 'c'\n}\n\nfunc (c *CompositeField) AnalyzedLength() int {\n\treturn c.totalLength\n}\n\nfunc (c *CompositeField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn c.compositeFrequencies\n}\n"
  },
  {
    "path": "document/field_datetime.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar dateTimeValueSeperator = []byte{'\\xff'}\n\nvar reflectStaticSizeDateTimeField int\n\nfunc init() {\n\tvar f DateTimeField\n\treflectStaticSizeDateTimeField = int(reflect.TypeOf(f).Size())\n}\n\nconst DefaultDateTimeIndexingOptions = index.StoreField | index.IndexField | index.DocValues\nconst DefaultDateTimePrecisionStep uint = 4\n\nvar MinTimeRepresentable = time.Unix(0, math.MinInt64)\nvar MaxTimeRepresentable = time.Unix(0, math.MaxInt64)\n\ntype DateTimeField struct {\n\tname              string\n\tarrayPositions    []uint64\n\toptions           index.FieldIndexingOptions\n\tvalue             numeric.PrefixCoded\n\tnumPlainTextBytes uint64\n\tlength            int\n\tfrequencies       index.TokenFrequencies\n}\n\nfunc (n *DateTimeField) Size() int {\n\tvar freqSize int\n\tif n.frequencies != nil {\n\t\tfreqSize = n.frequencies.Size()\n\t}\n\treturn reflectStaticSizeDateTimeField + size.SizeOfPtr +\n\t\tlen(n.name) +\n\t\tlen(n.arrayPositions)*size.SizeOfUint64 +\n\t\tlen(n.value) +\n\t\tfreqSize\n}\n\nfunc (n *DateTimeField) Name() string {\n\treturn n.name\n}\n\nfunc (n *DateTimeField) ArrayPositions() []uint64 {\n\treturn n.arrayPositions\n}\n\nfunc (n *DateTimeField) Options() index.FieldIndexingOptions {\n\treturn n.options\n}\n\nfunc (n *DateTimeField) EncodedFieldType() byte {\n\treturn 'd'\n}\n\nfunc (n *DateTimeField) AnalyzedLength() int {\n\treturn n.length\n}\n\nfunc (n *DateTimeField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn n.frequencies\n}\n\n// split the value into the prefix coded date and the layout\n// using the dateTimeValueSeperator as the split point\nfunc (n *DateTimeField) splitValue() (numeric.PrefixCoded, string) {\n\tparts := bytes.SplitN(n.value, dateTimeValueSeperator, 2)\n\tif len(parts) == 1 {\n\t\treturn numeric.PrefixCoded(parts[0]), \"\"\n\t}\n\treturn numeric.PrefixCoded(parts[0]), string(parts[1])\n}\n\nfunc (n *DateTimeField) Analyze() {\n\tvalueWithoutLayout, _ := n.splitValue()\n\ttokens := make(analysis.TokenStream, 0)\n\ttokens = append(tokens, &analysis.Token{\n\t\tStart:    0,\n\t\tEnd:      len(valueWithoutLayout),\n\t\tTerm:     valueWithoutLayout,\n\t\tPosition: 1,\n\t\tType:     analysis.DateTime,\n\t})\n\n\toriginal, err := valueWithoutLayout.Int64()\n\tif err == nil {\n\n\t\tshift := DefaultDateTimePrecisionStep\n\t\tfor shift < 64 {\n\t\t\tshiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ttoken := analysis.Token{\n\t\t\t\tStart:    0,\n\t\t\t\tEnd:      len(shiftEncoded),\n\t\t\t\tTerm:     shiftEncoded,\n\t\t\t\tPosition: 1,\n\t\t\t\tType:     analysis.DateTime,\n\t\t\t}\n\t\t\ttokens = append(tokens, &token)\n\t\t\tshift += DefaultDateTimePrecisionStep\n\t\t}\n\t}\n\n\tn.length = len(tokens)\n\tn.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)\n}\n\nfunc (n *DateTimeField) Value() []byte {\n\treturn n.value\n}\n\nfunc (n *DateTimeField) DateTime() (time.Time, string, error) {\n\tdate, layout := n.splitValue()\n\ti64, err := date.Int64()\n\tif err != nil {\n\t\treturn time.Time{}, \"\", err\n\t}\n\treturn time.Unix(0, i64).UTC(), layout, nil\n}\n\nfunc (n *DateTimeField) GoString() string {\n\treturn fmt.Sprintf(\"&document.DateField{Name:%s, Options: %s, Value: %s}\", n.name, n.options, n.value)\n}\n\nfunc (n *DateTimeField) NumPlainTextBytes() uint64 {\n\treturn n.numPlainTextBytes\n}\n\nfunc NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byte) *DateTimeField {\n\treturn &DateTimeField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\tvalue:             value,\n\t\toptions:           DefaultDateTimeIndexingOptions,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewDateTimeField(name string, arrayPositions []uint64, dt time.Time, layout string) (*DateTimeField, error) {\n\treturn NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, layout, DefaultDateTimeIndexingOptions)\n}\n\nfunc NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, layout string, options index.FieldIndexingOptions) (*DateTimeField, error) {\n\tif canRepresent(dt) {\n\t\tdtInt64 := dt.UnixNano()\n\t\tprefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0)\n\t\t// The prefixCoded value is combined with the layout.\n\t\t// This is necessary because the storage layer stores a fields value as a byte slice\n\t\t// without storing extra information like layout. So by making value = prefixCoded + layout,\n\t\t// both pieces of information are stored in the byte slice.\n\t\t// During a query, the layout is extracted from the byte slice stored to correctly\n\t\t// format the prefixCoded value.\n\t\tvalueWithLayout := append(prefixCoded, dateTimeValueSeperator...)\n\t\tvalueWithLayout = append(valueWithLayout, []byte(layout)...)\n\t\treturn &DateTimeField{\n\t\t\tname:           name,\n\t\t\tarrayPositions: arrayPositions,\n\t\t\tvalue:          valueWithLayout,\n\t\t\toptions:        options,\n\t\t\t// not correct, just a place holder until we revisit how fields are\n\t\t\t// represented and can fix this better\n\t\t\tnumPlainTextBytes: uint64(8),\n\t\t}, nil\n\t}\n\treturn nil, fmt.Errorf(\"cannot represent %s in this type\", dt)\n}\n\nfunc canRepresent(dt time.Time) bool {\n\tif dt.Before(MinTimeRepresentable) || dt.After(MaxTimeRepresentable) {\n\t\treturn false\n\t}\n\treturn true\n}\n"
  },
  {
    "path": "document/field_geopoint.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeGeoPointField int\n\nfunc init() {\n\tvar f GeoPointField\n\treflectStaticSizeGeoPointField = int(reflect.TypeOf(f).Size())\n}\n\nvar GeoPrecisionStep uint = 9\n\ntype GeoPointField struct {\n\tname              string\n\tarrayPositions    []uint64\n\toptions           index.FieldIndexingOptions\n\tvalue             numeric.PrefixCoded\n\tnumPlainTextBytes uint64\n\tlength            int\n\tfrequencies       index.TokenFrequencies\n\n\tspatialplugin index.SpatialAnalyzerPlugin\n}\n\nfunc (n *GeoPointField) Size() int {\n\tvar freqSize int\n\tif n.frequencies != nil {\n\t\tfreqSize = n.frequencies.Size()\n\t}\n\treturn reflectStaticSizeGeoPointField + size.SizeOfPtr +\n\t\tlen(n.name) +\n\t\tlen(n.arrayPositions)*size.SizeOfUint64 +\n\t\tlen(n.value) +\n\t\tfreqSize\n}\n\nfunc (n *GeoPointField) Name() string {\n\treturn n.name\n}\n\nfunc (n *GeoPointField) ArrayPositions() []uint64 {\n\treturn n.arrayPositions\n}\n\nfunc (n *GeoPointField) Options() index.FieldIndexingOptions {\n\treturn n.options\n}\n\nfunc (n *GeoPointField) EncodedFieldType() byte {\n\treturn 'g'\n}\n\nfunc (n *GeoPointField) AnalyzedLength() int {\n\treturn n.length\n}\n\nfunc (n *GeoPointField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn n.frequencies\n}\n\nfunc (n *GeoPointField) Analyze() {\n\ttokens := make(analysis.TokenStream, 0, 8)\n\ttokens = append(tokens, &analysis.Token{\n\t\tStart:    0,\n\t\tEnd:      len(n.value),\n\t\tTerm:     n.value,\n\t\tPosition: 1,\n\t\tType:     analysis.Numeric,\n\t})\n\n\tif n.spatialplugin != nil {\n\t\tlat, _ := n.Lat()\n\t\tlon, _ := n.Lon()\n\t\tp := &geo.Point{Lat: lat, Lon: lon}\n\t\tterms := n.spatialplugin.GetIndexTokens(p)\n\n\t\tfor _, term := range terms {\n\t\t\ttoken := analysis.Token{\n\t\t\t\tStart:    0,\n\t\t\t\tEnd:      len(term),\n\t\t\t\tTerm:     []byte(term),\n\t\t\t\tPosition: 1,\n\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t}\n\t\t\ttokens = append(tokens, &token)\n\t\t}\n\t} else {\n\t\toriginal, err := n.value.Int64()\n\t\tif err == nil {\n\n\t\t\tshift := GeoPrecisionStep\n\t\t\tfor shift < 64 {\n\t\t\t\tshiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)\n\t\t\t\tif err != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\ttoken := analysis.Token{\n\t\t\t\t\tStart:    0,\n\t\t\t\t\tEnd:      len(shiftEncoded),\n\t\t\t\t\tTerm:     shiftEncoded,\n\t\t\t\t\tPosition: 1,\n\t\t\t\t\tType:     analysis.Numeric,\n\t\t\t\t}\n\t\t\t\ttokens = append(tokens, &token)\n\t\t\t\tshift += GeoPrecisionStep\n\t\t\t}\n\t\t}\n\t}\n\n\tn.length = len(tokens)\n\tn.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)\n}\n\nfunc (n *GeoPointField) Value() []byte {\n\treturn n.value\n}\n\nfunc (n *GeoPointField) Lon() (float64, error) {\n\ti64, err := n.value.Int64()\n\tif err != nil {\n\t\treturn 0.0, err\n\t}\n\treturn geo.MortonUnhashLon(uint64(i64)), nil\n}\n\nfunc (n *GeoPointField) Lat() (float64, error) {\n\ti64, err := n.value.Int64()\n\tif err != nil {\n\t\treturn 0.0, err\n\t}\n\treturn geo.MortonUnhashLat(uint64(i64)), nil\n}\n\nfunc (n *GeoPointField) GoString() string {\n\treturn fmt.Sprintf(\"&document.GeoPointField{Name:%s, Options: %s, Value: %s}\", n.name, n.options, n.value)\n}\n\nfunc (n *GeoPointField) NumPlainTextBytes() uint64 {\n\treturn n.numPlainTextBytes\n}\n\nfunc NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField {\n\treturn &GeoPointField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\tvalue:             value,\n\t\toptions:           DefaultNumericIndexingOptions,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField {\n\treturn NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions)\n}\n\nfunc NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options index.FieldIndexingOptions) *GeoPointField {\n\tmhash := geo.MortonHash(lon, lat)\n\tprefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0)\n\n\t// docvalues are always enabled for geopoint fields, even if the\n\t// indexing options are set to not include docvalues.\n\t// snappy compression and chunking are always skipped for geopoint\n\t// to avoid mem copies and faster lookups.\n\toptions |= index.DocValues\n\toptions |= index.SkipDVChunking\n\toptions |= index.SkipDVCompression\n\n\treturn &GeoPointField{\n\t\tname:           name,\n\t\tarrayPositions: arrayPositions,\n\t\tvalue:          prefixCoded,\n\t\toptions:        options,\n\t\t// not correct, just a place holder until we revisit how fields are\n\t\t// represented and can fix this better\n\t\tnumPlainTextBytes: uint64(8),\n\t}\n}\n\n// SetSpatialAnalyzerPlugin implements the\n// index.TokenisableSpatialField interface.\nfunc (n *GeoPointField) SetSpatialAnalyzerPlugin(\n\tplugin index.SpatialAnalyzerPlugin) {\n\tn.spatialplugin = plugin\n}\n"
  },
  {
    "path": "document/field_geopoint_test.go",
    "content": "package document\n\nimport \"testing\"\n\nfunc TestGeoPointField(t *testing.T) {\n\tgf := NewGeoPointField(\"loc\", []uint64{}, 0.0015, 0.0015)\n\tgf.Analyze()\n\tnumTokens := gf.AnalyzedLength()\n\ttokenFreqs := gf.AnalyzedTokenFrequencies()\n\tif numTokens != 8 {\n\t\tt.Errorf(\"expected 8 tokens, got %d\", numTokens)\n\t}\n\tif len(tokenFreqs) != 8 {\n\t\tt.Errorf(\"expected 8 token freqs\")\n\t}\n}\n"
  },
  {
    "path": "document/field_geoshape.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"github.com/blevesearch/geo/geojson\"\n)\n\nvar reflectStaticSizeGeoShapeField int\n\nfunc init() {\n\tvar f GeoShapeField\n\treflectStaticSizeGeoShapeField = int(reflect.TypeOf(f).Size())\n}\n\nconst DefaultGeoShapeIndexingOptions = index.IndexField | index.DocValues\n\ntype GeoShapeField struct {\n\tname              string\n\tshape             index.GeoJSON\n\tarrayPositions    []uint64\n\toptions           index.FieldIndexingOptions\n\tnumPlainTextBytes uint64\n\tlength            int\n\tencodedValue      []byte\n\tvalue             []byte\n\n\tfrequencies index.TokenFrequencies\n}\n\nfunc (n *GeoShapeField) Size() int {\n\tvar freqSize int\n\tif n.frequencies != nil {\n\t\tfreqSize = n.frequencies.Size()\n\t}\n\treturn reflectStaticSizeGeoShapeField + size.SizeOfPtr +\n\t\tlen(n.name) +\n\t\tlen(n.arrayPositions)*size.SizeOfUint64 +\n\t\tlen(n.encodedValue) +\n\t\tlen(n.value) +\n\t\tfreqSize\n}\n\nfunc (n *GeoShapeField) Name() string {\n\treturn n.name\n}\n\nfunc (n *GeoShapeField) ArrayPositions() []uint64 {\n\treturn n.arrayPositions\n}\n\nfunc (n *GeoShapeField) Options() index.FieldIndexingOptions {\n\treturn n.options\n}\n\nfunc (n *GeoShapeField) EncodedFieldType() byte {\n\treturn 's'\n}\n\nfunc (n *GeoShapeField) AnalyzedLength() int {\n\treturn n.length\n}\n\nfunc (n *GeoShapeField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn n.frequencies\n}\n\nfunc (n *GeoShapeField) Analyze() {\n\t// compute the bytes representation for the coordinates\n\ttokens := make(analysis.TokenStream, 0)\n\n\trti := geo.GetSpatialAnalyzerPlugin(\"s2\")\n\tterms := rti.GetIndexTokens(n.shape)\n\n\tfor _, term := range terms {\n\t\ttoken := analysis.Token{\n\t\t\tStart:    0,\n\t\t\tEnd:      len(term),\n\t\t\tTerm:     []byte(term),\n\t\t\tPosition: 1,\n\t\t\tType:     analysis.AlphaNumeric,\n\t\t}\n\t\ttokens = append(tokens, &token)\n\t}\n\n\tn.length = len(tokens)\n\tn.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)\n}\n\nfunc (n *GeoShapeField) Value() []byte {\n\treturn n.value\n}\n\nfunc (n *GeoShapeField) GoString() string {\n\treturn fmt.Sprintf(\"&document.GeoShapeField{Name:%s, Options: %s, Value: %s}\",\n\t\tn.name, n.options, n.value)\n}\n\nfunc (n *GeoShapeField) NumPlainTextBytes() uint64 {\n\treturn n.numPlainTextBytes\n}\n\nfunc (n *GeoShapeField) EncodedShape() []byte {\n\treturn n.encodedValue\n}\n\nfunc NewGeoShapeField(name string, arrayPositions []uint64,\n\tcoordinates [][][][]float64, typ string) *GeoShapeField {\n\treturn NewGeoShapeFieldWithIndexingOptions(name, arrayPositions,\n\t\tcoordinates, typ, DefaultGeoShapeIndexingOptions)\n}\n\nfunc NewGeoShapeFieldFromBytes(name string, arrayPositions []uint64,\n\tvalue []byte) *GeoShapeField {\n\treturn &GeoShapeField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\tvalue:             value,\n\t\toptions:           DefaultGeoShapeIndexingOptions,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64,\n\tcoordinates [][][][]float64, typ string,\n\toptions index.FieldIndexingOptions) *GeoShapeField {\n\tshape := &geojson.GeoShape{\n\t\tCoordinates: coordinates,\n\t\tType:        typ,\n\t}\n\n\treturn NewGeoShapeFieldFromShapeWithIndexingOptions(name,\n\t\tarrayPositions, shape, options)\n}\n\nfunc NewGeoShapeFieldFromShapeWithIndexingOptions(name string, arrayPositions []uint64,\n\tgeoShape *geojson.GeoShape, options index.FieldIndexingOptions) *GeoShapeField {\n\n\tvar shape index.GeoJSON\n\tvar encodedValue []byte\n\tvar err error\n\n\tif geoShape.Type == geo.CircleType {\n\t\tshape, encodedValue, err = geo.NewGeoCircleShape(geoShape.Center, geoShape.Radius)\n\t} else {\n\t\tshape, encodedValue, err = geo.NewGeoJsonShape(geoShape.Coordinates, geoShape.Type)\n\t}\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\t// extra glue bytes to work around the term splitting logic from interfering\n\t// the custom encoding of the geoshape coordinates inside the docvalues.\n\tencodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...)\n\n\t// get the byte value for the geoshape.\n\tvalue, err := shape.Value()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\t// docvalues are always enabled for geoshape fields, even if the\n\t// indexing options are set to not include docvalues.\n\t// snappy compression and chunking are always skipped for geoshape\n\t// to avoid mem copies and faster lookups.\n\toptions |= index.DocValues\n\toptions |= index.SkipDVChunking\n\toptions |= index.SkipDVCompression\n\n\treturn &GeoShapeField{\n\t\tshape:             shape,\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\toptions:           options,\n\t\tencodedValue:      encodedValue,\n\t\tvalue:             value,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewGeometryCollectionFieldWithIndexingOptions(name string,\n\tarrayPositions []uint64, coordinates [][][][][]float64, types []string,\n\toptions index.FieldIndexingOptions) *GeoShapeField {\n\tif len(coordinates) != len(types) {\n\t\treturn nil\n\t}\n\n\tshapes := make([]*geojson.GeoShape, len(types))\n\tfor i := range coordinates {\n\t\tshapes[i] = &geojson.GeoShape{\n\t\t\tCoordinates: coordinates[i],\n\t\t\tType:        types[i],\n\t\t}\n\t}\n\n\treturn NewGeometryCollectionFieldFromShapesWithIndexingOptions(name,\n\t\tarrayPositions, shapes, options)\n}\n\nfunc NewGeometryCollectionFieldFromShapesWithIndexingOptions(name string,\n\tarrayPositions []uint64, geoShapes []*geojson.GeoShape,\n\toptions index.FieldIndexingOptions) *GeoShapeField {\n\tshape, encodedValue, err := geo.NewGeometryCollectionFromShapes(geoShapes)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\t// extra glue bytes to work around the term splitting logic from interfering\n\t// the custom encoding of the geoshape coordinates inside the docvalues.\n\tencodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...)\n\n\t// get the byte value for the geometryCollection.\n\tvalue, err := shape.Value()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\t// docvalues are always enabled for geoshape fields, even if the\n\t// indexing options are set to not include docvalues.\n\t// snappy compression and chunking are always skipped for geoshape\n\t// to avoid mem copies and faster lookups.\n\toptions |= index.DocValues\n\toptions |= index.SkipDVChunking\n\toptions |= index.SkipDVCompression\n\n\treturn &GeoShapeField{\n\t\tshape:             shape,\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\toptions:           options,\n\t\tencodedValue:      encodedValue,\n\t\tvalue:             value,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewGeoCircleFieldWithIndexingOptions(name string, arrayPositions []uint64,\n\tcenterPoint []float64, radius string,\n\toptions index.FieldIndexingOptions) *GeoShapeField {\n\n\tshape := &geojson.GeoShape{\n\t\tCenter: centerPoint,\n\t\tRadius: radius,\n\t\tType:   geo.CircleType,\n\t}\n\n\treturn NewGeoShapeFieldFromShapeWithIndexingOptions(name,\n\t\tarrayPositions, shape, options)\n}\n\n// GeoShape is an implementation of the index.GeoShapeField interface.\nfunc (n *GeoShapeField) GeoShape() (index.GeoJSON, error) {\n\treturn geojson.ParseGeoJSONShape(n.value)\n}\n"
  },
  {
    "path": "document/field_ip.go",
    "content": "//  Copyright (c) 2021 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeIPField int\n\nfunc init() {\n\tvar f IPField\n\treflectStaticSizeIPField = int(reflect.TypeOf(f).Size())\n}\n\nconst DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues\n\ntype IPField struct {\n\tname              string\n\tarrayPositions    []uint64\n\toptions           index.FieldIndexingOptions\n\tvalue             net.IP\n\tnumPlainTextBytes uint64\n\tlength            int\n\tfrequencies       index.TokenFrequencies\n}\n\nfunc (b *IPField) Size() int {\n\tvar freqSize int\n\tif b.frequencies != nil {\n\t\tfreqSize = b.frequencies.Size()\n\t}\n\treturn reflectStaticSizeIPField + size.SizeOfPtr +\n\t\tlen(b.name) +\n\t\tlen(b.arrayPositions)*size.SizeOfUint64 +\n\t\tlen(b.value) +\n\t\tfreqSize\n}\n\nfunc (b *IPField) Name() string {\n\treturn b.name\n}\n\nfunc (b *IPField) ArrayPositions() []uint64 {\n\treturn b.arrayPositions\n}\n\nfunc (b *IPField) Options() index.FieldIndexingOptions {\n\treturn b.options\n}\n\nfunc (n *IPField) EncodedFieldType() byte {\n\treturn 'i'\n}\n\nfunc (n *IPField) AnalyzedLength() int {\n\treturn n.length\n}\n\nfunc (n *IPField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn n.frequencies\n}\n\nfunc (b *IPField) Analyze() {\n\n\ttokens := analysis.TokenStream{\n\t\t&analysis.Token{\n\t\t\tStart:    0,\n\t\t\tEnd:      len(b.value),\n\t\t\tTerm:     b.value,\n\t\t\tPosition: 1,\n\t\t\tType:     analysis.IP,\n\t\t},\n\t}\n\tb.length = 1\n\tb.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options)\n}\n\nfunc (b *IPField) Value() []byte {\n\treturn b.value\n}\n\nfunc (b *IPField) IP() (net.IP, error) {\n\treturn net.IP(b.value), nil\n}\n\nfunc (b *IPField) GoString() string {\n\treturn fmt.Sprintf(\"&document.IPField{Name:%s, Options: %s, Value: %s}\", b.name, b.options, net.IP(b.value))\n}\n\nfunc (b *IPField) NumPlainTextBytes() uint64 {\n\treturn b.numPlainTextBytes\n}\n\nfunc NewIPFieldFromBytes(name string, arrayPositions []uint64, value []byte) *IPField {\n\treturn &IPField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\tvalue:             value,\n\t\toptions:           DefaultIPIndexingOptions,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewIPField(name string, arrayPositions []uint64, v net.IP) *IPField {\n\treturn NewIPFieldWithIndexingOptions(name, arrayPositions, v, DefaultIPIndexingOptions)\n}\n\nfunc NewIPFieldWithIndexingOptions(name string, arrayPositions []uint64, b net.IP, options index.FieldIndexingOptions) *IPField {\n\tv := b.To16()\n\n\treturn &IPField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\tvalue:             v,\n\t\toptions:           options,\n\t\tnumPlainTextBytes: net.IPv6len,\n\t}\n}\n"
  },
  {
    "path": "document/field_ip_test.go",
    "content": "//  Copyright (c) 2021 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"bytes\"\n\t\"net\"\n\t\"testing\"\n)\n\nfunc TestIPField(t *testing.T) {\n\tnf := NewIPField(\"ip\", []uint64{}, net.IPv4(192, 168, 1, 1))\n\tnf.Analyze()\n\tif nf.length != 1 {\n\t\tt.Errorf(\"expected 1 token\")\n\t}\n\tif len(nf.value) != 16 {\n\t\tt.Errorf(\"stored value should be in 16 byte ipv6 format\")\n\t}\n\tif !bytes.Equal(nf.value, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 168, 1, 1}) {\n\t\tt.Errorf(\"wrong value stored, expected 192.168.1.1, got %q\", nf.value.String())\n\t}\n\tif len(nf.frequencies) != 1 {\n\t\tt.Errorf(\"expected 1 token freqs\")\n\t}\n}\n"
  },
  {
    "path": "document/field_numeric.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeNumericField int\n\nfunc init() {\n\tvar f NumericField\n\treflectStaticSizeNumericField = int(reflect.TypeOf(f).Size())\n}\n\nconst DefaultNumericIndexingOptions = index.StoreField | index.IndexField | index.DocValues\n\nconst DefaultPrecisionStep uint = 4\n\ntype NumericField struct {\n\tname              string\n\tarrayPositions    []uint64\n\toptions           index.FieldIndexingOptions\n\tvalue             numeric.PrefixCoded\n\tnumPlainTextBytes uint64\n\tlength            int\n\tfrequencies       index.TokenFrequencies\n}\n\nfunc (n *NumericField) Size() int {\n\tvar freqSize int\n\tif n.frequencies != nil {\n\t\tfreqSize = n.frequencies.Size()\n\t}\n\treturn reflectStaticSizeNumericField + size.SizeOfPtr +\n\t\tlen(n.name) +\n\t\tlen(n.arrayPositions)*size.SizeOfUint64 +\n\t\tlen(n.value) +\n\t\tfreqSize\n}\n\nfunc (n *NumericField) Name() string {\n\treturn n.name\n}\n\nfunc (n *NumericField) ArrayPositions() []uint64 {\n\treturn n.arrayPositions\n}\n\nfunc (n *NumericField) Options() index.FieldIndexingOptions {\n\treturn n.options\n}\n\nfunc (n *NumericField) EncodedFieldType() byte {\n\treturn 'n'\n}\n\nfunc (n *NumericField) AnalyzedLength() int {\n\treturn n.length\n}\n\nfunc (n *NumericField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn n.frequencies\n}\n\nfunc (n *NumericField) Analyze() {\n\ttokens := make(analysis.TokenStream, 0)\n\ttokens = append(tokens, &analysis.Token{\n\t\tStart:    0,\n\t\tEnd:      len(n.value),\n\t\tTerm:     n.value,\n\t\tPosition: 1,\n\t\tType:     analysis.Numeric,\n\t})\n\n\toriginal, err := n.value.Int64()\n\tif err == nil {\n\n\t\tshift := DefaultPrecisionStep\n\t\tfor shift < 64 {\n\t\t\tshiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ttoken := analysis.Token{\n\t\t\t\tStart:    0,\n\t\t\t\tEnd:      len(shiftEncoded),\n\t\t\t\tTerm:     shiftEncoded,\n\t\t\t\tPosition: 1,\n\t\t\t\tType:     analysis.Numeric,\n\t\t\t}\n\t\t\ttokens = append(tokens, &token)\n\t\t\tshift += DefaultPrecisionStep\n\t\t}\n\t}\n\n\tn.length = len(tokens)\n\tn.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)\n}\n\nfunc (n *NumericField) Value() []byte {\n\treturn n.value\n}\n\nfunc (n *NumericField) Number() (float64, error) {\n\ti64, err := n.value.Int64()\n\tif err != nil {\n\t\treturn 0.0, err\n\t}\n\treturn numeric.Int64ToFloat64(i64), nil\n}\n\nfunc (n *NumericField) GoString() string {\n\treturn fmt.Sprintf(\"&document.NumericField{Name:%s, Options: %s, Value: %s}\", n.name, n.options, n.value)\n}\n\nfunc (n *NumericField) NumPlainTextBytes() uint64 {\n\treturn n.numPlainTextBytes\n}\n\nfunc NewNumericFieldFromBytes(name string, arrayPositions []uint64, value []byte) *NumericField {\n\treturn &NumericField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\tvalue:             value,\n\t\toptions:           DefaultNumericIndexingOptions,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewNumericField(name string, arrayPositions []uint64, number float64) *NumericField {\n\treturn NewNumericFieldWithIndexingOptions(name, arrayPositions, number, DefaultNumericIndexingOptions)\n}\n\nfunc NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options index.FieldIndexingOptions) *NumericField {\n\tnumberInt64 := numeric.Float64ToInt64(number)\n\tprefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0)\n\treturn &NumericField{\n\t\tname:           name,\n\t\tarrayPositions: arrayPositions,\n\t\tvalue:          prefixCoded,\n\t\toptions:        options,\n\t\t// not correct, just a place holder until we revisit how fields are\n\t\t// represented and can fix this better\n\t\tnumPlainTextBytes: uint64(8),\n\t}\n}\n"
  },
  {
    "path": "document/field_numeric_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"testing\"\n)\n\nfunc TestNumericField(t *testing.T) {\n\tnf := NewNumericField(\"age\", []uint64{}, 3.4)\n\tnf.Analyze()\n\tnumTokens := nf.AnalyzedLength()\n\ttokenFreqs := nf.AnalyzedTokenFrequencies()\n\tif numTokens != 16 {\n\t\tt.Errorf(\"expected 16 tokens\")\n\t}\n\tif len(tokenFreqs) != 16 {\n\t\tt.Errorf(\"expected 16 token freqs\")\n\t}\n}\n"
  },
  {
    "path": "document/field_synonym.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeSynonymField int\n\nfunc init() {\n\tvar f SynonymField\n\treflectStaticSizeSynonymField = int(reflect.TypeOf(f).Size())\n}\n\nconst DefaultSynonymIndexingOptions = index.IndexField\n\ntype SynonymField struct {\n\tname              string\n\tanalyzer          analysis.Analyzer\n\toptions           index.FieldIndexingOptions\n\tinput             []string\n\tsynonyms          []string\n\tnumPlainTextBytes uint64\n\n\t// populated during analysis\n\tsynonymMap map[string][]string\n}\n\nfunc (s *SynonymField) Size() int {\n\treturn reflectStaticSizeSynonymField + size.SizeOfPtr +\n\t\tlen(s.name)\n}\n\nfunc (s *SynonymField) Name() string {\n\treturn s.name\n}\n\nfunc (s *SynonymField) ArrayPositions() []uint64 {\n\treturn nil\n}\n\nfunc (s *SynonymField) Options() index.FieldIndexingOptions {\n\treturn s.options\n}\n\nfunc (s *SynonymField) NumPlainTextBytes() uint64 {\n\treturn s.numPlainTextBytes\n}\n\nfunc (s *SynonymField) AnalyzedLength() int {\n\treturn 0\n}\n\nfunc (s *SynonymField) EncodedFieldType() byte {\n\treturn 'y'\n}\n\nfunc (s *SynonymField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn nil\n}\n\nfunc (s *SynonymField) Analyze() {\n\tvar analyzedInput []string\n\tif len(s.input) > 0 {\n\t\tanalyzedInput = make([]string, 0, len(s.input))\n\t\tfor _, term := range s.input {\n\t\t\tanalyzedTerm := analyzeSynonymTerm(term, s.analyzer)\n\t\t\tif analyzedTerm != \"\" {\n\t\t\t\tanalyzedInput = append(analyzedInput, analyzedTerm)\n\t\t\t}\n\t\t}\n\t}\n\tanalyzedSynonyms := make([]string, 0, len(s.synonyms))\n\tfor _, syn := range s.synonyms {\n\t\tanalyzedTerm := analyzeSynonymTerm(syn, s.analyzer)\n\t\tif analyzedTerm != \"\" {\n\t\t\tanalyzedSynonyms = append(analyzedSynonyms, analyzedTerm)\n\t\t}\n\t}\n\ts.synonymMap = processSynonymData(analyzedInput, analyzedSynonyms)\n}\n\nfunc (s *SynonymField) Value() []byte {\n\treturn nil\n}\n\nfunc (s *SynonymField) IterateSynonyms(visitor func(term string, synonyms []string)) {\n\tfor term, synonyms := range s.synonymMap {\n\t\tvisitor(term, synonyms)\n\t}\n}\n\nfunc NewSynonymField(name string, analyzer analysis.Analyzer, input []string, synonyms []string) *SynonymField {\n\treturn &SynonymField{\n\t\tname:     name,\n\t\tanalyzer: analyzer,\n\t\toptions:  DefaultSynonymIndexingOptions,\n\t\tinput:    input,\n\t\tsynonyms: synonyms,\n\t}\n}\n\nfunc processSynonymData(input []string, synonyms []string) map[string][]string {\n\tvar synonymMap map[string][]string\n\tif len(input) > 0 {\n\t\t// Map each term to the same list of synonyms.\n\t\tsynonymMap = make(map[string][]string, len(input))\n\t\tfor _, term := range input {\n\t\t\tsynonymMap[term] = synonyms\n\t\t}\n\t} else {\n\t\tsynonymMap = make(map[string][]string, len(synonyms))\n\t\t// Precompute a map where each synonym points to all other synonyms.\n\t\tfor i, elem := range synonyms {\n\t\t\tsynonymMap[elem] = make([]string, 0, len(synonyms)-1)\n\t\t\tfor j, otherElem := range synonyms {\n\t\t\t\tif i != j {\n\t\t\t\t\tsynonymMap[elem] = append(synonymMap[elem], otherElem)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn synonymMap\n}\n\nfunc analyzeSynonymTerm(term string, analyzer analysis.Analyzer) string {\n\ttokenStream := analyzer.Analyze([]byte(term))\n\tif len(tokenStream) == 1 {\n\t\treturn string(tokenStream[0].Term)\n\t}\n\treturn \"\"\n}\n"
  },
  {
    "path": "document/field_text.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage document\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeTextField int\n\nfunc init() {\n\tvar f TextField\n\treflectStaticSizeTextField = int(reflect.TypeOf(f).Size())\n}\n\nconst DefaultTextIndexingOptions = index.IndexField | index.DocValues\n\ntype TextField struct {\n\tname              string\n\tarrayPositions    []uint64\n\toptions           index.FieldIndexingOptions\n\tanalyzer          analysis.Analyzer\n\tvalue             []byte\n\tnumPlainTextBytes uint64\n\tlength            int\n\tfrequencies       index.TokenFrequencies\n}\n\nfunc (t *TextField) Size() int {\n\tvar freqSize int\n\tif t.frequencies != nil {\n\t\tfreqSize = t.frequencies.Size()\n\t}\n\treturn reflectStaticSizeTextField + size.SizeOfPtr +\n\t\tlen(t.name) +\n\t\tlen(t.arrayPositions)*size.SizeOfUint64 +\n\t\tlen(t.value) +\n\t\tfreqSize\n}\n\nfunc (t *TextField) Name() string {\n\treturn t.name\n}\n\nfunc (t *TextField) ArrayPositions() []uint64 {\n\treturn t.arrayPositions\n}\n\nfunc (t *TextField) Options() index.FieldIndexingOptions {\n\treturn t.options\n}\n\nfunc (t *TextField) EncodedFieldType() byte {\n\treturn 't'\n}\n\nfunc (t *TextField) AnalyzedLength() int {\n\treturn t.length\n}\n\nfunc (t *TextField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn t.frequencies\n}\n\nfunc (t *TextField) Analyze() {\n\tvar tokens analysis.TokenStream\n\tif t.analyzer != nil {\n\t\tbytesToAnalyze := t.Value()\n\t\tif t.options.IsStored() {\n\t\t\t// need to copy\n\t\t\tbytesCopied := make([]byte, len(bytesToAnalyze))\n\t\t\tcopy(bytesCopied, bytesToAnalyze)\n\t\t\tbytesToAnalyze = bytesCopied\n\t\t}\n\t\ttokens = t.analyzer.Analyze(bytesToAnalyze)\n\t} else {\n\t\ttokens = analysis.TokenStream{\n\t\t\t&analysis.Token{\n\t\t\t\tStart:    0,\n\t\t\t\tEnd:      len(t.value),\n\t\t\t\tTerm:     t.value,\n\t\t\t\tPosition: 1,\n\t\t\t\tType:     analysis.AlphaNumeric,\n\t\t\t},\n\t\t}\n\t}\n\tt.length = len(tokens) // number of tokens in this doc field\n\tt.frequencies = analysis.TokenFrequency(tokens, t.arrayPositions, t.options)\n}\n\nfunc (t *TextField) Analyzer() analysis.Analyzer {\n\treturn t.analyzer\n}\n\nfunc (t *TextField) Value() []byte {\n\treturn t.value\n}\n\nfunc (t *TextField) Text() string {\n\treturn string(t.value)\n}\n\nfunc (t *TextField) GoString() string {\n\treturn fmt.Sprintf(\"&document.TextField{Name:%s, Options: %s, Analyzer: %v, Value: %s, ArrayPositions: %v}\", t.name, t.options, t.analyzer, t.value, t.arrayPositions)\n}\n\nfunc (t *TextField) NumPlainTextBytes() uint64 {\n\treturn t.numPlainTextBytes\n}\n\nfunc NewTextField(name string, arrayPositions []uint64, value []byte) *TextField {\n\treturn NewTextFieldWithIndexingOptions(name, arrayPositions, value, DefaultTextIndexingOptions)\n}\n\nfunc NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options index.FieldIndexingOptions) *TextField {\n\treturn &TextField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\toptions:           options,\n\t\tvalue:             value,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte, analyzer analysis.Analyzer) *TextField {\n\treturn &TextField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\toptions:           DefaultTextIndexingOptions,\n\t\tanalyzer:          analyzer,\n\t\tvalue:             value,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n\nfunc NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options index.FieldIndexingOptions, analyzer analysis.Analyzer) *TextField {\n\treturn &TextField{\n\t\tname:              name,\n\t\tarrayPositions:    arrayPositions,\n\t\toptions:           options,\n\t\tanalyzer:          analyzer,\n\t\tvalue:             value,\n\t\tnumPlainTextBytes: uint64(len(value)),\n\t}\n}\n"
  },
  {
    "path": "document/field_vector.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage document\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeVectorField int\n\nfunc init() {\n\tvar f VectorField\n\treflectStaticSizeVectorField = int(reflect.TypeOf(f).Size())\n}\n\nconst DefaultVectorIndexingOptions = index.IndexField\n\ntype VectorField struct {\n\tname                    string\n\tdims                    int    // Dimensionality of the vector\n\tsimilarity              string // Similarity metric to use for scoring\n\toptions                 index.FieldIndexingOptions\n\tvalue                   []float32\n\tnumPlainTextBytes       uint64\n\tvectorIndexOptimizedFor string // Optimization applied to this index.\n}\n\nfunc (n *VectorField) Size() int {\n\treturn reflectStaticSizeVectorField + size.SizeOfPtr +\n\t\tlen(n.name) +\n\t\tlen(n.similarity) +\n\t\tlen(n.vectorIndexOptimizedFor) +\n\t\tint(numBytesFloat32s(n.value))\n}\n\nfunc (n *VectorField) Name() string {\n\treturn n.name\n}\n\nfunc (n *VectorField) ArrayPositions() []uint64 {\n\treturn nil\n}\n\nfunc (n *VectorField) Options() index.FieldIndexingOptions {\n\treturn n.options\n}\n\nfunc (n *VectorField) NumPlainTextBytes() uint64 {\n\treturn n.numPlainTextBytes\n}\n\nfunc (n *VectorField) AnalyzedLength() int {\n\t// vectors aren't analyzed\n\treturn 0\n}\n\nfunc (n *VectorField) EncodedFieldType() byte {\n\treturn 'v'\n}\n\nfunc (n *VectorField) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\t// vectors aren't analyzed\n\treturn nil\n}\n\nfunc (n *VectorField) Analyze() {\n\t// vectors aren't analyzed\n}\n\nfunc (n *VectorField) Value() []byte {\n\treturn nil\n}\n\nfunc (n *VectorField) GoString() string {\n\treturn fmt.Sprintf(\"&document.VectorField{Name:%s, Options: %s, \"+\n\t\t\"Value: %+v}\", n.name, n.options, n.value)\n}\n\n// For the sake of not polluting the API, we are keeping arrayPositions as a\n// parameter, but it is not used.\nfunc NewVectorField(name string, arrayPositions []uint64,\n\tvector []float32, dims int, similarity, vectorIndexOptimizedFor string) *VectorField {\n\treturn NewVectorFieldWithIndexingOptions(name, arrayPositions,\n\t\tvector, dims, similarity, vectorIndexOptimizedFor,\n\t\tDefaultVectorIndexingOptions)\n}\n\n// For the sake of not polluting the API, we are keeping arrayPositions as a\n// parameter, but it is not used.\nfunc NewVectorFieldWithIndexingOptions(name string, arrayPositions []uint64,\n\tvector []float32, dims int, similarity, vectorIndexOptimizedFor string,\n\toptions index.FieldIndexingOptions) *VectorField {\n\t// ensure the options are set to not store/index term vectors/doc values\n\toptions &^= index.StoreField | index.IncludeTermVectors | index.DocValues\n\t// skip freq/norms for vector field\n\toptions |= index.SkipFreqNorm\n\n\t// bivf-sq8 indexes only supports hamming distance for the primary\n\t// binary index. Similarity here is used for the backing flat index,\n\t// which is set to cosine similarity for recall reasons\n\tif index.OptimizationRequiresBinaryIndex(vectorIndexOptimizedFor) {\n\t\tsimilarity = index.CosineSimilarity\n\t}\n\n\treturn &VectorField{\n\t\tname:                    name,\n\t\tdims:                    dims,\n\t\tsimilarity:              similarity,\n\t\toptions:                 options,\n\t\tvalue:                   vector,\n\t\tnumPlainTextBytes:       numBytesFloat32s(vector),\n\t\tvectorIndexOptimizedFor: vectorIndexOptimizedFor,\n\t}\n}\n\nfunc numBytesFloat32s(value []float32) uint64 {\n\treturn uint64(len(value) * size.SizeOfFloat32)\n}\n\n// -----------------------------------------------------------------------------\n// Following methods help in implementing the bleve_index_api's VectorField\n// interface.\n\nfunc (n *VectorField) Vector() []float32 {\n\treturn n.value\n}\n\nfunc (n *VectorField) Dims() int {\n\treturn n.dims\n}\n\nfunc (n *VectorField) Similarity() string {\n\treturn n.similarity\n}\n\nfunc (n *VectorField) IndexOptimizedFor() string {\n\treturn n.vectorIndexOptimizedFor\n}\n"
  },
  {
    "path": "document/field_vector_base64.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage document\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeVectorBase64Field int\n\nfunc init() {\n\tvar f VectorBase64Field\n\treflectStaticSizeVectorBase64Field = int(reflect.TypeOf(f).Size())\n}\n\ntype VectorBase64Field struct {\n\tvectorField    *VectorField\n\tbase64Encoding string\n}\n\nfunc (n *VectorBase64Field) Size() int {\n\tvar vecFieldSize int\n\tif n.vectorField != nil {\n\t\tvecFieldSize = n.vectorField.Size()\n\t}\n\treturn reflectStaticSizeVectorBase64Field + size.SizeOfPtr +\n\t\tlen(n.base64Encoding) +\n\t\tvecFieldSize\n}\n\nfunc (n *VectorBase64Field) Name() string {\n\treturn n.vectorField.Name()\n}\n\nfunc (n *VectorBase64Field) ArrayPositions() []uint64 {\n\treturn n.vectorField.ArrayPositions()\n}\n\nfunc (n *VectorBase64Field) Options() index.FieldIndexingOptions {\n\treturn n.vectorField.Options()\n}\n\nfunc (n *VectorBase64Field) NumPlainTextBytes() uint64 {\n\treturn n.vectorField.NumPlainTextBytes()\n}\n\nfunc (n *VectorBase64Field) AnalyzedLength() int {\n\treturn n.vectorField.AnalyzedLength()\n}\n\nfunc (n *VectorBase64Field) EncodedFieldType() byte {\n\treturn 'e'\n}\n\nfunc (n *VectorBase64Field) AnalyzedTokenFrequencies() index.TokenFrequencies {\n\treturn n.vectorField.AnalyzedTokenFrequencies()\n}\n\nfunc (n *VectorBase64Field) Analyze() {\n}\n\nfunc (n *VectorBase64Field) Value() []byte {\n\treturn n.vectorField.Value()\n}\n\nfunc (n *VectorBase64Field) GoString() string {\n\treturn fmt.Sprintf(\"&document.vectorFieldBase64Field{Name:%s, Options: %s, \"+\n\t\t\"Value: %+v}\", n.vectorField.Name(), n.vectorField.Options(), n.vectorField.Value())\n}\n\n// For the sake of not polluting the API, we are keeping arrayPositions as a\n// parameter, but it is not used.\nfunc NewVectorBase64Field(name string, arrayPositions []uint64, vectorBase64 string,\n\tdims int, similarity, vectorIndexOptimizedFor string) (*VectorBase64Field, error) {\n\n\tdecodedVector, err := DecodeVector(vectorBase64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &VectorBase64Field{\n\t\tvectorField: NewVectorFieldWithIndexingOptions(name, arrayPositions,\n\t\t\tdecodedVector, dims, similarity,\n\t\t\tvectorIndexOptimizedFor, DefaultVectorIndexingOptions),\n\n\t\tbase64Encoding: vectorBase64,\n\t}, nil\n}\n\n// This function takes a base64 encoded string and decodes it into\n// a vector.\nfunc DecodeVector(encodedValue string) ([]float32, error) {\n\t// We first decode the encoded string into a byte array.\n\tdecodedString, err := base64.StdEncoding.DecodeString(encodedValue)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// The array is expected to be divisible by 4 because each float32\n\t// should occupy 4 bytes\n\tif len(decodedString)%size.SizeOfFloat32 != 0 {\n\t\treturn nil, fmt.Errorf(\"decoded byte array not divisible by %d\", size.SizeOfFloat32)\n\t}\n\tdims := int(len(decodedString) / size.SizeOfFloat32)\n\n\tif dims <= 0 {\n\t\treturn nil, fmt.Errorf(\"unable to decode encoded vector\")\n\t}\n\n\tdecodedVector := make([]float32, dims)\n\n\t// We iterate through the array 4 bytes at a time and convert each of\n\t// them to a float32 value by reading them in a little endian notation\n\tfor i := 0; i < dims; i++ {\n\t\tbytes := decodedString[i*size.SizeOfFloat32 : (i+1)*size.SizeOfFloat32]\n\t\tentry := math.Float32frombits(binary.LittleEndian.Uint32(bytes))\n\t\tif !util.IsValidFloat32(float64(entry)) {\n\t\t\treturn nil, fmt.Errorf(\"invalid float32 value: %f\", entry)\n\t\t}\n\t\tdecodedVector[i] = entry\n\t}\n\n\treturn decodedVector, nil\n}\n\nfunc (n *VectorBase64Field) Vector() []float32 {\n\treturn n.vectorField.Vector()\n}\n\nfunc (n *VectorBase64Field) Dims() int {\n\treturn n.vectorField.Dims()\n}\n\nfunc (n *VectorBase64Field) Similarity() string {\n\treturn n.vectorField.Similarity()\n}\n\nfunc (n *VectorBase64Field) IndexOptimizedFor() string {\n\treturn n.vectorField.IndexOptimizedFor()\n}\n"
  },
  {
    "path": "document/field_vector_base64_test.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage document\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"testing\"\n)\n\nfunc TestDecodeVector(t *testing.T) {\n\tvec := make([]float32, 2048)\n\tfor i := range vec {\n\t\tvec[i] = rand.Float32()\n\t}\n\n\tvecBytes := bytifyVec(vec)\n\tencodedVec := base64.StdEncoding.EncodeToString(vecBytes)\n\n\tdecodedVector, err := DecodeVector(encodedVec)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif len(decodedVector) != len(vec) {\n\t\tt.Errorf(\"Decoded vector dimensions not same as original vector dimensions\")\n\t}\n\n\tfor i := range vec {\n\t\tif vec[i] != decodedVector[i] {\n\t\t\tt.Fatalf(\"Decoded vector not the same as original vector %v != %v\", vec[i], decodedVector[i])\n\t\t}\n\t}\n}\n\nfunc BenchmarkDecodeVector128(b *testing.B) {\n\tvec := make([]float32, 128)\n\tfor i := range vec {\n\t\tvec[i] = rand.Float32()\n\t}\n\n\tvecBytes := bytifyVec(vec)\n\tencodedVec := base64.StdEncoding.EncodeToString(vecBytes)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _ = DecodeVector(encodedVec)\n\t}\n}\n\nfunc BenchmarkDecodeVector784(b *testing.B) {\n\tvec := make([]float32, 784)\n\tfor i := range vec {\n\t\tvec[i] = rand.Float32()\n\t}\n\n\tvecBytes := bytifyVec(vec)\n\tencodedVec := base64.StdEncoding.EncodeToString(vecBytes)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _ = DecodeVector(encodedVec)\n\t}\n}\n\nfunc BenchmarkDecodeVector1536(b *testing.B) {\n\tvec := make([]float32, 1536)\n\tfor i := range vec {\n\t\tvec[i] = rand.Float32()\n\t}\n\n\tvecBytes := bytifyVec(vec)\n\tencodedVec := base64.StdEncoding.EncodeToString(vecBytes)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _ = DecodeVector(encodedVec)\n\t}\n}\n\nfunc bytifyVec(vec []float32) []byte {\n\tbuf := new(bytes.Buffer)\n\n\tfor _, v := range vec {\n\t\terr := binary.Write(buf, binary.LittleEndian, v)\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t}\n\t}\n\n\treturn buf.Bytes()\n}\n"
  },
  {
    "path": "error.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\n// Constant Error values which can be compared to determine the type of error\nconst (\n\tErrorIndexPathExists Error = iota\n\tErrorIndexPathDoesNotExist\n\tErrorIndexMetaMissing\n\tErrorIndexMetaCorrupt\n\tErrorIndexClosed\n\tErrorAliasMulti\n\tErrorAliasEmpty\n\tErrorUnknownIndexType\n\tErrorEmptyID\n\tErrorIndexReadInconsistency\n\tErrorTwoPhaseSearchInconsistency\n\tErrorSynonymSearchNotSupported\n)\n\n// Error represents a more strongly typed bleve error for detecting\n// and handling specific types of errors.\ntype Error int\n\nfunc (e Error) Error() string {\n\treturn errorMessages[e]\n}\n\nvar errorMessages = map[Error]string{\n\tErrorIndexPathExists:             \"cannot create new index, path already exists\",\n\tErrorIndexPathDoesNotExist:       \"cannot open index, path does not exist\",\n\tErrorIndexMetaMissing:            \"cannot open index, metadata missing\",\n\tErrorIndexMetaCorrupt:            \"cannot open index, metadata corrupt\",\n\tErrorIndexClosed:                 \"index is closed\",\n\tErrorAliasMulti:                  \"cannot perform single index operation on multiple index alias\",\n\tErrorAliasEmpty:                  \"cannot perform operation on empty alias\",\n\tErrorUnknownIndexType:            \"unknown index type\",\n\tErrorEmptyID:                     \"document ID cannot be empty\",\n\tErrorIndexReadInconsistency:      \"index read inconsistency detected\",\n\tErrorTwoPhaseSearchInconsistency: \"2-phase search failed, likely due to an overlapping topology change\",\n\tErrorSynonymSearchNotSupported:   \"synonym search not supported\",\n}\n"
  },
  {
    "path": "examples_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight/highlighter/ansi\"\n)\n\nvar indexMapping mapping.IndexMapping\nvar exampleIndex Index\nvar err error\n\nfunc TestMain(m *testing.M) {\n\terr = os.RemoveAll(\"path_to_index\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttoRun := m.Run()\n\tif exampleIndex != nil {\n\t\terr = exampleIndex.Close()\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\terr = os.RemoveAll(\"path_to_index\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tos.Exit(toRun)\n}\n\nfunc ExampleNew() {\n\tindexMapping = NewIndexMapping()\n\texampleIndex, err = New(\"path_to_index\", indexMapping)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tcount, err := exampleIndex.DocCount()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(count)\n\t// Output:\n\t// 0\n}\n\nfunc ExampleIndex_indexing() {\n\tdata := struct {\n\t\tName    string\n\t\tCreated time.Time\n\t\tAge     int\n\t}{Name: \"named one\", Created: time.Now(), Age: 50}\n\tdata2 := struct {\n\t\tName    string\n\t\tCreated time.Time\n\t\tAge     int\n\t}{Name: \"great nameless one\", Created: time.Now(), Age: 25}\n\n\t// index some data\n\terr = exampleIndex.Index(\"document id 1\", data)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\terr = exampleIndex.Index(\"document id 2\", data2)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t// 2 documents have been indexed\n\tcount, err := exampleIndex.DocCount()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(count)\n\t// Output:\n\t// 2\n}\n\n// Examples for query related functions\n\nfunc ExampleNewMatchQuery() {\n\t// finds documents with fields fully matching the given query text\n\tquery := NewMatchQuery(\"named one\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 1\n}\n\nfunc ExampleNewMatchAllQuery() {\n\t// finds all documents in the index\n\tquery := NewMatchAllQuery()\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(len(searchResults.Hits))\n\t// Output:\n\t// 2\n}\n\nfunc ExampleNewMatchNoneQuery() {\n\t// matches no documents in the index\n\tquery := NewMatchNoneQuery()\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(len(searchResults.Hits))\n\t// Output:\n\t// 0\n}\n\nfunc ExampleNewMatchPhraseQuery() {\n\t// finds all documents with the given phrase in the index\n\tquery := NewMatchPhraseQuery(\"nameless one\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 2\n}\n\nfunc ExampleNewNumericRangeQuery() {\n\tvalue1 := float64(11)\n\tvalue2 := float64(100)\n\tdata := struct{ Priority float64 }{Priority: float64(15)}\n\tdata2 := struct{ Priority float64 }{Priority: float64(10)}\n\n\terr = exampleIndex.Index(\"document id 3\", data)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\terr = exampleIndex.Index(\"document id 4\", data2)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tquery := NewNumericRangeQuery(&value1, &value2)\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 3\n}\n\nfunc ExampleNewNumericRangeInclusiveQuery() {\n\tvalue1 := float64(10)\n\tvalue2 := float64(100)\n\tv1incl := false\n\tv2incl := false\n\n\tquery := NewNumericRangeInclusiveQuery(&value1, &value2, &v1incl, &v2incl)\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 3\n}\n\nfunc ExampleNewPhraseQuery() {\n\t// finds all documents with the given phrases in the given field in the index\n\tquery := NewPhraseQuery([]string{\"nameless\", \"one\"}, \"Name\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 2\n}\n\nfunc ExampleNewPrefixQuery() {\n\t// finds all documents with terms having the given prefix in the index\n\tquery := NewPrefixQuery(\"name\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(len(searchResults.Hits))\n\t// Output:\n\t// 2\n}\n\nfunc ExampleNewQueryStringQuery() {\n\tquery := NewQueryStringQuery(\"+one -great\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 1\n}\n\nfunc ExampleNewTermQuery() {\n\tquery := NewTermQuery(\"great\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 2\n}\n\nfunc ExampleNewFacetRequest() {\n\tfacet := NewFacetRequest(\"Name\", 1)\n\tquery := NewMatchAllQuery()\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.AddFacet(\"facet name\", facet)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t// total number of terms\n\tfmt.Println(searchResults.Facets[\"facet name\"].Total)\n\t// number of docs with no value for this field\n\tfmt.Println(searchResults.Facets[\"facet name\"].Missing)\n\t// term with highest occurrences in field name\n\tfmt.Println(searchResults.Facets[\"facet name\"].Terms.Terms()[0].Term)\n\t// Output:\n\t// 5\n\t// 2\n\t// one\n}\n\nfunc ExampleFacetRequest_AddDateTimeRange() {\n\tfacet := NewFacetRequest(\"Created\", 1)\n\tfacet.AddDateTimeRange(\"range name\", time.Unix(0, 0), time.Now())\n\tquery := NewMatchAllQuery()\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.AddFacet(\"facet name\", facet)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t// dates in field Created since starting of unix time till now\n\tfmt.Println(searchResults.Facets[\"facet name\"].DateRanges[0].Count)\n\t// Output:\n\t// 2\n}\n\nfunc ExampleFacetRequest_AddNumericRange() {\n\tvalue1 := float64(11)\n\n\tfacet := NewFacetRequest(\"Priority\", 1)\n\tfacet.AddNumericRange(\"range name\", &value1, nil)\n\tquery := NewMatchAllQuery()\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.AddFacet(\"facet name\", facet)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t// number documents with field Priority in the given range\n\tfmt.Println(searchResults.Facets[\"facet name\"].NumericRanges[0].Count)\n\t// Output:\n\t// 1\n}\n\nfunc ExampleNewHighlight() {\n\tquery := NewMatchQuery(\"nameless\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.Highlight = NewHighlight()\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].Fragments[\"Name\"][0])\n\t// Output:\n\t// great <mark>nameless</mark> one\n}\n\nfunc ExampleNewHighlightWithStyle() {\n\tquery := NewMatchQuery(\"nameless\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.Highlight = NewHighlightWithStyle(ansi.Name)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].Fragments[\"Name\"][0])\n\t// Output:\n\t// great \u001b[43mnameless\u001b[0m one\n}\n\nfunc ExampleSearchRequest_AddFacet() {\n\tfacet := NewFacetRequest(\"Name\", 1)\n\tquery := NewMatchAllQuery()\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.AddFacet(\"facet name\", facet)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t// total number of terms\n\tfmt.Println(searchResults.Facets[\"facet name\"].Total)\n\t// number of docs with no value for this field\n\tfmt.Println(searchResults.Facets[\"facet name\"].Missing)\n\t// term with highest occurrences in field name\n\tfmt.Println(searchResults.Facets[\"facet name\"].Terms.Terms()[0].Term)\n\t// Output:\n\t// 5\n\t// 2\n\t// one\n}\n\nfunc ExampleNewSearchRequest() {\n\t// finds documents with fields fully matching the given query text\n\tquery := NewMatchQuery(\"named one\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 1\n}\n\nfunc ExampleNewBooleanQuery() {\n\tmust := NewMatchQuery(\"one\")\n\tmustNot := NewMatchQuery(\"great\")\n\tquery := NewBooleanQuery()\n\tquery.AddMust(must)\n\tquery.AddMustNot(mustNot)\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 1\n}\n\nfunc ExampleNewConjunctionQuery() {\n\tconjunct1 := NewMatchQuery(\"great\")\n\tconjunct2 := NewMatchQuery(\"one\")\n\tquery := NewConjunctionQuery(conjunct1, conjunct2)\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\t// Output:\n\t// document id 2\n}\n\nfunc ExampleNewDisjunctionQuery() {\n\tdisjunct1 := NewMatchQuery(\"great\")\n\tdisjunct2 := NewMatchQuery(\"named\")\n\tquery := NewDisjunctionQuery(disjunct1, disjunct2)\n\tsearchRequest := NewSearchRequest(query)\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(len(searchResults.Hits))\n\t// Output:\n\t// 2\n}\n\nfunc ExampleSearchRequest_SortBy() {\n\t// find docs containing \"one\", order by Age instead of score\n\tquery := NewMatchQuery(\"one\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.SortBy([]string{\"Age\"})\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\tfmt.Println(searchResults.Hits[1].ID)\n\t// Output:\n\t// document id 2\n\t// document id 1\n}\n\nfunc ExampleSearchRequest_SortByCustom() {\n\t// find all docs, order by Age, with docs missing Age field first\n\tquery := NewMatchAllQuery()\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.SortByCustom(search.SortOrder{\n\t\t&search.SortField{\n\t\t\tField:   \"Age\",\n\t\t\tMissing: search.SortFieldMissingFirst,\n\t\t},\n\t\t&search.SortDocID{},\n\t})\n\tsearchResults, err := exampleIndex.Search(searchRequest)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfmt.Println(searchResults.Hits[0].ID)\n\tfmt.Println(searchResults.Hits[1].ID)\n\tfmt.Println(searchResults.Hits[2].ID)\n\tfmt.Println(searchResults.Hits[3].ID)\n\t// Output:\n\t// document id 3\n\t// document id 4\n\t// document id 2\n\t// document id 1\n}\n"
  },
  {
    "path": "fusion/fusion.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fusion\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\ntype FusionResult struct {\n\tHits     search.DocumentMatchCollection\n\tTotal    uint64\n\tMaxScore float64\n}\n"
  },
  {
    "path": "fusion/rrf.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fusion\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\n// formatRRFMessage builds the explanation string for a single component of the\n// Reciprocal Rank Fusion calculation.\nfunc formatRRFMessage(weight float64, rank int, rankConstant int) string {\n\treturn fmt.Sprintf(\"rrf score (weight=%.3f, rank=%d, rank_constant=%d), normalized score of\", weight, rank, rankConstant)\n}\n\n// ReciprocalRankFusion applies Reciprocal Rank Fusion across the primary FTS\n// results and each KNN sub-query. Ranks are limited to `windowSize` per source,\n// weighted, and combined into a single fused score, with optional explanation\n// details.\nfunc ReciprocalRankFusion(hits search.DocumentMatchCollection, weights []float64, rankConstant int, windowSize int, numKNNQueries int, explain bool) *FusionResult {\n\tnHits := len(hits)\n\tif nHits == 0 || windowSize == 0 {\n\t\treturn &FusionResult{\n\t\t\tHits:     search.DocumentMatchCollection{},\n\t\t\tTotal:    0,\n\t\t\tMaxScore: 0.0,\n\t\t}\n\t}\n\n\tlimit := min(nHits, windowSize)\n\n\t// precompute rank+scores to prevent additional division ops later\n\trankReciprocals := make([]float64, limit)\n\tfor i := range rankReciprocals {\n\t\trankReciprocals[i] = 1.0 / float64(rankConstant+i+1)\n\t}\n\n\t// init explanations if required\n\tvar fusionExpl map[*search.DocumentMatch][]*search.Explanation\n\tif explain {\n\t\tfusionExpl = make(map[*search.DocumentMatch][]*search.Explanation, nHits)\n\t}\n\n\t// The code here mainly deals with obtaining rank/score for fts hits.\n\t// First sort hits by score\n\tsortDocMatchesByScore(hits)\n\n\t// Calculate fts rank+scores\n\tftsWeight := weights[0]\n\tfor i := 0; i < nHits; i++ {\n\t\tif i < windowSize {\n\t\t\thit := hits[i]\n\n\t\t\t// No fts scores from this hit onwards, break loop\n\t\t\tif hit.Score == 0.0 {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tcontrib := ftsWeight * rankReciprocals[i]\n\t\t\thit.Score = contrib\n\n\t\t\tif explain {\n\t\t\t\texpl := getFusionExplAt(\n\t\t\t\t\thit,\n\t\t\t\t\t0,\n\t\t\t\t\tcontrib,\n\t\t\t\t\tformatRRFMessage(ftsWeight, i+1, rankConstant),\n\t\t\t\t)\n\t\t\t\tfusionExpl[hit] = append(fusionExpl[hit], expl)\n\t\t\t}\n\t\t} else {\n\t\t\t// These FTS hits are not counted in the results, so set to 0\n\t\t\thits[i].Score = 0.0\n\t\t}\n\t}\n\n\t// Code from here is to calculate knn ranks and scores\n\t// iterate over each knn query and calculate knn rank+scores\n\tfor queryIdx := 0; queryIdx < numKNNQueries; queryIdx++ {\n\t\tknnWeight := weights[queryIdx+1]\n\t\t// Sorts hits in decreasing order of hit.ScoreBreakdown[i]\n\t\tsortDocMatchesByBreakdown(hits, queryIdx)\n\n\t\tfor i := 0; i < nHits; i++ {\n\t\t\t// break if score breakdown doesn't exist (sort function puts these hits at the end)\n\t\t\t// or if we go past the windowSize\n\t\t\t_, scoreBreakdownExists := scoreBreakdownForQuery(hits[i], queryIdx)\n\t\t\tif i >= windowSize || !scoreBreakdownExists {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\thit := hits[i]\n\t\t\tcontrib := knnWeight * rankReciprocals[i]\n\t\t\thit.Score += contrib\n\n\t\t\tif explain {\n\t\t\t\texpl := getFusionExplAt(\n\t\t\t\t\thit,\n\t\t\t\t\tqueryIdx+1,\n\t\t\t\t\tcontrib,\n\t\t\t\t\tformatRRFMessage(knnWeight, i+1, rankConstant),\n\t\t\t\t)\n\t\t\t\tfusionExpl[hit] = append(fusionExpl[hit], expl)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar maxScore float64\n\tfor _, hit := range hits {\n\t\tif explain {\n\t\t\tfinalizeFusionExpl(hit, fusionExpl[hit])\n\t\t}\n\t\thit.ScoreBreakdown = nil\n\n\t\tif hit.Score > maxScore {\n\t\t\tmaxScore = hit.Score\n\t\t}\n\t}\n\n\tsortDocMatchesByScore(hits)\n\tif nHits > windowSize {\n\t\thits = hits[:windowSize]\n\t}\n\treturn &FusionResult{\n\t\tHits:     hits,\n\t\tTotal:    uint64(len(hits)),\n\t\tMaxScore: maxScore,\n\t}\n}\n"
  },
  {
    "path": "fusion/rrf_test.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fusion\n\nimport (\n\t\"math\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\nconst epsilon float64 = 1e-3\n\nfunc nearlyEqual(a float64, b float64, epsilon float64) bool {\n\treturn math.Abs(a-b) < epsilon\n}\n\nfunc compareFusionResults(a, b FusionResult) bool {\n\tif a.Total != b.Total || !nearlyEqual(a.MaxScore, b.MaxScore, epsilon) || len(a.Hits) != len(b.Hits) {\n\t\treturn false\n\t}\n\tfor i := range a.Hits {\n\t\tif a.Hits[i].ID != b.Hits[i].ID || !nearlyEqual(a.Hits[i].Score, b.Hits[i].Score, epsilon) {\n\t\t\treturn false\n\t\t}\n\n\t\tif a.Hits[i].ScoreBreakdown != nil || b.Hits[i].ScoreBreakdown != nil {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc TestReciprocalRankFusion(t *testing.T) {\n\ttests := []struct {\n\t\tname          string\n\t\thits          search.DocumentMatchCollection\n\t\tweights       []float64\n\t\trank_constant int\n\t\twindow_size   int\n\t\tnumKNNQueries int\n\t\twant          FusionResult\n\t}{\n\t\t{\n\t\t\tname:          \"empty hits\",\n\t\t\thits:          search.DocumentMatchCollection{},\n\t\t\tweights:       []float64{0.5, 0.5},\n\t\t\trank_constant: 60,\n\t\t\twindow_size:   10,\n\t\t\tnumKNNQueries: 1,\n\t\t\twant: FusionResult{\n\t\t\t\tHits:     search.DocumentMatchCollection{},\n\t\t\t\tTotal:    0,\n\t\t\t\tMaxScore: 0.0,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"single knn query\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8}},\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}},\n\t\t\t\t{ID: \"c\", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.7}},\n\t\t\t},\n\t\t\tweights:       []float64{0.4, 0.6},\n\t\t\trank_constant: 1,\n\t\t\twindow_size:   3,\n\t\t\tnumKNNQueries: 1,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"b\", Score: 0.433},\n\t\t\t\t\t{ID: \"a\", Score: 0.4},\n\t\t\t\t\t{ID: \"c\", Score: 0.25},\n\t\t\t\t},\n\t\t\t\tTotal:    3,\n\t\t\t\tMaxScore: 0.433,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"multiple knn queries\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8, 1: 0.6}},\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9, 1: 0.5}},\n\t\t\t\t{ID: \"c\", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.7, 1: 0.7}},\n\t\t\t},\n\t\t\tweights:       []float64{0.3, 0.4, 0.3},\n\t\t\trank_constant: 1,\n\t\t\twindow_size:   3,\n\t\t\tnumKNNQueries: 2,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"a\", Score: 0.383},\n\t\t\t\t\t{ID: \"b\", Score: 0.375},\n\t\t\t\t\t{ID: \"c\", Score: 0.325},\n\t\t\t\t},\n\t\t\t\tTotal:    3,\n\t\t\t\tMaxScore: 0.383,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"window size smaller than hits\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.7}},\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}},\n\t\t\t\t{ID: \"c\", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.8}},\n\t\t\t},\n\t\t\tweights:       []float64{0.4, 0.6},\n\t\t\trank_constant: 1,\n\t\t\twindow_size:   2,\n\t\t\tnumKNNQueries: 1,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"b\", Score: 0.433},\n\t\t\t\t\t{ID: \"a\", Score: 0.2},\n\t\t\t\t},\n\t\t\t\tTotal:    2,\n\t\t\t\tMaxScore: 0.433,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"documents with partial scores missing KNN scores\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8}},         // has FTS and KNN query 0, missing KNN query 1\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{1: 0.7}},         // has FTS and KNN query 1, missing KNN query 0\n\t\t\t\t{ID: \"c\", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.6, 1: 0.9}}, // has all scores\n\t\t\t\t{ID: \"d\", Score: 0.6, ScoreBreakdown: map[int]float64{}},               // has only FTS, missing all KNN scores\n\t\t\t},\n\t\t\tweights:       []float64{0.3, 0.4, 0.3}, // FTS, KNN query 0, KNN query 1\n\t\t\trank_constant: 1,\n\t\t\twindow_size:   4,\n\t\t\tnumKNNQueries: 2,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"c\", Score: 0.358}, // FTS rank 3, KNN0 rank 2, KNN1 rank 1: 0.3/4 + 0.4/3 + 0.3/2 = 0.075 + 0.133 + 0.15 = 0.358\n\t\t\t\t\t{ID: \"a\", Score: 0.35},  // FTS rank 1, KNN0 rank 1, no KNN1: 0.3/2 + 0.4/2 + 0 = 0.15 + 0.2 + 0 = 0.35\n\t\t\t\t\t{ID: \"b\", Score: 0.2},   // FTS rank 2, no KNN0, KNN1 rank 2: 0.3/3 + 0 + 0.3/3 = 0.1 + 0 + 0.1 = 0.2\n\t\t\t\t\t{ID: \"d\", Score: 0.06},  // FTS rank 4, no KNN0, no KNN1: 0.3/5 + 0 + 0 = 0.06\n\t\t\t\t},\n\t\t\t\tTotal:    4,\n\t\t\t\tMaxScore: 0.358,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"documents with only KNN scores\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.9}},         // no FTS rank (Score 0.0), only KNN query 0\n\t\t\t\t{ID: \"b\", Score: 0.0, ScoreBreakdown: map[int]float64{1: 0.8}},         // no FTS rank (Score 0.0), only KNN query 1\n\t\t\t\t{ID: \"c\", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.7, 1: 0.6}}, // no FTS rank (Score 0.0), both KNN queries\n\t\t\t},\n\t\t\tweights:       []float64{0.5, 0.3, 0.2}, // FTS, KNN query 0, KNN query 1\n\t\t\trank_constant: 1,\n\t\t\twindow_size:   3,\n\t\t\tnumKNNQueries: 2,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"c\", Score: 0.167}, // no FTS rank, KNN0 rank 2, KNN1 rank 2: 0 + 0.3/3 + 0.2/3 = 0 + 0.1 + 0.067 = 0.167\n\t\t\t\t\t{ID: \"a\", Score: 0.15},  // no FTS rank, KNN0 rank 1, no KNN1: 0 + 0.3/2 + 0 = 0 + 0.15 + 0 = 0.15\n\t\t\t\t\t{ID: \"b\", Score: 0.1},   // no FTS rank, no KNN0, KNN1 rank 1: 0 + 0 + 0.2/2 = 0 + 0 + 0.1 = 0.1\n\t\t\t\t},\n\t\t\t\tTotal:    3,\n\t\t\t\tMaxScore: 0.167,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"mixed scenario with gaps in KNN queries\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.8, ScoreBreakdown: map[int]float64{1: 0.9}}, // has FTS and KNN query 1, missing KNN query 0\n\t\t\t\t{ID: \"b\", Score: 0.6, ScoreBreakdown: map[int]float64{0: 0.8}}, // has FTS and KNN query 0, missing KNN query 1\n\t\t\t\t{ID: \"c\", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.7}}, // no FTS rank (Score 0.0), only KNN query 0\n\t\t\t\t{ID: \"d\", Score: 0.4, ScoreBreakdown: map[int]float64{}},       // only FTS, no KNN scores\n\t\t\t},\n\t\t\tweights:       []float64{0.4, 0.3, 0.3}, // FTS, KNN query 0, KNN query 1\n\t\t\trank_constant: 1,\n\t\t\twindow_size:   4,\n\t\t\tnumKNNQueries: 2,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"a\", Score: 0.35},  // FTS rank 1, no KNN0, KNN1 rank 1: 0.4/2 + 0 + 0.3/2 = 0.2 + 0 + 0.15 = 0.35\n\t\t\t\t\t{ID: \"b\", Score: 0.283}, // FTS rank 2, KNN0 rank 1, no KNN1: 0.4/3 + 0.3/2 + 0 = 0.133 + 0.15 + 0 = 0.283\n\t\t\t\t\t{ID: \"d\", Score: 0.1},   // FTS rank 3, no KNN0, no KNN1: 0.4/4 + 0 + 0 = 0.1\n\t\t\t\t\t{ID: \"c\", Score: 0.1},   // no FTS rank, KNN0 rank 2, no KNN1: 0 + 0.3/3 + 0 = 0 + 0.1 + 0 = 0.1\n\t\t\t\t},\n\t\t\t\tTotal:    4,\n\t\t\t\tMaxScore: 0.35,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfor i, hit := range tt.hits {\n\t\t\t\thit.HitNumber = uint64(i)\n\t\t\t}\n\n\t\t\tif got := ReciprocalRankFusion(tt.hits, tt.weights, tt.rank_constant, tt.window_size, tt.numKNNQueries, false); !compareFusionResults(*got, tt.want) {\n\t\t\t\tt.Errorf(\"ReciprocalRankFusion() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "fusion/rsf.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fusion\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\n// formatRSFMessage builds the explanation string associated with a single\n// component of the Relative Score Fusion calculation.\nfunc formatRSFMessage(weight float64, normalizedScore float64, minScore float64, maxScore float64) string {\n\treturn fmt.Sprintf(\"rsf score (weight=%.3f, normalized=%.6f, min=%.6f, max=%.6f), normalized score of\",\n\t\tweight, normalizedScore, minScore, maxScore)\n}\n\n// RelativeScoreFusion normalizes the best-scoring documents from the primary\n// FTS query and each KNN query, scales those normalized values by the supplied\n// weights, and combines them into a single fused score. Only the top\n// `windowSize` documents per source are considered, and explanations are\n// materialized lazily when requested.\nfunc RelativeScoreFusion(hits search.DocumentMatchCollection, weights []float64, windowSize int, numKNNQueries int, explain bool) *FusionResult {\n\tnHits := len(hits)\n\tif nHits == 0 || windowSize == 0 {\n\t\treturn &FusionResult{\n\t\t\tHits:     search.DocumentMatchCollection{},\n\t\t\tTotal:    0,\n\t\t\tMaxScore: 0.0,\n\t\t}\n\t}\n\n\t// init explanations if required\n\tvar fusionExpl map[*search.DocumentMatch][]*search.Explanation\n\tif explain {\n\t\tfusionExpl = make(map[*search.DocumentMatch][]*search.Explanation, nHits)\n\t}\n\n\t// Code here for calculating fts results\n\t// Sort by fts scores\n\tsortDocMatchesByScore(hits)\n\n\t// ftsLimit holds the total number of fts hits to consider for rsf\n\tftsLimit := 0\n\tfor _, hit := range hits {\n\t\tif hit.Score == 0.0 {\n\t\t\tbreak\n\t\t}\n\t\tftsLimit++\n\t}\n\tftsLimit = min(ftsLimit, windowSize)\n\n\t// calculate fts scores\n\tif ftsLimit > 0 {\n\t\tmax := hits[0].Score\n\t\tmin := hits[ftsLimit-1].Score\n\t\tdenom := max - min\n\t\tweight := weights[0]\n\n\t\tfor i := 0; i < ftsLimit; i++ {\n\t\t\thit := hits[i]\n\t\t\tnorm := 1.0\n\t\t\tif denom > 0 {\n\t\t\t\tnorm = (hit.Score - min) / denom\n\t\t\t}\n\t\t\tcontrib := weight * norm\n\t\t\tif explain {\n\t\t\t\texpl := getFusionExplAt(\n\t\t\t\t\thit,\n\t\t\t\t\t0,\n\t\t\t\t\tnorm,\n\t\t\t\t\tformatRSFMessage(weight, norm, min, max),\n\t\t\t\t)\n\t\t\t\tfusionExpl[hit] = append(fusionExpl[hit], expl)\n\t\t\t}\n\t\t\thit.Score = contrib\n\t\t}\n\t\tfor i := ftsLimit; i < nHits; i++ {\n\t\t\t// These FTS hits are not counted in the results, so set to 0\n\t\t\thits[i].Score = 0.0\n\t\t}\n\t}\n\n\t// Code from here is for calculating knn scores\n\tfor queryIdx := 0; queryIdx < numKNNQueries; queryIdx++ {\n\t\tsortDocMatchesByBreakdown(hits, queryIdx)\n\n\t\t// knnLimit holds the total number of knn hits retrieved for a specific knn query\n\t\tknnLimit := 0\n\t\tfor _, hit := range hits {\n\t\t\tif _, ok := scoreBreakdownForQuery(hit, queryIdx); !ok {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tknnLimit++\n\t\t}\n\t\tknnLimit = min(knnLimit, windowSize)\n\n\t\t// if limit is 0, skip calculating\n\t\tif knnLimit == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tmax, _ := scoreBreakdownForQuery(hits[0], queryIdx)\n\t\tmin, _ := scoreBreakdownForQuery(hits[knnLimit-1], queryIdx)\n\t\tdenom := max - min\n\t\tweight := weights[queryIdx+1]\n\n\t\tfor i := 0; i < knnLimit; i++ {\n\t\t\thit := hits[i]\n\t\t\tscore, _ := scoreBreakdownForQuery(hit, queryIdx)\n\t\t\tnorm := 1.0\n\t\t\tif denom > 0 {\n\t\t\t\tnorm = (score - min) / denom\n\t\t\t}\n\t\t\tcontrib := weight * norm\n\t\t\tif explain {\n\t\t\t\texpl := getFusionExplAt(\n\t\t\t\t\thit,\n\t\t\t\t\tqueryIdx+1,\n\t\t\t\t\tnorm,\n\t\t\t\t\tformatRSFMessage(weight, norm, min, max),\n\t\t\t\t)\n\t\t\t\tfusionExpl[hit] = append(fusionExpl[hit], expl)\n\t\t\t}\n\t\t\thit.Score += contrib\n\t\t}\n\t}\n\n\t// Finalize scores\n\tvar maxScore float64\n\tfor _, hit := range hits {\n\t\tif explain {\n\t\t\tfinalizeFusionExpl(hit, fusionExpl[hit])\n\t\t}\n\t\tif hit.Score > maxScore {\n\t\t\tmaxScore = hit.Score\n\t\t}\n\t\thit.ScoreBreakdown = nil\n\t}\n\n\tsortDocMatchesByScore(hits)\n\n\tif nHits > windowSize {\n\t\thits = hits[:windowSize]\n\t}\n\n\treturn &FusionResult{\n\t\tHits:     hits,\n\t\tTotal:    uint64(len(hits)),\n\t\tMaxScore: maxScore,\n\t}\n}\n"
  },
  {
    "path": "fusion/rsf_test.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fusion\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\nfunc TestRelativeScoreFusion(t *testing.T) {\n\ttests := []struct {\n\t\tname          string\n\t\thits          search.DocumentMatchCollection\n\t\tweights       []float64\n\t\twindowSize    int\n\t\tnumKNNQueries int\n\t\twant          FusionResult\n\t}{\n\t\t{\n\t\t\tname:          \"empty hits\",\n\t\t\thits:          search.DocumentMatchCollection{},\n\t\t\tweights:       []float64{0.5, 0.5},\n\t\t\twindowSize:    10,\n\t\t\tnumKNNQueries: 1,\n\t\t\twant: FusionResult{\n\t\t\t\tHits:     search.DocumentMatchCollection{},\n\t\t\t\tTotal:    0,\n\t\t\t\tMaxScore: 0.0,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"single knn query\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8}},\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}},\n\t\t\t\t{ID: \"c\", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.7}},\n\t\t\t},\n\t\t\tweights:       []float64{0.4, 0.6},\n\t\t\twindowSize:    3,\n\t\t\tnumKNNQueries: 1,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"b\", Score: 0.8}, // FTS: (0.8-0.7)/(0.9-0.7) * 0.4 + KNN: (0.9-0.7)/(0.9-0.7) * 0.6 = 0.2 + 0.6 = 0.8\n\t\t\t\t\t{ID: \"a\", Score: 0.7}, // FTS: (0.9-0.7)/(0.9-0.7) * 0.4 + KNN: (0.8-0.7)/(0.9-0.7) * 0.6 = 0.4 + 0.3 = 0.7\n\t\t\t\t\t{ID: \"c\", Score: 0.0}, // FTS: (0.7-0.7)/(0.9-0.7) * 0.4 + KNN: (0.7-0.7)/(0.9-0.7) * 0.6 = 0.0 + 0.0 = 0.0\n\t\t\t\t},\n\t\t\t\tTotal:    3,\n\t\t\t\tMaxScore: 0.8,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"multiple knn queries\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8, 1: 0.6}},\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9, 1: 0.5}},\n\t\t\t\t{ID: \"c\", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.7, 1: 0.7}},\n\t\t\t},\n\t\t\tweights:       []float64{0.3, 0.4, 0.3},\n\t\t\twindowSize:    3,\n\t\t\tnumKNNQueries: 2,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"a\", Score: 0.65}, // FTS: (0.9-0.7)/(0.9-0.7)*0.3 + KNN0: (0.8-0.7)/(0.9-0.7)*0.4 + KNN1: (0.6-0.5)/(0.7-0.5)*0.3 = 1.0*0.3 + 0.5*0.4 + 0.5*0.3 = 0.65\n\t\t\t\t\t{ID: \"b\", Score: 0.55}, // FTS: (0.8-0.7)/(0.9-0.7)*0.3 + KNN0: (0.9-0.7)/(0.9-0.7)*0.4 + KNN1: (0.5-0.5)/(0.7-0.5)*0.3 = 0.5*0.3 + 1.0*0.4 + 0.0*0.3 = 0.55\n\t\t\t\t\t{ID: \"c\", Score: 0.3},  // FTS: (0.7-0.7)/(0.9-0.7)*0.3 + KNN0: (0.7-0.7)/(0.9-0.7)*0.4 + KNN1: (0.7-0.5)/(0.7-0.5)*0.3 = 0.0*0.3 + 0.0*0.4 + 1.0*0.3 = 0.3\n\t\t\t\t},\n\t\t\t\tTotal:    3,\n\t\t\t\tMaxScore: 0.65,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"all scores identical should normalize to 1.0\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}},\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}},\n\t\t\t\t{ID: \"c\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}},\n\t\t\t},\n\t\t\tweights:       []float64{0.4, 0.6},\n\t\t\twindowSize:    3,\n\t\t\tnumKNNQueries: 1,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"a\", Score: 1.0}, // All scores identical: 1.0 * 0.4 + 1.0 * 0.6 = 1.0\n\t\t\t\t\t{ID: \"b\", Score: 1.0},\n\t\t\t\t\t{ID: \"c\", Score: 1.0},\n\t\t\t\t},\n\t\t\t\tTotal:    3,\n\t\t\t\tMaxScore: 1.0,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"window size smaller than hits\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.7}},\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}},\n\t\t\t\t{ID: \"c\", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.8}},\n\t\t\t},\n\t\t\tweights:       []float64{0.4, 0.6},\n\t\t\twindowSize:    2,\n\t\t\tnumKNNQueries: 1,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"b\", Score: 0.6}, // Using top 2 for min/max: FTS min/max from [0.9,0.8] = [0.8,0.9], KNN min/max from [0.9,0.7] = [0.7,0.9]\n\t\t\t\t\t{ID: \"a\", Score: 0.4}, // FTS: (0.9-0.8)/(0.9-0.8) * 0.4 + KNN: (0.7-0.7)/(0.9-0.7) * 0.6 = 0.4 + 0 = 0.4\n\t\t\t\t},\n\t\t\t\tTotal:    2,\n\t\t\t\tMaxScore: 0.6,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"documents with partial scores missing KNN scores\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8}},         // has FTS and KNN query 0, missing KNN query 1\n\t\t\t\t{ID: \"b\", Score: 0.8, ScoreBreakdown: map[int]float64{1: 0.7}},         // has FTS and KNN query 1, missing KNN query 0\n\t\t\t\t{ID: \"c\", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.6, 1: 0.9}}, // has all scores\n\t\t\t\t{ID: \"d\", Score: 0.6, ScoreBreakdown: map[int]float64{}},               // has only FTS, missing all KNN scores\n\t\t\t},\n\t\t\tweights:       []float64{0.3, 0.4, 0.3}, // FTS, KNN query 0, KNN query 1\n\t\t\twindowSize:    4,\n\t\t\tnumKNNQueries: 2,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"a\", Score: 0.7}, // FTS: (0.9-0.6)/(0.9-0.6)*0.3 + KNN0: (0.8-0.6)/(0.8-0.6)*0.4 + KNN1: 0 = 1.0*0.3 + 1.0*0.4 + 0 = 0.7\n\t\t\t\t\t{ID: \"c\", Score: 0.4}, // FTS: (0.7-0.6)/(0.9-0.6)*0.3 + KNN0: (0.6-0.6)/(0.8-0.6)*0.4 + KNN1: (0.9-0.7)/(0.9-0.7)*0.3 = 0.33*0.3 + 0.0*0.4 + 1.0*0.3 = 0.1 + 0 + 0.3 = 0.4\n\t\t\t\t\t{ID: \"b\", Score: 0.2}, // FTS: (0.8-0.6)/(0.9-0.6)*0.3 + KNN0: 0 + KNN1: (0.7-0.7)/(0.9-0.7)*0.3 = 0.67*0.3 + 0 + 0.0*0.3 = 0.2 + 0 + 0 = 0.2\n\t\t\t\t\t{ID: \"d\", Score: 0.0}, // FTS: (0.6-0.6)/(0.9-0.6)*0.3 + KNN0: 0 + KNN1: 0 = 0.0*0.3 + 0 + 0 = 0\n\t\t\t\t},\n\t\t\t\tTotal:    4,\n\t\t\t\tMaxScore: 0.7,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"documents with only KNN scores\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.9}},         // no FTS rank (Score 0.0), only KNN query 0\n\t\t\t\t{ID: \"b\", Score: 0.0, ScoreBreakdown: map[int]float64{1: 0.8}},         // no FTS rank (Score 0.0), only KNN query 1\n\t\t\t\t{ID: \"c\", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.7, 1: 0.6}}, // no FTS rank (Score 0.0), both KNN queries\n\t\t\t},\n\t\t\tweights:       []float64{0.5, 0.3, 0.2}, // FTS, KNN query 0, KNN query 1\n\t\t\twindowSize:    3,\n\t\t\tnumKNNQueries: 2,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"a\", Score: 0.3}, // FTS: 0 + KNN0: 1.0 * 0.3 + KNN1: 0 = 0.3\n\t\t\t\t\t{ID: \"b\", Score: 0.2}, // FTS: 0 + KNN0: 0 + KNN1: 1.0 * 0.2 = 0.2\n\t\t\t\t\t{ID: \"c\", Score: 0.0}, // FTS: 0 + KNN0: 0 * 0.3 + KNN1: 0 * 0.2 = 0\n\t\t\t\t},\n\t\t\t\tTotal:    3,\n\t\t\t\tMaxScore: 0.3,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"mixed scenario with different score ranges\",\n\t\t\thits: search.DocumentMatchCollection{\n\t\t\t\t{ID: \"a\", Score: 1.0, ScoreBreakdown: map[int]float64{0: 0.1}}, // high FTS, low KNN\n\t\t\t\t{ID: \"b\", Score: 0.1, ScoreBreakdown: map[int]float64{0: 1.0}}, // low FTS, high KNN\n\t\t\t\t{ID: \"c\", Score: 0.5, ScoreBreakdown: map[int]float64{0: 0.5}}, // mid FTS, mid KNN\n\t\t\t},\n\t\t\tweights:       []float64{0.5, 0.5}, // Equal weights\n\t\t\twindowSize:    3,\n\t\t\tnumKNNQueries: 1,\n\t\t\twant: FusionResult{\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t{ID: \"a\", Score: 0.5},   // FTS: (1.0-0.1)/(1.0-0.1)*0.5 + KNN: (0.1-0.1)/(1.0-0.1)*0.5 = 1.0*0.5 + 0.0*0.5 = 0.5\n\t\t\t\t\t{ID: \"b\", Score: 0.5},   // FTS: (0.1-0.1)/(1.0-0.1)*0.5 + KNN: (1.0-0.1)/(1.0-0.1)*0.5 = 0.0*0.5 + 1.0*0.5 = 0.5\n\t\t\t\t\t{ID: \"c\", Score: 0.444}, // FTS: (0.5-0.1)/(1.0-0.1)*0.5 + KNN: (0.5-0.1)/(1.0-0.1)*0.5 = 0.444*0.5 + 0.444*0.5 = 0.444\n\t\t\t\t},\n\t\t\t\tTotal:    3,\n\t\t\t\tMaxScore: 0.5,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tfor i, hit := range tt.hits {\n\t\t\t\thit.HitNumber = uint64(i)\n\t\t\t}\n\n\t\t\tif got := RelativeScoreFusion(tt.hits, tt.weights, tt.windowSize, tt.numKNNQueries, false); !compareFusionResults(*got, tt.want) {\n\t\t\t\tt.Errorf(\"RelativeScoreFusion() = %v, want %v\", got, tt.want)\n\t\t\t\t// Print detailed comparison for debugging\n\t\t\t\tt.Logf(\"Got hits:\")\n\t\t\t\tfor i, hit := range got.Hits {\n\t\t\t\t\tt.Logf(\"  [%d] ID: %s, Score: %.6f\", i, hit.ID, hit.Score)\n\t\t\t\t}\n\t\t\t\tt.Logf(\"Want hits:\")\n\t\t\t\tfor i, hit := range tt.want.Hits {\n\t\t\t\t\tt.Logf(\"  [%d] ID: %s, Score: %.6f\", i, hit.ID, hit.Score)\n\t\t\t\t}\n\t\t\t\tt.Logf(\"Got Total: %d, MaxScore: %.6f\", got.Total, got.MaxScore)\n\t\t\t\tt.Logf(\"Want Total: %d, MaxScore: %.6f\", tt.want.Total, tt.want.MaxScore)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "fusion/util.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fusion\n\nimport (\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\n// sortDocMatchesByScore orders the provided collection in-place by the primary\n// score in descending order, breaking ties with the original `HitNumber` to\n// ensure deterministic output.\nfunc sortDocMatchesByScore(hits search.DocumentMatchCollection) {\n\tif len(hits) < 2 {\n\t\treturn\n\t}\n\n\tsort.Slice(hits, func(a, b int) bool {\n\t\ti := hits[a]\n\t\tj := hits[b]\n\t\tif i.Score == j.Score {\n\t\t\treturn i.HitNumber < j.HitNumber\n\t\t}\n\t\treturn i.Score > j.Score\n\t})\n}\n\n// scoreBreakdownForQuery fetches the score for a specific KNN query index from\n// the provided hit. The boolean return indicates whether the score is present.\nfunc scoreBreakdownForQuery(hit *search.DocumentMatch, idx int) (float64, bool) {\n\tif hit == nil || hit.ScoreBreakdown == nil {\n\t\treturn 0, false\n\t}\n\n\tscore, ok := hit.ScoreBreakdown[idx]\n\treturn score, ok\n}\n\n// sortDocMatchesByBreakdown orders the hits in-place using the KNN score for\n// the supplied query index (descending), breaking ties with `HitNumber` and\n// placing hits without a score at the end.\nfunc sortDocMatchesByBreakdown(hits search.DocumentMatchCollection, queryIdx int) {\n\tif len(hits) < 2 {\n\t\treturn\n\t}\n\n\tsort.SliceStable(hits, func(a, b int) bool {\n\t\tleft := hits[a]\n\t\tright := hits[b]\n\n\t\tvar leftScore float64\n\t\tleftOK := false\n\t\tif left != nil && left.ScoreBreakdown != nil {\n\t\t\tleftScore, leftOK = left.ScoreBreakdown[queryIdx]\n\t\t}\n\n\t\tvar rightScore float64\n\t\trightOK := false\n\t\tif right != nil && right.ScoreBreakdown != nil {\n\t\t\trightScore, rightOK = right.ScoreBreakdown[queryIdx]\n\t\t}\n\n\t\tif leftOK && rightOK {\n\t\t\tif leftScore == rightScore {\n\t\t\t\treturn left.HitNumber < right.HitNumber\n\t\t\t}\n\t\t\treturn leftScore > rightScore\n\t\t}\n\n\t\tif leftOK != rightOK {\n\t\t\treturn leftOK\n\t\t}\n\n\t\treturn left.HitNumber < right.HitNumber\n\t})\n}\n\n// getFusionExplAt copies the existing explanation child at the requested index\n// and wraps it in a new node describing how the fusion algorithm adjusted the\n// score.\nfunc getFusionExplAt(hit *search.DocumentMatch, i int, value float64, message string) *search.Explanation {\n\treturn &search.Explanation{\n\t\tValue:    value,\n\t\tMessage:  message,\n\t\tChildren: []*search.Explanation{hit.Expl.Children[i]},\n\t}\n}\n\n// finalizeFusionExpl installs the collection of fusion explanation children and\n// updates the root message so the caller sees the fused score as the sum of its\n// parts.\nfunc finalizeFusionExpl(hit *search.DocumentMatch, explChildren []*search.Explanation) {\n\thit.Expl.Children = explChildren\n\n\thit.Expl.Value = hit.Score\n\thit.Expl.Message = \"sum of\"\n}\n"
  },
  {
    "path": "geo/README.md",
    "content": "# Geo spatial search support in bleve\n\nLatest bleve spatial capabilities are powered by spatial hierarchical tokens generated from s2geometry.\nYou can find more details about the [s2geometry basics here](http://s2geometry.io/), and explore the extended functionality of our forked golang port of [s2geometry lib here](https://github.com/blevesearch/geo).\n\nUsers can continue to index and query `geopoint` field type and the existing queries like,\n\n- Point Distance\n- Bounded Rectangle\n- Bounded Polygon\n\nas before.\n\n## New Spatial Field Type - geoshape\n\nWe have introduced a field type (`geoshape`) for representing the new spatial types.\n\nUsing the new `geoshape` field type, users can unblock the spatial capabilities  \nfor the [geojson](https://datatracker.ietf.org/doc/html/rfc7946) shapes like,\n\n- Point\n- LineString\n- Polygon\n- MultiPoint\n- MultiLineString\n- MultiPolygon\n- GeometryCollection\n\nIn addition to these shapes, bleve will also support additional shapes like,\n\n- Circle\n- Envelope (Bounded box)\n\nTo specify GeoJSON data, use a nested field with:\n\n- a field named type that specifies the GeoJSON object type and the type value will be case-insensitive.\n- a field named coordinates that specifies the object's coordinates.\n\n```text\n         \"fieldName\": { \n              \"type\": \"GeoJSON Type\", \n              \"coordinates\": <coordinates> \n           }\n```\n\n- If specifying latitude and longitude coordinates, list the longitude first and then latitude.\n- Valid longitude values are between -180 and 180, both inclusive.\n- Valid latitude values are between -90 and 90, both inclusive.\n- Shapes would be internally represented as geodesics.\n- The GeoJSON specification strongly suggests splitting geometries so that neither of their parts crosses the antimeridian.\n\nExamples for the various geojson shapes representations are as below.\n\n## Point\n\nThe following specifies a [Point](https://tools.ietf.org/html/rfc7946#section-3.1.2) field in a document:\n\n```json\n{\n  \"type\": \"point\",\n  \"coordinates\": [75.05687713623047, 22.53539059204079]\n}\n```\n\n## Linestring\n\nThe following specifies a [Linestring](https://tools.ietf.org/html/rfc7946#section-3.1.4) field in a document:\n\n```json\n{\n  \"type\": \"linestring\",\n  \"coordinates\": [\n    [77.01416015625, 23.0797317624497],\n    [78.134765625, 20.385825381874263]\n  ]\n}\n```\n\n## Polygon\n\nThe following specifies a [Polygon](https://tools.ietf.org/html/rfc7946#section-3.1.6) field in a document:\n\n```json\n{\n  \"type\": \"polygon\",\n  \"coordinates\": [\n    [\n      [85.605, 57.207],\n      [86.396, 55.998],\n      [87.033, 56.716],\n      [85.605, 57.207]\n    ]\n  ]\n}\n```\n\nThe first and last coordinates must match in order to close the polygon.\nAnd the exterior coordinates have to be in Counter Clockwise Order in a polygon. (CCW)\n\n## MultiPoint\n\nThe following specifies a [Multipoint](https://tools.ietf.org/html/rfc7946#section-3.1.3) field in a document:\n\n```json\n{\n  \"type\": \"multipoint\",\n  \"coordinates\": [\n    [-115.8343505859375, 38.45789034424927],\n    [-115.81237792968749, 38.19502155795575],\n    [-120.80017089843749, 36.54053616262899],\n    [-120.67932128906249, 36.33725319397006]\n  ]\n}\n```\n\n## MultiLineString\n\nThe following specifies a [MultiLineString](https://tools.ietf.org/html/rfc7946#section-3.1.5) field in a document:\n\n```json\n{\n  \"type\": \"multilinestring\",\n  \"coordinates\": [\n    [\n      [-118.31726074, 35.250105158],\n      [-117.509765624, 35.3756141]\n    ],\n    [\n      [-118.696289, 34.624167789],\n      [-118.317260742, 35.03899204]\n    ],\n    [\n      [-117.9492187, 35.146862906],\n      [-117.6745605, 34.41144164]\n    ]\n  ]\n}\n```\n\n## MultiPolygon\n\nThe following specifies a [MultiPolygon](https://tools.ietf.org/html/rfc7946#section-3.1.7) field in a document:\n\n```json\n{\n  \"type\": \"multipolygon\",\n  \"coordinates\": [\n    [\n      [\n        [-73.958, 40.8003],\n        [-73.9498, 40.7968],\n        [-73.9737, 40.7648],\n        [-73.9814, 40.7681],\n        [-73.958, 40.8003]\n      ]\n    ],\n    [\n      [\n        [-73.958, 40.8003],\n        [-73.9498, 40.7968],\n        [-73.9737, 40.7648],\n        [-73.958, 40.8003]\n      ]\n    ]\n  ]\n}\n```\n\n## GeometryCollection\n\nThe following specifies a [GeometryCollection](https://tools.ietf.org/html/rfc7946#section-3.1.8) field in a document:\n\n```json\n{\n  \"type\": \"geometrycollection\",\n  \"geometries\": [\n    {\n      \"type\": \"multipoint\",\n      \"coordinates\": [\n        [-73.958, 40.8003],\n        [-73.9498, 40.7968],\n        [-73.9737, 40.7648],\n        [-73.9814, 40.7681]\n      ]\n    },\n    {\n      \"type\": \"multilinestring\",\n      \"coordinates\": [\n        [\n          [-73.96943, 40.78519],\n          [-73.96082, 40.78095]\n        ],\n        [\n          [-73.96415, 40.79229],\n          [-73.95544, 40.78854]\n        ],\n        [\n          [-73.97162, 40.78205],\n          [-73.96374, 40.77715]\n        ],\n        [\n          [-73.9788, 40.77247],\n          [-73.97036, 40.76811]\n        ]\n      ]\n    },\n    {\n      \"type\": \"polygon\",\n      \"coordinates\": [\n        [\n          [0, 0],\n          [3, 6],\n          [6, 1],\n          [0, 0]\n        ],\n        [\n          [2, 2],\n          [3, 3],\n          [4, 2],\n          [2, 2]\n        ]\n      ]\n    }\n  ]\n}\n```\n\n## Circle\n\nIf the user wishes to cover a circular region over the earth's surface, then they could use this shape.\nA  sample circular shape is as below.\n\n```json\n{\n  \"type\": \"circle\",\n  \"coordinates\": [75.05687713623047, 22.53539059204079],\n  \"radius\": \"1000m\"\n}\n```\n\nCircle is specified over the center point coordinates along with the radius.\nExample formats supported for radius are:\n\"5in\" , \"5inch\" , \"7yd\" , \"7yards\",  \"9ft\" , \"9feet\", \"11km\", \"11kilometers\", \"3nm\", \"3nauticalmiles\", \"13mm\" , \"13millimeters\",  \"15cm\", \"15centimeters\", \"17mi\", \"17miles\", \"19m\" or \"19meters\".\n\nIf the unit cannot be determined, the entire string is parsed and the unit of meters is assumed.\n\n## Envelope\n\nEnvelope type, which consists of coordinates for upper left and lower right points of the shape to represent a bounding rectangle in the format  [[minLon, maxLat], [maxLon, minLat]].\n\n```json\n{\n  \"type\": \"envelope\",\n  \"coordinates\": [\n    [72.83, 18.979],\n    [78.508, 17.4555]\n  ]\n}\n```\n\n## GeoShape Query\n\nGeoshape query support three types/filters of spatial querying capability across those heterogeneous types of documents indexed.\n\n### Query Structure\n\n```json\n{\n  \"query\": {\n    \"geometry\": {\n      \"shape\": {\n        \"type\": \"<shapeType>\",\n        \"coordinates\": [\n          [[]]\n        ]\n      },\n      \"relation\": \"<<filterName>>\"\n    }\n  }\n}\n```\n\n*shapeType* => can be any of the aforementioned types like Point, LineString, Polygon, MultiPoint,\nGeometrycollection, MultiLineString, MultiPolygon, Circle and Envelope.\n\n*filterName* => can be any of the 3 types like *intersects*, *contains* and *within*.\n\n### Relation\n\n|   FilterName   |  Description                                                             |\n|   :-----------:|  :-----------------------------------------------------------------:     |\n|   `intersects` |  Return all documents whose shape field intersects the query  geometry.  |\n|   `contains`   |  Return all documents whose shape field contains the query geometry      |\n|   `within`     |  Return all documents whose shape field is within the query geometry.    |\n\n------------------------------------------------------------------------------------------------------------------------\n\n### Older Implementation\n\nFirst, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html).\n\n## Notes\n\n- All of the APIs will use float64 for lon/lat values.\n- When describing a point in function arguments or return values, we always use the order lon, lat.\n- High level APIs will use TopLeft and BottomRight to describe bounding boxes. This may not map cleanly to min/max lon/lat when crossing the dateline. The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly.\n- Points and MultiPoints may only contain Points and MultiPoints.\n- LineStrings and MultiLineStrings may only contain Points and MultiPoints.\n- Polygons or MultiPolygons intersecting Polygons and MultiPolygons may return arbitrary results when the overlap is only an edge or a vertex.\n- Circles containing polygon will return a false positive result if all of the vertices of the polygon are within the circle, but the orientation of those points are clock-wise.\n- The edges of an Envelope follows the latitude and longitude lines instead of the shortest path on a globe.\n- Envelope intersecting queries with LineStrings, MultiLineStrings, Polygons and MultiPolygons implicitly converts the Envelope into a Polygon which changes the curvature of the edges causing inaccurate results for few edge cases.\n"
  },
  {
    "path": "geo/benchmark_geohash_test.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"testing\"\n)\n\nfunc BenchmarkGeoHashLen5NewDecode(b *testing.B) {\n\tb.ResetTimer()\n\thash := \"d3hn3\"\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _ = DecodeGeoHash(hash)\n\t}\n}\n\nfunc BenchmarkGeoHashLen6NewDecode(b *testing.B) {\n\tb.ResetTimer()\n\thash := \"u4pruy\"\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _ = DecodeGeoHash(hash)\n\t}\n}\n\nfunc BenchmarkGeoHashLen7NewDecode(b *testing.B) {\n\tb.ResetTimer()\n\thash := \"u4pruyd\"\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _ = DecodeGeoHash(hash)\n\t}\n}\n"
  },
  {
    "path": "geo/geo.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n)\n\n// GeoBits is the number of bits used for a single geo point\n// Currently this is 32bits for lon and 32bits for lat\nvar GeoBits uint = 32\n\nvar minLon = -180.0\nvar minLat = -90.0\nvar maxLon = 180.0\nvar maxLat = 90.0\nvar minLonRad = minLon * degreesToRadian\nvar minLatRad = minLat * degreesToRadian\nvar maxLonRad = maxLon * degreesToRadian\nvar maxLatRad = maxLat * degreesToRadian\nvar geoTolerance = 1e-6\nvar lonScale = float64((uint64(0x1)<<GeoBits)-1) / 360.0\nvar latScale = float64((uint64(0x1)<<GeoBits)-1) / 180.0\n\nvar geoHashMaxLength = 12\n\n// Point represents a geo point.\ntype Point struct {\n\tLon float64 `json:\"lon\"`\n\tLat float64 `json:\"lat\"`\n}\n\n// MortonHash computes the morton hash value for the provided geo point\n// This point is ordered as lon, lat.\nfunc MortonHash(lon, lat float64) uint64 {\n\treturn numeric.Interleave(scaleLon(lon), scaleLat(lat))\n}\n\nfunc scaleLon(lon float64) uint64 {\n\trv := uint64((lon - minLon) * lonScale)\n\treturn rv\n}\n\nfunc scaleLat(lat float64) uint64 {\n\trv := uint64((lat - minLat) * latScale)\n\treturn rv\n}\n\n// MortonUnhashLon extracts the longitude value from the provided morton hash.\nfunc MortonUnhashLon(hash uint64) float64 {\n\treturn unscaleLon(numeric.Deinterleave(hash))\n}\n\n// MortonUnhashLat extracts the latitude value from the provided morton hash.\nfunc MortonUnhashLat(hash uint64) float64 {\n\treturn unscaleLat(numeric.Deinterleave(hash >> 1))\n}\n\nfunc unscaleLon(lon uint64) float64 {\n\treturn (float64(lon) / lonScale) + minLon\n}\n\nfunc unscaleLat(lat uint64) float64 {\n\treturn (float64(lat) / latScale) + minLat\n}\n\n// compareGeo will compare two float values and see if they are the same\n// taking into consideration a known geo tolerance.\nfunc compareGeo(a, b float64) float64 {\n\tcompare := a - b\n\tif math.Abs(compare) <= geoTolerance {\n\t\treturn 0\n\t}\n\treturn compare\n}\n\n// RectIntersects checks whether rectangles a and b intersect\nfunc RectIntersects(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool {\n\treturn !(aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY)\n}\n\n// RectWithin checks whether box a is within box b\nfunc RectWithin(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool {\n\trv := !(aMinX < bMinX || aMinY < bMinY || aMaxX > bMaxX || aMaxY > bMaxY)\n\treturn rv\n}\n\n// BoundingBoxContains checks whether the lon/lat point is within the box\nfunc BoundingBoxContains(lon, lat, minLon, minLat, maxLon, maxLat float64) bool {\n\treturn compareGeo(lon, minLon) >= 0 && compareGeo(lon, maxLon) <= 0 &&\n\t\tcompareGeo(lat, minLat) >= 0 && compareGeo(lat, maxLat) <= 0\n}\n\nconst degreesToRadian = math.Pi / 180\nconst radiansToDegrees = 180 / math.Pi\n\n// DegreesToRadians converts an angle in degrees to radians\nfunc DegreesToRadians(d float64) float64 {\n\treturn d * degreesToRadian\n}\n\n// RadiansToDegrees converts an angle in radians to degrees\nfunc RadiansToDegrees(r float64) float64 {\n\treturn r * radiansToDegrees\n}\n\nvar earthMeanRadiusMeters = 6371008.7714\n\nfunc RectFromPointDistance(lon, lat, dist float64) (float64, float64, float64, float64, error) {\n\terr := checkLongitude(lon)\n\tif err != nil {\n\t\treturn 0, 0, 0, 0, err\n\t}\n\terr = checkLatitude(lat)\n\tif err != nil {\n\t\treturn 0, 0, 0, 0, err\n\t}\n\tradLon := DegreesToRadians(lon)\n\tradLat := DegreesToRadians(lat)\n\tradDistance := (dist + 7e-2) / earthMeanRadiusMeters\n\n\tminLatL := radLat - radDistance\n\tmaxLatL := radLat + radDistance\n\n\tvar minLonL, maxLonL float64\n\tif minLatL > minLatRad && maxLatL < maxLatRad {\n\t\tdeltaLon := math.Asin(math.Sin(radDistance) / math.Cos(radLat))\n\t\tminLonL = radLon - deltaLon\n\t\tif minLonL < minLonRad {\n\t\t\tminLonL += 2 * math.Pi\n\t\t}\n\t\tmaxLonL = radLon + deltaLon\n\t\tif maxLonL > maxLonRad {\n\t\t\tmaxLonL -= 2 * math.Pi\n\t\t}\n\t} else {\n\t\t// pole is inside distance\n\t\tminLatL = math.Max(minLatL, minLatRad)\n\t\tmaxLatL = math.Min(maxLatL, maxLatRad)\n\t\tminLonL = minLonRad\n\t\tmaxLonL = maxLonRad\n\t}\n\n\treturn RadiansToDegrees(minLonL),\n\t\tRadiansToDegrees(maxLatL),\n\t\tRadiansToDegrees(maxLonL),\n\t\tRadiansToDegrees(minLatL),\n\t\tnil\n}\n\nfunc checkLatitude(latitude float64) error {\n\tif math.IsNaN(latitude) || latitude < minLat || latitude > maxLat {\n\t\treturn fmt.Errorf(\"invalid latitude %f; must be between %f and %f\", latitude, minLat, maxLat)\n\t}\n\treturn nil\n}\n\nfunc checkLongitude(longitude float64) error {\n\tif math.IsNaN(longitude) || longitude < minLon || longitude > maxLon {\n\t\treturn fmt.Errorf(\"invalid longitude %f; must be between %f and %f\", longitude, minLon, maxLon)\n\t}\n\treturn nil\n}\n\nfunc BoundingRectangleForPolygon(polygon []Point) (\n\tfloat64, float64, float64, float64, error) {\n\terr := checkLongitude(polygon[0].Lon)\n\tif err != nil {\n\t\treturn 0, 0, 0, 0, err\n\t}\n\terr = checkLatitude(polygon[0].Lat)\n\tif err != nil {\n\t\treturn 0, 0, 0, 0, err\n\t}\n\tmaxY, minY := polygon[0].Lat, polygon[0].Lat\n\tmaxX, minX := polygon[0].Lon, polygon[0].Lon\n\tfor i := 1; i < len(polygon); i++ {\n\t\terr := checkLongitude(polygon[i].Lon)\n\t\tif err != nil {\n\t\t\treturn 0, 0, 0, 0, err\n\t\t}\n\t\terr = checkLatitude(polygon[i].Lat)\n\t\tif err != nil {\n\t\t\treturn 0, 0, 0, 0, err\n\t\t}\n\n\t\tmaxY = math.Max(maxY, polygon[i].Lat)\n\t\tminY = math.Min(minY, polygon[i].Lat)\n\n\t\tmaxX = math.Max(maxX, polygon[i].Lon)\n\t\tminX = math.Min(minX, polygon[i].Lon)\n\t}\n\n\treturn minX, maxY, maxX, minY, nil\n}\n"
  },
  {
    "path": "geo/geo_dist.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"strconv\"\n\t\"strings\"\n)\n\ntype distanceUnit struct {\n\tconv     float64\n\tsuffixes []string\n}\n\nvar inch = distanceUnit{0.0254, []string{\"in\", \"inch\"}}\nvar yard = distanceUnit{0.9144, []string{\"yd\", \"yards\"}}\nvar feet = distanceUnit{0.3048, []string{\"ft\", \"feet\"}}\nvar kilom = distanceUnit{1000, []string{\"km\", \"kilometers\"}}\nvar nauticalm = distanceUnit{1852.0, []string{\"nm\", \"nauticalmiles\"}}\nvar millim = distanceUnit{0.001, []string{\"mm\", \"millimeters\"}}\nvar centim = distanceUnit{0.01, []string{\"cm\", \"centimeters\"}}\nvar miles = distanceUnit{1609.344, []string{\"mi\", \"miles\"}}\nvar meters = distanceUnit{1, []string{\"m\", \"meters\"}}\n\nvar distanceUnits = []*distanceUnit{\n\t&inch, &yard, &feet, &kilom, &nauticalm, &millim, &centim, &miles, &meters,\n}\n\n// ParseDistance attempts to parse a distance string and return distance in\n// meters.  Example formats supported:\n// \"5in\" \"5inch\" \"7yd\" \"7yards\" \"9ft\" \"9feet\" \"11km\" \"11kilometers\"\n// \"3nm\" \"3nauticalmiles\" \"13mm\" \"13millimeters\" \"15cm\" \"15centimeters\"\n// \"17mi\" \"17miles\" \"19m\" \"19meters\"\n// If the unit cannot be determined, the entire string is parsed and the\n// unit of meters is assumed.\n// If the number portion cannot be parsed, 0 and the parse error are returned.\nfunc ParseDistance(d string) (float64, error) {\n\tfor _, unit := range distanceUnits {\n\t\tfor _, unitSuffix := range unit.suffixes {\n\t\t\tif strings.HasSuffix(d, unitSuffix) {\n\t\t\t\tparsedNum, err := strconv.ParseFloat(d[0:len(d)-len(unitSuffix)], 64)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn 0, err\n\t\t\t\t}\n\t\t\t\treturn parsedNum * unit.conv, nil\n\t\t\t}\n\t\t}\n\t}\n\t// no unit matched, try assuming meters?\n\tparsedNum, err := strconv.ParseFloat(d, 64)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\treturn parsedNum, nil\n}\n\n// ParseDistanceUnit attempts to parse a distance unit and return the\n// multiplier for converting this to meters.  If the unit cannot be parsed\n// then 0 and the error message is returned.\nfunc ParseDistanceUnit(u string) (float64, error) {\n\tfor _, unit := range distanceUnits {\n\t\tfor _, unitSuffix := range unit.suffixes {\n\t\t\tif u == unitSuffix {\n\t\t\t\treturn unit.conv, nil\n\t\t\t}\n\t\t}\n\t}\n\treturn 0, fmt.Errorf(\"unknown distance unit: %s\", u)\n}\n\n// Haversin computes the distance between two points.\n// This implementation uses the sloppy math implementations which trade off\n// accuracy for performance.  The distance returned is in kilometers.\nfunc Haversin(lon1, lat1, lon2, lat2 float64) float64 {\n\tx1 := lat1 * degreesToRadian\n\tx2 := lat2 * degreesToRadian\n\th1 := 1 - math.Cos(x1-x2)\n\th2 := 1 - math.Cos((lon1-lon2)*degreesToRadian)\n\th := (h1 + math.Cos(x1)*math.Cos(x2)*h2) / 2\n\tavgLat := (x1 + x2) / 2\n\tdiameter := earthDiameter(avgLat)\n\n\treturn diameter * math.Asin(math.Min(1, math.Sqrt(h)))\n}\n"
  },
  {
    "path": "geo/geo_dist_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestParseDistance(t *testing.T) {\n\ttests := []struct {\n\t\tdist    string\n\t\twant    float64\n\t\twantErr error\n\t}{\n\t\t{\"5mi\", 5 * 1609.344, nil},\n\t\t{\"3\", 3, nil},\n\t\t{\"3m\", 3, nil},\n\t\t{\"5km\", 5000, nil},\n\t\t{\"km\", 0, &strconv.NumError{Func: \"ParseFloat\", Num: \"\", Err: strconv.ErrSyntax}},\n\t\t{\"\", 0, &strconv.NumError{Func: \"ParseFloat\", Num: \"\", Err: strconv.ErrSyntax}},\n\t}\n\n\tfor _, test := range tests {\n\t\tgot, err := ParseDistance(test.dist)\n\t\tif !reflect.DeepEqual(err, test.wantErr) {\n\t\t\tt.Errorf(\"expected err: %v, got %v for %s\", test.wantErr, err, test.dist)\n\t\t}\n\t\tif got != test.want {\n\t\t\tt.Errorf(\"expected distance %f got %f for %s\", test.want, got, test.dist)\n\t\t}\n\t}\n}\n\nfunc TestParseDistanceUnit(t *testing.T) {\n\ttests := []struct {\n\t\tdist    string\n\t\twant    float64\n\t\twantErr error\n\t}{\n\t\t{\"mi\", 1609.344, nil},\n\t\t{\"m\", 1, nil},\n\t\t{\"km\", 1000, nil},\n\t\t{\"\", 0, fmt.Errorf(\"unknown distance unit: \")},\n\t\t{\"kam\", 0, fmt.Errorf(\"unknown distance unit: kam\")},\n\t}\n\n\tfor _, test := range tests {\n\t\tgot, err := ParseDistanceUnit(test.dist)\n\t\tif !reflect.DeepEqual(err, test.wantErr) {\n\t\t\tt.Errorf(\"expected err: %v, got %v for %s\", test.wantErr, err, test.dist)\n\t\t}\n\t\tif got != test.want {\n\t\t\tt.Errorf(\"expected distance %f got %f for %s\", test.want, got, test.dist)\n\t\t}\n\t}\n}\n\nfunc TestHaversinDistance(t *testing.T) {\n\tearthRadiusKMs := 6378.137\n\thalfCircle := earthRadiusKMs * math.Pi\n\n\ttests := []struct {\n\t\tlon1 float64\n\t\tlat1 float64\n\t\tlon2 float64\n\t\tlat2 float64\n\t\twant float64\n\t}{\n\t\t{1, 1, math.NaN(), 1, math.NaN()},\n\t\t{1, 1, 1, math.NaN(), math.NaN()},\n\t\t{1, math.NaN(), 1, 1, math.NaN()},\n\t\t{math.NaN(), 1, 1, 1, math.NaN()},\n\n\t\t{0, 0, 0, 0, 0},\n\t\t{-180, 0, -180, 0, 0},\n\t\t{-180, 0, 180, 0, 0},\n\t\t{180, 0, 180, 0, 0},\n\n\t\t{0, 90, 0, 90, 0},\n\t\t{-180, 90, -180, 90, 0},\n\t\t{-180, 90, 180, 90, 0},\n\t\t{180, 90, 180, 90, 0},\n\n\t\t{0, 0, 180, 0, halfCircle},\n\n\t\t{-74.0059731, 40.7143528, -74.0059731, 40.7143528, 0},\n\t\t{-74.0059731, 40.7143528, -73.9844722, 40.759011, 5.286},\n\t\t{-74.0059731, 40.7143528, -74.007819, 40.718266, 0.4621},\n\t\t{-74.0059731, 40.7143528, -74.0088305, 40.7051157, 1.055},\n\t\t{-74.0059731, 40.7143528, -74, 40.7247222, 1.258},\n\t\t{-74.0059731, 40.7143528, -73.9962255, 40.731033, 2.029},\n\t\t{-74.0059731, 40.7143528, -73.95, 40.65, 8.572},\n\t}\n\n\tfor _, test := range tests {\n\t\tgot := Haversin(test.lon1, test.lat1, test.lon2, test.lat2)\n\t\tif math.IsNaN(test.want) && !math.IsNaN(got) {\n\t\t\tt.Errorf(\"expected NaN, got %f\", got)\n\t\t}\n\t\tif !math.IsNaN(test.want) && math.Abs(got-test.want) > 1e-2 {\n\t\t\tt.Errorf(\"expected %f got %f\", test.want, got)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "geo/geo_s2plugin_impl.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"encoding/json\"\n\t\"sync\"\n\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"github.com/blevesearch/geo/geojson\"\n\t\"github.com/blevesearch/geo/s2\"\n)\n\nconst (\n\tPointType              = \"point\"\n\tMultiPointType         = \"multipoint\"\n\tLineStringType         = \"linestring\"\n\tMultiLineStringType    = \"multilinestring\"\n\tPolygonType            = \"polygon\"\n\tMultiPolygonType       = \"multipolygon\"\n\tGeometryCollectionType = \"geometrycollection\"\n\tCircleType             = \"circle\"\n\tEnvelopeType           = \"envelope\"\n)\n\n// spatialPluginsMap is spatial plugin cache.\nvar (\n\tspatialPluginsMap = make(map[string]index.SpatialAnalyzerPlugin)\n\tpluginsMapLock    = sync.RWMutex{}\n)\n\nfunc init() {\n\tregisterS2RegionTermIndexer()\n}\n\nfunc registerS2RegionTermIndexer() {\n\tspatialPlugin := S2SpatialAnalyzerPlugin{\n\t\ts2Indexer:                    s2.NewRegionTermIndexerWithOptions(initS2IndexerOptions()),\n\t\ts2Searcher:                   s2.NewRegionTermIndexerWithOptions(initS2SearcherOptions()),\n\t\ts2GeoPointsRegionTermIndexer: s2.NewRegionTermIndexerWithOptions(initS2OptionsForGeoPoints()),\n\t}\n\n\tRegisterSpatialAnalyzerPlugin(&spatialPlugin)\n}\n\n// RegisterSpatialAnalyzerPlugin registers the given plugin implementation.\nfunc RegisterSpatialAnalyzerPlugin(plugin index.SpatialAnalyzerPlugin) {\n\tpluginsMapLock.Lock()\n\tspatialPluginsMap[plugin.Type()] = plugin\n\tpluginsMapLock.Unlock()\n}\n\n// GetSpatialAnalyzerPlugin retrieves the given implementation type.\nfunc GetSpatialAnalyzerPlugin(typ string) index.SpatialAnalyzerPlugin {\n\tpluginsMapLock.RLock()\n\trv := spatialPluginsMap[typ]\n\tpluginsMapLock.RUnlock()\n\treturn rv\n}\n\n// initS2IndexerOptions returns the options for s2's region\n// term indexer for the index time tokens of geojson shapes.\nfunc initS2IndexerOptions() s2.Options {\n\toptions := s2.Options{}\n\t// maxLevel control the maximum size of the\n\t// S2Cells used to approximate regions.\n\toptions.SetMaxLevel(16)\n\n\t// minLevel control the minimum size of the\n\t// S2Cells used to approximate regions.\n\toptions.SetMinLevel(2)\n\n\t// levelMod value greater than 1 increases the effective branching\n\t// factor of the S2Cell hierarchy by skipping some levels.\n\toptions.SetLevelMod(1)\n\n\t// maxCells controls the maximum number of cells\n\t// when approximating each s2 region.\n\toptions.SetMaxCells(20)\n\n\treturn options\n}\n\n// initS2SearcherOptions returns the options for s2's region\n// term indexer for the query time tokens of geojson shapes.\nfunc initS2SearcherOptions() s2.Options {\n\toptions := s2.Options{}\n\t// maxLevel control the maximum size of the\n\t// S2Cells used to approximate regions.\n\toptions.SetMaxLevel(16)\n\n\t// minLevel control the minimum size of the\n\t// S2Cells used to approximate regions.\n\toptions.SetMinLevel(2)\n\n\t// levelMod value greater than 1 increases the effective branching\n\t// factor of the S2Cell hierarchy by skipping some levels.\n\toptions.SetLevelMod(1)\n\n\t// maxCells controls the maximum number of cells\n\t// when approximating each s2 region.\n\toptions.SetMaxCells(8)\n\n\treturn options\n}\n\n// initS2OptionsForGeoPoints returns the options for\n// s2's region term indexer for the original geopoints.\nfunc initS2OptionsForGeoPoints() s2.Options {\n\toptions := s2.Options{}\n\t// maxLevel control the maximum size of the\n\t// S2Cells used to approximate regions.\n\toptions.SetMaxLevel(16)\n\n\t// minLevel control the minimum size of the\n\t// S2Cells used to approximate regions.\n\toptions.SetMinLevel(4)\n\n\t// levelMod value greater than 1 increases the effective branching\n\t// factor of the S2Cell hierarchy by skipping some levels.\n\toptions.SetLevelMod(2)\n\n\t// maxCells controls the maximum number of cells\n\t// when approximating each s2 region.\n\toptions.SetMaxCells(8)\n\n\t// explicit for geo points.\n\toptions.SetPointsOnly(true)\n\n\treturn options\n}\n\n// S2SpatialAnalyzerPlugin is an implementation of\n// the index.SpatialAnalyzerPlugin interface.\ntype S2SpatialAnalyzerPlugin struct {\n\ts2Indexer                    *s2.RegionTermIndexer\n\ts2Searcher                   *s2.RegionTermIndexer\n\ts2GeoPointsRegionTermIndexer *s2.RegionTermIndexer\n}\n\nfunc (s *S2SpatialAnalyzerPlugin) Type() string {\n\treturn \"s2\"\n}\n\nfunc (s *S2SpatialAnalyzerPlugin) GetIndexTokens(queryShape index.GeoJSON) []string {\n\tvar rv []string\n\tshapes := []index.GeoJSON{queryShape}\n\tif gc, ok := queryShape.(*geojson.GeometryCollection); ok {\n\t\tshapes = gc.Shapes\n\t}\n\n\tfor _, shape := range shapes {\n\t\tif s2t, ok := shape.(s2Tokenizable); ok {\n\t\t\trv = append(rv, s2t.IndexTokens(s.s2Indexer)...)\n\t\t} else if s2t, ok := shape.(s2TokenizableEx); ok {\n\t\t\trv = append(rv, s2t.IndexTokens(s)...)\n\t\t}\n\t}\n\n\treturn geojson.DeduplicateTerms(rv)\n}\n\nfunc (s *S2SpatialAnalyzerPlugin) GetQueryTokens(queryShape index.GeoJSON) []string {\n\tvar rv []string\n\tshapes := []index.GeoJSON{queryShape}\n\tif gc, ok := queryShape.(*geojson.GeometryCollection); ok {\n\t\tshapes = gc.Shapes\n\t}\n\n\tfor _, shape := range shapes {\n\t\tif s2t, ok := shape.(s2Tokenizable); ok {\n\t\t\trv = append(rv, s2t.QueryTokens(s.s2Searcher)...)\n\t\t} else if s2t, ok := shape.(s2TokenizableEx); ok {\n\t\t\trv = append(rv, s2t.QueryTokens(s)...)\n\t\t}\n\t}\n\n\treturn geojson.DeduplicateTerms(rv)\n}\n\n// ------------------------------------------------------------------------\n// s2Tokenizable is an optional interface for shapes that support\n// the generation of s2 based tokens that can be used for both\n// indexing and querying.\n\ntype s2Tokenizable interface {\n\t// IndexTokens returns the tokens for indexing.\n\tIndexTokens(*s2.RegionTermIndexer) []string\n\n\t// QueryTokens returns the tokens for searching.\n\tQueryTokens(*s2.RegionTermIndexer) []string\n}\n\n// ------------------------------------------------------------------------\n// s2TokenizableEx is an optional interface for shapes that support\n// the generation of s2 based tokens that can be used for both\n// indexing and querying. This is intended for the older geopoint\n// indexing and querying.\ntype s2TokenizableEx interface {\n\t// IndexTokens returns the tokens for indexing.\n\tIndexTokens(*S2SpatialAnalyzerPlugin) []string\n\n\t// QueryTokens returns the tokens for searching.\n\tQueryTokens(*S2SpatialAnalyzerPlugin) []string\n}\n\n//----------------------------------------------------------------------------------\n\nfunc (p *Point) Type() string {\n\treturn PointType\n}\n\nfunc (p *Point) Value() ([]byte, error) {\n\treturn util.MarshalJSON(p)\n}\n\nfunc (p *Point) Intersects(s index.GeoJSON) (bool, error) {\n\t// placeholder implementation\n\treturn false, nil\n}\n\nfunc (p *Point) Contains(s index.GeoJSON) (bool, error) {\n\t// placeholder implementation\n\treturn false, nil\n}\n\nfunc (p *Point) IndexTokens(s *S2SpatialAnalyzerPlugin) []string {\n\treturn s.s2GeoPointsRegionTermIndexer.GetIndexTermsForPoint(s2.PointFromLatLng(\n\t\ts2.LatLngFromDegrees(p.Lat, p.Lon)), \"\")\n}\n\nfunc (p *Point) QueryTokens(s *S2SpatialAnalyzerPlugin) []string {\n\treturn nil\n}\n\n//----------------------------------------------------------------------------------\n\ntype boundedRectangle struct {\n\tminLat float64\n\tmaxLat float64\n\tminLon float64\n\tmaxLon float64\n}\n\nfunc NewBoundedRectangle(minLat, minLon, maxLat,\n\tmaxLon float64) *boundedRectangle {\n\treturn &boundedRectangle{minLat: minLat,\n\t\tmaxLat: maxLat, minLon: minLon, maxLon: maxLon}\n}\n\nfunc (br *boundedRectangle) Type() string {\n\t// placeholder implementation\n\treturn \"boundedRectangle\"\n}\n\nfunc (br *boundedRectangle) Value() ([]byte, error) {\n\treturn util.MarshalJSON(br)\n}\n\nfunc (p *boundedRectangle) Intersects(s index.GeoJSON) (bool, error) {\n\t// placeholder implementation\n\treturn false, nil\n}\n\nfunc (p *boundedRectangle) Contains(s index.GeoJSON) (bool, error) {\n\t// placeholder implementation\n\treturn false, nil\n}\n\nfunc (br *boundedRectangle) IndexTokens(s *S2SpatialAnalyzerPlugin) []string {\n\treturn nil\n}\n\nfunc (br *boundedRectangle) QueryTokens(s *S2SpatialAnalyzerPlugin) []string {\n\trect := s2.RectFromDegrees(br.minLat, br.minLon, br.maxLat, br.maxLon)\n\n\t// obtain the terms to be searched for the given bounding box.\n\tterms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion(rect, \"\")\n\n\treturn geojson.StripCoveringTerms(terms)\n}\n\n//----------------------------------------------------------------------------------\n\ntype boundedPolygon struct {\n\tcoordinates []Point\n}\n\nfunc NewBoundedPolygon(coordinates []Point) *boundedPolygon {\n\treturn &boundedPolygon{coordinates: coordinates}\n}\n\nfunc (bp *boundedPolygon) Type() string {\n\t// placeholder implementation\n\treturn \"boundedPolygon\"\n}\n\nfunc (bp *boundedPolygon) Value() ([]byte, error) {\n\treturn util.MarshalJSON(bp)\n}\n\nfunc (p *boundedPolygon) Intersects(s index.GeoJSON) (bool, error) {\n\t// placeholder implementation\n\treturn false, nil\n}\n\nfunc (p *boundedPolygon) Contains(s index.GeoJSON) (bool, error) {\n\t// placeholder implementation\n\treturn false, nil\n}\n\nfunc (bp *boundedPolygon) IndexTokens(s *S2SpatialAnalyzerPlugin) []string {\n\treturn nil\n}\n\nfunc (bp *boundedPolygon) QueryTokens(s *S2SpatialAnalyzerPlugin) []string {\n\tvertices := make([]s2.Point, len(bp.coordinates))\n\tfor i, point := range bp.coordinates {\n\t\tvertices[i] = s2.PointFromLatLng(\n\t\t\ts2.LatLngFromDegrees(point.Lat, point.Lon))\n\t}\n\ts2polygon := s2.PolygonFromOrientedLoops([]*s2.Loop{s2.LoopFromPoints(vertices)})\n\n\t// obtain the terms to be searched for the given polygon.\n\tterms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion(\n\t\ts2polygon.CapBound(), \"\")\n\n\treturn geojson.StripCoveringTerms(terms)\n}\n\n//----------------------------------------------------------------------------------\n\ntype pointDistance struct {\n\tdist      float64\n\tcenterLat float64\n\tcenterLon float64\n}\n\nfunc (p *pointDistance) Type() string {\n\t// placeholder implementation\n\treturn \"pointDistance\"\n}\n\nfunc (p *pointDistance) Value() ([]byte, error) {\n\treturn util.MarshalJSON(p)\n}\n\nfunc NewPointDistance(centerLat, centerLon,\n\tdist float64) *pointDistance {\n\treturn &pointDistance{centerLat: centerLat,\n\t\tcenterLon: centerLon, dist: dist}\n}\n\nfunc (p *pointDistance) Intersects(s index.GeoJSON) (bool, error) {\n\t// placeholder implementation\n\treturn false, nil\n}\n\nfunc (p *pointDistance) Contains(s index.GeoJSON) (bool, error) {\n\t// placeholder implementation\n\treturn false, nil\n}\n\nfunc (pd *pointDistance) IndexTokens(s *S2SpatialAnalyzerPlugin) []string {\n\treturn nil\n}\n\nfunc (pd *pointDistance) QueryTokens(s *S2SpatialAnalyzerPlugin) []string {\n\t// obtain the covering query region from the given points.\n\tqueryRegion := s2.CapFromCenterAndRadius(pd.centerLat,\n\t\tpd.centerLon, pd.dist)\n\n\t// obtain the query terms for the query region.\n\tterms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion(queryRegion, \"\")\n\n\treturn geojson.StripCoveringTerms(terms)\n}\n\n// ------------------------------------------------------------------------\n\n// NewGeometryCollection instantiate a geometrycollection\n// and prefix the byte contents with certain glue bytes that\n// can be used later while filering the doc values.\nfunc NewGeometryCollection(coordinates [][][][][]float64,\n\ttyps []string) (index.GeoJSON, []byte, error) {\n\tshapes := make([]*geojson.GeoShape, len(coordinates))\n\tfor i := range coordinates {\n\t\tshapes[i] = &geojson.GeoShape{\n\t\t\tCoordinates: coordinates[i],\n\t\t\tType:        typs[i],\n\t\t}\n\t}\n\n\treturn geojson.NewGeometryCollection(shapes)\n}\n\nfunc NewGeometryCollectionFromShapes(shapes []*geojson.GeoShape) (\n\tindex.GeoJSON, []byte, error) {\n\n\treturn geojson.NewGeometryCollection(shapes)\n}\n\n// NewGeoCircleShape instantiate a circle shape and\n// prefix the byte contents with certain glue bytes that\n// can be used later while filering the doc values.\nfunc NewGeoCircleShape(cp []float64,\n\tradius string) (index.GeoJSON, []byte, error) {\n\treturn geojson.NewGeoCircleShape(cp, radius)\n}\n\nfunc NewGeoJsonShape(coordinates [][][][]float64, typ string) (\n\tindex.GeoJSON, []byte, error) {\n\treturn geojson.NewGeoJsonShape(coordinates, typ)\n}\n\nfunc NewGeoJsonPoint(points []float64) index.GeoJSON {\n\treturn geojson.NewGeoJsonPoint(points)\n}\n\nfunc NewGeoJsonMultiPoint(points [][]float64) index.GeoJSON {\n\treturn geojson.NewGeoJsonMultiPoint(points)\n}\n\nfunc NewGeoJsonLinestring(points [][]float64) index.GeoJSON {\n\treturn geojson.NewGeoJsonLinestring(points)\n}\n\nfunc NewGeoJsonMultilinestring(points [][][]float64) index.GeoJSON {\n\treturn geojson.NewGeoJsonMultilinestring(points)\n}\n\nfunc NewGeoJsonPolygon(points [][][]float64) index.GeoJSON {\n\treturn geojson.NewGeoJsonPolygon(points)\n}\n\nfunc NewGeoJsonMultiPolygon(points [][][][]float64) index.GeoJSON {\n\treturn geojson.NewGeoJsonMultiPolygon(points)\n}\n\nfunc NewGeoCircle(points []float64, radius string) index.GeoJSON {\n\treturn geojson.NewGeoCircle(points, radius)\n}\n\nfunc NewGeoEnvelope(points [][]float64) index.GeoJSON {\n\treturn geojson.NewGeoEnvelope(points)\n}\n\nfunc ParseGeoJSONShape(input json.RawMessage) (index.GeoJSON, error) {\n\treturn geojson.ParseGeoJSONShape(input)\n}\n"
  },
  {
    "path": "geo/geo_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"math\"\n\t\"testing\"\n)\n\nfunc TestMortonHashMortonUnhash(t *testing.T) {\n\ttests := []struct {\n\t\tlon float64\n\t\tlat float64\n\t}{\n\t\t{-180.0, -90.0},\n\t\t{-5, 27.3},\n\t\t{0, 0},\n\t\t{1.0, 1.0},\n\t\t{24.7, -80.4},\n\t\t{180.0, 90.0},\n\t}\n\n\tfor _, test := range tests {\n\t\thash := MortonHash(test.lon, test.lat)\n\t\tlon := MortonUnhashLon(hash)\n\t\tlat := MortonUnhashLat(hash)\n\t\tif compareGeo(test.lon, lon) != 0 {\n\t\t\tt.Errorf(\"expected lon %f, got %f, hash %x\", test.lon, lon, hash)\n\t\t}\n\t\tif compareGeo(test.lat, lat) != 0 {\n\t\t\tt.Errorf(\"expected lat %f, got %f, hash %x\", test.lat, lat, hash)\n\t\t}\n\t}\n}\n\nfunc TestScaleLonUnscaleLon(t *testing.T) {\n\ttests := []struct {\n\t\tlon float64\n\t}{\n\t\t{-180.0},\n\t\t{0.0},\n\t\t{1.0},\n\t\t{180.0},\n\t}\n\n\tfor _, test := range tests {\n\t\ts := scaleLon(test.lon)\n\t\tlon := unscaleLon(s)\n\t\tif compareGeo(test.lon, lon) != 0 {\n\t\t\tt.Errorf(\"expected %f, got %f, scaled was %d\", test.lon, lon, s)\n\t\t}\n\t}\n}\n\nfunc TestScaleLatUnscaleLat(t *testing.T) {\n\ttests := []struct {\n\t\tlat float64\n\t}{\n\t\t{-90.0},\n\t\t{0.0},\n\t\t{1.0},\n\t\t{90.0},\n\t}\n\n\tfor _, test := range tests {\n\t\ts := scaleLat(test.lat)\n\t\tlat := unscaleLat(s)\n\t\tif compareGeo(test.lat, lat) != 0 {\n\t\t\tt.Errorf(\"expected %.16f, got %.16f, scaled was %d\", test.lat, lat, s)\n\t\t}\n\t}\n}\n\nfunc TestRectFromPointDistance(t *testing.T) {\n\t// at the equator 1 degree of latitude is about 110567 meters\n\t_, upperLeftLat, _, lowerRightLat, err := RectFromPointDistance(0, 0, 110567)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif math.Abs(upperLeftLat-1) > 1e-2 {\n\t\tt.Errorf(\"expected bounding box upper left lat to be almost 1, got %f\", upperLeftLat)\n\t}\n\tif math.Abs(lowerRightLat+1) > 1e-2 {\n\t\tt.Errorf(\"expected bounding box lower right lat to be almost -1, got %f\", lowerRightLat)\n\t}\n}\n\nfunc TestRectIntersects(t *testing.T) {\n\ttests := []struct {\n\t\taMinX float64\n\t\taMinY float64\n\t\taMaxX float64\n\t\taMaxY float64\n\t\tbMinX float64\n\t\tbMinY float64\n\t\tbMaxX float64\n\t\tbMaxY float64\n\t\twant  bool\n\t}{\n\t\t// clearly overlap\n\t\t{0, 0, 2, 2, 1, 1, 3, 3, true},\n\t\t// clearly do not overalp\n\t\t{0, 0, 1, 1, 2, 2, 3, 3, false},\n\t\t// share common point\n\t\t{0, 0, 1, 1, 1, 1, 2, 2, true},\n\t}\n\n\tfor _, test := range tests {\n\t\tgot := RectIntersects(test.aMinX, test.aMinY, test.aMaxX, test.aMaxY, test.bMinX, test.bMinY, test.bMaxX, test.bMaxY)\n\t\tif test.want != got {\n\t\t\tt.Errorf(\"expected intersects %t, got %t for %f %f %f %f %f %f %f %f\", test.want, got, test.aMinX, test.aMinY, test.aMaxX, test.aMaxY, test.bMinX, test.bMinY, test.bMaxX, test.bMaxY)\n\t\t}\n\t}\n}\n\nfunc TestRectWithin(t *testing.T) {\n\ttests := []struct {\n\t\taMinX float64\n\t\taMinY float64\n\t\taMaxX float64\n\t\taMaxY float64\n\t\tbMinX float64\n\t\tbMinY float64\n\t\tbMaxX float64\n\t\tbMaxY float64\n\t\twant  bool\n\t}{\n\t\t// clearly within\n\t\t{1, 1, 2, 2, 0, 0, 3, 3, true},\n\t\t// clearly not within\n\t\t{0, 0, 1, 1, 2, 2, 3, 3, false},\n\t\t// overlapping\n\t\t{0, 0, 2, 2, 1, 1, 3, 3, false},\n\t\t// share common point\n\t\t{0, 0, 1, 1, 1, 1, 2, 2, false},\n\t\t// within, but boxes reversed (b is within a, but not a within b)\n\t\t{0, 0, 3, 3, 1, 1, 2, 2, false},\n\t}\n\n\tfor _, test := range tests {\n\t\tgot := RectWithin(test.aMinX, test.aMinY, test.aMaxX, test.aMaxY, test.bMinX, test.bMinY, test.bMaxX, test.bMaxY)\n\t\tif test.want != got {\n\t\t\tt.Errorf(\"expected within %t, got %t for %f %f %f %f %f %f %f %f\", test.want, got, test.aMinX, test.aMinY, test.aMaxX, test.aMaxY, test.bMinX, test.bMinY, test.bMaxX, test.bMaxY)\n\t\t}\n\t}\n}\n\nfunc TestBoundingBoxContains(t *testing.T) {\n\ttests := []struct {\n\t\tlon  float64\n\t\tlat  float64\n\t\tminX float64\n\t\tminY float64\n\t\tmaxX float64\n\t\tmaxY float64\n\t\twant bool\n\t}{\n\t\t// clearly contains\n\t\t{1, 1, 0, 0, 2, 2, true},\n\t\t// clearly does not contain\n\t\t{0, 0, 1, 1, 2, 2, false},\n\t\t// on corner\n\t\t{0, 0, 0, 0, 2, 2, true},\n\t}\n\tfor _, test := range tests {\n\t\tgot := BoundingBoxContains(test.lon, test.lat, test.minX, test.minY, test.maxX, test.maxY)\n\t\tif test.want != got {\n\t\t\tt.Errorf(\"expected box contains %t, got %t for %f,%f in %f %f %f %f \", test.want, got, test.lon, test.lat, test.minX, test.minY, test.maxX, test.maxY)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "geo/geohash.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n// This implementation is inspired from the geohash-js\n// ref: https://github.com/davetroy/geohash-js\n\npackage geo\n\n// encoding encapsulates an encoding defined by a given base32 alphabet.\ntype encoding struct {\n\tenc string\n\tdec [256]byte\n}\n\n// newEncoding constructs a new encoding defined by the given alphabet,\n// which must be a 32-byte string.\nfunc newEncoding(encoder string) *encoding {\n\te := new(encoding)\n\te.enc = encoder\n\tfor i := 0; i < len(e.dec); i++ {\n\t\te.dec[i] = 0xff\n\t}\n\tfor i := 0; i < len(encoder); i++ {\n\t\te.dec[encoder[i]] = byte(i)\n\t}\n\treturn e\n}\n\n// base32encoding with the Geohash alphabet.\nvar base32encoding = newEncoding(\"0123456789bcdefghjkmnpqrstuvwxyz\")\n\nvar masks = []uint64{16, 8, 4, 2, 1}\n\n// DecodeGeoHash decodes the string geohash faster with\n// higher precision. This api is in experimental phase.\nfunc DecodeGeoHash(geoHash string) (float64, float64) {\n\teven := true\n\tlat := []float64{-90.0, 90.0}\n\tlon := []float64{-180.0, 180.0}\n\n\tfor i := 0; i < len(geoHash); i++ {\n\t\tcd := uint64(base32encoding.dec[geoHash[i]])\n\t\tfor j := 0; j < 5; j++ {\n\t\t\tif even {\n\t\t\t\tif cd&masks[j] > 0 {\n\t\t\t\t\tlon[0] = (lon[0] + lon[1]) / 2\n\t\t\t\t} else {\n\t\t\t\t\tlon[1] = (lon[0] + lon[1]) / 2\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif cd&masks[j] > 0 {\n\t\t\t\t\tlat[0] = (lat[0] + lat[1]) / 2\n\t\t\t\t} else {\n\t\t\t\t\tlat[1] = (lat[0] + lat[1]) / 2\n\t\t\t\t}\n\t\t\t}\n\t\t\teven = !even\n\t\t}\n\t}\n\n\treturn (lat[0] + lat[1]) / 2, (lon[0] + lon[1]) / 2\n}\n\nfunc EncodeGeoHash(lat, lon float64) string {\n\teven := true\n\tlats := []float64{-90.0, 90.0}\n\tlons := []float64{-180.0, 180.0}\n\tprecision := 12\n\tvar ch, bit uint64\n\tvar geoHash string\n\n\tfor len(geoHash) < precision {\n\t\tif even {\n\t\t\tmid := (lons[0] + lons[1]) / 2\n\t\t\tif lon > mid {\n\t\t\t\tch |= masks[bit]\n\t\t\t\tlons[0] = mid\n\t\t\t} else {\n\t\t\t\tlons[1] = mid\n\t\t\t}\n\t\t} else {\n\t\t\tmid := (lats[0] + lats[1]) / 2\n\t\t\tif lat > mid {\n\t\t\t\tch |= masks[bit]\n\t\t\t\tlats[0] = mid\n\t\t\t} else {\n\t\t\t\tlats[1] = mid\n\t\t\t}\n\t\t}\n\t\teven = !even\n\t\tif bit < 4 {\n\t\t\tbit++\n\t\t} else {\n\t\t\tgeoHash += string(base32encoding.enc[ch])\n\t\t\tch = 0\n\t\t\tbit = 0\n\t\t}\n\t}\n\n\treturn geoHash\n}\n"
  },
  {
    "path": "geo/geohash_test.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestDecodeGeoHash(t *testing.T) {\n\ttests := []struct {\n\t\thash string\n\t\tlon  float64\n\t\tlat  float64\n\t}{\n\t\t{\"d3hn3\", -73.059082, 6.745605},     // -73.05908203, 6.74560547 as per http://geohash.co/\n\t\t{\"u4pru\", 10.393066, 57.634277},     // 10.39306641, 57.63427734\n\t\t{\"u4pruy\", 10.409546, 57.648010},    // 10.40954590, 57.64801025\n\t\t{\"u4pruyd\", 10.407486, 57.648697},   // 10.40748596, 57.64869690\n\t\t{\"u4pruydqqvj\", 10.40744, 57.64911}, // 10.40743969, 57.64911063\n\t}\n\n\tfor _, test := range tests {\n\t\tlat, lon := DecodeGeoHash(test.hash)\n\n\t\tif compareGeo(test.lon, lon) != 0 {\n\t\t\tt.Errorf(\"expected lon %f, got %f, hash %s\", test.lon, lon, test.hash)\n\t\t}\n\t\tif compareGeo(test.lat, lat) != 0 {\n\t\t\tt.Errorf(\"expected lat %f, got %f, hash %s\", test.lat, lat, test.hash)\n\t\t}\n\t}\n}\n\nfunc TestEncodeGeoHash(t *testing.T) {\n\ttests := []struct {\n\t\tlon  float64\n\t\tlat  float64\n\t\thash string\n\t}{\n\t\t{2.29449034, 48.85841131, \"u09tunquc\"},\n\t\t{76.491540, 10.060349, \"t9y3hx7my0fp\"},\n\t}\n\n\tfor _, test := range tests {\n\t\thash := EncodeGeoHash(test.lat, test.lon)\n\n\t\tif !strings.HasPrefix(hash, test.hash) {\n\t\t\tt.Errorf(\"expected hash %s, got %s\", test.hash, hash)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "geo/parse.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/util\"\n\t\"github.com/blevesearch/geo/geojson\"\n)\n\n// ExtractGeoPoint takes an arbitrary interface{} and tries it's best to\n// interpret it is as geo point.  Supported formats:\n// Container:\n// slice length 2 (GeoJSON)\n//\n//\tfirst element lon, second element lat\n//\n// string (coordinates separated by comma, or a geohash)\n//\n//\tfirst element lat, second element lon\n//\n// map[string]interface{}\n//\n//\texact keys lat and lon or lng\n//\n// struct\n//\n//\tw/exported fields case-insensitive match on lat and lon or lng\n//\n// struct\n//\n//\tsatisfying Later and Loner or Lnger interfaces\n//\n// in all cases values must be some sort of numeric-like thing: int/uint/float\nfunc ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) {\n\tvar foundLon, foundLat bool\n\n\tthingVal := reflect.ValueOf(thing)\n\tif !thingVal.IsValid() {\n\t\treturn lon, lat, false\n\t}\n\n\tthingTyp := thingVal.Type()\n\n\t// is it a slice\n\tif thingVal.Kind() == reflect.Slice {\n\t\t// must be length 2\n\t\tif thingVal.Len() == 2 {\n\t\t\tfirst := thingVal.Index(0)\n\t\t\tif first.CanInterface() {\n\t\t\t\tfirstVal := first.Interface()\n\t\t\t\tlon, foundLon = util.ExtractNumericValFloat64(firstVal)\n\t\t\t}\n\t\t\tsecond := thingVal.Index(1)\n\t\t\tif second.CanInterface() {\n\t\t\t\tsecondVal := second.Interface()\n\t\t\t\tlat, foundLat = util.ExtractNumericValFloat64(secondVal)\n\t\t\t}\n\t\t}\n\t}\n\n\t// is it a string\n\tif thingVal.Kind() == reflect.String {\n\t\tgeoStr := thingVal.Interface().(string)\n\t\tif strings.Contains(geoStr, \",\") {\n\t\t\t// geo point with coordinates split by comma\n\t\t\tpoints := strings.Split(geoStr, \",\")\n\t\t\tfor i, point := range points {\n\t\t\t\t// trim any leading or trailing white spaces\n\t\t\t\tpoints[i] = strings.TrimSpace(point)\n\t\t\t}\n\t\t\tif len(points) == 2 {\n\t\t\t\tvar err error\n\t\t\t\tlat, err = strconv.ParseFloat(points[0], 64)\n\t\t\t\tif err == nil {\n\t\t\t\t\tfoundLat = true\n\t\t\t\t}\n\t\t\t\tlon, err = strconv.ParseFloat(points[1], 64)\n\t\t\t\tif err == nil {\n\t\t\t\t\tfoundLon = true\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\t// geohash\n\t\t\tif len(geoStr) <= geoHashMaxLength {\n\t\t\t\tlat, lon = DecodeGeoHash(geoStr)\n\t\t\t\tfoundLat = true\n\t\t\t\tfoundLon = true\n\t\t\t}\n\t\t}\n\t}\n\n\t// is it a map\n\tif l, ok := thing.(map[string]interface{}); ok {\n\t\tif lval, ok := l[\"lon\"]; ok {\n\t\t\tlon, foundLon = util.ExtractNumericValFloat64(lval)\n\t\t} else if lval, ok := l[\"lng\"]; ok {\n\t\t\tlon, foundLon = util.ExtractNumericValFloat64(lval)\n\t\t}\n\t\tif lval, ok := l[\"lat\"]; ok {\n\t\t\tlat, foundLat = util.ExtractNumericValFloat64(lval)\n\t\t}\n\t}\n\n\t// now try reflection on struct fields\n\tif thingVal.Kind() == reflect.Struct {\n\t\tfor i := 0; i < thingVal.NumField(); i++ {\n\t\t\tfieldName := thingTyp.Field(i).Name\n\t\t\tif strings.HasPrefix(strings.ToLower(fieldName), \"lon\") {\n\t\t\t\tif thingVal.Field(i).CanInterface() {\n\t\t\t\t\tfieldVal := thingVal.Field(i).Interface()\n\t\t\t\t\tlon, foundLon = util.ExtractNumericValFloat64(fieldVal)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif strings.HasPrefix(strings.ToLower(fieldName), \"lng\") {\n\t\t\t\tif thingVal.Field(i).CanInterface() {\n\t\t\t\t\tfieldVal := thingVal.Field(i).Interface()\n\t\t\t\t\tlon, foundLon = util.ExtractNumericValFloat64(fieldVal)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif strings.HasPrefix(strings.ToLower(fieldName), \"lat\") {\n\t\t\t\tif thingVal.Field(i).CanInterface() {\n\t\t\t\t\tfieldVal := thingVal.Field(i).Interface()\n\t\t\t\t\tlat, foundLat = util.ExtractNumericValFloat64(fieldVal)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// last hope, some interfaces\n\t// lon\n\tif l, ok := thing.(loner); ok {\n\t\tlon = l.Lon()\n\t\tfoundLon = true\n\t} else if l, ok := thing.(lnger); ok {\n\t\tlon = l.Lng()\n\t\tfoundLon = true\n\t}\n\t// lat\n\tif l, ok := thing.(later); ok {\n\t\tlat = l.Lat()\n\t\tfoundLat = true\n\t}\n\n\treturn lon, lat, foundLon && foundLat\n}\n\n// various support interfaces which can be used to find lat/lon\ntype loner interface {\n\tLon() float64\n}\n\ntype later interface {\n\tLat() float64\n}\n\ntype lnger interface {\n\tLng() float64\n}\n\n// GlueBytes primarily for quicker filtering of docvalues\n// during the filtering phase.\nvar GlueBytes = []byte(\"##\")\n\nvar GlueBytesOffset = len(GlueBytes)\n\nfunc extractCoordinates(thing interface{}) []float64 {\n\tthingVal := reflect.ValueOf(thing)\n\tif !thingVal.IsValid() {\n\t\treturn nil\n\t}\n\n\tif thingVal.Kind() == reflect.Slice {\n\t\t// must be length 2\n\t\tif thingVal.Len() == 2 {\n\t\t\tvar foundLon, foundLat bool\n\t\t\tvar lon, lat float64\n\t\t\tfirst := thingVal.Index(0)\n\t\t\tif first.CanInterface() {\n\t\t\t\tfirstVal := first.Interface()\n\t\t\t\tlon, foundLon = util.ExtractNumericValFloat64(firstVal)\n\t\t\t}\n\t\t\tsecond := thingVal.Index(1)\n\t\t\tif second.CanInterface() {\n\t\t\t\tsecondVal := second.Interface()\n\t\t\t\tlat, foundLat = util.ExtractNumericValFloat64(secondVal)\n\t\t\t}\n\n\t\t\tif !foundLon || !foundLat {\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\treturn []float64{lon, lat}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc extract2DCoordinates(thing interface{}) [][]float64 {\n\tthingVal := reflect.ValueOf(thing)\n\tif !thingVal.IsValid() {\n\t\treturn nil\n\t}\n\n\trv := make([][]float64, 0, 8)\n\tif thingVal.Kind() == reflect.Slice {\n\t\tfor j := 0; j < thingVal.Len(); j++ {\n\t\t\tedges := thingVal.Index(j).Interface()\n\t\t\tif es, ok := edges.([]interface{}); ok {\n\t\t\t\tv := extractCoordinates(es)\n\t\t\t\tif len(v) == 2 {\n\t\t\t\t\trv = append(rv, v)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn rv\n\t}\n\n\treturn nil\n}\n\nfunc extract3DCoordinates(thing interface{}) (c [][][]float64) {\n\tcoords := reflect.ValueOf(thing)\n\tif !coords.IsValid() {\n\t\treturn nil\n\t}\n\n\tif coords.Kind() == reflect.Slice {\n\t\tfor i := 0; i < coords.Len(); i++ {\n\t\t\tvals := coords.Index(i)\n\t\t\tedges := vals.Interface()\n\t\t\tif es, ok := edges.([]interface{}); ok {\n\t\t\t\tloop := extract2DCoordinates(es)\n\t\t\t\tif len(loop) > 0 {\n\t\t\t\t\tc = append(c, loop)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn c\n}\n\nfunc extract4DCoordinates(thing interface{}) (rv [][][][]float64) {\n\tthingVal := reflect.ValueOf(thing)\n\tif !thingVal.IsValid() {\n\t\treturn nil\n\t}\n\n\tif thingVal.Kind() == reflect.Slice {\n\t\tfor j := 0; j < thingVal.Len(); j++ {\n\t\t\tc := extract3DCoordinates(thingVal.Index(j).Interface())\n\t\t\trv = append(rv, c)\n\t\t}\n\t}\n\n\treturn rv\n}\n\nfunc ParseGeoShapeField(thing interface{}) (interface{}, string, error) {\n\tthingVal := reflect.ValueOf(thing)\n\tif !thingVal.IsValid() {\n\t\treturn nil, \"\", nil\n\t}\n\n\tvar shape string\n\tvar coordValue interface{}\n\n\tif thingVal.Kind() == reflect.Map {\n\t\titer := thingVal.MapRange()\n\t\tfor iter.Next() {\n\t\t\tif iter.Key().String() == \"type\" {\n\t\t\t\tshape = iter.Value().Interface().(string)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif iter.Key().String() == \"coordinates\" {\n\t\t\t\tcoordValue = iter.Value().Interface()\n\t\t\t}\n\t\t}\n\t}\n\n\treturn coordValue, strings.ToLower(shape), nil\n}\n\nfunc extractGeoShape(thing interface{}) (*geojson.GeoShape, bool) {\n\n\tcoordValue, typ, err := ParseGeoShapeField(thing)\n\tif err != nil {\n\t\treturn nil, false\n\t}\n\n\tif typ == CircleType {\n\t\treturn ExtractCircle(thing)\n\t}\n\n\treturn ExtractGeoShapeCoordinates(coordValue, typ)\n}\n\n// ExtractGeometryCollection takes an interface{} and tries it's best to\n// interpret all the member geojson shapes within it.\nfunc ExtractGeometryCollection(thing interface{}) ([]*geojson.GeoShape, bool) {\n\tthingVal := reflect.ValueOf(thing)\n\tif !thingVal.IsValid() {\n\t\treturn nil, false\n\t}\n\tvar rv []*geojson.GeoShape\n\tvar f bool\n\n\tif thingVal.Kind() == reflect.Map {\n\t\titer := thingVal.MapRange()\n\t\tfor iter.Next() {\n\n\t\t\tif iter.Key().String() == \"type\" {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif iter.Key().String() == \"geometries\" {\n\t\t\t\tcollection := iter.Value().Interface()\n\t\t\t\titems := reflect.ValueOf(collection)\n\n\t\t\t\tfor j := 0; j < items.Len(); j++ {\n\t\t\t\t\tshape, found := extractGeoShape(items.Index(j).Interface())\n\t\t\t\t\tif found {\n\t\t\t\t\t\tf = found\n\t\t\t\t\t\trv = append(rv, shape)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn rv, f\n}\n\n// ExtractCircle takes an interface{} and tries it's best to\n// interpret the center point coordinates and the radius for a\n// given circle shape.\nfunc ExtractCircle(thing interface{}) (*geojson.GeoShape, bool) {\n\tthingVal := reflect.ValueOf(thing)\n\tif !thingVal.IsValid() {\n\t\treturn nil, false\n\t}\n\n\trv := &geojson.GeoShape{\n\t\tType:   CircleType,\n\t\tCenter: make([]float64, 0, 2),\n\t}\n\n\tif thingVal.Kind() == reflect.Map {\n\t\titer := thingVal.MapRange()\n\t\tfor iter.Next() {\n\n\t\t\tif iter.Key().String() == \"radius\" {\n\t\t\t\trv.Radius = iter.Value().Interface().(string)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif iter.Key().String() == \"coordinates\" {\n\t\t\t\tlng, lat, found := ExtractGeoPoint(iter.Value().Interface())\n\t\t\t\tif !found {\n\t\t\t\t\treturn nil, false\n\t\t\t\t}\n\t\t\t\trv.Center = append(rv.Center, lng, lat)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn rv, true\n}\n\n// ExtractGeoShapeCoordinates takes an interface{} and tries it's best to\n// interpret the coordinates for any of the given geoshape typ like\n// a point, multipoint, linestring, multilinestring, polygon, multipolygon,\nfunc ExtractGeoShapeCoordinates(coordValue interface{},\n\ttyp string) (*geojson.GeoShape, bool) {\n\trv := &geojson.GeoShape{\n\t\tType: typ,\n\t}\n\n\tif typ == PointType {\n\t\tpoint := extractCoordinates(coordValue)\n\n\t\t// ignore the contents with invalid entry.\n\t\tif len(point) < 2 {\n\t\t\treturn nil, false\n\t\t}\n\n\t\trv.Coordinates = [][][][]float64{{{point}}}\n\t\treturn rv, true\n\t}\n\n\tif typ == MultiPointType || typ == LineStringType ||\n\t\ttyp == EnvelopeType {\n\t\tcoords := extract2DCoordinates(coordValue)\n\n\t\t// ignore the contents with invalid entry.\n\t\tif len(coords) == 0 {\n\t\t\treturn nil, false\n\t\t}\n\n\t\tif typ == EnvelopeType && len(coords) != 2 {\n\t\t\treturn nil, false\n\t\t}\n\n\t\tif typ == LineStringType && len(coords) < 2 {\n\t\t\treturn nil, false\n\t\t}\n\n\t\trv.Coordinates = [][][][]float64{{coords}}\n\t\treturn rv, true\n\t}\n\n\tif typ == PolygonType || typ == MultiLineStringType {\n\t\tcoords := extract3DCoordinates(coordValue)\n\n\t\t// ignore the contents with invalid entry.\n\t\tif len(coords) == 0 {\n\t\t\treturn nil, false\n\t\t}\n\n\t\tif typ == PolygonType && len(coords[0]) < 3 ||\n\t\t\ttyp == MultiLineStringType && len(coords[0]) < 2 {\n\t\t\treturn nil, false\n\t\t}\n\n\t\trv.Coordinates = [][][][]float64{coords}\n\t\treturn rv, true\n\t}\n\n\tif typ == MultiPolygonType {\n\t\tcoords := extract4DCoordinates(coordValue)\n\n\t\t// ignore the contents with invalid entry.\n\t\tif len(coords) == 0 || len(coords[0]) == 0 {\n\t\t\treturn nil, false\n\n\t\t}\n\n\t\tif len(coords[0][0]) < 3 {\n\t\t\treturn nil, false\n\t\t}\n\n\t\trv.Coordinates = coords\n\t\treturn rv, true\n\t}\n\n\treturn rv, false\n}\n"
  },
  {
    "path": "geo/parse_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"encoding/json\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestExtractGeoPoint(t *testing.T) {\n\ttests := []struct {\n\t\tin      interface{}\n\t\tlon     float64\n\t\tlat     float64\n\t\tsuccess bool\n\t}{\n\t\t// values are ints\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"lat\": 5,\n\t\t\t\t\"lon\": 5,\n\t\t\t},\n\t\t\tlon:     5,\n\t\t\tlat:     5,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// values are uints\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"lat\": uint(5),\n\t\t\t\t\"lon\": uint(5),\n\t\t\t},\n\t\t\tlon:     5,\n\t\t\tlat:     5,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// values float64 as with parsed JSON\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"lat\": 5.0,\n\t\t\t\t\"lon\": 5.0,\n\t\t\t},\n\t\t\tlon:     5,\n\t\t\tlat:     5,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// values are bool (not supported)\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"lat\": true,\n\t\t\t\t\"lon\": false,\n\t\t\t},\n\t\t\tlon:     0,\n\t\t\tlat:     0,\n\t\t\tsuccess: false,\n\t\t},\n\t\t// using lng variant of lon\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"lat\": 5.0,\n\t\t\t\t\"lng\": 5.0,\n\t\t\t},\n\t\t\tlon:     5,\n\t\t\tlat:     5,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// using struct\n\t\t{\n\t\t\tin: struct {\n\t\t\t\tLon float64\n\t\t\t\tLat float64\n\t\t\t}{\n\t\t\t\tLon: 3.0,\n\t\t\t\tLat: 7.5,\n\t\t\t},\n\t\t\tlon:     3.0,\n\t\t\tlat:     7.5,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// struct with lng alternate\n\t\t{\n\t\t\tin: struct {\n\t\t\t\tLng float64\n\t\t\t\tLat float64\n\t\t\t}{\n\t\t\t\tLng: 3.0,\n\t\t\t\tLat: 7.5,\n\t\t\t},\n\t\t\tlon:     3.0,\n\t\t\tlat:     7.5,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// test going through interface\n\t\t{\n\t\t\tin: &s11{\n\t\t\t\tlon: 4.0,\n\t\t\t\tlat: 6.9,\n\t\t\t},\n\t\t\tlon:     4.0,\n\t\t\tlat:     6.9,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// test going through interface with lng variant\n\t\t{\n\t\t\tin: &s12{\n\t\t\t\tlng: 4.0,\n\t\t\t\tlat: 6.9,\n\t\t\t},\n\t\t\tlon:     4.0,\n\t\t\tlat:     6.9,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// try GeoJSON slice\n\t\t{\n\t\t\tin:      []interface{}{3.4, 5.9},\n\t\t\tlon:     3.4,\n\t\t\tlat:     5.9,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// try GeoJSON slice too long\n\t\t{\n\t\t\tin:      []interface{}{3.4, 5.9, 9.4},\n\t\t\tlon:     0,\n\t\t\tlat:     0,\n\t\t\tsuccess: false,\n\t\t},\n\t\t// slice of floats\n\t\t{\n\t\t\tin:      []float64{3.4, 5.9},\n\t\t\tlon:     3.4,\n\t\t\tlat:     5.9,\n\t\t\tsuccess: true,\n\t\t},\n\t\t// values are nil (not supported)\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"lat\": nil,\n\t\t\t\t\"lon\": nil,\n\t\t\t},\n\t\t\tlon:     0,\n\t\t\tlat:     0,\n\t\t\tsuccess: false,\n\t\t},\n\t\t// input is nil\n\t\t{\n\t\t\tin:      nil,\n\t\t\tlon:     0,\n\t\t\tlat:     0,\n\t\t\tsuccess: false,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tlon, lat, success := ExtractGeoPoint(test.in)\n\t\tif success != test.success {\n\t\t\tt.Errorf(\"expected extract geo point %t, got %t for %v\", test.success, success, test.in)\n\t\t}\n\t\tif lon != test.lon {\n\t\t\tt.Errorf(\"expected lon %f, got %f for %v\", test.lon, lon, test.in)\n\t\t}\n\t\tif lat != test.lat {\n\t\t\tt.Errorf(\"expected lat %f, got %f for %v\", test.lat, lat, test.in)\n\t\t}\n\t}\n}\n\ntype s11 struct {\n\tlon float64\n\tlat float64\n}\n\nfunc (s *s11) Lon() float64 {\n\treturn s.lon\n}\n\nfunc (s *s11) Lat() float64 {\n\treturn s.lat\n}\n\ntype s12 struct {\n\tlng float64\n\tlat float64\n}\n\nfunc (s *s12) Lng() float64 {\n\treturn s.lng\n}\n\nfunc (s *s12) Lat() float64 {\n\treturn s.lat\n}\n\nfunc TestExtractGeoShape(t *testing.T) {\n\ttests := []struct {\n\t\tin          interface{}\n\t\tresTyp      string\n\t\tcoordinates [][][][]float64\n\t\tcenter      []float64\n\t\tsuccess     bool\n\t}{\n\t\t// valid point slice\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": []interface{}{3.4, 5.9},\n\t\t\t\t\"type\":        \"Point\",\n\t\t\t},\n\t\t\tresTyp:      \"point\",\n\t\t\tcoordinates: [][][][]float64{{{{3.4, 5.9}}}},\n\t\t\tsuccess:     true,\n\t\t},\n\t\t// invalid point slice\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": []interface{}{3.4},\n\t\t\t\t\"type\":        \"point\"},\n\n\t\t\tresTyp:      \"point\",\n\t\t\tcoordinates: nil,\n\t\t\tsuccess:     false,\n\t\t},\n\t\t// valid multipoint slice containing single point\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 5.9}},\n\t\t\t\t\"type\":        \"multipoint\"},\n\t\t\tresTyp:      \"multipoint\",\n\t\t\tcoordinates: [][][][]float64{{{{3.4, 5.9}}}},\n\t\t\tsuccess:     true,\n\t\t},\n\t\t// valid multipoint slice\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 5.9}, {6.7, 9.8}},\n\t\t\t\t\"type\":        \"multipoint\"},\n\t\t\tresTyp:      \"multipoint\",\n\t\t\tcoordinates: [][][][]float64{{{{3.4, 5.9}, {6.7, 9.8}}}},\n\t\t\tsuccess:     true,\n\t\t},\n\t\t// valid multipoint slice containing one invalid entry\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 5.9}, {6.7}},\n\t\t\t\t\"type\":        \"multipoint\"},\n\t\t\tresTyp:      \"multipoint\",\n\t\t\tcoordinates: [][][][]float64{{{{3.4, 5.9}}}},\n\t\t\tsuccess:     true,\n\t\t},\n\t\t// invalid multipoint slice\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4}},\n\t\t\t\t\"type\":        \"multipoint\"},\n\t\t\tresTyp:      \"multipoint\",\n\t\t\tcoordinates: nil,\n\t\t\tsuccess:     false,\n\t\t},\n\t\t// valid linestring slice\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 4.4}, {8.4, 9.4}},\n\t\t\t\t\"type\":        \"linestring\"},\n\t\t\tresTyp:      \"linestring\",\n\t\t\tcoordinates: [][][][]float64{{{{3.4, 4.4}, {8.4, 9.4}}}},\n\t\t\tsuccess:     true,\n\t\t},\n\t\t// valid linestring slice\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 4.4}, {8.4, 9.4}, {10.1, 12.3}},\n\t\t\t\t\"type\":        \"linestring\"},\n\t\t\tresTyp:      \"linestring\",\n\t\t\tcoordinates: [][][][]float64{{{{3.4, 4.4}, {8.4, 9.4}, {10.1, 12.3}}}},\n\t\t\tsuccess:     true,\n\t\t},\n\t\t// invalid linestring slice with single entry\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 4.4}},\n\t\t\t\t\"type\":        \"linestring\"},\n\t\t\tresTyp:      \"linestring\",\n\t\t\tcoordinates: nil,\n\t\t\tsuccess:     false,\n\t\t},\n\t\t// invalid linestring slice with wrong parenthesis\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][][]interface{}{{{3.4, 4.4}, {8.4, 9.4}}},\n\t\t\t\t\"type\":        \"linestring\"},\n\t\t\tresTyp:      \"linestring\",\n\t\t\tcoordinates: nil,\n\t\t\tsuccess:     false,\n\t\t},\n\t\t// valid envelope\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 4.4}, {8.4, 9.4}},\n\t\t\t\t\"type\":        \"envelope\"},\n\t\t\tresTyp:      \"envelope\",\n\t\t\tcoordinates: [][][][]float64{{{{3.4, 4.4}, {8.4, 9.4}}}},\n\t\t\tsuccess:     true,\n\t\t},\n\t\t// invalid envelope\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 4.4}},\n\t\t\t\t\"type\":        \"envelope\"},\n\t\t\tresTyp:      \"envelope\",\n\t\t\tcoordinates: nil,\n\t\t\tsuccess:     false,\n\t\t},\n\t\t// invalid envelope\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][][]interface{}{{{3.4, 4.4}, {8.4, 9.4}}},\n\t\t\t\t\"type\":        \"envelope\"},\n\t\t\tresTyp:      \"envelope\",\n\t\t\tcoordinates: nil,\n\t\t\tsuccess:     false,\n\t\t},\n\t\t// invalid envelope with >2 vertices\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": [][]interface{}{{3.4, 4.4}, {5.6, 6.4}, {7.4, 7.4}},\n\t\t\t\t\"type\":        \"envelope\"},\n\t\t\tresTyp:      \"envelope\",\n\t\t\tcoordinates: nil,\n\t\t\tsuccess:     false,\n\t\t},\n\t\t// valid circle\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": []interface{}{4.4, 5.0},\n\t\t\t\t\"radius\":      \"200m\",\n\t\t\t\t\"type\":        \"circle\"},\n\t\t\tresTyp:  \"circle\",\n\t\t\tcenter:  []float64{4.4, 5.0},\n\t\t\tsuccess: true,\n\t\t},\n\t\t// invalid circle\n\t\t{\n\t\t\tin: map[string]interface{}{\n\t\t\t\t\"coordinates\": []interface{}{4.4},\n\t\t\t\t\"radius\":      \"200m\",\n\t\t\t\t\"type\":        \"circle\"},\n\t\t\tresTyp:  \"circle\",\n\t\t\tsuccess: false,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tres, success := extractGeoShape(test.in)\n\t\tif success != test.success {\n\t\t\tt.Errorf(\"expected extract geo point: %t, got: %t for: %v\", test.success, success, test.in)\n\t\t}\n\t\tif success && res.Type != test.resTyp {\n\t\t\tt.Errorf(\"expected shape type: %v, got: %v for input: %v\", test.resTyp, res.Type, test.in)\n\t\t}\n\t\tif success && !reflect.DeepEqual(test.coordinates, res.Coordinates) {\n\t\t\tt.Errorf(\"expected result %+v, got %+v for %v\", test.coordinates, res.Coordinates, test.in)\n\t\t}\n\t\tif success && test.center != nil && !reflect.DeepEqual(test.center, res.Center) {\n\t\t\tt.Errorf(\"expected center %+v, got %+v for %v\", test.center, res.Center, test.in)\n\t\t}\n\t}\n}\n\nfunc TestExtractGeoShapeCoordinates(t *testing.T) {\n\ttests := []struct {\n\t\tx        []byte\n\t\ttyp      string\n\t\texpectOK bool\n\t}{\n\t\t{\n\t\t\tx: []byte(`[\n\t\t\t\t[\n\t\t\t\t\t[77.58894681930542,12.976498523818783],\n\t\t\t\t\t[77.58677959442139,12.974533005048169],\n\t\t\t\t\t[77.58894681930542,12.976498523818783]\n\t\t\t\t]\n\t\t\t]`),\n\t\t\ttyp:      PolygonType,\n\t\t\texpectOK: true,\n\t\t},\n\t\t{ // Invalid construct, but handled\n\t\t\tx: []byte(`[\n\t\t\t\t[\n\t\t\t\t\t{\"lon\":77.58894681930542,\"lat\":12.976498523818783},\n\t\t\t\t\t{\"lon\":77.58677959442139,\"lat\":12.974533005048169},\n\t\t\t\t\t{\"lon\":77.58894681930542,\"lat\":12.976498523818783}\n\t\t\t\t]\n\t\t\t]`),\n\t\t\ttyp:      PolygonType,\n\t\t\texpectOK: false,\n\t\t},\n\t\t{ // Invalid construct causes panic (within extract3DCoordinates), fix MB-65807\n\t\t\tx: []byte(`{\n\t\t\t\t\"coordinates\": [\n\t\t\t\t\t[77.58894681930542,12.976498523818783],\n\t\t\t\t\t[77.58677959442139,12.974533005048169],\n\t\t\t\t\t[77.58894681930542,12.976498523818783]\n\t\t\t\t]\n\t\t\t}`),\n\t\t\ttyp:      PolygonType,\n\t\t\texpectOK: false,\n\t\t},\n\t\t{\n\t\t\tx: []byte(`[\n\t\t\t\t[\n\t\t\t\t\t[\n\t\t\t\t\t\t[-0.163421630859375,51.531600743186644],\n\t\t\t\t\t\t[-0.15277862548828125,51.52455221546295],\n\t\t\t\t\t\t[-0.15895843505859375,51.53693981046689],\n\t\t\t\t\t\t[-0.163421630859375,51.531600743186644]\n\t\t\t\t\t]\n\t\t\t\t],\n\t\t\t\t[\n\t\t\t\t\t[\n\t\t\t\t\t\t[-0.1902008056640625,51.5091698216777],\n\t\t\t\t\t\t[-0.1599884033203125,51.51322956905176],\n\t\t\t\t\t\t[-0.1902008056640625,51.5091698216777]\n\t\t\t\t\t]\n\t\t\t\t]\n\t\t\t]`),\n\t\t\ttyp:      MultiPolygonType,\n\t\t\texpectOK: true,\n\t\t},\n\t\t{ // Invalid construct causes panic (within extract3DCoordinates), fix MB-65807\n\t\t\tx: []byte(`[\n\t\t\t\t{\n\t\t\t\t\t\"coordinates\": [\n\t\t\t\t\t\t[-0.163421630859375,51.531600743186644],\n\t\t\t\t\t\t[-0.15277862548828125,51.52455221546295],\n\t\t\t\t\t\t[-0.15895843505859375,51.53693981046689],\n\t\t\t\t\t\t[-0.163421630859375,51.531600743186644]\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"coordinates\": [\n\t\t\t\t\t\t[-0.1902008056640625,51.5091698216777],\n\t\t\t\t\t\t[-0.1599884033203125,51.51322956905176],\n\t\t\t\t\t\t[-0.1902008056640625,51.5091698216777]\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t]`),\n\t\t\ttyp:      MultiPolygonType,\n\t\t\texpectOK: false,\n\t\t},\n\t}\n\n\tfor i := range tests {\n\t\tvar x interface{}\n\t\tif err := json.Unmarshal(tests[i].x, &x); err != nil {\n\t\t\tt.Fatalf(\"[%d] JSON err: %v\", i+1, err)\n\t\t}\n\n\t\tres, ok := ExtractGeoShapeCoordinates(x, tests[i].typ)\n\t\tif ok != tests[i].expectOK {\n\t\t\tt.Errorf(\"[%d] expected ok %t, got %t\", i+1, tests[i].expectOK, ok)\n\t\t}\n\n\t\tif ok && res.Type != tests[i].typ {\n\t\t\tt.Errorf(\"[%d] expected type %s, got %s\", i+1, tests[i].typ, res.Type)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "geo/sloppy.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"math\"\n)\n\nvar earthDiameterPerLatitude []float64\n\nconst (\n\tradiusTabsSize = (1 << 10) + 1\n\tradiusDelta    = (math.Pi / 2) / (radiusTabsSize - 1)\n\tradiusIndexer  = 1 / radiusDelta\n)\n\nfunc init() {\n\t// initializes the tables used for the sloppy math functions\n\n\t// earth radius\n\ta := 6378137.0\n\tb := 6356752.31420\n\ta2 := a * a\n\tb2 := b * b\n\tearthDiameterPerLatitude = make([]float64, radiusTabsSize)\n\tearthDiameterPerLatitude[0] = 2.0 * a / 1000\n\tearthDiameterPerLatitude[radiusTabsSize-1] = 2.0 * b / 1000\n\tfor i := 1; i < radiusTabsSize-1; i++ {\n\t\tlat := math.Pi * float64(i) / (2*radiusTabsSize - 1)\n\t\tone := math.Pow(a2*math.Cos(lat), 2)\n\t\ttwo := math.Pow(b2*math.Sin(lat), 2)\n\t\tthree := math.Pow(float64(a)*math.Cos(lat), 2)\n\t\tfour := math.Pow(b*math.Sin(lat), 2)\n\t\tradius := math.Sqrt((one + two) / (three + four))\n\t\tearthDiameterPerLatitude[i] = 2 * radius / 1000\n\t}\n}\n\n// earthDiameter returns an estimation of the earth's diameter at the specified\n// latitude in kilometers\nfunc earthDiameter(lat float64) float64 {\n\tindex := math.Mod(math.Abs(lat)*radiusIndexer+0.5, float64(len(earthDiameterPerLatitude)))\n\tif math.IsNaN(index) {\n\t\treturn 0\n\t}\n\treturn earthDiameterPerLatitude[int(index)]\n}\n"
  },
  {
    "path": "geo/versus_test.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage geo\n\nimport (\n\t\"testing\"\n)\n\n// This test basically confirms the dimensions of the\n// bounded box computed between the DecodeGeoHash method\n// and the popular implementation from\n// https://github.com/mmcloughlin/geohash.\n// DecodeGeoHash method returns the centre of the rectangle\n// than returning the box dimensions.\n// This test verifies that the returned rectangle centre matches\n// the centre for the box dimensions defined in the original\n// implementation tests here:\n// https://github.com/mmcloughlin/geohash/blob/master/decodecases_test.go\nfunc TestDecodeGeoHashVersus(t *testing.T) {\n\ttests := []struct {\n\t\thash string\n\t\tbox  []float64\n\t}{\n\t\t{\"91rc\", []float64{7.20703125, 7.3828125, -124.1015625, -123.75}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"0fuz\", []float64{-73.30078125, -73.125, -139.5703125, -139.21875}},\n\t\t{\"dwfcndf\", []float64{38.1596374512, 38.1610107422, -63.3444213867, -63.3430480957}},\n\t\t{\"2z7\", []float64{-4.21875, -2.8125, -142.03125, -140.625}},\n\t\t{\"7spw2w\", []float64{-21.3684082031, -21.3629150391, -11.9311523438, -11.9201660156}},\n\t\t{\"eq\", []float64{33.75, 39.375, -33.75, -22.5}},\n\t\t{\"mgff0\", []float64{-23.5546875, -23.5107421875, 82.6171875, 82.6611328125}},\n\t\t{\"dp7k386jtk0\", []float64{41.5306591988, 41.5306605399, -85.3607976437, -85.3607963026}},\n\t\t{\"pjb\", []float64{-57.65625, -56.25, 135.0, 136.40625}},\n\t\t{\"jkc7uh9\", []float64{-62.5973510742, -62.5959777832, 58.184967041, 58.186340332}},\n\t\t{\"1gdp9\", []float64{-68.994140625, -68.9501953125, -98.3935546875, -98.349609375}},\n\t\t{\"z9yj14mmnxte\", []float64{55.7359149121, 55.7359150797, 165.988941416, 165.988941751}},\n\t\t{\"2brk\", []float64{-42.890625, -42.71484375, -136.0546875, -135.703125}},\n\t\t{\"dhv5t2qh59\", []float64{27.3360496759, 27.3360550404, -82.7296471596, -82.7296364307}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"3fgd9k15b5k\", []float64{-29.0691630542, -29.0691617131, -96.2718147039, -96.2718133628}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"b4\", []float64{56.25, 61.875, -180.0, -168.75}},\n\t\t{\"sb38\", []float64{1.40625, 1.58203125, 35.859375, 36.2109375}},\n\t\t{\"puqeug\", []float64{-65.4180908203, -65.4125976562, 178.099365234, 178.110351562}},\n\t\t{\"45\", []float64{-73.125, -67.5, -90.0, -78.75}},\n\t\t{\"34b\", []float64{-29.53125, -28.125, -135.0, -133.59375}},\n\t\t{\"tqb8jzn9dfqn\", []float64{38.0074727163, 38.0074728839, 57.2148630023, 57.2148633376}},\n\t\t{\"9x\", []float64{39.375, 45.0, -112.5, -101.25}},\n\t\t{\"tybf7\", []float64{38.3642578125, 38.408203125, 79.9365234375, 79.98046875}},\n\t\t{\"9nc\", []float64{37.96875, 39.375, -133.59375, -132.1875}},\n\t\t{\"pp21\", []float64{-49.04296875, -48.8671875, 135.0, 135.3515625}},\n\t\t{\"s6wjfu76v\", []float64{15.0970602036, 15.0971031189, 19.8130273819, 19.8130702972}},\n\t\t{\"wxh8ped7\", []float64{39.3947410583, 39.3949127197, 119.160804749, 119.161148071}},\n\t\t{\"8gr\", []float64{18.28125, 19.6875, -136.40625, -135.0}},\n\t\t{\"ug6hf\", []float64{64.1162109375, 64.16015625, 36.650390625, 36.6943359375}},\n\t\t{\"pb\", []float64{-90.0, -84.375, 168.75, 180.0}},\n\t\t{\"nmhvpv\", []float64{-60.9686279297, -60.9631347656, 108.270263672, 108.28125}},\n\t\t{\"rxgthm\", []float64{-0.499877929688, -0.494384765625, 162.608642578, 162.619628906}},\n\t\t{\"mj8t\", []float64{-13.18359375, -13.0078125, 45.703125, 46.0546875}},\n\t\t{\"rkvw\", []float64{-17.2265625, -17.05078125, 153.984375, 154.3359375}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"u4ryw22k\", []float64{58.8008880615, 58.8010597229, 11.1734390259, 11.1737823486}},\n\t\t{\"96bf6jfr\", []float64{15.8970451355, 15.8972167969, -122.60433197, -122.603988647}},\n\t\t{\"ubhnbn2n1jvj\", []float64{46.2219173647, 46.2219175324, 39.3750496209, 39.3750499561}},\n\t\t{\"3gmczz\", []float64{-26.3726806641, -26.3671875, -92.8234863281, -92.8125}},\n\t\t{\"yb4jh3u7px0\", []float64{45.8890718222, 45.8890731633, 126.75542593, 126.755427271}},\n\t\t{\"9ex7rkbw1duf\", []float64{20.2859266475, 20.2859268151, -101.985326596, -101.98532626}},\n\t\t{\"xrkyg3mj\", []float64{41.9754981995, 41.9756698608, 153.079376221, 153.079719543}},\n\t\t{\"z4hn2yfzryjq\", []float64{57.3869894072, 57.3869895749, 140.662075169, 140.662075505}},\n\t\t{\"x357\", []float64{6.15234375, 6.328125, 150.8203125, 151.171875}},\n\t\t{\"v3pew\", []float64{51.240234375, 51.2841796875, 67.060546875, 67.1044921875}},\n\t\t{\"j1\", []float64{-84.375, -78.75, 45.0, 56.25}},\n\t\t{\"ec1bkph03\", []float64{5.70744037628, 5.70748329163, -8.60774517059, -8.60770225525}},\n\t\t{\"q4t85\", []float64{-30.9375, -30.8935546875, 97.8662109375, 97.91015625}},\n\t\t{\"k26z8hf\", []float64{-42.2492980957, -42.2479248047, 15.119934082, 15.121307373}},\n\t\t{\"p6fyq2u63\", []float64{-73.4281110764, -73.428068161, 150.397725105, 150.397768021}},\n\t\t{\"uqu5w2yu\", []float64{83.5887908936, 83.5889625549, 17.1589279175, 17.1592712402}},\n\t\t{\"terbkv\", []float64{18.3526611328, 18.3581542969, 78.6071777344, 78.6181640625}},\n\t\t{\"8v2nwkp\", []float64{30.6958007812, 30.6971740723, -145.96572876, -145.964355469}},\n\t\t{\"rbktxk2enhr\", []float64{-42.6030693948, -42.6030680537, 175.397682041, 175.397683382}},\n\t\t{\"r1pnfct8\", []float64{-38.1802368164, -38.180065155, 144.97215271, 144.972496033}},\n\t\t{\"rcmxg\", []float64{-36.6064453125, -36.5625, 176.616210938, 176.66015625}},\n\t\t{\"jqw7xjdt8\", []float64{-52.7911090851, -52.7910661697, 65.350112915, 65.3501558304}},\n\t\t{\"7mt737xd\", []float64{-13.4716415405, -13.4714698792, -26.3019561768, -26.301612854}},\n\t\t{\"2dprx5rg57es\", []float64{-32.4132534117, -32.413253244, -146.986283138, -146.986282803}},\n\t\t{\"fpr5d98ws0\", []float64{86.40583992, 86.4058452845, -80.0455284119, -80.045517683}},\n\t\t{\"z4cmm3\", []float64{61.3970947266, 61.4025878906, 136.988525391, 136.999511719}},\n\t\t{\"gz3b8j\", []float64{85.8966064453, 85.9020996094, -8.7890625, -8.77807617188}},\n\t\t{\"svfztv42f2k\", []float64{33.6897052824, 33.6897066236, 37.8730648756, 37.8730662167}},\n\t\t{\"z6f3yb6rum\", []float64{60.7790976763, 60.7791030407, 149.713965654, 149.713976383}},\n\t\t{\"wvqv397\", []float64{30.4609680176, 30.4623413086, 133.312225342, 133.313598633}},\n\t\t{\"r0ce65wshm\", []float64{-40.1900213957, -40.1900160313, 137.206374407, 137.206385136}},\n\t\t{\"crqvbtfbjb09\", []float64{86.8235780485, 86.8235782161, -114.23181586, -114.231815524}},\n\t\t{\"ptz0vc5z87\", []float64{-57.5176173449, -57.5176119804, 167.601596117, 167.601606846}},\n\t\t{\"x806byp\", []float64{0.516357421875, 0.517730712891, 157.894134521, 157.895507812}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"kb23vjn10\", []float64{-43.2584953308, -43.2584524155, 34.3295288086, 34.3295717239}},\n\t\t{\"n82kv\", []float64{-87.7587890625, -87.71484375, 113.071289062, 113.115234375}},\n\t\t{\"dkhzmmqfg\", []float64{23.8037252426, 23.803768158, -71.830201149, -71.8301582336}},\n\t\t{\"y1v\", []float64{54.84375, 56.25, 97.03125, 98.4375}},\n\t\t{\"q977h9\", []float64{-37.4359130859, -37.4304199219, 117.268066406, 117.279052734}},\n\t\t{\"sdffzu5qzu\", []float64{15.9753012657, 15.9753066301, 26.7125594616, 26.7125701904}},\n\t\t{\"n3q\", []float64{-82.96875, -81.5625, 109.6875, 111.09375}},\n\t\t{\"fr99zgs7e\", []float64{87.5149440765, 87.5149869919, -76.2940835953, -76.2940406799}},\n\t\t{\"4d9cv4cbh\", []float64{-75.6147766113, -75.614733696, -64.8167610168, -64.8167181015}},\n\t\t{\"np896wq5\", []float64{-47.557926178, -47.5577545166, 90.8212280273, 90.8215713501}},\n\t\t{\"45\", []float64{-73.125, -67.5, -90.0, -78.75}},\n\t\t{\"c5srsy\", []float64{66.0388183594, 66.0443115234, -128.814697266, -128.803710938}},\n\t\t{\"vcshr\", []float64{54.1845703125, 54.228515625, 84.6826171875, 84.7265625}},\n\t\t{\"3zbbs1wnn5\", []float64{-1.30907356739, -1.30906820297, -100.011034012, -100.011023283}},\n\t\t{\"jtdvxn\", []float64{-58.0627441406, -58.0572509766, 71.6748046875, 71.6857910156}},\n\t\t{\"yz27wu0e\", []float64{86.4189720154, 86.4191436768, 124.398880005, 124.399223328}},\n\t\t{\"utb0ck65h9\", []float64{77.4994522333, 77.4994575977, 22.5578713417, 22.5578820705}},\n\t\t{\"1t1dvq5jj\", []float64{-61.3577842712, -61.3577413559, -110.15557766, -110.155534744}},\n\t\t{\"yzwgy5hvwz\", []float64{87.8641408682, 87.8641462326, 133.512672186, 133.512682915}},\n\t\t{\"8dx38y6vby\", []float64{14.3615233898, 14.3615287542, -147.267919779, -147.26790905}},\n\t\t{\"9bh\", []float64{0.0, 1.40625, -95.625, -94.21875}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"x14ve\", []float64{6.591796875, 6.6357421875, 138.999023438, 139.04296875}},\n\t\t{\"3603v\", []float64{-33.4423828125, -33.3984375, -123.178710938, -123.134765625}},\n\t\t{\"bsuje4cn6\", []float64{72.7017259598, 72.7017688751, -151.741704941, -151.741662025}},\n\t\t{\"9nnut36epqhn\", []float64{34.5484302565, 34.5484304242, -125.273349881, -125.273349546}},\n\t\t{\"7q27jg\", []float64{-9.29992675781, -9.29443359375, -33.1457519531, -33.134765625}},\n\t\t{\"933zzd6\", []float64{8.40591430664, 8.40728759766, -120.956726074, -120.955352783}},\n\t\t{\"5743y\", []float64{-72.8173828125, -72.7734375, -30.322265625, -30.2783203125}},\n\t\t{\"7m94uc9\", []float64{-13.5708618164, -13.5694885254, -32.1336364746, -32.1322631836}},\n\t\t{\"780\", []float64{-45.0, -43.59375, -22.5, -21.09375}},\n\t\t{\"sqpqx2w0hh\", []float64{34.8953461647, 34.8953515291, 21.7723274231, 21.7723381519}},\n\t\t{\"1ep\", []float64{-73.125, -71.71875, -102.65625, -101.25}},\n\t\t{\"k0j\", []float64{-45.0, -43.59375, 7.03125, 8.4375}},\n\t\t{\"z1zgr0g440\", []float64{55.4195022583, 55.4195076227, 146.210260391, 146.21027112}},\n\t\t{\"681bf\", []float64{-44.8681640625, -44.82421875, -64.951171875, -64.9072265625}},\n\t\t{\"dn6j\", []float64{36.03515625, 36.2109375, -87.1875, -86.8359375}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"t1m\", []float64{7.03125, 8.4375, 52.03125, 53.4375}},\n\t\t{\"9qkshh67xe3\", []float64{35.8833391964, 35.8833405375, -117.242680639, -117.242679298}},\n\t\t{\"4507gyj\", []float64{-72.4328613281, -72.4314880371, -89.476776123, -89.475402832}},\n\t\t{\"bb844h\", []float64{48.1860351562, 48.1915283203, -146.162109375, -146.151123047}},\n\t\t{\"trn6yz\", []float64{39.8968505859, 39.90234375, 65.3356933594, 65.3466796875}},\n\t\t{\"vb99\", []float64{47.98828125, 48.1640625, 80.859375, 81.2109375}},\n\t\t{\"dxpbksmed\", []float64{39.4428920746, 39.4429349899, -56.3961696625, -56.3961267471}},\n\t\t{\"zc\", []float64{50.625, 56.25, 168.75, 180.0}},\n\t\t{\"3webp3s6hdeb\", []float64{-8.42890352011, -8.42890335247, -106.901924349, -106.901924014}},\n\t\t{\"8qg\", []float64{37.96875, 39.375, -164.53125, -163.125}},\n\t\t{\"fgwzk\", []float64{65.9619140625, 66.005859375, -46.58203125, -46.5380859375}},\n\t\t{\"1q2mxvd\", []float64{-53.8467407227, -53.8453674316, -123.055114746, -123.053741455}},\n\t\t{\"sd\", []float64{11.25, 16.875, 22.5, 33.75}},\n\t\t{\"8hhwv\", []float64{23.6865234375, 23.73046875, -173.452148438, -173.408203125}},\n\t\t{\"1bw\", []float64{-87.1875, -85.78125, -92.8125, -91.40625}},\n\t\t{\"66zkh0ex724\", []float64{-28.824133873, -28.8241325319, -68.3739575744, -68.3739562333}},\n\t\t{\"w6qpy8\", []float64{14.0185546875, 14.0240478516, 109.973144531, 109.984130859}},\n\t\t{\"nux13fvfr\", []float64{-64.4522809982, -64.4522380829, 133.678851128, 133.678894043}},\n\t\t{\"cj2\", []float64{74.53125, 75.9375, -135.0, -133.59375}},\n\t\t{\"1qrumeqp\", []float64{-54.0776252747, -54.0774536133, -112.601623535, -112.601280212}},\n\t\t{\"hhm0\", []float64{-66.09375, -65.91796875, 7.03125, 7.3828125}},\n\t\t{\"50cp37u6w86\", []float64{-84.4858060777, -84.4858047366, -43.5327002406, -43.5326988995}},\n\t\t{\"0vjjfe5w\", []float64{-60.8467483521, -60.8465766907, -139.1040802, -139.103736877}},\n\t\t{\"xm67\", []float64{30.05859375, 30.234375, 149.4140625, 149.765625}},\n\t\t{\"1jmnqz1\", []float64{-59.3316650391, -59.330291748, -127.67074585, -127.669372559}},\n\t\t{\"jjfh69u78\", []float64{-56.8989658356, -56.8989229202, 47.9281997681, 47.9282426834}},\n\t\t{\"9xvnjtj3ytr\", []float64{44.6762318909, 44.676233232, -105.219552666, -105.219551325}},\n\t\t{\"ebwk44\", []float64{3.52661132812, 3.53210449219, -2.373046875, -2.36206054688}},\n\t\t{\"xej0p\", []float64{16.875, 16.9189453125, 164.838867188, 164.8828125}},\n\t\t{\"hm4m\", []float64{-60.99609375, -60.8203125, 14.4140625, 14.765625}},\n\t\t{\"71d1uhzq\", []float64{-36.2277603149, -36.2275886536, -42.0017623901, -42.0014190674}},\n\t\t{\"u7d8cgc9h4w\", []float64{64.8401203752, 64.8401217163, 14.8447689414, 14.8447702825}},\n\t\t{\"zwud\", []float64{83.3203125, 83.49609375, 163.828125, 164.1796875}},\n\t\t{\"p4nbh7wnwsb\", []float64{-78.7296326458, -78.7296313047, 144.687473774, 144.687475115}},\n\t\t{\"ev\", []float64{28.125, 33.75, -11.25, 0.0}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"ytj8jhg3vmke\", []float64{73.1514216028, 73.1514217705, 120.458796099, 120.458796434}},\n\t\t{\"xn\", []float64{33.75, 39.375, 135.0, 146.25}},\n\t\t{\"1t7f\", []float64{-60.1171875, -59.94140625, -107.2265625, -106.875}},\n\t\t{\"wjt\", []float64{30.9375, 32.34375, 97.03125, 98.4375}},\n\t\t{\"f7j3\", []float64{62.05078125, 62.2265625, -71.3671875, -71.015625}},\n\t\t{\"62bd4mvcg\", []float64{-40.3978013992, -40.3977584839, -77.9399728775, -77.9399299622}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"577epg1nh0be\", []float64{-71.1738922633, -71.1738920957, -28.4860032052, -28.4860028699}},\n\t\t{\"pvyz2qtjw6kc\", []float64{-56.3451739959, -56.3451738283, 178.260314874, 178.26031521}},\n\t\t{\"2hrf6fbj\", []float64{-20.6822776794, -20.6821060181, -168.980712891, -168.980369568}},\n\t\t{\"z1yz7p6dc3h8\", []float64{56.1584669352, 56.1584671028, 144.627516344, 144.627516679}},\n\t\t{\"24v5r460td50\", []float64{-28.9475047588, -28.9475045912, -172.658146173, -172.658145837}},\n\t\t{\"rdtnyvudc4mu\", []float64{-29.7189060599, -29.7189058922, 164.834111296, 164.834111631}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"e7he\", []float64{17.40234375, 17.578125, -27.421875, -27.0703125}},\n\t\t{\"9yrkg\", []float64{35.9912109375, 36.03515625, -90.9228515625, -90.87890625}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"yppvw0\", []float64{85.341796875, 85.3472900391, 101.162109375, 101.173095703}},\n\t\t{\"n3876wh\", []float64{-80.9582519531, -80.9568786621, 101.716918945, 101.718292236}},\n\t\t{\"m1vjqf4\", []float64{-34.2224121094, -34.2210388184, 52.3306274414, 52.3320007324}},\n\t\t{\"kev\", []float64{-23.90625, -22.5, 29.53125, 30.9375}},\n\t\t{\"4qc\", []float64{-52.03125, -50.625, -77.34375, -75.9375}},\n\t\t{\"5f4\", []float64{-78.75, -77.34375, -8.4375, -7.03125}},\n\t\t{\"ug\", []float64{61.875, 67.5, 33.75, 45.0}},\n\t\t{\"g5\", []float64{61.875, 67.5, -45.0, -33.75}},\n\t\t{\"7wvx\", []float64{-5.80078125, -5.625, -14.765625, -14.4140625}},\n\t\t{\"rx\", []float64{-5.625, 0.0, 157.5, 168.75}},\n\t\t{\"jdg0t21qzr\", []float64{-74.4421631098, -74.4421577454, 71.9514906406, 71.9515013695}},\n\t\t{\"606yg\", []float64{-42.4072265625, -42.36328125, -86.0009765625, -85.95703125}},\n\t\t{\"nxt6zwhgqd\", []float64{-47.2955739498, -47.2955685854, 120.219204426, 120.219215155}},\n\t\t{\"e71g7w8dr\", []float64{17.482380867, 17.4824237823, -31.1342668533, -31.134223938}},\n\t\t{\"n6u\", []float64{-74.53125, -73.125, 106.875, 108.28125}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"zfcehd0d3\", []float64{61.0074663162, 61.0075092316, 171.057858467, 171.057901382}},\n\t\t{\"hgx5efc24r5e\", []float64{-69.68212137, -69.6821212023, 43.760362789, 43.7603631243}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"q\", []float64{-45.0, 0.0, 90.0, 135.0}},\n\t\t{\"zbzg87qu07g\", []float64{49.8525439203, 49.8525452614, 179.668708295, 179.668709636}},\n\t\t{\"02urcn\", []float64{-84.3859863281, -84.3804931641, -162.729492188, -162.718505859}},\n\t\t{\"p4\", []float64{-78.75, -73.125, 135.0, 146.25}},\n\t\t{\"j6xjzcpsk78\", []float64{-74.9205163121, -74.920514971, 66.4448082447, 66.4448095858}},\n\t\t{\"svchwhvd08d\", []float64{33.1612041593, 33.1612055004, 35.4274991155, 35.4275004566}},\n\t\t{\"yughcgqek2\", []float64{72.5721216202, 72.5721269846, 128.054763079, 128.054773808}},\n\t\t{\"18tc1b\", []float64{-87.01171875, -87.0062255859, -104.337158203, -104.326171875}},\n\t\t{\"es43c\", []float64{22.8076171875, 22.8515625, -19.2919921875, -19.248046875}},\n\t\t{\"zdcephk\", []float64{61.0194396973, 61.0208129883, 159.922485352, 159.923858643}},\n\t\t{\"2zqh\", []float64{-3.515625, -3.33984375, -137.8125, -137.4609375}},\n\t\t{\"sh9bfh31\", []float64{25.4678535461, 25.4680252075, 2.55020141602, 2.55054473877}},\n\t\t{\"62vfgv\", []float64{-40.2703857422, -40.2648925781, -70.4992675781, -70.48828125}},\n\t\t{\"u2yn4\", []float64{50.2734375, 50.3173828125, 19.775390625, 19.8193359375}},\n\t\t{\"qx5wuf\", []float64{-4.42749023438, -4.42199707031, 117.630615234, 117.641601562}},\n\t\t{\"9y163\", []float64{34.1455078125, 34.189453125, -99.4482421875, -99.404296875}},\n\t\t{\"7ygu094y\", []float64{-6.32160186768, -6.3214302063, -5.95081329346, -5.9504699707}},\n\t\t{\"nkfj9vxh9e6\", []float64{-62.2834508121, -62.283449471, 104.149084389, 104.14908573}},\n\t\t{\"n220y9m\", []float64{-88.4550476074, -88.4536743164, 101.542510986, 101.543884277}},\n\t\t{\"472p7k4d\", []float64{-70.4220199585, -70.4218482971, -78.6037445068, -78.6034011841}},\n\t\t{\"7p\", []float64{-5.625, 0.0, -45.0, -33.75}},\n\t\t{\"b1r1n2zp\", []float64{52.2123527527, 52.2125244141, -169.87197876, -169.871635437}},\n\t\t{\"neu8fxsk8k4\", []float64{-68.7324213982, -68.7324200571, 118.943838179, 118.94383952}},\n\t\t{\"m33fn\", []float64{-37.6171875, -37.5732421875, 58.974609375, 59.0185546875}},\n\t\t{\"u6z5je\", []float64{61.0125732422, 61.0180664062, 21.3354492188, 21.3464355469}},\n\t\t{\"5e6csnf\", []float64{-71.4179992676, -71.4166259766, -18.454284668, -18.452911377}},\n\t\t{\"7f7k4\", []float64{-31.640625, -31.5966796875, -6.591796875, -6.5478515625}},\n\t\t{\"ykd07ytm4\", []float64{70.3930091858, 70.3930521011, 104.23459053, 104.234633446}},\n\t\t{\"ubyx97\", []float64{50.5535888672, 50.5590820312, 42.9455566406, 42.9565429688}},\n\t\t{\"r\", []float64{-45.0, 0.0, 135.0, 180.0}},\n\t\t{\"ungw16\", []float64{84.0344238281, 84.0399169922, 4.97680664062, 4.98779296875}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"bhfkepk0n\", []float64{72.5495910645, 72.5496339798, -176.698350906, -176.698307991}},\n\t\t{\"d3\", []float64{5.625, 11.25, -78.75, -67.5}},\n\t\t{\"wsenn\", []float64{26.3671875, 26.4111328125, 116.982421875, 117.026367188}},\n\t\t{\"b4b6qpqk\", []float64{60.9047698975, 60.9049415588, -179.376182556, -179.375839233}},\n\t\t{\"zwjx2wv3\", []float64{80.0616645813, 80.0618362427, 165.263557434, 165.263900757}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"3sgq6\", []float64{-17.1826171875, -17.138671875, -107.841796875, -107.797851562}},\n\t\t{\"9chbq\", []float64{5.6689453125, 5.712890625, -94.306640625, -94.2626953125}},\n\t\t{\"37nehnbt\", []float64{-27.5597190857, -27.5595474243, -114.432907104, -114.432563782}},\n\t\t{\"uhr5w71b\", []float64{69.5379638672, 69.5381355286, 10.1208114624, 10.1211547852}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"81\", []float64{5.625, 11.25, -180.0, -168.75}},\n\t\t{\"vyxn\", []float64{82.6171875, 82.79296875, 88.59375, 88.9453125}},\n\t\t{\"73jvv\", []float64{-38.3642578125, -38.3203125, -25.4443359375, -25.400390625}},\n\t\t{\"nmw1ysg2f23\", []float64{-58.7286601961, -58.728658855, 109.977705628, 109.977706969}},\n\t\t{\"5sv6js2nphq\", []float64{-62.9052887857, -62.9052874446, -14.8751798272, -14.8751784861}},\n\t\t{\"289\", []float64{-42.1875, -40.78125, -156.09375, -154.6875}},\n\t\t{\"cq9\", []float64{81.5625, 82.96875, -122.34375, -120.9375}},\n\t\t{\"mm\", []float64{-16.875, -11.25, 56.25, 67.5}},\n\t\t{\"49378fm9v\", []float64{-82.3408555984, -82.3408126831, -65.7014608383, -65.701417923}},\n\t\t{\"ee079be\", []float64{17.492980957, 17.494354248, -22.0674133301, -22.0660400391}},\n\t\t{\"4m5cs3h\", []float64{-61.6058349609, -61.6044616699, -73.2843017578, -73.2829284668}},\n\t\t{\"hpdre\", []float64{-46.494140625, -46.4501953125, 3.2958984375, 3.33984375}},\n\t\t{\"06\", []float64{-78.75, -73.125, -168.75, -157.5}},\n\t\t{\"3x3r4evkpp1q\", []float64{-2.9669566825, -2.96695651487, -110.624812357, -110.624812022}},\n\t\t{\"95074fyh23t\", []float64{17.4181875587, 17.4181888998, -134.51933071, -134.519329369}},\n\t\t{\"5tdj7sf\", []float64{-58.1135559082, -58.1121826172, -19.5309448242, -19.5295715332}},\n\t\t{\"8r9z\", []float64{43.41796875, 43.59375, -166.2890625, -165.9375}},\n\t\t{\"nc\", []float64{-84.375, -78.75, 123.75, 135.0}},\n\t\t{\"t2mb\", []float64{1.40625, 1.58203125, 64.3359375, 64.6875}},\n\t\t{\"hcs1g2vbkq0g\", []float64{-81.2506873347, -81.250687167, 39.525902085, 39.5259024203}},\n\t\t{\"pduzpxt\", []float64{-73.2595825195, -73.2582092285, 164.516143799, 164.51751709}},\n\t\t{\"mw9u700\", []float64{-7.6904296875, -7.68905639648, 70.0927734375, 70.0941467285}},\n\t\t{\"9y0ttsfr\", []float64{34.7440910339, 34.7442626953, -100.302085876, -100.301742554}},\n\t\t{\"ztmu17de4\", []float64{75.2541160583, 75.2541589737, 165.644388199, 165.644431114}},\n\t\t{\"1dqmehx\", []float64{-76.3522338867, -76.3508605957, -103.569488525, -103.568115234}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"6ydvkt6cp\", []float64{-7.48563766479, -7.48559474945, -52.180981636, -52.1809387207}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"706tn\", []float64{-42.71484375, -42.6708984375, -41.220703125, -41.1767578125}},\n\t\t{\"6z89j2x0gun\", []float64{-2.63382196426, -2.63382062316, -55.3063800931, -55.306378752}},\n\t\t{\"mt1t\", []float64{-15.99609375, -15.8203125, 69.609375, 69.9609375}},\n\t\t{\"95ypdkpcd\", []float64{22.4343395233, 22.4343824387, -126.452894211, -126.452851295}},\n\t\t{\"nr1wc\", []float64{-49.4384765625, -49.39453125, 103.403320312, 103.447265625}},\n\t\t{\"rb2uxf4\", []float64{-42.7917480469, -42.7903747559, 170.148010254, 170.149383545}},\n\t\t{\"gzz5w20e1wk\", []float64{89.2095328867, 89.2095342278, -1.13083541393, -1.13083407283}},\n\t\t{\"mtuddkh7my\", []float64{-12.1942341328, -12.1942287683, 73.9330852032, 73.933095932}},\n\t\t{\"r15def4\", []float64{-38.9245605469, -38.9231872559, 140.089416504, 140.090789795}},\n\t\t{\"9fwcy0d\", []float64{14.3728637695, 14.3742370605, -91.491394043, -91.490020752}},\n\t\t{\"6vx9z80hkd\", []float64{-13.7541425228, -13.7541371584, -45.3733420372, -45.3733313084}},\n\t\t{\"qqxt55re\", []float64{-7.54022598267, -7.54005432129, 111.93901062, 111.939353943}},\n\t\t{\"4hreqy\", []float64{-65.4895019531, -65.4840087891, -79.1564941406, -79.1455078125}},\n\t\t{\"9nv6\", []float64{38.3203125, 38.49609375, -127.6171875, -127.265625}},\n\t\t{\"k980q\", []float64{-36.5185546875, -36.474609375, 22.763671875, 22.8076171875}},\n\t\t{\"zd6qmdhsrtce\", []float64{58.7666300498, 58.7666302174, 160.912265405, 160.91226574}},\n\t\t{\"ucz57k\", []float64{55.4370117188, 55.4425048828, 43.7365722656, 43.7475585938}},\n\t\t{\"u5pq\", []float64{62.9296875, 63.10546875, 10.1953125, 10.546875}},\n\t\t{\"fey0v9\", []float64{66.2310791016, 66.2365722656, -58.8208007812, -58.8098144531}},\n\t\t{\"mdtggek9fmh5\", []float64{-30.2601397969, -30.2601396292, 75.7460278273, 75.7460281625}},\n\t\t{\"y6nfy\", []float64{56.7333984375, 56.77734375, 111.005859375, 111.049804688}},\n\t\t{\"f\", []float64{45.0, 90.0, -90.0, -45.0}},\n\t\t{\"33hz5xbsw\", []float64{-38.1011867523, -38.101143837, -116.915559769, -116.915516853}},\n\t\t{\"2wnb71890np2\", []float64{-11.1976110935, -11.1976109259, -147.875280194, -147.875279859}},\n\t\t{\"7d\", []float64{-33.75, -28.125, -22.5, -11.25}},\n\t\t{\"71vsdbh\", []float64{-34.365234375, -34.363861084, -37.1392822266, -37.1379089355}},\n\t\t{\"nk\", []float64{-67.5, -61.875, 101.25, 112.5}},\n\t\t{\"pn6uks\", []float64{-54.0747070312, -54.0692138672, 139.064941406, 139.075927734}},\n\t\t{\"t0\", []float64{0.0, 5.625, 45.0, 56.25}},\n\t\t{\"ze7c4z3e\", []float64{63.4973716736, 63.497543335, 162.896347046, 162.896690369}},\n\t\t{\"ujq6txghfjdv\", []float64{75.0141208805, 75.0141210482, 9.03497111052, 9.0349714458}},\n\t\t{\"40muh9sfm7\", []float64{-87.8819829226, -87.8819775581, -81.7095601559, -81.709549427}},\n\t\t{\"0y4kk1\", []float64{-55.4974365234, -55.4919433594, -142.91015625, -142.899169922}},\n\t\t{\"q8btfscp1g\", []float64{-39.7431975603, -39.7431921959, 113.314436674, 113.314447403}},\n\t\t{\"yg2mxt\", []float64{64.2755126953, 64.2810058594, 124.431152344, 124.442138672}},\n\t\t{\"7r5kh\", []float64{-4.921875, -4.8779296875, -29.00390625, -28.9599609375}},\n\t\t{\"cvg6t\", []float64{77.783203125, 77.8271484375, -96.4599609375, -96.416015625}},\n\t\t{\"msj4q7e3db\", []float64{-22.0850086212, -22.0850032568, 74.8104894161, 74.810500145}},\n\t\t{\"1452n4\", []float64{-78.7390136719, -78.7335205078, -130.166015625, -130.155029297}},\n\t\t{\"xj3kvwr2d\", []float64{30.4006290436, 30.4006719589, 137.009553909, 137.009596825}},\n\t\t{\"2wtvb\", []float64{-7.4267578125, -7.3828125, -149.4140625, -149.370117188}},\n\t\t{\"c5x5dhk\", []float64{65.3260803223, 65.3274536133, -125.062866211, -125.06149292}},\n\t\t{\"eph56u6\", []float64{39.9696350098, 39.9710083008, -39.2514038086, -39.2500305176}},\n\t\t{\"5dhyg0hckf\", []float64{-77.5632512569, -77.5632458925, -15.6817495823, -15.6817388535}},\n\t\t{\"9vrb\", []float64{29.53125, 29.70703125, -90.3515625, -90.0}},\n\t\t{\"b7nsys\", []float64{62.7319335938, 62.7374267578, -159.323730469, -159.312744141}},\n\t\t{\"kmenbsk0\", []float64{-12.8526306152, -12.8524589539, 15.4962158203, 15.4965591431}},\n\t\t{\"j35\", []float64{-84.375, -82.96875, 60.46875, 61.875}},\n\t\t{\"e7sx\", []float64{20.91796875, 21.09375, -27.421875, -27.0703125}},\n\t\t{\"h87mpg\", []float64{-87.6983642578, -87.6928710938, 27.4108886719, 27.421875}},\n\t\t{\"qjbtp\", []float64{-11.77734375, -11.7333984375, 91.0107421875, 91.0546875}},\n\t\t{\"4zqs2pndx\", []float64{-48.4327983856, -48.4327554703, -47.100148201, -47.1001052856}},\n\t\t{\"1fsc5v3\", []float64{-75.7328796387, -75.7315063477, -94.4041442871, -94.4027709961}},\n\t\t{\"kp6hptx\", []float64{-3.48541259766, -3.48403930664, 3.15170288086, 3.15307617188}},\n\t\t{\"3n77\", []float64{-9.31640625, -9.140625, -130.4296875, -130.078125}},\n\t\t{\"q347uc\", []float64{-38.7103271484, -38.7048339844, 104.622802734, 104.633789062}},\n\t\t{\"n8gckvg3\", []float64{-85.5297660828, -85.5295944214, 117.98664093, 117.986984253}},\n\t\t{\"p7szbr6ceq\", []float64{-68.9100801945, -68.9100748301, 152.944589853, 152.944600582}},\n\t\t{\"8w7n\", []float64{36.2109375, 36.38671875, -153.28125, -152.9296875}},\n\t\t{\"k4s3ndj8\", []float64{-30.7507324219, -30.7505607605, 6.26976013184, 6.27010345459}},\n\t\t{\"fh38ev\", []float64{69.0216064453, 69.0270996094, -87.7258300781, -87.71484375}},\n\t\t{\"rzebrsw\", []float64{-2.74383544922, -2.7424621582, 174.36126709, 174.362640381}},\n\t\t{\"un\", []float64{78.75, 84.375, 0.0, 11.25}},\n\t\t{\"27u3d4ybbkt\", []float64{-23.6273190379, -23.6273176968, -162.676259726, -162.676258385}},\n\t\t{\"5hk2\", []float64{-66.09375, -65.91796875, -39.0234375, -38.671875}},\n\t\t{\"62f6wqsbfc6n\", []float64{-40.3059548512, -40.3059546836, -75.3046354651, -75.3046351299}},\n\t\t{\"r6jvqxczv\", []float64{-32.7832460403, -32.783203125, 154.624199867, 154.624242783}},\n\t\t{\"wyg1es5yx\", []float64{38.2555103302, 38.2555532455, 128.128008842, 128.128051758}},\n\t\t{\"smp9qbcpnt\", []float64{28.3500748873, 28.3500802517, 22.0951581001, 22.095168829}},\n\t\t{\"4q46h\", []float64{-55.8984375, -55.8544921875, -75.41015625, -75.3662109375}},\n\t\t{\"9u6v9g2ebcm5\", []float64{24.8915505968, 24.8915507644, -97.3051826656, -97.3051823303}},\n\t\t{\"25mwbs3\", []float64{-25.5088806152, -25.5075073242, -172.242279053, -172.240905762}},\n\t\t{\"kwf\", []float64{-7.03125, -5.625, 25.3125, 26.71875}},\n\t\t{\"ekqgkknev\", []float64{24.5001554489, 24.5001983643, -24.0619039536, -24.0618610382}},\n\t\t{\"y9974617cb12\", []float64{53.9764738083, 53.9764739759, 114.358482845, 114.35848318}},\n\t\t{\"htuu1sxcpd\", []float64{-56.9282233715, -56.9282180071, 29.2565703392, 29.256581068}},\n\t\t{\"150sv8q\", []float64{-72.2886657715, -72.2872924805, -134.046936035, -134.045562744}},\n\t\t{\"j36p6wnmqm\", []float64{-81.6604489088, -81.6604435444, 59.181214571, 59.1812252998}},\n\t\t{\"py\", []float64{-56.25, -50.625, 168.75, 180.0}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"y87\", []float64{46.40625, 47.8125, 116.71875, 118.125}},\n\t\t{\"6v90bwn999\", []float64{-13.8974422216, -13.8974368572, -54.8127865791, -54.8127758503}},\n\t\t{\"8kyq\", []float64{27.7734375, 27.94921875, -159.9609375, -159.609375}},\n\t\t{\"8cht765b92zg\", []float64{6.55892824754, 6.55892841518, -139.773838855, -139.77383852}},\n\t\t{\"nu5jwk23v7f\", []float64{-66.5095366538, -66.5095353127, 128.243979514, 128.243980855}},\n\t\t{\"0m10969\", []float64{-61.7733764648, -61.7720031738, -167.287445068, -167.286071777}},\n\t\t{\"s29h49frdp\", []float64{3.52656304836, 3.52656841278, 12.7692890167, 12.7692997456}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"brmmeg2z8r\", []float64{86.7672246695, 86.7672300339, -161.201351881, -161.201341152}},\n\t\t{\"r3bngg619d\", []float64{-33.9516055584, -33.951600194, 146.417605877, 146.417616606}},\n\t\t{\"rmz76wced8c\", []float64{-12.0472772419, -12.0472759008, 156.557344347, 156.557345688}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"2h\", []float64{-22.5, -16.875, -180.0, -168.75}},\n\t\t{\"ty2764x\", []float64{35.7412719727, 35.7426452637, 79.1990661621, 79.2004394531}},\n\t\t{\"5yh3330r\", []float64{-56.0235786438, -56.0234069824, -5.21816253662, -5.21781921387}},\n\t\t{\"9szz\", []float64{27.94921875, 28.125, -101.6015625, -101.25}},\n\t\t{\"x7d41b\", []float64{20.0390625, 20.0445556641, 149.139404297, 149.150390625}},\n\t\t{\"dw\", []float64{33.75, 39.375, -67.5, -56.25}},\n\t\t{\"gnd4cw\", []float64{82.0788574219, 82.0843505859, -42.1215820312, -42.1105957031}},\n\t\t{\"k9bxc2n8\", []float64{-33.7939453125, -33.7937736511, 23.2669830322, 23.267326355}},\n\t\t{\"hump4nk\", []float64{-64.8289489746, -64.8275756836, 40.8746337891, 40.8760070801}},\n\t\t{\"gkz\", []float64{71.71875, 73.125, -23.90625, -22.5}},\n\t\t{\"g9e08yt\", []float64{53.5610961914, 53.5624694824, -18.2414245605, -18.2400512695}},\n\t\t{\"3eyuyzpm\", []float64{-23.0319786072, -23.0318069458, -102.701225281, -102.700881958}},\n\t\t{\"utpuc59p4m\", []float64{73.9804154634, 73.9804208279, 33.443852663, 33.4438633919}},\n\t\t{\"cnqt\", []float64{81.03515625, 81.2109375, -125.859375, -125.5078125}},\n\t\t{\"z05xfy72gk0\", []float64{46.3967871666, 46.3967885077, 140.04732728, 140.047328621}},\n\t\t{\"skr4zd\", []float64{24.4006347656, 24.4061279297, 21.4233398438, 21.4343261719}},\n\t\t{\"h2fe8mdnq\", []float64{-85.1347303391, -85.1346874237, 14.7796154022, 14.7796583176}},\n\t\t{\"z18b\", []float64{53.4375, 53.61328125, 136.0546875, 136.40625}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"vh\", []float64{67.5, 73.125, 45.0, 56.25}},\n\t\t{\"v64zfspngxw\", []float64{57.6354762912, 57.6354776323, 60.2368220687, 60.2368234098}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"d800\", []float64{0.0, 0.17578125, -67.5, -67.1484375}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"bnugeje1\", []float64{83.6143684387, 83.6145401001, -173.184356689, -173.184013367}},\n\t\t{\"46\", []float64{-78.75, -73.125, -78.75, -67.5}},\n\t\t{\"8rsncsbz2mx4\", []float64{43.4013903514, 43.401390519, -163.058031946, -163.058031611}},\n\t\t{\"t4w9wh7f98je\", []float64{14.3499474786, 14.3499476463, 54.4095184654, 54.4095188007}},\n\t\t{\"nsr1tp2\", []float64{-65.7902526855, -65.7888793945, 122.563476562, 122.564849854}},\n\t\t{\"9me\", []float64{30.9375, 32.34375, -119.53125, -118.125}},\n\t\t{\"t8250bh2y\", []float64{1.93372249603, 1.93376541138, 67.5390529633, 67.5390958786}},\n\t\t{\"1\", []float64{-90.0, -45.0, -135.0, -90.0}},\n\t\t{\"uqty0nmvvj0c\", []float64{82.652533818, 82.6525339857, 19.3440495059, 19.3440498412}},\n\t\t{\"7nkxt\", []float64{-8.525390625, -8.4814453125, -38.4521484375, -38.408203125}},\n\t\t{\"jzev\", []float64{-46.93359375, -46.7578125, 84.0234375, 84.375}},\n\t\t{\"dmtj1fktxe\", []float64{31.8297261, 31.8297314644, -71.6353440285, -71.6353332996}},\n\t\t{\"0r\", []float64{-50.625, -45.0, -168.75, -157.5}},\n\t\t{\"5hqv273y\", []float64{-65.152015686, -65.1518440247, -35.4944229126, -35.4940795898}},\n\t\t{\"78k\", []float64{-43.59375, -42.1875, -16.875, -15.46875}},\n\t\t{\"f2krt5\", []float64{47.7410888672, 47.7465820312, -72.5537109375, -72.5427246094}},\n\t\t{\"ffw63hhzm2u\", []float64{59.481229037, 59.4812303782, -47.4102383852, -47.4102370441}},\n\t\t{\"mv3z5k\", []float64{-14.2163085938, -14.2108154297, 81.3537597656, 81.3647460938}},\n\t\t{\"f15\", []float64{50.625, 52.03125, -85.78125, -84.375}},\n\t\t{\"j710re25dhzs\", []float64{-73.0625749379, -73.0625747703, 57.9859357327, 57.985936068}},\n\t\t{\"rtt328k93\", []float64{-13.8411855698, -13.8411426544, 164.911007881, 164.911050797}},\n\t\t{\"d2x003t048\", []float64{2.82073974609, 2.82074511051, -68.8882899284, -68.8882791996}},\n\t\t{\"22uy0\", []float64{-39.7265625, -39.6826171875, -162.0703125, -162.026367188}},\n\t\t{\"tuzxx\", []float64{28.037109375, 28.0810546875, 89.6044921875, 89.6484375}},\n\t\t{\"su44fdqr2pv1\", []float64{22.9970443435, 22.9970445111, 36.6809530556, 36.6809533909}},\n\t\t{\"yttq\", []float64{76.9921875, 77.16796875, 119.8828125, 120.234375}},\n\t\t{\"9fu5vyr\", []float64{16.1622619629, 16.1636352539, -95.362701416, -95.361328125}},\n\t\t{\"zwmzk37wsh97\", []float64{81.4386709593, 81.438671127, 165.777684934, 165.77768527}},\n\t\t{\"hd777ygzewj\", []float64{-76.7340624332, -76.7340610921, 27.2404141724, 27.2404155135}},\n\t\t{\"0b1\", []float64{-90.0, -88.59375, -144.84375, -143.4375}},\n\t\t{\"ejmgd\", []float64{30.146484375, 30.1904296875, -36.826171875, -36.7822265625}},\n\t\t{\"rzxh6\", []float64{-2.0654296875, -2.021484375, 178.681640625, 178.725585938}},\n\t\t{\"0rf4f4u\", []float64{-45.9077453613, -45.9063720703, -165.844116211, -165.84274292}},\n\t\t{\"1m23ggbv8\", []float64{-60.1395893097, -60.1395463943, -123.23261261, -123.232569695}},\n\t\t{\"gdvt2y6wfv\", []float64{61.4271193743, 61.4271247387, -14.7291147709, -14.7291040421}},\n\t\t{\"wxb3eqeug0y0\", []float64{43.8939468563, 43.8939470239, 112.9996714, 112.999671735}},\n\t\t{\"516kngbm6\", []float64{-82.2441244125, -82.2440814972, -41.5388774872, -41.5388345718}},\n\t\t{\"xtuwe2zu\", []float64{33.4911346436, 33.4913063049, 163.981590271, 163.981933594}},\n\t\t{\"dhb1c2jrz6c1\", []float64{27.027712483, 27.0277126506, -89.9375461042, -89.9375457689}},\n\t\t{\"23c397n4v8g\", []float64{-34.8756225407, -34.8756211996, -166.928776056, -166.928774714}},\n\t\t{\"1w81bnmc3\", []float64{-53.0953359604, -53.095293045, -112.492060661, -112.492017746}},\n\t\t{\"03hu77r\", []float64{-83.6100769043, -83.6087036133, -161.917877197, -161.916503906}},\n\t\t{\"z2vrm\", []float64{50.4931640625, 50.537109375, 153.852539062, 153.896484375}},\n\t\t{\"q630e\", []float64{-32.255859375, -32.2119140625, 102.788085938, 102.83203125}},\n\t\t{\"h9uzt\", []float64{-78.837890625, -78.7939453125, 29.3994140625, 29.443359375}},\n\t\t{\"x09c9jz\", []float64{3.10775756836, 3.10913085938, 137.51449585, 137.515869141}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"xv9ee3gq2j\", []float64{31.5634471178, 31.5634524822, 171.006660461, 171.00667119}},\n\t\t{\"6e8xjp4\", []float64{-24.0435791016, -24.0422058105, -66.5744018555, -66.5730285645}},\n\t\t{\"0m8sj439bys\", []float64{-58.3466801047, -58.3466787636, -167.82505095, -167.825049609}},\n\t\t{\"khf7yq8js6\", []float64{-17.5854098797, -17.5854045153, 3.43890309334, 3.43891382217}},\n\t\t{\"hh\", []float64{-67.5, -61.875, 0.0, 11.25}},\n\t\t{\"kcyx9b0rd65e\", []float64{-33.8365919329, -33.8365917653, 42.967973873, 42.9679742083}},\n\t\t{\"qcy4pees\", []float64{-34.7847747803, -34.7846031189, 132.521896362, 132.522239685}},\n\t\t{\"tc6\", []float64{7.03125, 8.4375, 81.5625, 82.96875}},\n\t\t{\"mxhnk2fkh\", []float64{-4.52156066895, -4.5215177536, 73.3150291443, 73.3150720596}},\n\t\t{\"1mggmg5x0k\", []float64{-57.067258358, -57.0672529936, -118.219059706, -118.219048977}},\n\t\t{\"f1udt102z\", []float64{55.2888250351, 55.2888679504, -83.4515047073, -83.451461792}},\n\t\t{\"jjz\", []float64{-57.65625, -56.25, 54.84375, 56.25}},\n\t\t{\"1q1dg8peze2\", []float64{-55.765940398, -55.7659390569, -121.476194859, -121.476193517}},\n\t\t{\"604unch\", []float64{-44.2913818359, -44.2900085449, -85.8306884766, -85.8293151855}},\n\t\t{\"kt\", []float64{-16.875, -11.25, 22.5, 33.75}},\n\t\t{\"wpbgc\", []float64{44.2529296875, 44.296875, 91.0986328125, 91.142578125}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"tz2pky3c\", []float64{42.0901679993, 42.0903396606, 78.9611434937, 78.9614868164}},\n\t\t{\"j96v3y\", []float64{-82.0129394531, -82.0074462891, 71.4440917969, 71.455078125}},\n\t\t{\"vs974dv\", []float64{70.8549499512, 70.8563232422, 69.3745422363, 69.3759155273}},\n\t\t{\"dunwchdt7d6\", []float64{23.712155968, 23.7121573091, -47.061843574, -47.0618422329}},\n\t\t{\"h2u2nkksw\", []float64{-85.7571315765, -85.7570886612, 17.5076580048, 17.5077009201}},\n\t\t{\"z8qtu1\", []float64{47.4224853516, 47.4279785156, 166.81640625, 166.827392578}},\n\t\t{\"2677dsdngh\", []float64{-31.7026162148, -31.7026108503, -164.066948891, -164.066938162}},\n\t\t{\"4yy\", []float64{-52.03125, -50.625, -47.8125, -46.40625}},\n\t\t{\"uym\", []float64{80.15625, 81.5625, 40.78125, 42.1875}},\n\t\t{\"m2mssd\", []float64{-42.7917480469, -42.7862548828, 64.1821289062, 64.1931152344}},\n\t\t{\"xed3b\", []float64{19.9951171875, 20.0390625, 160.6640625, 160.708007812}},\n\t\t{\"rfp4ky\", []float64{-33.3215332031, -33.3160400391, 178.802490234, 178.813476562}},\n\t\t{\"83fmm9e\", []float64{10.7748413086, 10.7762145996, -165.340118408, -165.338745117}},\n\t\t{\"tr7z2dqh\", []float64{42.0687103271, 42.0688819885, 61.5536499023, 61.5539932251}},\n\t\t{\"crsh\", []float64{87.890625, 88.06640625, -118.125, -117.7734375}},\n\t\t{\"f\", []float64{45.0, 90.0, -90.0, -45.0}},\n\t\t{\"761m6h\", []float64{-32.8051757812, -32.7996826172, -31.904296875, -31.8933105469}},\n\t\t{\"7p\", []float64{-5.625, 0.0, -45.0, -33.75}},\n\t\t{\"8qu9r5\", []float64{38.2049560547, 38.2104492188, -162.114257812, -162.103271484}},\n\t\t{\"5z\", []float64{-50.625, -45.0, -11.25, 0.0}},\n\t\t{\"kbz\", []float64{-40.78125, -39.375, 43.59375, 45.0}},\n\t\t{\"zdftsw1rbpsf\", []float64{61.4698768035, 61.4698769711, 161.21510189, 161.215102226}},\n\t\t{\"n1m105r\", []float64{-82.7751159668, -82.7737426758, 97.0408630371, 97.0422363281}},\n\t\t{\"xf4kktnxsz\", []float64{12.0258611441, 12.0258665085, 172.120946646, 172.120957375}},\n\t\t{\"kzqyq4m0qtyu\", []float64{-3.10768313706, -3.10768296942, 43.5130138323, 43.5130141675}},\n\t\t{\"1j1vy57w7\", []float64{-60.8453321457, -60.8452892303, -132.27045536, -132.270412445}},\n\t\t{\"nq4u3\", []float64{-55.5029296875, -55.458984375, 105.161132812, 105.205078125}},\n\t\t{\"bhcy\", []float64{72.7734375, 72.94921875, -177.5390625, -177.1875}},\n\t\t{\"vjr\", []float64{74.53125, 75.9375, 54.84375, 56.25}},\n\t\t{\"uc99nrdsntv\", []float64{53.6551974714, 53.6551988125, 36.1377520859, 36.137753427}},\n\t\t{\"3e8zmnz\", []float64{-24.0010070801, -23.9996337891, -111.2159729, -111.214599609}},\n\t\t{\"j0yzdvs9\", []float64{-84.4325065613, -84.4323348999, 54.6192169189, 54.6195602417}},\n\t\t{\"8chvm4\", []float64{6.55883789062, 6.56433105469, -139.350585938, -139.339599609}},\n\t\t{\"ywf6099wx\", []float64{83.329668045, 83.3297109604, 115.6883955, 115.688438416}},\n\t\t{\"b\", []float64{45.0, 90.0, -180.0, -135.0}},\n\t\t{\"zrc\", []float64{88.59375, 90.0, 147.65625, 149.0625}},\n\t\t{\"zq\", []float64{78.75, 84.375, 146.25, 157.5}},\n\t\t{\"7xznu50ru\", []float64{-0.201916694641, -0.201873779297, -12.4799537659, -12.4799108505}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"7r\", []float64{-5.625, 0.0, -33.75, -22.5}},\n\t\t{\"f27k\", []float64{47.109375, 47.28515625, -74.1796875, -73.828125}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"b4s13188yv\", []float64{59.2906218767, 59.2906272411, -174.330078363, -174.330067635}},\n\t\t{\"q8\", []float64{-45.0, -39.375, 112.5, 123.75}},\n\t\t{\"skn7w\", []float64{23.115234375, 23.1591796875, 20.302734375, 20.3466796875}},\n\t\t{\"1tzqwzyh5vp\", []float64{-56.4703863859, -56.4703850448, -101.999646574, -101.999645233}},\n\t\t{\"r52cmn\", []float64{-26.4660644531, -26.4605712891, 136.274414062, 136.285400391}},\n\t\t{\"7bwvbbbru\", []float64{-41.1713075638, -41.1712646484, -1.72433853149, -1.72429561615}},\n\t\t{\"ruv0b1n\", []float64{-18.1439208984, -18.1425476074, 175.789489746, 175.790863037}},\n\t\t{\"vwf01ujm\", []float64{82.9915809631, 82.9917526245, 70.3966140747, 70.3969573975}},\n\t\t{\"0metgxjtrm91\", []float64{-58.0123747699, -58.0123746023, -163.666450828, -163.666450493}},\n\t\t{\"2w\", []float64{-11.25, -5.625, -157.5, -146.25}},\n\t\t{\"8kmch3\", []float64{24.0875244141, 24.0930175781, -160.477294922, -160.466308594}},\n\t\t{\"g6m\", []float64{57.65625, 59.0625, -26.71875, -25.3125}},\n\t\t{\"t6v4k2\", []float64{15.8642578125, 15.8697509766, 63.4680175781, 63.4790039062}},\n\t\t{\"zr02vfju8hd\", []float64{84.5186188817, 84.5186202228, 146.862147152, 146.862148494}},\n\t\t{\"kb8jn751\", []float64{-41.2919425964, -41.2917709351, 34.0287780762, 34.0291213989}},\n\t\t{\"mj76\", []float64{-15.1171875, -14.94140625, 49.5703125, 49.921875}},\n\t\t{\"f0hwcbkwjnr\", []float64{46.1889602244, 46.1889615655, -83.5885669291, -83.588565588}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"6rt9btpxj2p1\", []float64{-2.47621519491, -2.47621502727, -70.9831179678, -70.9831176326}},\n\t\t{\"wh0s6yb6j\", []float64{23.2844924927, 23.284535408, 90.8245325089, 90.8245754242}},\n\t\t{\"65b\", []float64{-23.90625, -22.5, -90.0, -88.59375}},\n\t\t{\"r\", []float64{-45.0, 0.0, 135.0, 180.0}},\n\t\t{\"z0tpc5d\", []float64{49.1940307617, 49.1954040527, 142.077941895, 142.079315186}},\n\t\t{\"fgs2w0\", []float64{64.775390625, 64.7808837891, -50.009765625, -49.9987792969}},\n\t\t{\"vfus35qgp\", []float64{61.2341880798, 61.2342309952, 85.1316404343, 85.1316833496}},\n\t\t{\"hywnkqdye\", []float64{-52.3020458221, -52.3020029068, 42.3781728745, 42.3782157898}},\n\t\t{\"qpwxwun0b\", []float64{-1.47203922272, -1.47199630737, 99.4454956055, 99.4455385208}},\n\t\t{\"v88u9rqr24\", []float64{48.6445963383, 48.6446017027, 68.6182022095, 68.6182129383}},\n\t\t{\"17x06x4fyw\", []float64{-70.2295982838, -70.2295929193, -113.792331219, -113.79232049}},\n\t\t{\"3ykkhjyhrt\", []float64{-9.1082829237, -9.10827755928, -95.0890946388, -95.08908391}},\n\t\t{\"mbzkuqw5\", []float64{-39.910068512, -39.9098968506, 89.1403198242, 89.140663147}},\n\t\t{\"yqjf0p3mn\", []float64{79.1422462463, 79.1422891617, 109.337911606, 109.337954521}},\n\t\t{\"2f\", []float64{-33.75, -28.125, -146.25, -135.0}},\n\t\t{\"hqsm3\", []float64{-52.5146484375, -52.470703125, 17.2705078125, 17.314453125}},\n\t\t{\"gp1y4wtft1tp\", []float64{85.4658314399, 85.4658316076, -42.4210815132, -42.4210811779}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"kxxruvh\", []float64{-1.42272949219, -1.42135620117, 32.9095458984, 32.9109191895}},\n\t\t{\"08h0ft4vk97r\", []float64{-89.839789141, -89.8397889733, -151.761162691, -151.761162356}},\n\t\t{\"y9u8\", []float64{54.84375, 55.01953125, 118.828125, 119.1796875}},\n\t\t{\"3zhsk\", []float64{-4.8779296875, -4.833984375, -94.74609375, -94.7021484375}},\n\t\t{\"x2\", []float64{0.0, 5.625, 146.25, 157.5}},\n\t\t{\"mr7nnvr\", []float64{-3.13522338867, -3.13385009766, 60.7749938965, 60.7763671875}},\n\t\t{\"d3g0pn54hr\", []float64{9.87708985806, 9.87709522247, -74.2193305492, -74.2193198204}},\n\t\t{\"jv173u\", []float64{-61.2817382812, -61.2762451172, 80.5847167969, 80.595703125}},\n\t\t{\"vte\", []float64{75.9375, 77.34375, 71.71875, 73.125}},\n\t\t{\"303un61j2s\", []float64{-42.878715992, -42.8787106276, -132.263009548, -132.262998819}},\n\t\t{\"m528h\", []float64{-26.71875, -26.6748046875, 45.87890625, 45.9228515625}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"p8h\", []float64{-90.0, -88.59375, 163.125, 164.53125}},\n\t\t{\"tzusgm6v33y\", []float64{44.4584606588, 44.4584619999, 85.2247855067, 85.2247868478}},\n\t\t{\"26sre\", []float64{-29.619140625, -29.5751953125, -162.641601562, -162.59765625}},\n\t\t{\"q1tcnkmh55b\", []float64{-36.3626660407, -36.3626646996, 98.3675909042, 98.3675922453}},\n\t\t{\"xs4t1g3\", []float64{23.3967590332, 23.3981323242, 161.093902588, 161.095275879}},\n\t\t{\"td\", []float64{11.25, 16.875, 67.5, 78.75}},\n\t\t{\"xdvwxsmsjd\", []float64{16.6353714466, 16.635376811, 165.571753979, 165.571764708}},\n\t\t{\"dnw\", []float64{36.5625, 37.96875, -81.5625, -80.15625}},\n\t\t{\"u7nu2ff43vx\", []float64{62.6375922561, 62.6375935972, 20.777977556, 20.7779788971}},\n\t\t{\"0jy2\", []float64{-57.65625, -57.48046875, -171.2109375, -170.859375}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"2yx2k\", []float64{-8.3935546875, -8.349609375, -135.87890625, -135.834960938}},\n\t\t{\"g3\", []float64{50.625, 56.25, -33.75, -22.5}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"vrrpw34jfjs\", []float64{87.1061190963, 87.1061204374, 66.3712459803, 66.3712473214}},\n\t\t{\"g2gkzeutg\", []float64{50.0752973557, 50.075340271, -28.8437891006, -28.8437461853}},\n\t\t{\"z8ett4zh\", []float64{48.7950897217, 48.7952613831, 162.6512146, 162.651557922}},\n\t\t{\"cyspv2k0\", []float64{82.9261779785, 82.9263496399, -95.3887939453, -95.3884506226}},\n\t\t{\"fqu7qbshmr\", []float64{83.5435527563, 83.5435581207, -72.471088171, -72.4710774422}},\n\t\t{\"578u4ww7\", []float64{-69.5731544495, -69.5729827881, -32.5768661499, -32.5765228271}},\n\t\t{\"4gzkwxrzb\", []float64{-68.0740785599, -68.0740356445, -45.7583999634, -45.758357048}},\n\t\t{\"g0z038npm1ym\", []float64{49.2639500834, 49.263950251, -35.0818693265, -35.0818689913}},\n\t\t{\"vrse4ey62b1y\", []float64{87.7358303592, 87.7358305268, 62.6966058835, 62.6966062188}},\n\t\t{\"7x2fkbbj6\", []float64{-3.81822109222, -3.81817817688, -21.2364864349, -21.2364435196}},\n\t\t{\"tuvfdxy7b\", []float64{27.2014188766, 27.201461792, 86.9543838501, 86.9544267654}},\n\t\t{\"9qzggb\", []float64{38.6279296875, 38.6334228516, -112.686767578, -112.67578125}},\n\t\t{\"pupf6\", []float64{-67.1044921875, -67.060546875, 179.736328125, 179.780273438}},\n\t\t{\"fknyrbe41k0w\", []float64{68.6017451808, 68.6017453484, -68.9130621403, -68.9130618051}},\n\t\t{\"591vk5jz7x4v\", []float64{-83.4343860112, -83.4343858436, -19.8552309349, -19.8552305996}},\n\t\t{\"n4psdv\", []float64{-77.9315185547, -77.9260253906, 100.667724609, 100.678710938}},\n\t\t{\"zyw2\", []float64{81.5625, 81.73828125, 177.5390625, 177.890625}},\n\t\t{\"r0e29\", []float64{-42.099609375, -42.0556640625, 139.614257812, 139.658203125}},\n\t\t{\"j9cq7f\", []float64{-79.0466308594, -79.0411376953, 69.4226074219, 69.43359375}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"m42x20h\", []float64{-31.0693359375, -31.0679626465, 45.7086181641, 45.7099914551}},\n\t\t{\"405zg3kz2j\", []float64{-88.6295574903, -88.6295521259, -84.5772171021, -84.5772063732}},\n\t\t{\"7vu7httsdm\", []float64{-12.0978945494, -12.097889185, -5.06803393364, -5.0680232048}},\n\t\t{\"prznz91\", []float64{-45.2142333984, -45.2128601074, 156.424713135, 156.426086426}},\n\t\t{\"46dupvqwbsuj\", []float64{-75.2043508552, -75.2043506876, -74.5332831144, -74.5332827792}},\n\t\t{\"7s0\", []float64{-22.5, -21.09375, -22.5, -21.09375}},\n\t\t{\"5nhz5er2nnm5\", []float64{-55.0016444363, -55.0016442686, -38.1562833488, -38.1562830135}},\n\t\t{\"5qqp\", []float64{-53.61328125, -53.4375, -25.3125, -24.9609375}},\n\t\t{\"t4dqn2\", []float64{15.1171875, 15.1226806641, 48.4387207031, 48.4497070312}},\n\t\t{\"gmxt\", []float64{76.81640625, 76.9921875, -23.203125, -22.8515625}},\n\t\t{\"syuh0ke1syh\", []float64{38.6968839169, 38.696885258, 39.3903154135, 39.3903167546}},\n\t\t{\"nvr0\", []float64{-60.46875, -60.29296875, 133.59375, 133.9453125}},\n\t\t{\"g4\", []float64{56.25, 61.875, -45.0, -33.75}},\n\t\t{\"gnd2kb6w0s32\", []float64{81.6088713706, 81.6088715382, -41.623740904, -41.6237405688}},\n\t\t{\"x7hptvsgdvu\", []float64{18.2242034376, 18.2242047787, 152.134332061, 152.134333402}},\n\t\t{\"2fc8\", []float64{-29.53125, -29.35546875, -144.140625, -143.7890625}},\n\t\t{\"e3fsu48yte7\", []float64{10.693577081, 10.6935784221, -30.057323724, -30.0573223829}},\n\t\t{\"pbqhjvqh\", []float64{-87.8610992432, -87.8609275818, 177.448425293, 177.448768616}},\n\t\t{\"sqx1nwgn\", []float64{36.7763900757, 36.7765617371, 21.3835144043, 21.3838577271}},\n\t\t{\"e6g\", []float64{15.46875, 16.875, -29.53125, -28.125}},\n\t\t{\"dt\", []float64{28.125, 33.75, -67.5, -56.25}},\n\t\t{\"02s64yzk\", []float64{-86.7981719971, -86.7980003357, -162.642631531, -162.642288208}},\n\t\t{\"z9\", []float64{50.625, 56.25, 157.5, 168.75}},\n\t\t{\"fjv\", []float64{77.34375, 78.75, -82.96875, -81.5625}},\n\t\t{\"5yc0\", []float64{-52.03125, -51.85546875, -9.84375, -9.4921875}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"k3x3j1pmdnfp\", []float64{-36.3802440651, -36.3802438974, 21.6750839353, 21.6750842705}},\n\t\t{\"g2pjds\", []float64{45.9887695312, 45.9942626953, -23.7963867188, -23.7854003906}},\n\t\t{\"ppw0p3q2gzbk\", []float64{-47.8054625541, -47.8054623865, 143.764847852, 143.764848188}},\n\t\t{\"9xwp0kprs0x6\", []float64{43.4412318841, 43.4412320517, -104.041375928, -104.041375592}},\n\t\t{\"0gzww7fv7gj\", []float64{-67.7421551943, -67.7421538532, -135.424522609, -135.424521267}},\n\t\t{\"sgpt2\", []float64{17.7978515625, 17.841796875, 44.296875, 44.3408203125}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"rr\", []float64{-5.625, 0.0, 146.25, 157.5}},\n\t\t{\"wkqrtvj5\", []float64{25.2525901794, 25.2527618408, 110.298614502, 110.298957825}},\n\t\t{\"ngtfyx77u\", []float64{-69.7886323929, -69.7885894775, 132.126216888, 132.126259804}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"58y\", []float64{-85.78125, -84.375, -14.0625, -12.65625}},\n\t\t{\"mvjrxvx\", []float64{-15.5264282227, -15.5250549316, 86.483001709, 86.484375}},\n\t\t{\"jgve4ttu\", []float64{-68.3480072021, -68.3478355408, 86.6021347046, 86.6024780273}},\n\t\t{\"h9\", []float64{-84.375, -78.75, 22.5, 33.75}},\n\t\t{\"ytf9sb6mj27\", []float64{77.609654814, 77.6096561551, 116.227684468, 116.227685809}},\n\t\t{\"rk9kv3q\", []float64{-18.8456726074, -18.8442993164, 148.246765137, 148.248138428}},\n\t\t{\"kbq8t\", []float64{-43.505859375, -43.4619140625, 43.1103515625, 43.154296875}},\n\t\t{\"j8prqp8kx\", []float64{-88.6836147308, -88.6835718155, 77.9596281052, 77.9596710205}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"mq53k\", []float64{-11.0302734375, -10.986328125, 60.99609375, 61.0400390625}},\n\t\t{\"d95fw\", []float64{6.064453125, 6.1083984375, -61.962890625, -61.9189453125}},\n\t\t{\"x15j3\", []float64{6.5478515625, 6.591796875, 139.262695312, 139.306640625}},\n\t\t{\"x0yg7k2b\", []float64{4.81338500977, 4.81355667114, 144.636039734, 144.636383057}},\n\t\t{\"dx8x9yxwprk5\", []float64{43.5426343046, 43.5426344723, -66.7093545198, -66.7093541846}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"429ptwcscc3\", []float64{-85.8312396705, -85.8312383294, -77.0999144018, -77.0999130607}},\n\t\t{\"ncknjt\", []float64{-81.8865966797, -81.8811035156, 129.616699219, 129.627685547}},\n\t\t{\"7ce8p70yc7h\", []float64{-36.5448457003, -36.5448443592, -6.00843250751, -6.00843116641}},\n\t\t{\"un1\", []float64{78.75, 80.15625, 1.40625, 2.8125}},\n\t\t{\"b6rv0shhd5\", []float64{58.5579174757, 58.5579228401, -157.824010849, -157.82400012}},\n\t\t{\"qyg8d\", []float64{-6.943359375, -6.8994140625, 128.759765625, 128.803710938}},\n\t\t{\"3p7f2mvx6\", []float64{-3.79041194916, -3.79036903381, -129.707937241, -129.707894325}},\n\t\t{\"u0vj\", []float64{50.09765625, 50.2734375, 7.03125, 7.3828125}},\n\t\t{\"49m\", []float64{-82.96875, -81.5625, -60.46875, -59.0625}},\n\t\t{\"7vhu8hp00j2\", []float64{-16.0619835556, -16.0619822145, -4.56069946289, -4.56069812179}},\n\t\t{\"3sh\", []float64{-22.5, -21.09375, -106.875, -105.46875}},\n\t\t{\"sexg40hc\", []float64{20.2150154114, 20.2151870728, 33.4928512573, 33.4931945801}},\n\t\t{\"4uwxw8u\", []float64{-63.365020752, -63.3636474609, -46.8182373047, -46.8168640137}},\n\t\t{\"mr\", []float64{-5.625, 0.0, 56.25, 67.5}},\n\t\t{\"1kymkgft79d3\", []float64{-62.3368896358, -62.3368894681, -114.748610817, -114.748610482}},\n\t\t{\"3tj\", []float64{-16.875, -15.46875, -105.46875, -104.0625}},\n\t\t{\"vxfzth5\", []float64{89.9340820312, 89.9354553223, 71.5910339355, 71.5924072266}},\n\t\t{\"h\", []float64{-90.0, -45.0, 0.0, 45.0}},\n\t\t{\"e5r9\", []float64{18.45703125, 18.6328125, -34.453125, -34.1015625}},\n\t\t{\"hq9bc8pmfk\", []float64{-53.3046555519, -53.3046501875, 13.7869083881, 13.786919117}},\n\t\t{\"05t\", []float64{-70.3125, -68.90625, -172.96875, -171.5625}},\n\t\t{\"ye6yg\", []float64{64.4677734375, 64.51171875, 116.499023438, 116.54296875}},\n\t\t{\"gg4k\", []float64{62.578125, 62.75390625, -8.0859375, -7.734375}},\n\t\t{\"50q2fcp\", []float64{-88.4564208984, -88.4550476074, -36.0804748535, -36.0791015625}},\n\t\t{\"2hu8d3131\", []float64{-18.1876945496, -18.1876516342, -173.571238518, -173.571195602}},\n\t\t{\"08gcdmy84ewg\", []float64{-85.4859731533, -85.4859729856, -152.118642814, -152.118642479}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"t05v4wzmqmet\", []float64{0.91691667214, 0.916916839778, 50.3935300559, 50.3935303912}},\n\t\t{\"3nc\", []float64{-7.03125, -5.625, -133.59375, -132.1875}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"3gvk6zm1869\", []float64{-23.1190833449, -23.1190820038, -93.7394593656, -93.7394580245}},\n\t\t{\"39rruq\", []float64{-36.5734863281, -36.5679931641, -102.117919922, -102.106933594}},\n\t\t{\"jhkxvqt7q5z\", []float64{-64.6951617301, -64.6951603889, 51.5663145483, 51.5663158894}},\n\t\t{\"bb\", []float64{45.0, 50.625, -146.25, -135.0}},\n\t\t{\"es26styfpb\", []float64{24.3776321411, 24.3776375055, -21.9410812855, -21.9410705566}},\n\t\t{\"500q98r9\", []float64{-88.8558769226, -88.8557052612, -44.5722198486, -44.5718765259}},\n\t\t{\"kv\", []float64{-16.875, -11.25, 33.75, 45.0}},\n\t\t{\"njf\", []float64{-57.65625, -56.25, 92.8125, 94.21875}},\n\t\t{\"whk5vu\", []float64{24.5874023438, 24.5928955078, 95.8776855469, 95.888671875}},\n\t\t{\"w9pq4yk4p4qf\", []float64{6.71437550336, 6.714375671, 122.821964733, 122.821965069}},\n\t\t{\"yt\", []float64{73.125, 78.75, 112.5, 123.75}},\n\t\t{\"ccztdek\", []float64{55.8283996582, 55.8297729492, -90.5877685547, -90.5863952637}},\n\t\t{\"ej33gfth2\", []float64{29.8533296585, 29.8533725739, -43.070526123, -43.0704832077}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"hqyfjm\", []float64{-51.6522216797, -51.6467285156, 20.9729003906, 20.9838867188}},\n\t\t{\"njcr\", []float64{-56.42578125, -56.25, 91.7578125, 92.109375}},\n\t\t{\"48ejbxwrk6\", []float64{-86.1343038082, -86.1342984438, -63.2505118847, -63.2505011559}},\n\t\t{\"78f1r09e5v8\", []float64{-40.558232367, -40.5582310259, -19.3776619434, -19.3776606023}},\n\t\t{\"ged8fvuhk31\", []float64{64.8516565561, 64.8516578972, -18.8578484952, -18.8578471541}},\n\t\t{\"8ss3fn\", []float64{25.6530761719, 25.6585693359, -151.435546875, -151.424560547}},\n\t\t{\"v90sp4e6\", []float64{51.3422012329, 51.3423728943, 68.5152053833, 68.5155487061}},\n\t\t{\"bx00h848\", []float64{84.375, 84.3751716614, -157.298812866, -157.298469543}},\n\t\t{\"9y3\", []float64{35.15625, 36.5625, -99.84375, -98.4375}},\n\t\t{\"ehpkg7\", []float64{23.3514404297, 23.3569335938, -34.6618652344, -34.6508789062}},\n\t\t{\"r38623wxhs\", []float64{-36.1575293541, -36.1575239897, 146.621668339, 146.621679068}},\n\t\t{\"x6yex2zx\", []float64{16.0893058777, 16.0894775391, 155.719528198, 155.719871521}},\n\t\t{\"r\", []float64{-45.0, 0.0, 135.0, 180.0}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"6w\", []float64{-11.25, -5.625, -67.5, -56.25}},\n\t\t{\"mx6g2d\", []float64{-3.63647460938, -3.63098144531, 71.3891601562, 71.4001464844}},\n\t\t{\"vmsh9b6f\", []float64{76.7302322388, 76.7304039001, 61.9556808472, 61.9560241699}},\n\t\t{\"7uqbsm728bjw\", []float64{-20.9769334272, -20.9769332595, -1.56654216349, -1.56654182822}},\n\t\t{\"nvtqqh0jc3\", []float64{-57.9409021139, -57.9408967495, 131.396538019, 131.396548748}},\n\t\t{\"x8\", []float64{0.0, 5.625, 157.5, 168.75}},\n\t\t{\"nqx5n\", []float64{-52.91015625, -52.8662109375, 111.357421875, 111.401367188}},\n\t\t{\"c4wmv60xtud\", []float64{60.0855401158, 60.0855414569, -125.979288518, -125.979287177}},\n\t\t{\"t9\", []float64{5.625, 11.25, 67.5, 78.75}},\n\t\t{\"e3nqrsk9fqdu\", []float64{6.74731470644, 6.74731487408, -24.6250675991, -24.6250672638}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"7x9\", []float64{-2.8125, -1.40625, -21.09375, -19.6875}},\n\t\t{\"760qb6yxf\", []float64{-32.5470399857, -32.5469970703, -33.3784389496, -33.3783960342}},\n\t\t{\"x7skftff\", []float64{20.5543899536, 20.554561615, 152.340202332, 152.340545654}},\n\t\t{\"jv774u\", []float64{-59.9194335938, -59.9139404297, 83.4411621094, 83.4521484375}},\n\t\t{\"phh7\", []float64{-66.97265625, -66.796875, 140.9765625, 141.328125}},\n\t\t{\"hv4tt0pymy\", []float64{-60.9070980549, -60.9070926905, 37.4962413311, 37.4962520599}},\n\t\t{\"3zemdehyubj\", []float64{-1.82806491852, -1.82806357741, -96.563090533, -96.5630891919}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"dep80ett\", []float64{16.8950843811, 16.8952560425, -56.9235992432, -56.9232559204}},\n\t\t{\"bbj\", []float64{45.0, 46.40625, -139.21875, -137.8125}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"9k1t0ecqh3v3\", []float64{23.4005451389, 23.4005453065, -121.616746299, -121.616745964}},\n\t\t{\"mm5\", []float64{-16.875, -15.46875, 60.46875, 61.875}},\n\t\t{\"jz4zmmv2\", []float64{-49.3190002441, -49.3188285828, 82.8551101685, 82.8554534912}},\n\t\t{\"bpshcry\", []float64{88.065032959, 88.06640625, -174.311828613, -174.310455322}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"e87b\", []float64{1.40625, 1.58203125, -17.2265625, -16.875}},\n\t\t{\"pcphekd1ph0\", []float64{-83.5590720177, -83.5590706766, 178.739619255, 178.739620596}},\n\t\t{\"nrwq0\", []float64{-46.7578125, -46.7138671875, 110.0390625, 110.083007812}},\n\t\t{\"6k3e7rk823cj\", []float64{-20.4825823568, -20.4825821891, -76.4916108549, -76.4916105196}},\n\t\t{\"kkv059k5v\", []float64{-18.2737398148, -18.2736968994, 18.4407663345, 18.4408092499}},\n\t\t{\"ep7x2t4t6x\", []float64{42.084068656, 42.0840740204, -40.0526118279, -40.052601099}},\n\t\t{\"43hh0\", []float64{-83.671875, -83.6279296875, -73.125, -73.0810546875}},\n\t\t{\"rdhfdg0qee\", []float64{-33.2929354906, -33.2929301262, 164.301030636, 164.301041365}},\n\t\t{\"znku45j7p\", []float64{80.8763694763, 80.8764123917, 141.77508831, 141.775131226}},\n\t\t{\"ju\", []float64{-67.5, -61.875, 78.75, 90.0}},\n\t\t{\"b6zckuz\", []float64{60.7145690918, 60.7159423828, -157.633209229, -157.631835938}},\n\t\t{\"fm7m\", []float64{75.41015625, 75.5859375, -74.1796875, -73.828125}},\n\t\t{\"8xg\", []float64{43.59375, 45.0, -153.28125, -151.875}},\n\t\t{\"wfk7q\", []float64{13.2275390625, 13.271484375, 129.990234375, 130.034179688}},\n\t\t{\"6r1p9yz6z9\", []float64{-4.26908433437, -4.26907896996, -77.2565674782, -77.2565567493}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"fuzw5\", []float64{72.7734375, 72.8173828125, -45.5712890625, -45.52734375}},\n\t\t{\"m33ehjv\", []float64{-37.4098205566, -37.4084472656, 58.5420227051, 58.5433959961}},\n\t\t{\"s6stp\", []float64{14.94140625, 14.9853515625, 17.8857421875, 17.9296875}},\n\t\t{\"tjxuvxh\", []float64{31.8109130859, 31.812286377, 56.1456298828, 56.1470031738}},\n\t\t{\"0vgxezccsf\", []float64{-56.2950503826, -56.2950450182, -141.160722971, -141.160712242}},\n\t\t{\"0h\", []float64{-67.5, -61.875, -180.0, -168.75}},\n\t\t{\"ge6bb\", []float64{63.4130859375, 63.45703125, -18.6328125, -18.5888671875}},\n\t\t{\"h2gyy9\", []float64{-84.5892333984, -84.5837402344, 16.8090820312, 16.8200683594}},\n\t\t{\"g0f7xg\", []float64{49.8504638672, 49.8559570312, -41.4953613281, -41.484375}},\n\t\t{\"ujfk5\", []float64{78.046875, 78.0908203125, 3.2958984375, 3.33984375}},\n\t\t{\"q0b6rwm0\", []float64{-40.3514099121, -40.3512382507, 90.6880187988, 90.6883621216}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"0nyhwsx2g5\", []float64{-51.2153702974, -51.215364933, -171.266770363, -171.266759634}},\n\t\t{\"mjpe\", []float64{-16.34765625, -16.171875, 55.546875, 55.8984375}},\n\t\t{\"mt\", []float64{-16.875, -11.25, 67.5, 78.75}},\n\t\t{\"z49vj0d4zz\", []float64{59.9446624517, 59.9446678162, 137.683743238, 137.683753967}},\n\t\t{\"zry97vd3gs\", []float64{88.8440108299, 88.8440161943, 155.55866003, 155.558670759}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"v3syrvc9\", []float64{54.5678901672, 54.5680618286, 63.2723236084, 63.2726669312}},\n\t\t{\"nrer\", []float64{-46.58203125, -46.40625, 105.8203125, 106.171875}},\n\t\t{\"2hqt8nf65v5e\", []float64{-20.0895036198, -20.0895034522, -170.856119469, -170.856119134}},\n\t\t{\"wekgdxc\", []float64{18.9390563965, 18.9404296875, 119.290924072, 119.292297363}},\n\t\t{\"6bh43q\", []float64{-44.5715332031, -44.5660400391, -50.5700683594, -50.5590820312}},\n\t\t{\"d564\", []float64{18.6328125, 18.80859375, -87.1875, -86.8359375}},\n\t\t{\"m85\", []float64{-45.0, -43.59375, 71.71875, 73.125}},\n\t\t{\"x4tj5rb77r\", []float64{14.9845737219, 14.9845790863, 142.174555063, 142.174565792}},\n\t\t{\"0bwycz8vr\", []float64{-85.9588766098, -85.9588336945, -136.679577827, -136.679534912}},\n\t\t{\"ehnu2e\", []float64{23.2635498047, 23.2690429688, -35.4858398438, -35.4748535156}},\n\t\t{\"jbe8mk\", []float64{-87.1215820312, -87.1160888672, 83.9025878906, 83.9135742188}},\n\t\t{\"ss\", []float64{22.5, 28.125, 22.5, 33.75}},\n\t\t{\"8edywpv3ygj3\", []float64{20.8729668148, 20.8729669824, -153.361634128, -153.361633793}},\n\t\t{\"xpxu1erq390\", []float64{42.9095560312, 42.9095573723, 145.974376202, 145.974377543}},\n\t\t{\"4043qb91kj\", []float64{-89.7772854567, -89.7772800922, -86.5377616882, -86.5377509594}},\n\t\t{\"2n7q0j0r\", []float64{-8.76039505005, -8.76022338867, -175.429344177, -175.429000854}},\n\t\t{\"cm0n96q3vn8t\", []float64{74.2802738585, 74.2802740261, -123.686270043, -123.686269708}},\n\t\t{\"50hgpxg\", []float64{-89.4300842285, -89.4287109375, -37.9866027832, -37.9852294922}},\n\t\t{\"2m\", []float64{-16.875, -11.25, -168.75, -157.5}},\n\t\t{\"w4s1zj8n9\", []float64{14.4014453888, 14.4014883041, 95.9326601028, 95.9327030182}},\n\t\t{\"hxzh\", []float64{-45.703125, -45.52734375, 32.34375, 32.6953125}},\n\t\t{\"cn\", []float64{78.75, 84.375, -135.0, -123.75}},\n\t\t{\"dpt5k8\", []float64{42.7587890625, 42.7642822266, -82.7709960938, -82.7600097656}},\n\t\t{\"gz\", []float64{84.375, 90.0, -11.25, 0.0}},\n\t\t{\"d09knt\", []float64{3.54309082031, 3.54858398438, -87.9565429688, -87.9455566406}},\n\t\t{\"mrucw\", []float64{-1.142578125, -1.0986328125, 63.193359375, 63.2373046875}},\n\t\t{\"055egqf4y\", []float64{-72.4282693863, -72.4282264709, -174.93229866, -174.932255745}},\n\t\t{\"qphu\", []float64{-4.921875, -4.74609375, 96.6796875, 97.03125}},\n\t\t{\"dfeh\", []float64{14.765625, 14.94140625, -52.03125, -51.6796875}},\n\t\t{\"00qfn4ctp57\", []float64{-88.2262055576, -88.2262042165, -170.241776258, -170.241774917}},\n\t\t{\"q9xx27p2trn\", []float64{-35.2714830637, -35.2714817226, 123.06805104, 123.068052381}},\n\t\t{\"10bhfgfsj8m\", []float64{-84.9250017107, -84.9250003695, -134.875474423, -134.875473082}},\n\t\t{\"6k\", []float64{-22.5, -16.875, -78.75, -67.5}},\n\t\t{\"zyvjr4m37\", []float64{83.9041757584, 83.9042186737, 176.096205711, 176.096248627}},\n\t\t{\"0c8k7gb\", []float64{-80.7948303223, -80.7934570312, -145.733642578, -145.732269287}},\n\t\t{\"n7k\", []float64{-71.71875, -70.3125, 106.875, 108.28125}},\n\t\t{\"fpj4eecz\", []float64{84.8362541199, 84.8364257812, -82.812538147, -82.8121948242}},\n\t\t{\"ev\", []float64{28.125, 33.75, -11.25, 0.0}},\n\t\t{\"8y\", []float64{33.75, 39.375, -146.25, -135.0}},\n\t\t{\"x9xd1q\", []float64{8.82202148438, 8.82751464844, 168.101806641, 168.112792969}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"d1pmefm0t\", []float64{6.60424232483, 6.60428524017, -79.6328115463, -79.632768631}},\n\t\t{\"tmy\", []float64{32.34375, 33.75, 64.6875, 66.09375}},\n\t\t{\"phu6jzv0z\", []float64{-62.8869867325, -62.8869438171, 141.236414909, 141.236457825}},\n\t\t{\"zv7pjxpuwjbq\", []float64{75.8009752259, 75.8009753935, 173.221350051, 173.221350387}},\n\t\t{\"9gegu1\", []float64{20.3521728516, 20.3576660156, -95.80078125, -95.7897949219}},\n\t\t{\"tq33\", []float64{35.33203125, 35.5078125, 58.0078125, 58.359375}},\n\t\t{\"e\", []float64{0.0, 45.0, -45.0, 0.0}},\n\t\t{\"y6z1sy\", []float64{60.7653808594, 60.7708740234, 111.302490234, 111.313476562}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"q\", []float64{-45.0, 0.0, 90.0, 135.0}},\n\t\t{\"mtyyw7\", []float64{-11.4971923828, -11.4916992188, 77.2668457031, 77.2778320312}},\n\t\t{\"2sqmb\", []float64{-20.0830078125, -20.0390625, -148.7109375, -148.666992188}},\n\t\t{\"yp487k\", []float64{84.4409179688, 84.4464111328, 93.6584472656, 93.6694335938}},\n\t\t{\"nyn5k1fc\", []float64{-55.668926239, -55.6687545776, 132.3670578, 132.367401123}},\n\t\t{\"gnqk77v\", []float64{80.9239196777, 80.9252929688, -36.0612487793, -36.0598754883}},\n\t\t{\"yw12\", []float64{78.75, 78.92578125, 114.2578125, 114.609375}},\n\t\t{\"vcgqqmd\", []float64{55.9725952148, 55.9739685059, 83.5977172852, 83.5990905762}},\n\t\t{\"27zp3q\", []float64{-22.5988769531, -22.5933837891, -158.851318359, -158.840332031}},\n\t\t{\"cg4tb6\", []float64{62.8967285156, 62.9022216797, -97.7233886719, -97.7124023438}},\n\t\t{\"w5njb9rnp5e\", []float64{17.8936573863, 17.8936587274, 98.4693901241, 98.4693914652}},\n\t\t{\"9y6rp4pptv\", []float64{36.3990193605, 36.399024725, -97.7684605122, -97.7684497833}},\n\t\t{\"pjup17j08hup\", []float64{-56.4091892727, -56.409189105, 140.68680346, 140.686803795}},\n\t\t{\"vz52f33z9mu\", []float64{84.5150206983, 84.5150220394, 83.421651721, 83.4216530621}},\n\t\t{\"zd\", []float64{56.25, 61.875, 157.5, 168.75}},\n\t\t{\"q\", []float64{-45.0, 0.0, 90.0, 135.0}},\n\t\t{\"dm46s\", []float64{28.564453125, 28.6083984375, -75.41015625, -75.3662109375}},\n\t\t{\"d7dnvbu\", []float64{20.8781433105, 20.8795166016, -75.6793212891, -75.677947998}},\n\t\t{\"02ercsvx0h\", []float64{-85.7978796959, -85.7978743315, -164.106216431, -164.106205702}},\n\t\t{\"hnfx5f\", []float64{-50.7897949219, -50.7843017578, 3.68041992188, 3.69140625}},\n\t\t{\"evb677f6t\", []float64{32.7602863312, 32.7603292465, -10.7523107529, -10.7522678375}},\n\t\t{\"sg43r7p151\", []float64{17.1113830805, 17.1113884449, 37.2424077988, 37.2424185276}},\n\t\t{\"441mdz5\", []float64{-77.7447509766, -77.7433776855, -88.1172180176, -88.1158447266}},\n\t\t{\"k2qrsc7hr81\", []float64{-42.2677946091, -42.267793268, 20.2522458136, 20.2522471547}},\n\t\t{\"d0ydzxhjwr\", []float64{4.74158227444, 4.74158763885, -80.5240237713, -80.5240130424}},\n\t\t{\"4cn\", []float64{-84.375, -82.96875, -47.8125, -46.40625}},\n\t\t{\"ds2\", []float64{23.90625, 25.3125, -67.5, -66.09375}},\n\t\t{\"rxvywy\", []float64{-0.230712890625, -0.225219726562, 165.882568359, 165.893554688}},\n\t\t{\"zs2k\", []float64{69.609375, 69.78515625, 157.8515625, 158.203125}},\n\t\t{\"kg63\", []float64{-26.54296875, -26.3671875, 36.9140625, 37.265625}},\n\t\t{\"hmxh64j\", []float64{-58.3044433594, -58.3030700684, 21.1885070801, 21.1898803711}},\n\t\t{\"d5v3\", []float64{21.26953125, 21.4453125, -82.6171875, -82.265625}},\n\t\t{\"ddg1\", []float64{15.64453125, 15.8203125, -63.28125, -62.9296875}},\n\t\t{\"stf4tug\", []float64{32.8092956543, 32.8106689453, 25.5693054199, 25.5706787109}},\n\t\t{\"vgc\", []float64{66.09375, 67.5, 80.15625, 81.5625}},\n\t\t{\"jby3xf\", []float64{-85.5065917969, -85.5010986328, 87.8796386719, 87.890625}},\n\t\t{\"9b1f419tdjf\", []float64{0.360777229071, 0.360778570175, -98.6990234256, -98.6990220845}},\n\t\t{\"0zqp4\", []float64{-47.98828125, -47.9443359375, -137.724609375, -137.680664062}},\n\t\t{\"gg292c4wd\", []float64{63.5075855255, 63.5076284409, -10.5103969574, -10.5103540421}},\n\t\t{\"zy96qbt\", []float64{81.9607543945, 81.9621276855, 170.811309814, 170.812683105}},\n\t\t{\"tz9x7f5c70\", []float64{43.4731149673, 43.4731203318, 81.0294485092, 81.0294592381}},\n\t\t{\"rq\", []float64{-11.25, -5.625, 146.25, 157.5}},\n\t\t{\"94tt\", []float64{14.94140625, 15.1171875, -127.265625, -126.9140625}},\n\t\t{\"h7vnwf8\", []float64{-67.7499389648, -67.7485656738, 18.5778808594, 18.5792541504}},\n\t\t{\"4f\", []float64{-78.75, -73.125, -56.25, -45.0}},\n\t\t{\"kj3t\", []float64{-14.58984375, -14.4140625, 2.109375, 2.4609375}},\n\t\t{\"qspj\", []float64{-21.62109375, -21.4453125, 122.34375, 122.6953125}},\n\t\t{\"4y9\", []float64{-53.4375, -52.03125, -54.84375, -53.4375}},\n\t\t{\"b05kqcvsm8\", []float64{45.7574129105, 45.7574182749, -175.125267506, -175.125256777}},\n\t\t{\"p5zq4f\", []float64{-67.8405761719, -67.8350830078, 145.316162109, 145.327148438}},\n\t\t{\"1cgx\", []float64{-78.92578125, -78.75, -96.328125, -95.9765625}},\n\t\t{\"m2\", []float64{-45.0, -39.375, 56.25, 67.5}},\n\t\t{\"j150xkd492\", []float64{-84.2619609833, -84.2619556189, 49.5401537418, 49.5401644707}},\n\t\t{\"05rs\", []float64{-71.015625, -70.83984375, -169.453125, -169.1015625}},\n\t\t{\"ve8\", []float64{64.6875, 66.09375, 67.5, 68.90625}},\n\t\t{\"r9tv\", []float64{-35.68359375, -35.5078125, 165.5859375, 165.9375}},\n\t\t{\"d71r07\", []float64{18.1219482422, 18.1274414062, -76.9812011719, -76.9702148438}},\n\t\t{\"b6hepfqk6\", []float64{56.79043293, 56.7904758453, -162.072629929, -162.072587013}},\n\t\t{\"md8y86mqjd\", []float64{-29.7815215588, -29.7815161943, 68.5731196404, 68.5731303692}},\n\t\t{\"bcgcyq\", []float64{55.1843261719, 55.1898193359, -140.701904297, -140.690917969}},\n\t\t{\"e3hpu9352\", []float64{6.99472904205, 6.9947719574, -27.9258728027, -27.9258298874}},\n\t\t{\"hsbsq2rkmg\", []float64{-62.5320607424, -62.532055378, 23.4879863262, 23.4879970551}},\n\t\t{\"ub2r\", []float64{47.63671875, 47.8125, 34.1015625, 34.453125}},\n\t\t{\"d8\", []float64{0.0, 5.625, -67.5, -56.25}},\n\t\t{\"gexm9j6y088\", []float64{65.6841686368, 65.6841699779, -12.2569441795, -12.2569428384}},\n\t\t{\"15cdq\", []float64{-68.5107421875, -68.466796875, -132.626953125, -132.583007812}},\n\t\t{\"9zud17gy\", []float64{43.9669418335, 43.9671134949, -94.8617935181, -94.8614501953}},\n\t\t{\"4q3y\", []float64{-53.7890625, -53.61328125, -76.2890625, -75.9375}},\n\t\t{\"gph138\", []float64{84.5947265625, 84.6002197266, -39.3090820312, -39.2980957031}},\n\t\t{\"m09d8ju2b\", []float64{-41.7163324356, -41.7162895203, 47.1152114868, 47.1152544022}},\n\t\t{\"8mszup4gk\", []float64{32.3388147354, 32.3388576508, -161.890583038, -161.890540123}},\n\t\t{\"dyrfvtvqh\", []float64{35.6722640991, 35.6723070145, -45.102481842, -45.1024389267}},\n\t\t{\"3h9tgkp\", []float64{-18.6547851562, -18.6534118652, -132.738189697, -132.736816406}},\n\t\t{\"66gdty14u\", []float64{-29.0583658218, -29.0583229065, -73.5738945007, -73.5738515854}},\n\t\t{\"83zp1d\", []float64{11.0852050781, 11.0906982422, -158.840332031, -158.829345703}},\n\t\t{\"e7gp0\", []float64{22.32421875, 22.3681640625, -29.53125, -29.4873046875}},\n\t\t{\"s0ykfgceytk\", []float64{5.07498219609, 5.0749835372, 8.91225636005, 8.91225770116}},\n\t\t{\"zfe7\", []float64{59.58984375, 59.765625, 173.3203125, 173.671875}},\n\t\t{\"cr9\", []float64{87.1875, 88.59375, -122.34375, -120.9375}},\n\t\t{\"9ugr3kq\", []float64{28.0165100098, 28.0178833008, -96.6165161133, -96.6151428223}},\n\t\t{\"2grcq0\", []float64{-26.4990234375, -26.4935302734, -135.087890625, -135.076904297}},\n\t\t{\"50bb31vkds3p\", []float64{-85.726895202, -85.7268950343, -43.8940487802, -43.8940484449}},\n\t\t{\"qhxdv1hcqe3\", []float64{-19.1983763874, -19.1983750463, 100.773404986, 100.773406327}},\n\t\t{\"k2cv0gp2vk\", []float64{-39.8857140541, -39.8857086897, 13.7540781498, 13.7540888786}},\n\t\t{\"y5jd\", []float64{62.2265625, 62.40234375, 97.734375, 98.0859375}},\n\t\t{\"gnvg3p\", []float64{83.5784912109, 83.583984375, -36.8701171875, -36.8591308594}},\n\t\t{\"9g70w\", []float64{18.369140625, 18.4130859375, -96.767578125, -96.7236328125}},\n\t\t{\"9g7qsb\", []float64{19.423828125, 19.4293212891, -96.4709472656, -96.4599609375}},\n\t\t{\"1zq2u\", []float64{-49.0869140625, -49.04296875, -92.28515625, -92.2412109375}},\n\t\t{\"tr\", []float64{39.375, 45.0, 56.25, 67.5}},\n\t\t{\"4wmddz9mgk38\", []float64{-54.3620882928, -54.3620881252, -59.6429172903, -59.6429169551}},\n\t\t{\"wkcdsn8nbz\", []float64{27.1951049566, 27.195110321, 103.535188437, 103.535199165}},\n\t\t{\"1r198wxj\", []float64{-50.3247642517, -50.3245925903, -121.609039307, -121.608695984}},\n\t\t{\"eu\", []float64{22.5, 28.125, -11.25, 0.0}},\n\t\t{\"k2y7mk6sd0wz\", []float64{-40.1858386584, -40.1858384907, 20.2733035013, 20.2733038366}},\n\t\t{\"gms9ytw4v5\", []float64{76.2758177519, 76.2758231163, -27.1277761459, -27.1277654171}},\n\t\t{\"2vdkc\", []float64{-13.2275390625, -13.18359375, -143.041992188, -142.998046875}},\n\t\t{\"bke7fx3\", []float64{71.011505127, 71.012878418, -164.068450928, -164.067077637}},\n\t\t{\"tvnxu7jt8\", []float64{29.5047283173, 29.5047712326, 88.0849456787, 88.0849885941}},\n\t\t{\"f864yp3c\", []float64{46.9296455383, 46.9298171997, -64.4214248657, -64.421081543}},\n\t\t{\"g8hxr7x150d7\", []float64{46.2938149832, 46.2938151509, -15.8435266837, -15.8435263485}},\n\t\t{\"zmk4kmh\", []float64{74.9542236328, 74.9555969238, 152.067260742, 152.068634033}},\n\t\t{\"gtqsvep4\", []float64{75.3830337524, 75.3832054138, -13.1080627441, -13.1077194214}},\n\t\t{\"trsvy0\", []float64{43.1982421875, 43.2037353516, 63.193359375, 63.2043457031}},\n\t\t{\"bevjfs\", []float64{67.1264648438, 67.1319580078, -150.358886719, -150.347900391}},\n\t\t{\"ktrb\", []float64{-15.46875, -15.29296875, 33.3984375, 33.75}},\n\t\t{\"dn20q1pv\", []float64{35.2065467834, 35.2067184448, -89.7256851196, -89.7253417969}},\n\t\t{\"8n3wy5g2\", []float64{36.3633728027, 36.3635444641, -177.622489929, -177.622146606}},\n\t\t{\"vyzft\", []float64{83.408203125, 83.4521484375, 89.8681640625, 89.912109375}},\n\t\t{\"gwuedjbs8222\", []float64{83.6163438857, 83.6163440533, -16.0832866654, -16.0832863301}},\n\t\t{\"fpb89dkktj83\", []float64{88.6948023923, 88.6948025599, -89.2249056324, -89.2249052972}},\n\t\t{\"wjjk3\", []float64{28.8720703125, 28.916015625, 97.4267578125, 97.470703125}},\n\t\t{\"wx6\", []float64{40.78125, 42.1875, 115.3125, 116.71875}},\n\t\t{\"yzpuuv3w0pw\", []float64{85.2398702502, 85.2398715913, 134.859245718, 134.859247059}},\n\t\t{\"k2518ucy\", []float64{-44.7092056274, -44.7090339661, 15.5041122437, 15.5044555664}},\n\t\t{\"hkjk1075\", []float64{-66.7949867249, -66.7948150635, 18.6808776855, 18.6812210083}},\n\t\t{\"btmd\", []float64{74.8828125, 75.05859375, -149.765625, -149.4140625}},\n\t\t{\"ucbvmyuw9z\", []float64{55.8048337698, 55.8048391342, 35.0636279583, 35.0636386871}},\n\t\t{\"wf86ytd34\", []float64{14.5762825012, 14.5763254166, 124.390382767, 124.390425682}},\n\t\t{\"9zjbws3u\", []float64{39.4869232178, 39.4870948792, -92.8760147095, -92.8756713867}},\n\t\t{\"rqc\", []float64{-7.03125, -5.625, 147.65625, 149.0625}},\n\t\t{\"pwqw8gh8x74\", []float64{-53.6845904589, -53.6845891178, 166.680077612, 166.680078954}},\n\t\t{\"5ekn\", []float64{-70.6640625, -70.48828125, -16.875, -16.5234375}},\n\t\t{\"mxx0b3mzwxp\", []float64{-2.67247259617, -2.67247125506, 77.3629210889, 77.36292243}},\n\t\t{\"tn\", []float64{33.75, 39.375, 45.0, 56.25}},\n\t\t{\"ju59u9b8grr\", []float64{-67.1826021373, -67.1826007962, 83.8704644144, 83.8704657555}},\n\t\t{\"hmte6v6jzq84\", []float64{-58.4613495693, -58.4613494016, 19.1082823277, 19.1082826629}},\n\t\t{\"t3\", []float64{5.625, 11.25, 56.25, 67.5}},\n\t\t{\"es9g6\", []float64{25.8837890625, 25.927734375, -19.951171875, -19.9072265625}},\n\t\t{\"2bwcdk6y\", []float64{-41.8994522095, -41.8992805481, -136.655158997, -136.654815674}},\n\t\t{\"4ew3jn8umh\", []float64{-70.1002621651, -70.1002568007, -58.4899663925, -58.4899556637}},\n\t\t{\"0meekufm4x3\", []float64{-58.4642212093, -58.4642198682, -163.616186231, -163.61618489}},\n\t\t{\"02\", []float64{-90.0, -84.375, -168.75, -157.5}},\n\t\t{\"9yuv0t43sv0\", []float64{38.8754063845, 38.8754077256, -94.5450460911, -94.54504475}},\n\t\t{\"u0g7y8444\", []float64{49.8782730103, 49.8783159256, 4.85878944397, 4.85883235931}},\n\t\t{\"r4\", []float64{-33.75, -28.125, 135.0, 146.25}},\n\t\t{\"1ps631dwde\", []float64{-47.4076205492, -47.4076151848, -128.975951672, -128.975940943}},\n\t\t{\"vbmfdm\", []float64{46.8731689453, 46.8786621094, 86.9348144531, 86.9458007812}},\n\t\t{\"s7gmxm8vhn5v\", []float64{22.0916506089, 22.0916507766, 16.1401226744, 16.1401230097}},\n\t\t{\"mcwrh68t\", []float64{-35.317440033, -35.3172683716, 87.7265167236, 87.7268600464}},\n\t\t{\"9yt5645jq9r9\", []float64{37.145683486, 37.1456836537, -94.1264504939, -94.1264501587}},\n\t\t{\"61t1x3z\", []float64{-36.2892150879, -36.2878417969, -82.6405334473, -82.6391601562}},\n\t\t{\"wmzkmbm\", []float64{33.0921936035, 33.0935668945, 111.704864502, 111.706237793}},\n\t\t{\"jy\", []float64{-56.25, -50.625, 78.75, 90.0}},\n\t\t{\"9ctckntmvgkr\", []float64{8.69393778965, 8.69393795729, -92.9808190092, -92.980818674}},\n\t\t{\"8rnugbss4gd\", []float64{40.2134129405, 40.2134142816, -159.086717069, -159.086715728}},\n\t\t{\"ek8\", []float64{25.3125, 26.71875, -33.75, -32.34375}},\n\t\t{\"nvqegbu\", []float64{-59.8054504395, -59.8040771484, 133.060913086, 133.062286377}},\n\t\t{\"h0u7v6pujp5\", []float64{-85.1103597879, -85.1103584468, 6.21813699603, 6.21813833714}},\n\t\t{\"2rj\", []float64{-5.625, -4.21875, -161.71875, -160.3125}},\n\t\t{\"jk38\", []float64{-66.09375, -65.91796875, 58.359375, 58.7109375}},\n\t\t{\"c67qe30m\", []float64{58.8051795959, 58.8053512573, -119.036521912, -119.036178589}},\n\t\t{\"xhtwk\", []float64{26.4111328125, 26.455078125, 142.91015625, 142.954101562}},\n\t\t{\"pw6ue97rjn11\", []float64{-54.0446339361, -54.0446337685, 161.525675207, 161.525675543}},\n\t\t{\"xpmw424pk6\", []float64{41.8371927738, 41.8371981382, 142.836180925, 142.836191654}},\n\t\t{\"rk\", []float64{-22.5, -16.875, 146.25, 157.5}},\n\t\t{\"hmfmn5uwdh2\", []float64{-56.755605787, -56.7556044459, 14.6840000153, 14.6840013564}},\n\t\t{\"defvfj\", []float64{22.1319580078, 22.1374511719, -63.544921875, -63.5339355469}},\n\t\t{\"sg7jd89w\", []float64{19.2518234253, 19.2519950867, 38.0806732178, 38.0810165405}},\n\t\t{\"yn0\", []float64{78.75, 80.15625, 90.0, 91.40625}},\n\t\t{\"3n7re6gv2e92\", []float64{-8.50936442614, -8.5093642585, -130.281692259, -130.281691924}},\n\t\t{\"vyj\", []float64{78.75, 80.15625, 85.78125, 87.1875}},\n\t\t{\"9cntsz\", []float64{6.63024902344, 6.6357421875, -91.9006347656, -91.8896484375}},\n\t\t{\"w4uwq3d\", []float64{16.5756225586, 16.5769958496, 96.6055297852, 96.6069030762}},\n\t\t{\"zffrf\", []float64{61.8310546875, 61.875, 172.001953125, 172.045898438}},\n\t\t{\"hq6142s\", []float64{-54.665222168, -54.663848877, 14.1668701172, 14.1682434082}},\n\t\t{\"m7srp977\", []float64{-24.0746498108, -24.0744781494, 62.5606155396, 62.5609588623}},\n\t\t{\"8v6hxy3sz\", []float64{30.3574132919, 30.3574562073, -143.094563484, -143.094520569}},\n\t\t{\"rf4snrh70h\", []float64{-33.0078864098, -33.0078810453, 172.54611969, 172.546130419}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"ssr6y\", []float64{24.3896484375, 24.43359375, 32.958984375, 33.0029296875}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"yz\", []float64{84.375, 90.0, 123.75, 135.0}},\n\t\t{\"n2\", []float64{-90.0, -84.375, 101.25, 112.5}},\n\t\t{\"vfe\", []float64{59.0625, 60.46875, 82.96875, 84.375}},\n\t\t{\"h\", []float64{-90.0, -45.0, 0.0, 45.0}},\n\t\t{\"z3\", []float64{50.625, 56.25, 146.25, 157.5}},\n\t\t{\"z2web5usz\", []float64{48.4930944443, 48.4931373596, 155.397105217, 155.397148132}},\n\t\t{\"8gkc\", []float64{18.45703125, 18.6328125, -139.5703125, -139.21875}},\n\t\t{\"17de4\", []float64{-69.78515625, -69.7412109375, -120.146484375, -120.102539062}},\n\t\t{\"ky71b3fwd2vv\", []float64{-9.52539911494, -9.5253989473, 37.9832738265, 37.9832741618}},\n\t\t{\"e4gfcm\", []float64{15.9796142578, 15.9851074219, -39.6716308594, -39.6606445312}},\n\t\t{\"kdgwxnnb\", []float64{-28.3557128906, -28.3555412292, 27.7387619019, 27.7391052246}},\n\t\t{\"4nyn2chy\", []float64{-50.9260940552, -50.9259223938, -81.5230178833, -81.5226745605}},\n\t\t{\"u6je3kk57\", []float64{56.8451929092, 56.8452358246, 19.0449285507, 19.0449714661}},\n\t\t{\"nrs9j1hj5tj\", []float64{-47.630340457, -47.6303391159, 107.803501636, 107.803502977}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"krqrcc2hm9uq\", []float64{-2.84883890301, -2.84883873537, 20.116208531, 20.1162088662}},\n\t\t{\"fmceu\", []float64{78.0029296875, 78.046875, -76.46484375, -76.4208984375}},\n\t\t{\"q3w576\", []float64{-35.9802246094, -35.9747314453, 109.830322266, 109.841308594}},\n\t\t{\"19ehn\", []float64{-80.859375, -80.8154296875, -108.017578125, -107.973632812}},\n\t\t{\"zpkvjk\", []float64{86.6821289062, 86.6876220703, 141.910400391, 141.921386719}},\n\t\t{\"7cgwy9jm\", []float64{-33.9633750916, -33.9632034302, -6.03527069092, -6.03492736816}},\n\t\t{\"jju1cefk5v1\", []float64{-57.3273199797, -57.3273186386, 50.6941701472, 50.6941714883}},\n\t\t{\"5tfpq6\", []float64{-56.3708496094, -56.3653564453, -19.4128417969, -19.4018554688}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"dm0vh8\", []float64{29.00390625, 29.0093994141, -77.4975585938, -77.4865722656}},\n\t\t{\"jt5c8hhd58b\", []float64{-61.5890081227, -61.5890067816, 72.7797675133, 72.7797688544}},\n\t\t{\"4ttr\", []float64{-57.83203125, -57.65625, -60.1171875, -59.765625}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"sp2d3v6wz\", []float64{41.2067556381, 41.2067985535, 0.783762931824, 0.783805847168}},\n\t\t{\"up1yhbspqy\", []float64{85.4337108135, 85.4337161779, 2.67546057701, 2.67547130585}},\n\t\t{\"1z4bx8dp1ex1\", []float64{-50.5331422202, -50.5331420526, -97.0504023135, -97.0504019782}},\n\t\t{\"76qbnz5n1937\", []float64{-32.3042606749, -32.3042605072, -23.9569957182, -23.9569953829}},\n\t\t{\"0w1bvcjb40xd\", []float64{-56.112667881, -56.1126677133, -154.778384641, -154.778384306}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"yqbjwyvsmc\", []float64{83.9733606577, 83.9733660221, 101.554430723, 101.554441452}},\n\t\t{\"0hdp2tdy5\", []float64{-63.3818435669, -63.3818006516, -177.161622047, -177.161579132}},\n\t\t{\"s8yv34v\", []float64{5.15670776367, 5.15808105469, 32.0429992676, 32.0443725586}},\n\t\t{\"uc60k7w6\", []float64{52.0947647095, 52.0949363708, 36.757850647, 36.7581939697}},\n\t\t{\"en6h7me8\", []float64{35.9335327148, 35.9337043762, -42.0398712158, -42.0395278931}},\n\t\t{\"1bks34\", []float64{-87.8356933594, -87.8302001953, -94.8779296875, -94.8669433594}},\n\t\t{\"65\", []float64{-28.125, -22.5, -90.0, -78.75}},\n\t\t{\"qwrquvr\", []float64{-8.62838745117, -8.62701416016, 122.913665771, 122.915039062}},\n\t\t{\"3dmchcusw83\", []float64{-32.1575818956, -32.1575805545, -104.198862165, -104.198860824}},\n\t\t{\"urfeh\", []float64{89.12109375, 89.1650390625, 14.94140625, 14.9853515625}},\n\t\t{\"d3g6rs\", []float64{10.2612304688, 10.2667236328, -73.8500976562, -73.8391113281}},\n\t\t{\"wx0v\", []float64{40.25390625, 40.4296875, 113.5546875, 113.90625}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"et\", []float64{28.125, 33.75, -22.5, -11.25}},\n\t\t{\"dqtd0\", []float64{36.9140625, 36.9580078125, -71.015625, -70.9716796875}},\n\t\t{\"vtzuwhxhgjv8\", []float64{78.1603311002, 78.1603312679, 78.6718585342, 78.6718588695}},\n\t\t{\"bn9\", []float64{81.5625, 82.96875, -178.59375, -177.1875}},\n\t\t{\"685n\", []float64{-43.9453125, -43.76953125, -63.28125, -62.9296875}},\n\t\t{\"fqy4nhy\", []float64{83.3464050293, 83.3477783203, -70.0405883789, -70.0392150879}},\n\t\t{\"5q1dw\", []float64{-55.810546875, -55.7666015625, -31.376953125, -31.3330078125}},\n\t\t{\"0sv8m11dgkf\", []float64{-63.2313139737, -63.2313126326, -149.543696344, -149.543695003}},\n\t\t{\"vp435nn\", []float64{84.5837402344, 84.5851135254, 48.3041381836, 48.3055114746}},\n\t\t{\"wydj\", []float64{37.44140625, 37.6171875, 126.5625, 126.9140625}},\n\t\t{\"ebg8\", []float64{4.21875, 4.39453125, -6.328125, -5.9765625}},\n\t\t{\"ksmb6pfxe4\", []float64{-21.0059344769, -21.0059291124, 30.6773900986, 30.6774008274}},\n\t\t{\"8bv63qv96dp\", []float64{4.65156197548, 4.65156331658, -138.804586083, -138.804584742}},\n\t\t{\"0s0d53hd\", []float64{-67.1426010132, -67.1424293518, -156.647872925, -156.647529602}},\n\t\t{\"1yjf34ubpm\", []float64{-55.8393591642, -55.8393537998, -93.1132829189, -93.1132721901}},\n\t\t{\"823y79hmfe\", []float64{2.51137912273, 2.51138448715, -166.129310131, -166.129299402}},\n\t\t{\"xynnrv\", []float64{34.8760986328, 34.8815917969, 177.528076172, 177.5390625}},\n\t\t{\"9b9ejqcqqdu\", []float64{3.37801024318, 3.37801158428, -98.9079111814, -98.9079098403}},\n\t\t{\"cuuhfkd\", []float64{72.5784301758, 72.5798034668, -95.5233764648, -95.5220031738}},\n\t\t{\"khwceh\", []float64{-19.4018554688, -19.3963623047, 9.6240234375, 9.63500976562}},\n\t\t{\"z8vub32sy6\", []float64{50.061403513, 50.0614088774, 165.597878695, 165.597889423}},\n\t\t{\"4s\", []float64{-67.5, -61.875, -67.5, -56.25}},\n\t\t{\"bsrb4qeqt7eq\", []float64{68.9430911466, 68.9430913143, -146.497992687, -146.497992352}},\n\t\t{\"b1x0sc\", []float64{53.5308837891, 53.5363769531, -169.947509766, -169.936523438}},\n\t\t{\"1ngn2bn2vub\", []float64{-50.9324629605, -50.9324616194, -130.739461184, -130.739459842}},\n\t\t{\"bsm\", []float64{68.90625, 70.3125, -150.46875, -149.0625}},\n\t\t{\"xyzd7\", []float64{38.3642578125, 38.408203125, 179.428710938, 179.47265625}},\n\t\t{\"cvvf1tqs\", []float64{77.7248382568, 77.7250099182, -93.0892181396, -93.0888748169}},\n\t\t{\"fz2tpb1xj\", []float64{86.6613578796, 86.661400795, -55.2040243149, -55.2039813995}},\n\t\t{\"r4zqr7\", []float64{-28.4161376953, -28.4106445312, 145.513916016, 145.524902344}},\n\t\t{\"wth\", []float64{28.125, 29.53125, 118.125, 119.53125}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"4j0ff5xs58h\", []float64{-61.3716888428, -61.3716875017, -88.8469666243, -88.8469652832}},\n\t\t{\"c15\", []float64{50.625, 52.03125, -130.78125, -129.375}},\n\t\t{\"tnkgyg\", []float64{35.8319091797, 35.8374023438, 51.9763183594, 51.9873046875}},\n\t\t{\"e99\", []float64{8.4375, 9.84375, -21.09375, -19.6875}},\n\t\t{\"bcrz69\", []float64{53.3111572266, 53.3166503906, -135.241699219, -135.230712891}},\n\t\t{\"e5w29f6\", []float64{19.7877502441, 19.7891235352, -36.1312866211, -36.1299133301}},\n\t\t{\"gykjjcq656b5\", []float64{81.0423812829, 81.0423814505, -5.36359190941, -5.36359157413}},\n\t\t{\"7j62nzypd3be\", []float64{-15.4248806275, -15.4248804599, -41.5309696645, -41.5309693292}},\n\t\t{\"rrsgbbvwt9r7\", []float64{-2.14807743207, -2.14807726443, 152.970445342, 152.970445678}},\n\t\t{\"93kub\", []float64{7.8662109375, 7.91015625, -117.0703125, -117.026367188}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"02bcjtw5f84\", []float64{-85.5746126175, -85.5746112764, -167.445263565, -167.445262223}},\n\t\t{\"262dqsqs1\", []float64{-31.9242095947, -31.9241666794, -167.752261162, -167.752218246}},\n\t\t{\"185ss806c\", []float64{-89.2085123062, -89.2084693909, -107.379984856, -107.37994194}},\n\t\t{\"9s6\", []float64{23.90625, 25.3125, -109.6875, -108.28125}},\n\t\t{\"ych1d25e\", []float64{50.8891868591, 50.8893585205, 129.478683472, 129.479026794}},\n\t\t{\"7f2qcykht0d\", []float64{-31.1221191287, -31.1221177876, -10.8158227801, -10.815821439}},\n\t\t{\"5gk7qw\", []float64{-71.1145019531, -71.1090087891, -4.98779296875, -4.97680664062}},\n\t\t{\"7kjutjpu98\", []float64{-21.6807460785, -21.6807407141, -25.4336285591, -25.4336178303}},\n\t\t{\"h64\", []float64{-78.75, -77.34375, 14.0625, 15.46875}},\n\t\t{\"uy57\", []float64{79.27734375, 79.453125, 38.3203125, 38.671875}},\n\t\t{\"r5dtqkydk796\", []float64{-24.3631505594, -24.3631503917, 138.799393661, 138.799393997}},\n\t\t{\"5j\", []float64{-61.875, -56.25, -45.0, -33.75}},\n\t\t{\"xzbszzeu\", []float64{44.4705963135, 44.4707679749, 169.798851013, 169.799194336}},\n\t\t{\"wqjz0fj5e\", []float64{34.9920558929, 34.9920988083, 109.375891685, 109.375934601}},\n\t\t{\"dekz\", []float64{19.51171875, 19.6875, -60.8203125, -60.46875}},\n\t\t{\"bbyux\", []float64{50.009765625, 50.0537109375, -136.450195312, -136.40625}},\n\t\t{\"rctysz36\", []float64{-35.3797531128, -35.3795814514, 177.046394348, 177.046737671}},\n\t\t{\"xmhvqm1wcw7\", []float64{29.0765096247, 29.0765109658, 153.206474036, 153.206475377}},\n\t\t{\"nhw6c\", []float64{-64.2041015625, -64.16015625, 98.8330078125, 98.876953125}},\n\t\t{\"u9d3gdu\", []float64{53.7602233887, 53.7615966797, 25.8233642578, 25.8247375488}},\n\t\t{\"xenu1tyd0q\", []float64{17.6100862026, 17.610091567, 167.067042589, 167.067053318}},\n\t\t{\"qm70t\", []float64{-15.380859375, -15.3369140625, 105.688476562, 105.732421875}},\n\t\t{\"3g0\", []float64{-28.125, -26.71875, -101.25, -99.84375}},\n\t\t{\"fg\", []float64{61.875, 67.5, -56.25, -45.0}},\n\t\t{\"jq1tn6nn0hfs\", []float64{-55.3590513021, -55.3590511344, 58.642276302, 58.6422766373}},\n\t\t{\"b9kmw\", []float64{52.998046875, 53.0419921875, -151.259765625, -151.215820312}},\n\t\t{\"z7f9pj6\", []float64{66.2983703613, 66.2997436523, 150.07598877, 150.077362061}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"hqbvrz4x\", []float64{-51.0687446594, -51.068572998, 12.6486968994, 12.6490402222}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"81es\", []float64{9.140625, 9.31640625, -175.078125, -174.7265625}},\n\t\t{\"nyn6mf2\", []float64{-55.8421325684, -55.8407592773, 132.791748047, 132.793121338}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"46hfqnuv\", []float64{-78.3165550232, -78.3163833618, -71.8001174927, -71.7997741699}},\n\t\t{\"7tne\", []float64{-16.34765625, -16.171875, -13.359375, -13.0078125}},\n\t\t{\"z7pkduh4g2c\", []float64{62.6884643734, 62.6884657145, 156.571796089, 156.571797431}},\n\t\t{\"xgmm\", []float64{19.16015625, 19.3359375, 176.1328125, 176.484375}},\n\t\t{\"he054x\", []float64{-72.5592041016, -72.5537109375, 22.6098632812, 22.6208496094}},\n\t\t{\"rqnu21r8g\", []float64{-10.4959344864, -10.495891571, 155.752615929, 155.752658844}},\n\t\t{\"k8xg8n36\", []float64{-41.5375900269, -41.5374183655, 33.4001541138, 33.4004974365}},\n\t\t{\"d60md\", []float64{12.216796875, 12.2607421875, -78.310546875, -78.2666015625}},\n\t\t{\"50tqctv\", []float64{-85.9693908691, -85.9680175781, -37.5444030762, -37.5430297852}},\n\t\t{\"yxknv54eqnz\", []float64{86.984847039, 86.9848483801, 118.34842667, 118.348428011}},\n\t\t{\"5zpczbcjgj\", []float64{-50.3122490644, -50.3122437, -0.00948429107666, -0.0094735622406}},\n\t\t{\"yp7nz0rr7d8\", []float64{86.9704046845, 86.9704060256, 94.5364737511, 94.5364750922}},\n\t\t{\"kfjb9s772f\", []float64{-33.6381947994, -33.638189435, 41.9063508511, 41.9063615799}},\n\t\t{\"q\", []float64{-45.0, 0.0, 90.0, 135.0}},\n\t\t{\"cx1f\", []float64{84.7265625, 84.90234375, -110.0390625, -109.6875}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"hdtcdg62524\", []float64{-75.6559753418, -75.6559740007, 30.7100191712, 30.7100205123}},\n\t\t{\"fpy32\", []float64{88.8134765625, 88.857421875, -81.2109375, -81.1669921875}},\n\t\t{\"256qs\", []float64{-25.576171875, -25.5322265625, -176.66015625, -176.616210938}},\n\t\t{\"6rgzd89v3r48\", []float64{-0.0842052698135, -0.0842051021755, -73.3642389625, -73.3642386273}},\n\t\t{\"c8430hd23\", []float64{45.2005434036, 45.200586319, -109.33280468, -109.332761765}},\n\t\t{\"xtn\", []float64{28.125, 29.53125, 165.9375, 167.34375}},\n\t\t{\"1u2s9\", []float64{-65.302734375, -65.2587890625, -100.502929688, -100.458984375}},\n\t\t{\"5c80k35m35p\", []float64{-81.512144208, -81.5121428668, -11.058716923, -11.0587155819}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"u7c0rfsdjn\", []float64{66.1518037319, 66.1518090963, 13.0032205582, 13.003231287}},\n\t\t{\"f4ce8gy\", []float64{61.1045837402, 61.1059570312, -87.8494262695, -87.8480529785}},\n\t\t{\"pbd4y02\", []float64{-86.7027282715, -86.7013549805, 171.826171875, 171.827545166}},\n\t\t{\"9n\", []float64{33.75, 39.375, -135.0, -123.75}},\n\t\t{\"ztybk5c8mw\", []float64{77.4083697796, 77.408375144, 167.170264721, 167.17027545}},\n\t\t{\"ks8pv0cv5u6j\", []float64{-18.3201934956, -18.320193328, 22.7222934365, 22.7222937718}},\n\t\t{\"nuh\", []float64{-67.5, -66.09375, 129.375, 130.78125}},\n\t\t{\"6khcf5xumxz\", []float64{-22.1723856032, -22.1723842621, -71.9715334475, -71.9715321064}},\n\t\t{\"nj2\", []float64{-60.46875, -59.0625, 90.0, 91.40625}},\n\t\t{\"mdd459nebt64\", []float64{-30.5797721073, -30.5797719397, 70.4752591252, 70.4752594605}},\n\t\t{\"e7ujxh\", []float64{22.0825195312, 22.0880126953, -27.8173828125, -27.8063964844}},\n\t\t{\"eb1zvy9n\", []float64{1.39904022217, 1.39921188354, -8.53500366211, -8.53466033936}},\n\t\t{\"1huxmt\", []float64{-61.9793701172, -61.9738769531, -128.430175781, -128.419189453}},\n\t\t{\"v4cyrgyg\", []float64{61.5884971619, 61.5886688232, 47.8107833862, 47.811126709}},\n\t\t{\"j6h0dvut\", []float64{-78.6296653748, -78.6294937134, 62.0020294189, 62.0023727417}},\n\t\t{\"r8j\", []float64{-45.0, -43.59375, 164.53125, 165.9375}},\n\t\t{\"feyj018b7\", []float64{66.9809389114, 66.9809818268, -59.0613412857, -59.0612983704}},\n\t\t{\"tw33887htf7j\", []float64{35.4220805503, 35.422080718, 69.2841558158, 69.2841561511}},\n\t\t{\"3rbgkj8z4cy8\", []float64{-0.803537517786, -0.803537350148, -122.518374547, -122.518374212}},\n\t\t{\"gq30gnn9dg\", []float64{80.3213185072, 80.3213238716, -32.2028696537, -32.2028589249}},\n\t\t{\"rek4qy2\", []float64{-26.2889099121, -26.2875366211, 163.421630859, 163.42300415}},\n\t\t{\"j8\", []float64{-90.0, -84.375, 67.5, 78.75}},\n\t\t{\"f2rev1d7\", []float64{47.0741844177, 47.0743560791, -67.9803085327, -67.97996521}},\n\t\t{\"rw1sg2f78x\", []float64{-10.4102808237, -10.4102754593, 159.755308628, 159.755319357}},\n\t\t{\"r\", []float64{-45.0, 0.0, 135.0, 180.0}},\n\t\t{\"8xfw\", []float64{44.6484375, 44.82421875, -153.984375, -153.6328125}},\n\t\t{\"fj20\", []float64{74.53125, 74.70703125, -90.0, -89.6484375}},\n\t\t{\"m76w867pt9yj\", []float64{-25.5625145696, -25.562514402, 59.7809752822, 59.7809756175}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"wdb4v9vwt4ez\", []float64{15.9628918581, 15.9628920257, 112.749471925, 112.74947226}},\n\t\t{\"m53rvw\", []float64{-25.3234863281, -25.3179931641, 46.9995117188, 47.0104980469}},\n\t\t{\"f33\", []float64{52.03125, 53.4375, -77.34375, -75.9375}},\n\t\t{\"0t36bun98\", []float64{-59.9631214142, -59.9630784988, -155.700302124, -155.700259209}},\n\t\t{\"ezs1\", []float64{42.36328125, 42.5390625, -5.625, -5.2734375}},\n\t\t{\"jb\", []float64{-90.0, -84.375, 78.75, 90.0}},\n\t\t{\"vd0k\", []float64{56.953125, 57.12890625, 67.8515625, 68.203125}},\n\t\t{\"39cqr9bd\", []float64{-34.0476608276, -34.0474891663, -110.411911011, -110.411567688}},\n\t\t{\"n8r\", []float64{-88.59375, -87.1875, 122.34375, 123.75}},\n\t\t{\"1b6\", []float64{-88.59375, -87.1875, -98.4375, -97.03125}},\n\t\t{\"358wn7v3tch\", []float64{-24.2369502783, -24.2369489372, -134.014754891, -134.01475355}},\n\t\t{\"d506n2v77qf\", []float64{17.2312764823, 17.2312778234, -89.366427362, -89.3664260209}},\n\t\t{\"qb\", []float64{-45.0, -39.375, 123.75, 135.0}},\n\t\t{\"sn\", []float64{33.75, 39.375, 0.0, 11.25}},\n\t\t{\"5rj\", []float64{-50.625, -49.21875, -26.71875, -25.3125}},\n\t\t{\"0y51c\", []float64{-55.9423828125, -55.8984375, -141.987304688, -141.943359375}},\n\t\t{\"r85z8\", []float64{-43.681640625, -43.6376953125, 162.7734375, 162.817382812}},\n\t\t{\"rk5z\", []float64{-21.26953125, -21.09375, 151.5234375, 151.875}},\n\t\t{\"vzn2\", []float64{84.375, 84.55078125, 87.5390625, 87.890625}},\n\t\t{\"bjvk\", []float64{78.046875, 78.22265625, -172.6171875, -172.265625}},\n\t\t{\"b9f\", []float64{54.84375, 56.25, -154.6875, -153.28125}},\n\t\t{\"q0u1y3mu4k\", []float64{-40.4660582542, -40.4660528898, 95.907651186, 95.9076619148}},\n\t\t{\"r7\", []float64{-28.125, -22.5, 146.25, 157.5}},\n\t\t{\"cv90dz31\", []float64{76.0653877258, 76.0655593872, -99.7215270996, -99.7211837769}},\n\t\t{\"gxfeekgv9sng\", []float64{89.2360430025, 89.2360431701, -18.8363294676, -18.8363291323}},\n\t\t{\"92t\", []float64{2.8125, 4.21875, -116.71875, -115.3125}},\n\t\t{\"m06\", []float64{-43.59375, -42.1875, 47.8125, 49.21875}},\n\t\t{\"n27p3f3c\", []float64{-87.306804657, -87.3066329956, 105.548057556, 105.548400879}},\n\t\t{\"jjptjwvt2\", []float64{-60.9581136703, -60.958070755, 55.7961273193, 55.7961702347}},\n\t\t{\"u258hxwr\", []float64{45.0424003601, 45.0425720215, 16.3782119751, 16.3785552979}},\n\t\t{\"t47\", []float64{12.65625, 14.0625, 49.21875, 50.625}},\n\t\t{\"e\", []float64{0.0, 45.0, -45.0, 0.0}},\n\t\t{\"0s\", []float64{-67.5, -61.875, -157.5, -146.25}},\n\t\t{\"4drwu7v94g3\", []float64{-76.1364381015, -76.1364367604, -56.758684963, -56.7586836219}},\n\t\t{\"wk14pc\", []float64{22.8570556641, 22.8625488281, 102.996826172, 103.0078125}},\n\t\t{\"w5xg\", []float64{20.21484375, 20.390625, 100.8984375, 101.25}},\n\t\t{\"f2z2dvf7\", []float64{49.3387413025, 49.3389129639, -68.4307479858, -68.4304046631}},\n\t\t{\"duk0212sgtp\", []float64{23.9579039812, 23.9579053223, -50.6241537631, -50.624152422}},\n\t\t{\"h7z\", []float64{-68.90625, -67.5, 21.09375, 22.5}},\n\t\t{\"31k02yzy90\", []float64{-37.8866100311, -37.8866046667, -129.331355095, -129.331344366}},\n\t\t{\"vus0k\", []float64{70.3564453125, 70.400390625, 84.55078125, 84.5947265625}},\n\t\t{\"5m0sjm4rrvf\", []float64{-61.1431337893, -61.1431324482, -32.8127369285, -32.8127355874}},\n\t\t{\"4syd3n9\", []float64{-62.8500366211, -62.8486633301, -58.3140563965, -58.3126831055}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"g\", []float64{45.0, 90.0, -45.0, 0.0}},\n\t\t{\"ntn69zcu\", []float64{-61.392288208, -61.3921165466, 121.368370056, 121.368713379}},\n\t\t{\"vnc4m\", []float64{83.3642578125, 83.408203125, 46.6259765625, 46.669921875}},\n\t\t{\"sy848jtk7wdj\", []float64{37.0329307951, 37.0329309627, 33.7573626637, 33.757362999}},\n\t\t{\"ry7y\", []float64{-8.7890625, -8.61328125, 174.0234375, 174.375}},\n\t\t{\"5k7gesef08j\", []float64{-65.453453064, -65.4534517229, -28.3175759017, -28.3175745606}},\n\t\t{\"n5gnev\", []float64{-67.7362060547, -67.7307128906, 94.3835449219, 94.39453125}},\n\t\t{\"kz3eeg1trf7\", []float64{-3.58612284064, -3.58612149954, 36.0265664756, 36.0265678167}},\n\t\t{\"t55rxfr7\", []float64{18.2062339783, 18.2064056396, 49.9208450317, 49.9211883545}},\n\t\t{\"tm9\", []float64{30.9375, 32.34375, 57.65625, 59.0625}},\n\t\t{\"pjw\", []float64{-59.0625, -57.65625, 143.4375, 144.84375}},\n\t\t{\"f\", []float64{45.0, 90.0, -90.0, -45.0}},\n\t\t{\"2my\", []float64{-12.65625, -11.25, -160.3125, -158.90625}},\n\t\t{\"3w6bwyt\", []float64{-9.72015380859, -9.71878051758, -108.329315186, -108.327941895}},\n\t\t{\"v4s8r3q\", []float64{59.1133117676, 59.1146850586, 51.6549682617, 51.6563415527}},\n\t\t{\"ggx8be488kyn\", []float64{64.8359277472, 64.8359279148, -0.677700340748, -0.677700005472}},\n\t\t{\"95w\", []float64{19.6875, 21.09375, -126.5625, -125.15625}},\n\t\t{\"9jck7spubf\", []float64{33.1136190891, 33.1136244535, -133.077703714, -133.077692986}},\n\t\t{\"f1zfe\", []float64{55.283203125, 55.3271484375, -78.9697265625, -78.92578125}},\n\t\t{\"p5ycnk9j\", []float64{-68.7048912048, -68.7047195435, 144.768218994, 144.768562317}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"zk\", []float64{67.5, 73.125, 146.25, 157.5}},\n\t\t{\"rkq7z\", []float64{-20.4345703125, -20.390625, 155.346679688, 155.390625}},\n\t\t{\"j5wsc3t4\", []float64{-69.4689559937, -69.4687843323, 54.2024230957, 54.2027664185}},\n\t\t{\"56szw\", []float64{-74.619140625, -74.5751953125, -26.806640625, -26.7626953125}},\n\t\t{\"6xp21s7\", []float64{-5.60165405273, -5.60028076172, -57.2346496582, -57.2332763672}},\n\t\t{\"gbmymdgc5x\", []float64{47.520198226, 47.5202035904, -2.91706323624, -2.9170525074}},\n\t\t{\"bmm935md5m\", []float64{74.7691994905, 74.769204855, -160.963987112, -160.963976383}},\n\t\t{\"z2v55d8\", []float64{49.7598266602, 49.7611999512, 153.435058594, 153.436431885}},\n\t\t{\"q232vgb0\", []float64{-43.4413146973, -43.4411430359, 103.260498047, 103.26084137}},\n\t\t{\"b7kf7b5uz4\", []float64{63.6775839329, 63.6775892973, -161.900067329, -161.900056601}},\n\t\t{\"y6\", []float64{56.25, 61.875, 101.25, 112.5}},\n\t\t{\"3xdf2ts4d\", []float64{-2.38635063171, -2.38630771637, -108.605260849, -108.605217934}},\n\t\t{\"0szy1551\", []float64{-62.2099113464, -62.2097396851, -146.553497314, -146.553153992}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"kk86s\", []float64{-19.248046875, -19.2041015625, 11.77734375, 11.8212890625}},\n\t\t{\"tqq\", []float64{35.15625, 36.5625, 64.6875, 66.09375}},\n\t\t{\"3gcy97b\", []float64{-22.7430725098, -22.7416992188, -98.7341308594, -98.7327575684}},\n\t\t{\"5dnz5z1d\", []float64{-77.4807357788, -77.4805641174, -12.8409576416, -12.8406143188}},\n\t\t{\"eumv1pe\", []float64{24.8263549805, 24.8277282715, -3.11599731445, -3.11462402344}},\n\t\t{\"2kxmbgqtfc3\", []float64{-18.6579112709, -18.6579099298, -158.512682766, -158.512681425}},\n\t\t{\"hc5uf211rpb\", []float64{-83.5397829115, -83.5397815704, 39.1239881516, 39.1239894927}},\n\t\t{\"p4khbd21\", []float64{-76.496257782, -76.4960861206, 140.646972656, 140.647315979}},\n\t\t{\"xuq\", []float64{23.90625, 25.3125, 177.1875, 178.59375}},\n\t\t{\"gn63cf91nhf\", []float64{80.47779724, 80.4777985811, -41.7573997378, -41.7573983967}},\n\t\t{\"5hd1nh3\", []float64{-64.4883728027, -64.4869995117, -41.922454834, -41.921081543}},\n\t\t{\"ustjn5\", []float64{71.2078857422, 71.2133789062, 29.794921875, 29.8059082031}},\n\t\t{\"btv2\", []float64{77.34375, 77.51953125, -150.1171875, -149.765625}},\n\t\t{\"7ds5rkh3u1\", []float64{-30.3439325094, -30.343927145, -16.5503883362, -16.5503776073}},\n\t\t{\"54\", []float64{-78.75, -73.125, -45.0, -33.75}},\n\t\t{\"7sr\", []float64{-21.09375, -19.6875, -12.65625, -11.25}},\n\t\t{\"kr552fbb\", []float64{-5.03860473633, -5.03843307495, 15.5027389526, 15.5030822754}},\n\t\t{\"8uc453nv4qq4\", []float64{27.0766978338, 27.0766980015, -144.691553414, -144.691553079}},\n\t\t{\"6n3m\", []float64{-8.96484375, -8.7890625, -88.2421875, -87.890625}},\n\t\t{\"rptzf8\", []float64{-1.4501953125, -1.44470214844, 143.195800781, 143.206787109}},\n\t\t{\"x63b\", []float64{12.65625, 12.83203125, 148.7109375, 149.0625}},\n\t\t{\"qwj7353\", []float64{-10.6608581543, -10.6594848633, 119.928131104, 119.929504395}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"v6\", []float64{56.25, 61.875, 56.25, 67.5}},\n\t\t{\"5mpwtp\", []float64{-60.6939697266, -60.6884765625, -22.9833984375, -22.9724121094}},\n\t\t{\"ukgs\", []float64{72.421875, 72.59765625, 16.171875, 16.5234375}},\n\t\t{\"qxyb4g6qf\", []float64{-1.3872385025, -1.38719558716, 122.116212845, 122.11625576}},\n\t\t{\"qhww7zdb5v\", []float64{-18.5476416349, -18.5476362705, 99.3093574047, 99.3093681335}},\n\t\t{\"wfwffcw5vd\", []float64{14.5547926426, 14.554798007, 133.37151289, 133.371523619}},\n\t\t{\"rynp\", []float64{-10.01953125, -9.84375, 177.1875, 177.5390625}},\n\t\t{\"fb57ykxryn82\", []float64{45.6852641702, 45.6852643378, -51.3948151097, -51.3948147744}},\n\t\t{\"30sq\", []float64{-41.1328125, -40.95703125, -129.0234375, -128.671875}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"e5gjz7\", []float64{22.1209716797, 22.1264648438, -40.4626464844, -40.4516601562}},\n\t\t{\"t6vkbx\", []float64{16.3421630859, 16.34765625, 63.6547851562, 63.6657714844}},\n\t\t{\"3e\", []float64{-28.125, -22.5, -112.5, -101.25}},\n\t\t{\"th\", []float64{22.5, 28.125, 45.0, 56.25}},\n\t\t{\"7j\", []float64{-16.875, -11.25, -45.0, -33.75}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"1r15z2z5\", []float64{-49.9611854553, -49.9610137939, -122.015533447, -122.015190125}},\n\t\t{\"k9e57kg2f3\", []float64{-35.9649842978, -35.9649789333, 26.866132021, 26.8661427498}},\n\t\t{\"4w\", []float64{-56.25, -50.625, -67.5, -56.25}},\n\t\t{\"82c582qhsx7\", []float64{4.83616903424, 4.83617037535, -167.324326783, -167.324325442}},\n\t\t{\"qzdzkwwdc\", []float64{-1.50190830231, -1.50186538696, 127.823910713, 127.823953629}},\n\t\t{\"6xn26p6fnr0\", []float64{-5.54084837437, -5.54084703326, -58.6190021038, -58.6190007627}},\n\t\t{\"wm4y0xu8ee\", []float64{29.2223614454, 29.2223668098, 105.14549017, 105.145500898}},\n\t\t{\"rke5rewv\", []float64{-19.0961265564, -19.095954895, 150.807609558, 150.807952881}},\n\t\t{\"0bn3vwcjj\", []float64{-89.6544456482, -89.6544027328, -137.217650414, -137.217607498}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"red\", []float64{-25.3125, -23.90625, 160.3125, 161.71875}},\n\t\t{\"r2\", []float64{-45.0, -39.375, 146.25, 157.5}},\n\t\t{\"v7qptt2u5k\", []float64{64.6291565895, 64.6291619539, 64.9303686619, 64.9303793907}},\n\t\t{\"pey\", []float64{-68.90625, -67.5, 165.9375, 167.34375}},\n\t\t{\"r7cg5cz8\", []float64{-23.3692932129, -23.3691215515, 148.886032104, 148.886375427}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"qqr5mxz\", []float64{-9.22988891602, -9.228515625, 111.345062256, 111.346435547}},\n\t\t{\"wuuvdz7x2\", []float64{27.7266168594, 27.7266597748, 130.555343628, 130.555386543}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"4mgwkd9yp3r5\", []float64{-56.5428471006, -56.542846933, -73.6276473105, -73.6276469752}},\n\t\t{\"dpk6jbemhyvh\", []float64{41.1364542693, 41.1364544369, -83.7660782039, -83.7660778686}},\n\t\t{\"768py6th4818\", []float64{-29.5607757568, -29.5607755892, -33.4683660418, -33.4683657065}},\n\t\t{\"q8\", []float64{-45.0, -39.375, 112.5, 123.75}},\n\t\t{\"xmgvmq9cf\", []float64{33.3026075363, 33.3026504517, 151.756639481, 151.756682396}},\n\t\t{\"6rhhdy3\", []float64{-4.79965209961, -4.79827880859, -73.0027770996, -73.0014038086}},\n\t\t{\"05dd6myj4xqj\", []float64{-69.884508457, -69.8845082894, -176.377142966, -176.377142631}},\n\t\t{\"yw6\", []float64{80.15625, 81.5625, 115.3125, 116.71875}},\n\t\t{\"6mt6h4jrw\", []float64{-13.6986637115, -13.6986207962, -71.1839389801, -71.1838960648}},\n\t\t{\"86ymqv2s\", []float64{16.4211273193, 16.4212989807, -159.663619995, -159.663276672}},\n\t\t{\"vygq9vr6umc\", []float64{84.1406701505, 84.1406714916, 83.4073568881, 83.4073582292}},\n\t\t{\"89g6sp8p81\", []float64{10.3256946802, 10.3257000446, -152.75390625, -152.753895521}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"6st\", []float64{-19.6875, -18.28125, -60.46875, -59.0625}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"sk\", []float64{22.5, 28.125, 11.25, 22.5}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"f3pf69r61v4\", []float64{51.0277444124, 51.0277457535, -67.7316650748, -67.7316637337}},\n\t\t{\"r9qde0cj\", []float64{-37.5243186951, -37.5241470337, 166.773834229, 166.774177551}},\n\t\t{\"nbm5pqh\", []float64{-88.0334472656, -88.0320739746, 131.10534668, 131.106719971}},\n\t\t{\"u7f9fh2dzpw9\", []float64{66.4252256043, 66.425225772, 14.8545113951, 14.8545117304}},\n\t\t{\"pnzr19j29\", []float64{-50.7952022552, -50.7951593399, 145.268483162, 145.268526077}},\n\t\t{\"3e7rf8ww4fe5\", []float64{-25.3526548482, -25.3526546806, -107.810775787, -107.810775451}},\n\t\t{\"1x\", []float64{-50.625, -45.0, -112.5, -101.25}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"4h4swejxpzv\", []float64{-66.6912616789, -66.6912603378, -86.1908380687, -86.1908367276}},\n\t\t{\"hy6gnf0u\", []float64{-54.3047332764, -54.304561615, 37.9148483276, 37.9151916504}},\n\t\t{\"xnycujudf\", []float64{38.3084249496, 38.308467865, 144.67423439, 144.674277306}},\n\t\t{\"t6bypmux9z54\", []float64{16.5563485399, 16.5563487075, 57.6295499504, 57.6295502856}},\n\t\t{\"ufw1c6xp2t\", []float64{59.3851214647, 59.3851268291, 42.2520661354, 42.2520768642}},\n\t\t{\"42jpxwe9byn\", []float64{-88.6456024647, -88.6456011236, -71.3843134046, -71.3843120635}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"tj0de\", []float64{28.564453125, 28.6083984375, 45.8349609375, 45.87890625}},\n\t\t{\"e0kqd\", []float64{2.548828125, 2.5927734375, -38.935546875, -38.8916015625}},\n\t\t{\"bzysgj0rr\", []float64{89.4574213028, 89.4574642181, -136.976895332, -136.976852417}},\n\t\t{\"bxdp8ycgtcqt\", []float64{88.543546591, 88.5435467586, -154.651882276, -154.651881941}},\n\t\t{\"k0kx2785\", []float64{-42.2995948792, -42.2994232178, 6.33911132812, 6.33945465088}},\n\t\t{\"75ugg\", []float64{-23.2470703125, -23.203125, -38.1884765625, -38.14453125}},\n\t\t{\"sbbsbv2r\", []float64{5.08375167847, 5.08392333984, 34.4864273071, 34.4867706299}},\n\t\t{\"u7vunvq7rn\", []float64{66.8263041973, 66.8263095617, 19.6414518356, 19.6414625645}},\n\t\t{\"w4m7uexcx\", []float64{13.3349132538, 13.3349561691, 97.591509819, 97.5915527344}},\n\t\t{\"350g\", []float64{-27.59765625, -27.421875, -133.9453125, -133.59375}},\n\t\t{\"p\", []float64{-90.0, -45.0, 135.0, 180.0}},\n\t\t{\"t8w2g1m1\", []float64{2.95137405396, 2.95154571533, 76.4277648926, 76.4281082153}},\n\t\t{\"96k738f\", []float64{13.2316589355, 13.2330322266, -117.704772949, -117.703399658}},\n\t\t{\"c26nv174\", []float64{47.5999832153, 47.6001548767, -120.713653564, -120.713310242}},\n\t\t{\"s67g9ehvds\", []float64{13.2889294624, 13.2889348269, 16.5959858894, 16.5959966183}},\n\t\t{\"4ybt4sw\", []float64{-51.1276245117, -51.1262512207, -55.4287719727, -55.4273986816}},\n\t\t{\"5jqnz9zqyk12\", []float64{-59.2714333534, -59.2714331858, -36.2226838991, -36.2226835638}},\n\t\t{\"31d5nguy\", []float64{-36.0135269165, -36.0133552551, -131.884346008, -131.884002686}},\n\t\t{\"m4bbcg\", []float64{-29.3829345703, -29.3774414062, 46.1315917969, 46.142578125}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"mkx0hzzq\", []float64{-19.6438980103, -19.6437263489, 66.3124465942, 66.312789917}},\n\t\t{\"9bv7x0\", []float64{4.833984375, 4.83947753906, -93.5595703125, -93.5485839844}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"0x7pv0083h\", []float64{-47.8563809395, -47.8563755751, -153.060793877, -153.060783148}},\n\t\t{\"ruqzw0ztgw\", []float64{-19.7702515125, -19.7702461481, 178.516309261, 178.51631999}},\n\t\t{\"xpy5c3c\", []float64{44.2625427246, 44.2639160156, 143.493804932, 143.495178223}},\n\t\t{\"bnn6fxqm\", []float64{79.2740821838, 79.2742538452, -171.09249115, -171.092147827}},\n\t\t{\"fnm9u1t\", []float64{80.4721069336, 80.4734802246, -82.0829772949, -82.0816040039}},\n\t\t{\"jfvn54\", []float64{-73.4655761719, -73.4600830078, 85.9130859375, 85.9240722656}},\n\t\t{\"dj\", []float64{28.125, 33.75, -90.0, -78.75}},\n\t\t{\"mxfstk2s\", []float64{-0.591201782227, -0.59103012085, 71.2470245361, 71.2473678589}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"4xy4p\", []float64{-46.0546875, -46.0107421875, -58.7548828125, -58.7109375}},\n\t\t{\"cbdg47d\", []float64{48.3590698242, 48.3604431152, -97.2811889648, -97.2798156738}},\n\t\t{\"1ddv9\", []float64{-74.970703125, -74.9267578125, -108.588867188, -108.544921875}},\n\t\t{\"4cn07cd\", []float64{-84.3228149414, -84.3214416504, -47.6449584961, -47.6435852051}},\n\t\t{\"dq8fjtv64n\", []float64{36.9460237026, 36.946029067, -77.4463176727, -77.4463069439}},\n\t\t{\"gx2qc4\", []float64{86.9787597656, 86.9842529297, -22.1044921875, -22.0935058594}},\n\t\t{\"yx\", []float64{84.375, 90.0, 112.5, 123.75}},\n\t\t{\"44\", []float64{-78.75, -73.125, -90.0, -78.75}},\n\t\t{\"zz679sbs\", []float64{86.4232635498, 86.4234352112, 171.980667114, 171.981010437}},\n\t\t{\"2fdh2u769u7y\", []float64{-30.1666307822, -30.1666306145, -143.399997689, -143.399997354}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"11cxx\", []float64{-78.837890625, -78.7939453125, -132.583007812, -132.5390625}},\n\t\t{\"ygf2\", []float64{66.09375, 66.26953125, 126.9140625, 127.265625}},\n\t\t{\"m0uscc\", []float64{-39.9407958984, -39.9353027344, 51.4050292969, 51.416015625}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"kn2m\", []float64{-8.96484375, -8.7890625, 0.3515625, 0.703125}},\n\t\t{\"k56d043m45\", []float64{-26.3539534807, -26.3539481163, 3.51742744446, 3.51743817329}},\n\t\t{\"m7\", []float64{-28.125, -22.5, 56.25, 67.5}},\n\t\t{\"pwe\", []float64{-53.4375, -52.03125, 161.71875, 163.125}},\n\t\t{\"1j7q0hs8u\", []float64{-59.3892145157, -59.3891716003, -130.423336029, -130.423293114}},\n\t\t{\"f264077wzn\", []float64{46.776856184, 46.7768615484, -75.9214067459, -75.9213960171}},\n\t\t{\"2jvn\", []float64{-11.6015625, -11.42578125, -172.96875, -172.6171875}},\n\t\t{\"4d4ghmzzsjb9\", []float64{-78.1897520833, -78.1897519156, -63.4352295846, -63.4352292493}},\n\t\t{\"1h2n6vef0we\", []float64{-64.9645265937, -64.9645252526, -134.873975068, -134.873973727}},\n\t\t{\"vnfc1v2yh\", []float64{83.1744003296, 83.1744432449, 48.9452934265, 48.9453363419}},\n\t\t{\"2b5brk\", []float64{-44.9340820312, -44.9285888672, -140.657958984, -140.646972656}},\n\t\t{\"yntb\", []float64{81.5625, 81.73828125, 98.0859375, 98.4375}},\n\t\t{\"5fj85yy\", []float64{-78.7129211426, -78.7115478516, -3.34259033203, -3.34121704102}},\n\t\t{\"cm36wevqn\", []float64{74.9923324585, 74.9923753738, -121.699075699, -121.699032784}},\n\t\t{\"7hf52n\", []float64{-17.6770019531, -17.6715087891, -42.1875, -42.1765136719}},\n\t\t{\"dh1sz\", []float64{23.3349609375, 23.37890625, -87.5830078125, -87.5390625}},\n\t\t{\"h\", []float64{-90.0, -45.0, 0.0, 45.0}},\n\t\t{\"7ny5k0k28\", []float64{-6.4585018158, -6.45845890045, -36.3808822632, -36.3808393478}},\n\t\t{\"w7\", []float64{16.875, 22.5, 101.25, 112.5}},\n\t\t{\"f9j0296wjz\", []float64{50.6768792868, 50.6768846512, -60.443097353, -60.4430866241}},\n\t\t{\"v316n9gu7y5\", []float64{50.9869372845, 50.9869386256, 58.2987718284, 58.2987731695}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"0jt97bxt\", []float64{-58.8391685486, -58.8389968872, -172.090530396, -172.090187073}},\n\t\t{\"cfdbvrc\", []float64{59.236907959, 59.23828125, -97.1507263184, -97.1493530273}},\n\t\t{\"95d\", []float64{19.6875, 21.09375, -132.1875, -130.78125}},\n\t\t{\"my\", []float64{-11.25, -5.625, 78.75, 90.0}},\n\t\t{\"5t\", []float64{-61.875, -56.25, -22.5, -11.25}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"k2j\", []float64{-45.0, -43.59375, 18.28125, 19.6875}},\n\t\t{\"rjygh\", []float64{-12.12890625, -12.0849609375, 144.66796875, 144.711914062}},\n\t\t{\"xs79f\", []float64{24.2138671875, 24.2578125, 162.509765625, 162.553710938}},\n\t\t{\"nmz3x305vedq\", []float64{-57.3864214495, -57.3864212818, 111.764155068, 111.764155403}},\n\t\t{\"u1hq\", []float64{51.6796875, 51.85546875, 5.9765625, 6.328125}},\n\t\t{\"2v8jpgs92zxm\", []float64{-13.1641120277, -13.1641118601, -145.903202109, -145.903201774}},\n\t\t{\"f54n04uq\", []float64{62.9458236694, 62.9459953308, -87.1816635132, -87.1813201904}},\n\t\t{\"8zkv13p8\", []float64{41.6656494141, 41.6658210754, -139.505081177, -139.504737854}},\n\t\t{\"z03j6ktm2\", []float64{47.354722023, 47.3547649384, 136.512336731, 136.512379646}},\n\t\t{\"qd266u\", []float64{-31.9262695312, -31.9207763672, 112.972412109, 112.983398438}},\n\t\t{\"783q5\", []float64{-42.5390625, -42.4951171875, -20.6103515625, -20.56640625}},\n\t\t{\"ynqe7fy5s9m6\", []float64{80.7432531193, 80.7432532869, 99.3138598278, 99.3138601631}},\n\t\t{\"pp\", []float64{-50.625, -45.0, 135.0, 146.25}},\n\t\t{\"bd\", []float64{56.25, 61.875, -157.5, -146.25}},\n\t\t{\"8573xzmt2qy\", []float64{18.5856847465, 18.5856860876, -175.081539452, -175.081538111}},\n\t\t{\"gwhbzrftu0\", []float64{78.9253950119, 78.9254003763, -15.4981040955, -15.4980933666}},\n\t\t{\"h42w9e805\", []float64{-76.1819458008, -76.1819028854, 0.769171714783, 0.769214630127}},\n\t\t{\"uzbhd66135d1\", []float64{89.397358764, 89.3973589316, 33.8516691327, 33.851669468}},\n\t\t{\"zf\", []float64{56.25, 61.875, 168.75, 180.0}},\n\t\t{\"6q2r\", []float64{-8.61328125, -8.4375, -78.3984375, -78.046875}},\n\t\t{\"qtxy7v4w9\", []float64{-12.9352855682, -12.9352426529, 123.566708565, 123.56675148}},\n\t\t{\"58\", []float64{-90.0, -84.375, -22.5, -11.25}},\n\t\t{\"r\", []float64{-45.0, 0.0, 135.0, 180.0}},\n\t\t{\"8u9qjb5qw\", []float64{26.368303299, 26.3683462143, -144.234781265, -144.23473835}},\n\t\t{\"48sx\", []float64{-85.95703125, -85.78125, -61.171875, -60.8203125}},\n\t\t{\"690tdt6\", []float64{-38.3793640137, -38.3779907227, -66.6842651367, -66.6828918457}},\n\t\t{\"qm8\", []float64{-14.0625, -12.65625, 101.25, 102.65625}},\n\t\t{\"2mj\", []float64{-16.875, -15.46875, -161.71875, -160.3125}},\n\t\t{\"3e5\", []float64{-28.125, -26.71875, -108.28125, -106.875}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"f6dg1tndqxy\", []float64{59.6177373827, 59.6177387238, -74.8076811433, -74.8076798022}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"9f3eqkhmf\", []float64{13.2504987717, 13.250541687, -98.8600444794, -98.860001564}},\n\t\t{\"yuqghk1x5z\", []float64{69.4568055868, 69.4568109512, 133.431175947, 133.431186676}},\n\t\t{\"w30rynrjmy\", []float64{7.02257037163, 7.02257573605, 101.875094175, 101.875104904}},\n\t\t{\"m25ergh6\", []float64{-44.4118881226, -44.4117164612, 61.5182876587, 61.5186309814}},\n\t\t{\"jqznhq\", []float64{-50.9436035156, -50.9381103516, 66.2805175781, 66.2915039062}},\n\t\t{\"3u4\", []float64{-22.5, -21.09375, -98.4375, -97.03125}},\n\t\t{\"fzr\", []float64{85.78125, 87.1875, -46.40625, -45.0}},\n\t\t{\"je\", []float64{-73.125, -67.5, 67.5, 78.75}},\n\t\t{\"pmztf5q7e7d4\", []float64{-56.6270351037, -56.6270349361, 156.893490851, 156.893491186}},\n\t\t{\"xdknsz\", []float64{13.8372802734, 13.8427734375, 163.333740234, 163.344726562}},\n\t\t{\"736h1c7d5h\", []float64{-37.2583937645, -37.2583884001, -30.8556604385, -30.8556497097}},\n\t\t{\"c57pmmv9u\", []float64{64.5875501633, 64.5875930786, -130.542812347, -130.542769432}},\n\t\t{\"80qnjsh\", []float64{2.48291015625, 2.48428344727, -171.315307617, -171.313934326}},\n\t\t{\"kp\", []float64{-5.625, 0.0, 0.0, 11.25}},\n\t\t{\"u6yufbthbdw\", []float64{61.3072863221, 61.3072876632, 20.8699330688, 20.8699344099}},\n\t\t{\"c4yj93f16ys\", []float64{61.4454093575, 61.4454106987, -126.504698396, -126.504697055}},\n\t\t{\"ygkp\", []float64{64.51171875, 64.6875, 129.375, 129.7265625}},\n\t\t{\"h1ypf\", []float64{-78.7939453125, -78.75, 8.525390625, 8.5693359375}},\n\t\t{\"hz76my5h4qe\", []float64{-48.7895616889, -48.7895603478, 38.5772185028, 38.5772198439}},\n\t\t{\"zh1cw\", []float64{67.763671875, 67.8076171875, 137.724609375, 137.768554688}},\n\t\t{\"00\", []float64{-90.0, -84.375, -180.0, -168.75}},\n\t\t{\"h9be5u0k\", []float64{-79.6062469482, -79.6060752869, 23.3682632446, 23.3686065674}},\n\t\t{\"4btv\", []float64{-86.30859375, -86.1328125, -48.1640625, -47.8125}},\n\t\t{\"42hz7pm83dt\", []float64{-88.6857041717, -88.6857028306, -71.9308523834, -71.9308510423}},\n\t\t{\"qv49\", []float64{-16.69921875, -16.5234375, 127.265625, 127.6171875}},\n\t\t{\"0nwzwd\", []float64{-52.1081542969, -52.1026611328, -170.222167969, -170.211181641}},\n\t\t{\"jhkmc\", []float64{-65.0830078125, -65.0390625, 51.0205078125, 51.064453125}},\n\t\t{\"sysens0py\", []float64{37.1131467819, 37.1131896973, 40.3640270233, 40.3640699387}},\n\t\t{\"5q792kbf\", []float64{-54.5975875854, -54.5974159241, -28.8161087036, -28.8157653809}},\n\t\t{\"624gbe\", []float64{-44.3243408203, -44.3188476562, -74.8608398438, -74.8498535156}},\n\t\t{\"gtkjfqg\", []float64{75.5790710449, 75.5804443359, -16.7720031738, -16.7706298828}},\n\t\t{\"nv4\", []float64{-61.875, -60.46875, 126.5625, 127.96875}},\n\t\t{\"dcwv6uu0\", []float64{9.3864440918, 9.38661575317, -46.6314697266, -46.6311264038}},\n\t\t{\"vvgtyfv9\", []float64{78.36977005, 78.3699417114, 83.97605896, 83.9764022827}},\n\t\t{\"53rjpqr6zt9\", []float64{-82.0550099015, -82.0550085604, -23.5773669183, -23.5773655772}},\n\t\t{\"vmyp4dnemp6\", []float64{78.5858018696, 78.5858032107, 64.8065069318, 64.8065082729}},\n\t\t{\"t9xqjxjpv\", []float64{9.53197002411, 9.53201293945, 77.9440927505, 77.9441356659}},\n\t\t{\"sby32e\", []float64{4.45495605469, 4.46044921875, 42.5610351562, 42.5720214844}},\n\t\t{\"sjfgy\", []float64{33.0029296875, 33.046875, 4.130859375, 4.1748046875}},\n\t\t{\"k7q0z2b2du\", []float64{-26.5826869011, -26.5826815367, 20.0065648556, 20.0065755844}},\n\t\t{\"nt\", []float64{-61.875, -56.25, 112.5, 123.75}},\n\t\t{\"1\", []float64{-90.0, -45.0, -135.0, -90.0}},\n\t\t{\"mpfpfd32e\", []float64{-0.0314998626709, -0.0314569473267, 47.9242086411, 47.9242515564}},\n\t\t{\"hqjqn\", []float64{-55.1953125, -55.1513671875, 18.896484375, 18.9404296875}},\n\t\t{\"9q7chj6u2uw\", []float64{35.3616240621, 35.3616254032, -118.296964467, -118.296963125}},\n\t\t{\"0wsf47v9c\", []float64{-53.0650377274, -53.064994812, -150.713839531, -150.713796616}},\n\t\t{\"kdv72env8xke\", []float64{-28.9424979128, -28.9424977452, 29.9140823632, 29.9140826985}},\n\t\t{\"trfx\", []float64{44.82421875, 45.0, 59.765625, 60.1171875}},\n\t\t{\"02uttm\", []float64{-84.7869873047, -84.7814941406, -162.191162109, -162.180175781}},\n\t\t{\"hhjgb5s3vv39\", []float64{-66.8212655, -66.8212653324, 8.0920227617, 8.09202309698}},\n\t\t{\"r16\", []float64{-37.96875, -36.5625, 137.8125, 139.21875}},\n\t\t{\"4xy44eer4t3\", []float64{-46.0342316329, -46.0342302918, -58.9480648935, -58.9480635524}},\n\t\t{\"8b\", []float64{0.0, 5.625, -146.25, -135.0}},\n\t\t{\"zd\", []float64{56.25, 61.875, 157.5, 168.75}},\n\t\t{\"z0x\", []float64{47.8125, 49.21875, 144.84375, 146.25}},\n\t\t{\"4967\", []float64{-82.44140625, -82.265625, -64.3359375, -63.984375}},\n\t\t{\"2vf4\", []float64{-12.3046875, -12.12890625, -143.4375, -143.0859375}},\n\t\t{\"tzp3t0rtg\", []float64{39.6410322189, 39.6410751343, 89.1754674911, 89.1755104065}},\n\t\t{\"75yry\", []float64{-22.5439453125, -22.5, -35.947265625, -35.9033203125}},\n\t\t{\"bdgtu\", []float64{61.4794921875, 61.5234375, -152.40234375, -152.358398438}},\n\t\t{\"u1\", []float64{50.625, 56.25, 0.0, 11.25}},\n\t\t{\"rz2bgp7hds\", []float64{-4.04629468918, -4.04628932476, 169.940750599, 169.940761328}},\n\t\t{\"g\", []float64{45.0, 90.0, -45.0, 0.0}},\n\t\t{\"psptppx32e\", []float64{-66.5796643496, -66.5796589851, 168.364470005, 168.364480734}},\n\t\t{\"gshfctpwf1\", []float64{68.0120283365, 68.0120337009, -15.7440090179, -15.7439982891}},\n\t\t{\"3yq\", []float64{-9.84375, -8.4375, -92.8125, -91.40625}},\n\t\t{\"685zv36e0fd2\", []float64{-43.6303004622, -43.6303002946, -61.9923811778, -61.9923808426}},\n\t\t{\"7gf5fd\", []float64{-23.2360839844, -23.2305908203, -8.32763671875, -8.31665039062}},\n\t\t{\"bmmzfq\", []float64{75.9265136719, 75.9320068359, -160.565185547, -160.554199219}},\n\t\t{\"m40\", []float64{-33.75, -32.34375, 45.0, 46.40625}},\n\t\t{\"tx45501g7v\", []float64{39.9029284716, 39.902933836, 70.4469001293, 70.4469108582}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"ej054jn\", []float64{28.6798095703, 28.6811828613, -44.9038696289, -44.9024963379}},\n\t\t{\"n1g7d\", []float64{-79.541015625, -79.4970703125, 94.658203125, 94.7021484375}},\n\t\t{\"nn6ejehuf\", []float64{-54.2991113663, -54.2990684509, 93.7639331818, 93.7639760971}},\n\t\t{\"qs8e93xwrrc\", []float64{-19.0629114211, -19.06291008, 113.268668801, 113.268670142}},\n\t\t{\"f2\", []float64{45.0, 50.625, -78.75, -67.5}},\n\t\t{\"gm\", []float64{73.125, 78.75, -33.75, -22.5}},\n\t\t{\"npp4rnm\", []float64{-50.1951599121, -50.1937866211, 100.158233643, 100.159606934}},\n\t\t{\"6t\", []float64{-16.875, -11.25, -67.5, -56.25}},\n\t\t{\"2f4fe3d5\", []float64{-33.3017921448, -33.3016204834, -142.237243652, -142.23690033}},\n\t\t{\"s7r00k0196\", []float64{18.3034908772, 18.3034962416, 21.1047899723, 21.1048007011}},\n\t\t{\"st084\", []float64{28.125, 28.1689453125, 23.291015625, 23.3349609375}},\n\t\t{\"p6f3bv9f1\", []float64{-74.1930770874, -74.1930341721, 149.449467659, 149.449510574}},\n\t\t{\"fgk5j\", []float64{63.80859375, 63.8525390625, -50.4052734375, -50.361328125}},\n\t\t{\"yeu22jjtp\", []float64{66.1660194397, 66.166062355, 118.484416008, 118.484458923}},\n\t\t{\"3bcn7gkusn25\", []float64{-39.6639578976, -39.6639577299, -99.6722602844, -99.6722599491}},\n\t\t{\"1\", []float64{-90.0, -45.0, -135.0, -90.0}},\n\t\t{\"6q4dh71sqn\", []float64{-10.8811962605, -10.881190896, -75.0452899933, -75.0452792645}},\n\t\t{\"p\", []float64{-90.0, -45.0, 135.0, 180.0}},\n\t\t{\"6uy2kbef9yg\", []float64{-18.2340927422, -18.2340914011, -47.2469682992, -47.246966958}},\n\t\t{\"58j\", []float64{-90.0, -88.59375, -15.46875, -14.0625}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"mh6x\", []float64{-19.86328125, -19.6875, 48.515625, 48.8671875}},\n\t\t{\"cgq4xrjz\", []float64{63.7603569031, 63.7605285645, -92.486000061, -92.4856567383}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"ppqg570sk3n\", []float64{-48.6741918325, -48.6741904914, 144.635886848, 144.635888189}},\n\t\t{\"1suyu1k\", []float64{-62.0878601074, -62.0864868164, -105.639038086, -105.637664795}},\n\t\t{\"xm4xkzsnvux\", []float64{29.4417956471, 29.4417969882, 149.980114549, 149.980115891}},\n\t\t{\"8p3xj\", []float64{42.01171875, 42.0556640625, -177.670898438, -177.626953125}},\n\t\t{\"ef92nkk\", []float64{14.0858459473, 14.0872192383, -9.21203613281, -9.2106628418}},\n\t\t{\"qnrf101\", []float64{-9.4921875, -9.49081420898, 100.943756104, 100.945129395}},\n\t\t{\"2qt\", []float64{-8.4375, -7.03125, -161.71875, -160.3125}},\n\t\t{\"c2q7e\", []float64{47.021484375, 47.0654296875, -114.829101562, -114.78515625}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"j6t0bwpjpusg\", []float64{-75.7718221284, -75.7718219608, 63.3131746575, 63.3131749928}},\n\t\t{\"wrq\", []float64{40.78125, 42.1875, 109.6875, 111.09375}},\n\t\t{\"xvf2jk\", []float64{32.3657226562, 32.3712158203, 172.144775391, 172.155761719}},\n\t\t{\"xy0p5y\", []float64{35.0134277344, 35.0189208984, 168.914794922, 168.92578125}},\n\t\t{\"bsh9xbd\", []float64{67.766418457, 67.767791748, -150.828552246, -150.827178955}},\n\t\t{\"g675yc\", []float64{58.3209228516, 58.3264160156, -29.2346191406, -29.2236328125}},\n\t\t{\"dkrnq7\", []float64{25.0213623047, 25.0268554688, -68.6315917969, -68.6206054688}},\n\t\t{\"6q4uk3dyk5\", []float64{-10.4936009645, -10.4935956001, -74.6920967102, -74.6920859814}},\n\t\t{\"t58tp1kxb54p\", []float64{20.5746203475, 20.5746205151, 46.0169246793, 46.0169250146}},\n\t\t{\"bbw73yzeuy\", []float64{48.4215438366, 48.421549201, -137.373529673, -137.373518944}},\n\t\t{\"gnq\", []float64{80.15625, 81.5625, -36.5625, -35.15625}},\n\t\t{\"3j\", []float64{-16.875, -11.25, -135.0, -123.75}},\n\t\t{\"7dx2c\", []float64{-30.8056640625, -30.76171875, -12.2607421875, -12.216796875}},\n\t\t{\"vn9\", []float64{81.5625, 82.96875, 46.40625, 47.8125}},\n\t\t{\"4kj\", []float64{-67.5, -66.09375, -71.71875, -70.3125}},\n\t\t{\"cuvj4trg5nb8\", []float64{72.6270465553, 72.6270467229, -94.0981142968, -94.0981139615}},\n\t\t{\"uetmbuswe\", []float64{65.7240772247, 65.7241201401, 29.92208004, 29.9221229553}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"f\", []float64{45.0, 90.0, -90.0, -45.0}},\n\t\t{\"jg\", []float64{-73.125, -67.5, 78.75, 90.0}},\n\t\t{\"ycz\", []float64{54.84375, 56.25, 133.59375, 135.0}},\n\t\t{\"pevtd\", []float64{-67.939453125, -67.8955078125, 165.322265625, 165.366210938}},\n\t\t{\"gf7fm3hmb8\", []float64{58.0582380295, 58.0582433939, -5.73999166489, -5.73998093605}},\n\t\t{\"w7zwjnh24qd\", []float64{22.1814313531, 22.1814326942, 112.022537291, 112.022538632}},\n\t\t{\"nfesgy\", []float64{-75.0695800781, -75.0640869141, 128.836669922, 128.84765625}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"efq0q0dg0g0n\", []float64{12.7034739777, 12.7034741454, -2.5450193882, -2.54501905292}},\n\t\t{\"kkucr8pk\", []float64{-18.060836792, -18.0606651306, 18.2692337036, 18.2695770264}},\n\t\t{\"5zdg4zvz\", []float64{-47.2413825989, -47.2412109375, -7.25406646729, -7.25372314453}},\n\t\t{\"fuw\", []float64{70.3125, 71.71875, -47.8125, -46.40625}},\n\t\t{\"x51mnftp8\", []float64{17.7689266205, 17.7689695358, 137.061309814, 137.06135273}},\n\t\t{\"y0\", []float64{45.0, 50.625, 90.0, 101.25}},\n\t\t{\"ndufku4sr\", []float64{-74.1130399704, -74.1129970551, 119.392161369, 119.392204285}},\n\t\t{\"ydwndhywhg8\", []float64{60.232219398, 60.2322207391, 121.034520864, 121.034522206}},\n\t\t{\"gj6ehkq0\", []float64{75.0819396973, 75.0821113586, -41.2893676758, -41.289024353}},\n\t\t{\"m3hfct0\", []float64{-38.8641357422, -38.8627624512, 62.9956054688, 62.9969787598}},\n\t\t{\"6745yupp70qu\", []float64{-27.4426010996, -27.442600932, -75.631118305, -75.6311179698}},\n\t\t{\"d7b0m9dzj213\", []float64{21.1471368559, 21.1471370235, -78.504297249, -78.5042969137}},\n\t\t{\"py\", []float64{-56.25, -50.625, 168.75, 180.0}},\n\t\t{\"4vhrpypw\", []float64{-60.6105422974, -60.610370636, -49.9225616455, -49.9222183228}},\n\t\t{\"xwyyj1xz5kzw\", []float64{39.0329053625, 39.0329055302, 167.222706601, 167.222706936}},\n\t\t{\"18ht0j2w8ste\", []float64{-89.0911141969, -89.0911140293, -106.171159521, -106.171159185}},\n\t\t{\"vynwqurve\", []float64{79.8729228973, 79.8729658127, 88.1980276108, 88.1980705261}},\n\t\t{\"s77hhn\", []float64{19.0173339844, 19.0228271484, 15.64453125, 15.6555175781}},\n\t\t{\"hj66tgs86\", []float64{-60.0100278854, -60.0099849701, 3.42301368713, 3.42305660248}},\n\t\t{\"e5nh4k\", []float64{17.6000976562, 17.6055908203, -36.4636230469, -36.4526367188}},\n\t\t{\"jk\", []float64{-67.5, -61.875, 56.25, 67.5}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"f0p3v5\", []float64{45.3240966797, 45.3295898438, -79.5849609375, -79.5739746094}},\n\t\t{\"numc175r1\", []float64{-65.9002876282, -65.9002447128, 131.895375252, 131.895418167}},\n\t\t{\"7pc\", []float64{-1.40625, 0.0, -43.59375, -42.1875}},\n\t\t{\"b7qw82mfqc4\", []float64{64.4255930185, 64.4255943596, -159.590199888, -159.590198547}},\n\t\t{\"qfe\", []float64{-30.9375, -29.53125, 127.96875, 129.375}},\n\t\t{\"mw9kj6nrue61\", []float64{-7.72204069421, -7.72204052657, 69.4973042607, 69.497304596}},\n\t\t{\"6en5d6psj\", []float64{-27.4980926514, -27.498049736, -58.9531087875, -58.9530658722}},\n\t\t{\"mk80\", []float64{-19.6875, -19.51171875, 56.25, 56.6015625}},\n\t\t{\"d2fbpmpyjv\", []float64{4.24727261066, 4.24727797508, -74.5533192158, -74.5533084869}},\n\t\t{\"pf84wbwguh9\", []float64{-75.4946324229, -75.4946310818, 169.056073576, 169.056074917}},\n\t\t{\"ncj8287\", []float64{-84.3296813965, -84.3283081055, 131.510467529, 131.51184082}},\n\t\t{\"smd4t\", []float64{31.376953125, 31.4208984375, 14.2822265625, 14.326171875}},\n\t\t{\"4ryj3jjxrd\", []float64{-45.4546773434, -45.454671979, -70.2606797218, -70.260668993}},\n\t\t{\"udffsxnn8\", []float64{60.9477710724, 60.9478139877, 26.5731811523, 26.5732240677}},\n\t\t{\"cub7vr\", []float64{72.4163818359, 72.421875, -100.667724609, -100.656738281}},\n\t\t{\"y7c6s4\", []float64{66.5441894531, 66.5496826172, 103.18359375, 103.194580078}},\n\t\t{\"t253\", []float64{0.17578125, 0.3515625, 60.8203125, 61.171875}},\n\t\t{\"1e2bhmk9ybw\", []float64{-71.6896077991, -71.6896064579, -111.252067387, -111.252066046}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"99r75\", []float64{7.55859375, 7.6025390625, -102.172851562, -102.12890625}},\n\t\t{\"knbr2kzz1791\", []float64{-5.72952283546, -5.72952266783, 0.373246818781, 0.373247154057}},\n\t\t{\"v8h\", []float64{45.0, 46.40625, 73.125, 74.53125}},\n\t\t{\"sm6xvf3bc\", []float64{30.9060430527, 30.906085968, 15.0207567215, 15.0207996368}},\n\t\t{\"vu\", []float64{67.5, 73.125, 78.75, 90.0}},\n\t\t{\"w56htc6\", []float64{19.0791320801, 19.0805053711, 93.0679321289, 93.0693054199}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"5hm57j8\", []float64{-65.4922485352, -65.4908752441, -37.8369140625, -37.8355407715}},\n\t\t{\"k8qwr0e\", []float64{-42.4923706055, -42.4909973145, 31.9523620605, 31.9537353516}},\n\t\t{\"716e0\", []float64{-37.44140625, -37.3974609375, -41.484375, -41.4404296875}},\n\t\t{\"wz71b\", []float64{41.0888671875, 41.1328125, 127.96875, 128.012695312}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"89x2\", []float64{8.4375, 8.61328125, -147.3046875, -146.953125}},\n\t\t{\"rcr37p\", []float64{-37.7105712891, -37.705078125, 179.077148438, 179.088134766}},\n\t\t{\"4xzjc7vmstr\", []float64{-45.3739361465, -45.3739348054, -57.5939060748, -57.5939047337}},\n\t\t{\"tv07ndf8\", []float64{28.6674499512, 28.6676216125, 79.3906402588, 79.3909835815}},\n\t\t{\"qb2z\", []float64{-42.36328125, -42.1875, 124.8046875, 125.15625}},\n\t\t{\"xjq0fjferr\", []float64{29.6952670813, 29.6952724457, 143.529134989, 143.529145718}},\n\t\t{\"zwn6\", []float64{79.1015625, 79.27734375, 166.2890625, 166.640625}},\n\t\t{\"7xc\", []float64{-1.40625, 0.0, -21.09375, -19.6875}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"vw4sy8z\", []float64{79.5890808105, 79.5904541016, 71.3108825684, 71.3122558594}},\n\t\t{\"djg8s3pre\", []float64{32.4384212494, 32.4384641647, -84.881272316, -84.8812294006}},\n\t\t{\"vpn8t\", []float64{84.462890625, 84.5068359375, 54.3603515625, 54.404296875}},\n\t\t{\"1sse8x6\", []float64{-64.0324401855, -64.0310668945, -106.147155762, -106.145782471}},\n\t\t{\"snm4\", []float64{35.5078125, 35.68359375, 7.03125, 7.3828125}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"5rb\", []float64{-46.40625, -45.0, -33.75, -32.34375}},\n\t\t{\"q7\", []float64{-28.125, -22.5, 101.25, 112.5}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"hn5sw9rbvjk3\", []float64{-55.4519608431, -55.4519606754, 5.21838281304, 5.21838314831}},\n\t\t{\"y0f7w5tep1f\", []float64{49.8537348211, 49.8537361622, 93.4355905652, 93.4355919063}},\n\t\t{\"xts0uk\", []float64{31.0913085938, 31.0968017578, 163.311767578, 163.322753906}},\n\t\t{\"ftybqfey\", []float64{77.4024581909, 77.4026298523, -57.7060317993, -57.7056884766}},\n\t\t{\"eqvhf\", []float64{38.8037109375, 38.84765625, -26.630859375, -26.5869140625}},\n\t\t{\"ctpbp73\", []float64{73.1428527832, 73.1442260742, -101.281585693, -101.280212402}},\n\t\t{\"15czhy35\", []float64{-67.6409339905, -67.6407623291, -132.328948975, -132.328605652}},\n\t\t{\"1\", []float64{-90.0, -45.0, -135.0, -90.0}},\n\t\t{\"vk6mwwrysbf\", []float64{69.9084989727, 69.9085003138, 59.7105565667, 59.7105579078}},\n\t\t{\"4yfcbqvevqy\", []float64{-51.6858740151, -51.685872674, -52.3640397191, -52.364038378}},\n\t\t{\"d6qm1q41g\", []float64{13.5684156418, 13.5684585571, -69.9031305313, -69.903087616}},\n\t\t{\"kjqew1cty\", []float64{-14.842915535, -14.8428726196, 9.40661430359, 9.40665721893}},\n\t\t{\"hf9zn39ewerv\", []float64{-74.6981724165, -74.6981722489, 36.4879449829, 36.4879453182}},\n\t\t{\"1j\", []float64{-61.875, -56.25, -135.0, -123.75}},\n\t\t{\"u41\", []float64{56.25, 57.65625, 1.40625, 2.8125}},\n\t\t{\"pd8sbu5sk\", []float64{-75.0798368454, -75.0797939301, 158.241062164, 158.24110508}},\n\t\t{\"k7\", []float64{-28.125, -22.5, 11.25, 22.5}},\n\t\t{\"fx6xcm\", []float64{87.1710205078, 87.1765136719, -63.9294433594, -63.9184570312}},\n\t\t{\"k1nwc4mun\", []float64{-38.1754302979, -38.1753873825, 9.19272422791, 9.19276714325}},\n\t\t{\"nechx1mg\", []float64{-68.1078529358, -68.1076812744, 114.221763611, 114.222106934}},\n\t\t{\"8et6dbj4g\", []float64{20.1274251938, 20.1274681091, -149.98934269, -149.989299774}},\n\t\t{\"7e\", []float64{-28.125, -22.5, -22.5, -11.25}},\n\t\t{\"vqcthybtw0\", []float64{83.885679245, 83.8856846094, 58.5690593719, 58.5690701008}},\n\t\t{\"r6qdv32n\", []float64{-31.8524551392, -31.8522834778, 155.621337891, 155.621681213}},\n\t\t{\"tbhh\", []float64{0.703125, 0.87890625, 84.375, 84.7265625}},\n\t\t{\"0c5fpu\", []float64{-84.0014648438, -83.9959716797, -140.635986328, -140.625}},\n\t\t{\"7b\", []float64{-45.0, -39.375, -11.25, 0.0}},\n\t\t{\"9vzmkfvug\", []float64{33.2825231552, 33.2825660706, -90.8379220963, -90.8378791809}},\n\t\t{\"68t\", []float64{-42.1875, -40.78125, -60.46875, -59.0625}},\n\t\t{\"ef1szshm45h\", []float64{12.1078079939, 12.107809335, -8.80510747433, -8.80510613322}},\n\t\t{\"21dgj4\", []float64{-36.0241699219, -36.0186767578, -175.913085938, -175.902099609}},\n\t\t{\"109q9yt\", []float64{-86.0092163086, -86.0078430176, -133.158416748, -133.157043457}},\n\t\t{\"nhj3b9vc\", []float64{-67.182598114, -67.1824264526, 97.4126815796, 97.4130249023}},\n\t\t{\"nye5uzgb\", []float64{-52.735748291, -52.7355766296, 128.182640076, 128.182983398}},\n\t\t{\"dhz5f\", []float64{27.3779296875, 27.421875, -80.068359375, -80.0244140625}},\n\t\t{\"g1verehd\", []float64{55.4318618774, 55.4320335388, -36.9298553467, -36.9295120239}},\n\t\t{\"jtr\", []float64{-60.46875, -59.0625, 77.34375, 78.75}},\n\t\t{\"m5nbruj\", []float64{-28.0590820312, -28.0577087402, 54.839630127, 54.841003418}},\n\t\t{\"p\", []float64{-90.0, -45.0, 135.0, 180.0}},\n\t\t{\"h\", []float64{-90.0, -45.0, 0.0, 45.0}},\n\t\t{\"bm3gr\", []float64{75.1025390625, 75.146484375, -165.981445312, -165.9375}},\n\t\t{\"e7my1m0qp\", []float64{19.3644332886, 19.3644762039, -25.6084871292, -25.6084442139}},\n\t\t{\"fzue\", []float64{89.12109375, 89.296875, -49.921875, -49.5703125}},\n\t\t{\"q70\", []float64{-28.125, -26.71875, 101.25, 102.65625}},\n\t\t{\"sjeed\", []float64{31.552734375, 31.5966796875, 5.009765625, 5.0537109375}},\n\t\t{\"cvsuyyw\", []float64{76.8081665039, 76.8095397949, -94.2654418945, -94.2640686035}},\n\t\t{\"7dnp\", []float64{-32.51953125, -32.34375, -14.0625, -13.7109375}},\n\t\t{\"tf9kr1u\", []float64{14.8191833496, 14.8205566406, 80.8209228516, 80.8222961426}},\n\t\t{\"j38nduwqew\", []float64{-80.3940546513, -80.3940492868, 56.3795828819, 56.3795936108}},\n\t\t{\"444y82r\", []float64{-77.606048584, -77.604675293, -86.1122131348, -86.1108398438}},\n\t\t{\"1rwzsww\", []float64{-46.4584350586, -46.4570617676, -114.051818848, -114.050445557}},\n\t\t{\"98vu\", []float64{4.921875, 5.09765625, -104.4140625, -104.0625}},\n\t\t{\"f0hu79k2y84\", []float64{45.7540655136, 45.7540668547, -83.1603857875, -83.1603844464}},\n\t\t{\"35399zm2gnn\", []float64{-26.415091753, -26.4150904119, -132.806374133, -132.806372792}},\n\t\t{\"qzxy\", []float64{-1.7578125, -1.58203125, 134.6484375, 135.0}},\n\t\t{\"7gpr25\", []float64{-26.8341064453, -26.8286132812, -1.0546875, -1.04370117188}},\n\t\t{\"xucdp\", []float64{27.0703125, 27.1142578125, 171.166992188, 171.2109375}},\n\t\t{\"db3mpnz89zq\", []float64{2.32235983014, 2.32236117125, -54.1741874814, -54.1741861403}},\n\t\t{\"p\", []float64{-90.0, -45.0, 135.0, 180.0}},\n\t\t{\"94m52\", []float64{13.2275390625, 13.271484375, -127.96875, -127.924804688}},\n\t\t{\"u7ucp\", []float64{66.26953125, 66.3134765625, 18.2373046875, 18.28125}},\n\t\t{\"81qq43p\", []float64{8.09143066406, 8.09280395508, -171.10244751, -171.101074219}},\n\t\t{\"f80w8\", []float64{46.142578125, 46.1865234375, -66.796875, -66.7529296875}},\n\t\t{\"8j5z\", []float64{29.35546875, 29.53125, -174.7265625, -174.375}},\n\t\t{\"56q\", []float64{-77.34375, -75.9375, -25.3125, -23.90625}},\n\t\t{\"b72vvhj\", []float64{64.3139648438, 64.3153381348, -167.468719482, -167.467346191}},\n\t\t{\"5j\", []float64{-61.875, -56.25, -45.0, -33.75}},\n\t\t{\"42hm9tj0rn\", []float64{-89.0056622028, -89.0056568384, -72.7003526688, -72.7003419399}},\n\t\t{\"cbxx9q2c89\", []float64{49.1654545069, 49.1654598713, -90.6471419334, -90.6471312046}},\n\t\t{\"43\", []float64{-84.375, -78.75, -78.75, -67.5}},\n\t\t{\"rvmw\", []float64{-14.4140625, -14.23828125, 176.484375, 176.8359375}},\n\t\t{\"jwmeyr4hj7\", []float64{-54.1454154253, -54.1454100609, 75.5120050907, 75.5120158195}},\n\t\t{\"b3y\", []float64{54.84375, 56.25, -160.3125, -158.90625}},\n\t\t{\"4y3n0e7u8j\", []float64{-53.7704104185, -53.7704050541, -54.8166275024, -54.8166167736}},\n\t\t{\"m0k0x\", []float64{-43.505859375, -43.4619140625, 50.9326171875, 50.9765625}},\n\t\t{\"2zc1v219ev8z\", []float64{-1.09834464267, -1.09834447503, -144.610815234, -144.610814899}},\n\t\t{\"3rvj3ezyffez\", []float64{-0.46162577346, -0.461625605822, -116.64206598, -116.642065644}},\n\t\t{\"35bdpq\", []float64{-23.5217285156, -23.5162353516, -133.978271484, -133.967285156}},\n\t\t{\"qdqrzp2e7b\", []float64{-30.9410619736, -30.9410566092, 121.597527266, 121.597537994}},\n\t\t{\"vmrsrejf\", []float64{75.2951431274, 75.2953147888, 67.1343612671, 67.1347045898}},\n\t\t{\"up\", []float64{84.375, 90.0, 0.0, 11.25}},\n\t\t{\"bzy\", []float64{88.59375, 90.0, -137.8125, -136.40625}},\n\t\t{\"3rnm42gs62\", []float64{-4.7412443161, -4.74123895168, -114.857157469, -114.85714674}},\n\t\t{\"yhekty3621c\", []float64{71.1382435262, 71.1382448673, 94.8247160017, 94.8247173429}},\n\t\t{\"ektx\", []float64{26.54296875, 26.71875, -26.015625, -25.6640625}},\n\t\t{\"9nxkb4u9f6\", []float64{37.4128782749, 37.4128836393, -124.798411131, -124.798400402}},\n\t\t{\"fg\", []float64{61.875, 67.5, -56.25, -45.0}},\n\t\t{\"66e4x10k68\", []float64{-30.4918241501, -30.4918187857, -74.2231822014, -74.2231714725}},\n\t\t{\"me\", []float64{-28.125, -22.5, 67.5, 78.75}},\n\t\t{\"r385f5q9\", []float64{-35.8852958679, -35.8851242065, 146.346817017, 146.347160339}},\n\t\t{\"xbdc8wgn0\", []float64{3.11428070068, 3.11432361603, 172.643280029, 172.643322945}},\n\t\t{\"74s\", []float64{-30.9375, -29.53125, -39.375, -37.96875}},\n\t\t{\"dg8t7\", []float64{20.6103515625, 20.654296875, -55.4150390625, -55.37109375}},\n\t\t{\"nf7\", []float64{-77.34375, -75.9375, 127.96875, 129.375}},\n\t\t{\"6nzfqpxy\", []float64{-6.59351348877, -6.59334182739, -78.8272476196, -78.8269042969}},\n\t\t{\"0ux06p9jktht\", []float64{-64.6014270745, -64.6014269069, -136.31678693, -136.316786595}},\n\t\t{\"nb8pxznpjh\", []float64{-85.8294653893, -85.8294600248, 124.099030495, 124.099041224}},\n\t\t{\"6qks2x97hzm\", []float64{-9.05492708087, -9.05492573977, -72.3979751766, -72.3979738355}},\n\t\t{\"us6qrd43em\", []float64{70.0161534548, 70.0161588192, 25.9968817234, 25.9968924522}},\n\t\t{\"tp2eh\", []float64{41.30859375, 41.3525390625, 45.87890625, 45.9228515625}},\n\t\t{\"vcgf16q2\", []float64{55.2076721191, 55.2078437805, 84.0869522095, 84.0872955322}},\n\t\t{\"qt15nkc82kz\", []float64{-16.3214953244, -16.3214939833, 114.182988256, 114.182989597}},\n\t\t{\"t6t\", []float64{14.0625, 15.46875, 63.28125, 64.6875}},\n\t\t{\"yx53b3kj32\", []float64{84.6903848648, 84.6903902292, 117.086845636, 117.086856365}},\n\t\t{\"twqdxev4cx\", []float64{35.6168121099, 35.6168174744, 76.9771456718, 76.9771564007}},\n\t\t{\"p\", []float64{-90.0, -45.0, 135.0, 180.0}},\n\t\t{\"4gty084hgddy\", []float64{-69.2569826916, -69.2569825239, -48.13918937, -48.1391890347}},\n\t\t{\"c7\", []float64{61.875, 67.5, -123.75, -112.5}},\n\t\t{\"ywffc641\", []float64{83.463306427, 83.4634780884, 116.424865723, 116.425209045}},\n\t\t{\"k0km\", []float64{-42.71484375, -42.5390625, 5.9765625, 6.328125}},\n\t\t{\"17k4fg\", []float64{-71.2188720703, -71.2133789062, -118.004150391, -117.993164062}},\n\t\t{\"9fr\", []float64{12.65625, 14.0625, -91.40625, -90.0}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"h08scsx\", []float64{-86.3278198242, -86.3264465332, 0.778656005859, 0.780029296875}},\n\t\t{\"8f8nq48\", []float64{15.1748657227, 15.1762390137, -145.986328125, -145.984954834}},\n\t\t{\"hecr6qx49k0\", []float64{-67.59567976, -67.5956784189, 24.3663561344, 24.3663574755}},\n\t\t{\"jn\", []float64{-56.25, -50.625, 45.0, 56.25}},\n\t\t{\"qwx7j\", []float64{-7.91015625, -7.8662109375, 122.915039062, 122.958984375}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"wxcj4\", []float64{44.47265625, 44.5166015625, 113.994140625, 114.038085938}},\n\t\t{\"gw63h\", []float64{80.33203125, 80.3759765625, -19.16015625, -19.1162109375}},\n\t\t{\"hp7b6f27tq6p\", []float64{-49.1618095525, -49.1618093848, 5.3948584199, 5.39485875517}},\n\t\t{\"kd\", []float64{-33.75, -28.125, 22.5, 33.75}},\n\t\t{\"fbweu\", []float64{48.4716796875, 48.515625, -46.93359375, -46.8896484375}},\n\t\t{\"m1fcuue7nm1g\", []float64{-34.8233712651, -34.8233710974, 49.080661498, 49.0806618333}},\n\t\t{\"h9j\", []float64{-84.375, -82.96875, 29.53125, 30.9375}},\n\t\t{\"n9d3g5uv\", []float64{-81.2334251404, -81.233253479, 115.80242157, 115.802764893}},\n\t\t{\"nhpp1spf\", []float64{-66.247215271, -66.2470436096, 99.9203109741, 99.9206542969}},\n\t\t{\"7jg2w13b23h\", []float64{-12.5614446402, -12.5614432991, -40.1635962725, -40.1635949314}},\n\t\t{\"6q4z0ebf3\", []float64{-9.99854564667, -9.99850273132, -74.8597669601, -74.8597240448}},\n\t\t{\"sv9vqgeecr\", []float64{31.8802589178, 31.8802642822, 36.5124285221, 36.5124392509}},\n\t\t{\"wqd4\", []float64{36.9140625, 37.08984375, 104.0625, 104.4140625}},\n\t\t{\"bwqgj\", []float64{80.68359375, 80.7275390625, -147.788085938, -147.744140625}},\n\t\t{\"hk73qx\", []float64{-65.8355712891, -65.830078125, 16.1059570312, 16.1169433594}},\n\t\t{\"gdx1d6hr76v\", []float64{59.3384175003, 59.3384188414, -12.5513903797, -12.5513890386}},\n\t\t{\"47czd\", []float64{-67.587890625, -67.5439453125, -76.201171875, -76.1572265625}},\n\t\t{\"1kpebdk50\", []float64{-66.8279457092, -66.8279027939, -113.17565918, -113.175616264}},\n\t\t{\"g3z7\", []float64{55.37109375, 55.546875, -23.5546875, -23.203125}},\n\t\t{\"8x\", []float64{39.375, 45.0, -157.5, -146.25}},\n\t\t{\"nczvrs\", []float64{-79.2114257812, -79.2059326172, 134.978027344, 134.989013672}},\n\t\t{\"fbyjmwc6\", []float64{50.1790237427, 50.1791954041, -47.5690841675, -47.5687408447}},\n\t\t{\"yhz41tus5wm\", []float64{72.1026183665, 72.1026197076, 99.9160046875, 99.9160060287}},\n\t\t{\"uktff1pp0\", []float64{70.8025932312, 70.8026361465, 19.4334411621, 19.4334840775}},\n\t\t{\"hrt8\", []float64{-47.8125, -47.63671875, 18.984375, 19.3359375}},\n\t\t{\"b5vkzheshz3\", []float64{66.9541557133, 66.9541570544, -172.304558605, -172.304557264}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"h6mvgwv1kmp\", []float64{-76.2956875563, -76.2956862152, 19.4968043268, 19.4968056679}},\n\t\t{\"etzq7udz\", []float64{33.4683036804, 33.4684753418, -12.1361160278, -12.1357727051}},\n\t\t{\"rf2x5pd6\", []float64{-31.0717391968, -31.0715675354, 169.588050842, 169.588394165}},\n\t\t{\"k8kquxqbt2\", []float64{-42.3673152924, -42.3673099279, 28.6838114262, 28.683822155}},\n\t\t{\"bz91jncb2c7\", []float64{87.4004097283, 87.4004110694, -144.621583968, -144.621582627}},\n\t\t{\"3uk4y5r6sjwr\", []float64{-20.5920389481, -20.5920387805, -95.3511917219, -95.3511913866}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"y95n1hd\", []float64{51.7044067383, 51.7057800293, 116.765441895, 116.766815186}},\n\t\t{\"629k5b3kbkc\", []float64{-41.4821608365, -41.4821594954, -76.8256638944, -76.8256625533}},\n\t\t{\"pp42st7vp5\", []float64{-50.5073958635, -50.5073904991, 138.367266655, 138.367277384}},\n\t\t{\"u17e\", []float64{52.55859375, 52.734375, 4.921875, 5.2734375}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"ex5w6znubms\", []float64{40.5129298568, 40.5129311979, -17.447989583, -17.4479882419}},\n\t\t{\"8jsn\", []float64{31.9921875, 32.16796875, -174.375, -174.0234375}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"mht9efkj\", []float64{-19.410610199, -19.4104385376, 52.9046630859, 52.9050064087}},\n\t\t{\"kkbcqqfcsz\", []float64{-18.0241495371, -18.0241441727, 12.5833261013, 12.5833368301}},\n\t\t{\"866rppwznm\", []float64{13.9291459322, 13.9291512966, -165.268782377, -165.268771648}},\n\t\t{\"96wj53wczp0c\", []float64{14.9499841221, 14.9499842897, -115.160106607, -115.160106272}},\n\t\t{\"9ctzrj\", []float64{9.73937988281, 9.74487304688, -92.8564453125, -92.8454589844}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"wfq5hd\", []float64{13.1945800781, 13.2000732422, 132.385253906, 132.396240234}},\n\t\t{\"9y6vu2v8wm5\", []float64{36.1712247133, 36.1712260544, -97.1882195771, -97.188218236}},\n\t\t{\"6xcpg\", []float64{-0.0439453125, 0.0, -65.9619140625, -65.91796875}},\n\t\t{\"rxqgqmc\", []float64{-3.61587524414, -3.61450195312, 167.268218994, 167.269592285}},\n\t\t{\"yye\", []float64{81.5625, 82.96875, 127.96875, 129.375}},\n\t\t{\"r3\", []float64{-39.375, -33.75, 146.25, 157.5}},\n\t\t{\"x7t\", []float64{19.6875, 21.09375, 153.28125, 154.6875}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"g9mmh9sku8h\", []float64{52.9192113876, 52.9192127287, -14.9133986235, -14.9133972824}},\n\t\t{\"v2q6qu3\", []float64{46.8251037598, 46.8264770508, 65.3370666504, 65.3384399414}},\n\t\t{\"7j9ckmu00j\", []float64{-13.8111609221, -13.8111555576, -42.3468017578, -42.346791029}},\n\t\t{\"4q3td\", []float64{-53.876953125, -53.8330078125, -76.552734375, -76.5087890625}},\n\t\t{\"9ve2c92z33ke\", []float64{31.077454146, 31.0774543136, -96.6126798838, -96.6126795486}},\n\t\t{\"9sscvm0mw1kw\", []float64{25.6485348567, 25.6485350244, -105.58899276, -105.588992424}},\n\t\t{\"9u\", []float64{22.5, 28.125, -101.25, -90.0}},\n\t\t{\"nv4j7yb8mjjy\", []float64{-60.9149988368, -60.9149986692, 126.728203855, 126.728204191}},\n\t\t{\"80w8\", []float64{2.8125, 2.98828125, -170.859375, -170.5078125}},\n\t\t{\"q06ch78z1\", []float64{-43.3975410461, -43.3974981308, 94.0550279617, 94.0550708771}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"gzw\", []float64{87.1875, 88.59375, -2.8125, -1.40625}},\n\t\t{\"1\", []float64{-90.0, -45.0, -135.0, -90.0}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"furb\", []float64{68.90625, 69.08203125, -45.3515625, -45.0}},\n\t\t{\"xen8pyc36\", []float64{16.9122934341, 16.9123363495, 166.983003616, 166.983046532}},\n\t\t{\"n1gc29sd\", []float64{-79.9279403687, -79.9277687073, 95.3015899658, 95.3019332886}},\n\t\t{\"cjvu\", []float64{78.046875, 78.22265625, -126.9140625, -126.5625}},\n\t\t{\"w7x53f7fb0\", []float64{20.2716207504, 20.2716261148, 111.175804138, 111.175814867}},\n\t\t{\"hz\", []float64{-50.625, -45.0, 33.75, 45.0}},\n\t\t{\"8tz14\", []float64{32.51953125, 32.5634765625, -147.568359375, -147.524414062}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"pyfunm\", []float64{-51.3006591797, -51.2951660156, 172.891845703, 172.902832031}},\n\t\t{\"b\", []float64{45.0, 90.0, -180.0, -135.0}},\n\t\t{\"7xre21ceuxv\", []float64{-3.63716259599, -3.63716125488, -11.9508652389, -11.9508638978}},\n\t\t{\"17\", []float64{-73.125, -67.5, -123.75, -112.5}},\n\t\t{\"ru\", []float64{-22.5, -16.875, 168.75, 180.0}},\n\t\t{\"bdjs6y77m\", []float64{57.0319604874, 57.0320034027, -149.640097618, -149.640054703}},\n\t\t{\"u1vsgx2\", []float64{55.718536377, 55.719909668, 7.88818359375, 7.88955688477}},\n\t\t{\"pj5e80uz\", []float64{-61.2544441223, -61.2542724609, 139.928398132, 139.928741455}},\n\t\t{\"ju01xcg9\", []float64{-67.2265434265, -67.2263717651, 79.0953826904, 79.0957260132}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"k05kz1\", []float64{-44.1595458984, -44.1540527344, 4.8779296875, 4.88891601562}},\n\t\t{\"ru2h1xc\", []float64{-20.3480529785, -20.3466796875, 168.81729126, 168.818664551}},\n\t\t{\"ud2s6zr\", []float64{58.443145752, 58.444519043, 23.3335876465, 23.3349609375}},\n\t\t{\"mrq5fm1zrqp\", []float64{-3.5308277607, -3.53082641959, 64.7891007364, 64.7891020775}},\n\t\t{\"0y2u0dg2rg\", []float64{-54.1254597902, -54.1254544258, -145.168544054, -145.168533325}},\n\t\t{\"9nkt0rnyx3\", []float64{36.0747295618, 36.0747349262, -128.651307821, -128.651297092}},\n\t\t{\"77q\", []float64{-26.71875, -25.3125, -25.3125, -23.90625}},\n\t\t{\"ng76t7\", []float64{-71.2628173828, -71.2573242188, 128.551025391, 128.562011719}},\n\t\t{\"4ewypr0y27up\", []float64{-69.2182661779, -69.2182660103, -57.6881629229, -57.6881625876}},\n\t\t{\"ge7vkm2x73\", []float64{64.2341905832, 64.2341959476, -17.0389688015, -17.0389580727}},\n\t\t{\"3qm4d\", []float64{-9.404296875, -9.3603515625, -116.630859375, -116.586914062}},\n\t\t{\"6gqw9\", []float64{-25.576171875, -25.5322265625, -47.0654296875, -47.021484375}},\n\t\t{\"32\", []float64{-45.0, -39.375, -123.75, -112.5}},\n\t\t{\"ns85\", []float64{-64.16015625, -63.984375, 112.5, 112.8515625}},\n\t\t{\"hzy00b4j5tcj\", []float64{-46.4053600095, -46.4053598419, 42.2233571112, 42.2233574465}},\n\t\t{\"7qrk5nt\", []float64{-9.10491943359, -9.10354614258, -23.4159851074, -23.4146118164}},\n\t\t{\"vd4219t7th\", []float64{56.2588620186, 56.258867383, 70.7374048233, 70.7374155521}},\n\t\t{\"g\", []float64{45.0, 90.0, -45.0, 0.0}},\n\t\t{\"pq1p16t\", []float64{-55.0057983398, -55.0044250488, 147.718048096, 147.719421387}},\n\t\t{\"gryfsc3wgn\", []float64{89.0412604809, 89.0412658453, -24.0468835831, -24.0468728542}},\n\t\t{\"np0u\", []float64{-49.921875, -49.74609375, 91.0546875, 91.40625}},\n\t\t{\"u87\", []float64{46.40625, 47.8125, 26.71875, 28.125}},\n\t\t{\"9qz2\", []float64{37.96875, 38.14453125, -113.5546875, -113.203125}},\n\t\t{\"xunf\", []float64{22.8515625, 23.02734375, 178.2421875, 178.59375}},\n\t\t{\"ve\", []float64{61.875, 67.5, 67.5, 78.75}},\n\t\t{\"s2c1tf2bvp4s\", []float64{4.49494846165, 4.49494862929, 12.9101834446, 12.9101837799}},\n\t\t{\"5znd0f\", []float64{-50.2624511719, -50.2569580078, -2.07641601562, -2.0654296875}},\n\t\t{\"dn90ug\", []float64{36.7108154297, 36.7163085938, -88.3850097656, -88.3740234375}},\n\t\t{\"24bg3\", []float64{-28.9599609375, -28.916015625, -178.901367188, -178.857421875}},\n\t\t{\"x46xpb2\", []float64{13.888092041, 13.889465332, 138.856201172, 138.857574463}},\n\t\t{\"83q\", []float64{7.03125, 8.4375, -160.3125, -158.90625}},\n\t\t{\"2pup20sqj\", []float64{-0.128059387207, -0.128016471863, -174.368948936, -174.368906021}},\n\t\t{\"07\", []float64{-73.125, -67.5, -168.75, -157.5}},\n\t\t{\"jj7nh21g4zk\", []float64{-59.4135086238, -59.4135072827, 49.408044219, 49.4080455601}},\n\t\t{\"19up4rw9\", []float64{-78.8844108582, -78.8842391968, -106.767196655, -106.766853333}},\n\t\t{\"2j\", []float64{-16.875, -11.25, -180.0, -168.75}},\n\t\t{\"14uexty\", []float64{-73.8844299316, -73.8830566406, -128.33404541, -128.332672119}},\n\t\t{\"t3wtb\", []float64{9.4482421875, 9.4921875, 65.390625, 65.4345703125}},\n\t\t{\"wv0z\", []float64{29.35546875, 29.53125, 124.8046875, 125.15625}},\n\t\t{\"jj6gcte19u\", []float64{-59.7790789604, -59.779073596, 48.9373004436, 48.9373111725}},\n\t\t{\"xz0wc0te1bbe\", []float64{40.5647895299, 40.5647896975, 169.504699185, 169.504699521}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"zyejp1um86c\", []float64{82.4519781768, 82.4519795179, 173.282215744, 173.282217085}},\n\t\t{\"ft915xruxj8n\", []float64{76.1539096758, 76.1539098434, -65.9289979935, -65.9289976582}},\n\t\t{\"vchvx0yp5c\", []float64{51.5971237421, 51.5971291065, 85.7457053661, 85.745716095}},\n\t\t{\"x5x\", []float64{19.6875, 21.09375, 144.84375, 146.25}},\n\t\t{\"0ykju58\", []float64{-53.8137817383, -53.8124084473, -140.44921875, -140.447845459}},\n\t\t{\"d2yk35fd\", []float64{4.98676300049, 4.98693466187, -69.91355896, -69.9132156372}},\n\t\t{\"6ymr7k8\", []float64{-8.54461669922, -8.5432434082, -48.7243652344, -48.7229919434}},\n\t\t{\"pjsxb9f\", []float64{-57.6905822754, -57.6892089844, 141.352844238, 141.354217529}},\n\t\t{\"trydkh27gn2t\", []float64{44.0132818557, 44.0132820234, 65.5668789893, 65.5668793246}},\n\t\t{\"k72c0uqqw2\", []float64{-26.5185070038, -26.5185016394, 12.3464977741, 12.346508503}},\n\t\t{\"s8trjfz\", []float64{4.05807495117, 4.05944824219, 30.145111084, 30.146484375}},\n\t\t{\"57ffkwfnrh1\", []float64{-68.4725689888, -68.4725676477, -29.6820102632, -29.6820089221}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"u2k\", []float64{46.40625, 47.8125, 16.875, 18.28125}},\n\t\t{\"nndqt\", []float64{-52.294921875, -52.2509765625, 93.3837890625, 93.427734375}},\n\t\t{\"w7\", []float64{16.875, 22.5, 101.25, 112.5}},\n\t\t{\"r6c7x00p6\", []float64{-28.91477108, -28.9147281647, 148.315515518, 148.315558434}},\n\t\t{\"mdrgz\", []float64{-31.6845703125, -31.640625, 78.7060546875, 78.75}},\n\t\t{\"f1dzhud0j\", []float64{54.6926879883, 54.6927309036, -85.9211111069, -85.9210681915}},\n\t\t{\"yqh\", []float64{78.75, 80.15625, 106.875, 108.28125}},\n\t\t{\"9jp43kj\", []float64{28.5424804688, 28.5438537598, -125.094451904, -125.093078613}},\n\t\t{\"14pb4v5q\", []float64{-78.7215042114, -78.72133255, -123.976249695, -123.975906372}},\n\t\t{\"bjzkjzr9hsvc\", []float64{78.0868977495, 78.0868979171, -169.54150144, -169.541501105}},\n\t\t{\"svjbhs9e9g8\", []float64{28.1503388286, 28.1503401697, 42.0358264446, 42.0358277857}},\n\t\t{\"guuxc8c5v\", []float64{73.0858182907, 73.0858612061, -4.85436916351, -4.85432624817}},\n\t\t{\"utu2603h0ru5\", []float64{77.3897973262, 77.3897974938, 28.5658425093, 28.5658428445}},\n\t\t{\"bq\", []float64{78.75, 84.375, -168.75, -157.5}},\n\t\t{\"kk\", []float64{-22.5, -16.875, 11.25, 22.5}},\n\t\t{\"6vxq65mhx\", []float64{-12.9452419281, -12.9451990128, -45.9596300125, -45.9595870972}},\n\t\t{\"f4sb\", []float64{59.0625, 59.23828125, -83.3203125, -82.96875}},\n\t\t{\"y5p4\", []float64{62.2265625, 62.40234375, 99.84375, 100.1953125}},\n\t\t{\"bs6cju\", []float64{69.1040039062, 69.1094970703, -153.380126953, -153.369140625}},\n\t\t{\"5j\", []float64{-61.875, -56.25, -45.0, -33.75}},\n\t\t{\"4e8z0qpsppx\", []float64{-69.048345387, -69.0483440459, -66.4237166941, -66.423715353}},\n\t\t{\"nbyg\", []float64{-85.25390625, -85.078125, 133.2421875, 133.59375}},\n\t\t{\"8jn2dnxvbd\", []float64{28.2495939732, 28.2495993376, -171.112382412, -171.112371683}},\n\t\t{\"0h6ej9g\", []float64{-65.5567932129, -65.5554199219, -176.238555908, -176.237182617}},\n\t\t{\"9j18njf9mc\", []float64{28.1568056345, 28.1568109989, -132.623273134, -132.623262405}},\n\t\t{\"pf93jtqd2xm9\", []float64{-75.7324543409, -75.7324541733, 170.758466944, 170.758467279}},\n\t\t{\"hc2\", []float64{-82.96875, -81.5625, 33.75, 35.15625}},\n\t\t{\"g\", []float64{45.0, 90.0, -45.0, 0.0}},\n\t\t{\"cewhub\", []float64{65.5224609375, 65.5279541016, -103.853759766, -103.842773438}},\n\t\t{\"2vcrfbgsv2sx\", []float64{-11.2890061922, -11.2890060246, -144.366300032, -144.366299696}},\n\t\t{\"mxdsnv\", []float64{-2.08190917969, -2.07641601562, 71.3122558594, 71.3232421875}},\n\t\t{\"03\", []float64{-84.375, -78.75, -168.75, -157.5}},\n\t\t{\"u73kybuxbq\", []float64{64.1216933727, 64.1216987371, 13.3106338978, 13.3106446266}},\n\t\t{\"uz2\", []float64{85.78125, 87.1875, 33.75, 35.15625}},\n\t\t{\"3w\", []float64{-11.25, -5.625, -112.5, -101.25}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"q8kttcg9t\", []float64{-42.6170825958, -42.6170396805, 119.085831642, 119.085874557}},\n\t\t{\"j8w8vhmh9d\", []float64{-87.0315349102, -87.0315295458, 76.8672823906, 76.8672931194}},\n\t\t{\"qxn54fn91g\", []float64{-5.08648216724, -5.08647680283, 121.067351103, 121.067361832}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"3h2p4vmu\", []float64{-19.8337554932, -19.8335838318, -134.871253967, -134.870910645}},\n\t\t{\"j4re6xcw\", []float64{-76.7288589478, -76.7286872864, 55.6587982178, 55.6591415405}},\n\t\t{\"ugkmrc1vj\", []float64{64.2104530334, 64.2104959488, 40.0697565079, 40.0697994232}},\n\t\t{\"n1mj98dq\", []float64{-81.9981765747, -81.9980049133, 97.1002578735, 97.1006011963}},\n\t\t{\"r6c\", []float64{-29.53125, -28.125, 147.65625, 149.0625}},\n\t\t{\"9uksmr\", []float64{24.6917724609, 24.697265625, -94.6911621094, -94.6801757812}},\n\t\t{\"dmve\", []float64{32.87109375, 33.046875, -71.015625, -70.6640625}},\n\t\t{\"jgrdkvuq\", []float64{-71.2906265259, -71.2904548645, 89.5114517212, 89.5117950439}},\n\t\t{\"94g\", []float64{15.46875, 16.875, -130.78125, -129.375}},\n\t\t{\"2vj4gt\", []float64{-16.3641357422, -16.3586425781, -139.064941406, -139.053955078}},\n\t\t{\"q39q2pm5kxt\", []float64{-35.4234436154, -35.4234422743, 103.01487878, 103.014880121}},\n\t\t{\"qcuy493hpwdf\", []float64{-34.0939741954, -34.0939740278, 130.541249625, 130.541249961}},\n\t\t{\"nhfpjqpyqnv3\", []float64{-62.0167130046, -62.0167128369, 93.0541204289, 93.0541207641}},\n\t\t{\"838kk\", []float64{9.1845703125, 9.228515625, -168.22265625, -168.178710938}},\n\t\t{\"zx2fdg\", []float64{86.2371826172, 86.2426757812, 158.675537109, 158.686523438}},\n\t\t{\"j7ktd1g4\", []float64{-70.7419967651, -70.7418251038, 62.670135498, 62.6704788208}},\n\t\t{\"yzp\", []float64{84.375, 85.78125, 133.59375, 135.0}},\n\t\t{\"76kf7tcsnb94\", []float64{-31.9159668311, -31.9159666635, -26.91415295, -26.9141526148}},\n\t\t{\"9jb\", []float64{32.34375, 33.75, -135.0, -133.59375}},\n\t\t{\"6w\", []float64{-11.25, -5.625, -67.5, -56.25}},\n\t\t{\"f2zs58\", []float64{49.921875, 49.9273681641, -68.0493164062, -68.0383300781}},\n\t\t{\"f0\", []float64{45.0, 50.625, -90.0, -78.75}},\n\t\t{\"mnqum74bk\", []float64{-9.08015727997, -9.08011436462, 54.7268486023, 54.7268915176}},\n\t\t{\"t6rhggbn5\", []float64{13.512840271, 13.5128831863, 66.2586736679, 66.2587165833}},\n\t\t{\"g9q\", []float64{52.03125, 53.4375, -14.0625, -12.65625}},\n\t\t{\"7vr\", []float64{-15.46875, -14.0625, -1.40625, 0.0}},\n\t\t{\"t6sr47h\", []float64{15.3094482422, 15.3108215332, 62.3309326172, 62.3323059082}},\n\t\t{\"076vzrw\", []float64{-70.666809082, -70.665435791, -164.555969238, -164.554595947}},\n\t\t{\"s2\", []float64{0.0, 5.625, 11.25, 22.5}},\n\t\t{\"gd7350xxrrs\", []float64{57.8360626101, 57.8360639513, -17.7872353792, -17.7872340381}},\n\t\t{\"0p8sgbg\", []float64{-46.9734191895, -46.9720458984, -179.127960205, -179.126586914}},\n\t\t{\"5zsn39\", []float64{-46.7083740234, -46.7028808594, -5.55908203125, -5.54809570312}},\n\t\t{\"80f\", []float64{4.21875, 5.625, -177.1875, -175.78125}},\n\t\t{\"cymr4xm05c0\", []float64{81.4265495539, 81.426550895, -93.7502968311, -93.75029549}},\n\t\t{\"qmjmz\", []float64{-15.8642578125, -15.8203125, 108.940429688, 108.984375}},\n\t\t{\"39c0\", []float64{-35.15625, -34.98046875, -111.09375, -110.7421875}},\n\t\t{\"pgxnue0jpfn\", []float64{-69.1086280346, -69.1086266935, 178.791844547, 178.791845888}},\n\t\t{\"nytxjp\", []float64{-52.1685791016, -52.1630859375, 131.704101562, 131.715087891}},\n\t\t{\"q1mvpgze\", []float64{-37.0687294006, -37.0685577393, 98.4368133545, 98.4371566772}},\n\t\t{\"tqjgn4h\", []float64{34.2883300781, 34.2897033691, 64.6051025391, 64.6064758301}},\n\t\t{\"tjn18qrtdk\", []float64{28.4239697456, 28.4239751101, 53.4588825703, 53.4588932991}},\n\t\t{\"qq3w12r\", []float64{-8.78768920898, -8.78631591797, 103.423919678, 103.425292969}},\n\t\t{\"zzdu8g6\", []float64{87.9963684082, 87.9977416992, 172.652893066, 172.654266357}},\n\t\t{\"mz72xeccms3t\", []float64{-4.11002179608, -4.11002162844, 83.6525436491, 83.6525439844}},\n\t\t{\"s7fnw5nzhbs4\", []float64{22.2540122643, 22.2540124319, 14.3356508017, 14.3356511369}},\n\t\t{\"76rtxb6nkdm\", []float64{-31.3744948804, -31.3744935393, -22.8596024215, -22.8596010804}},\n\t\t{\"znbejer93dr\", []float64{83.5141731799, 83.514174521, 135.955197662, 135.955199003}},\n\t\t{\"smk7u7bz27\", []float64{30.212289691, 30.2122950554, 17.4143707752, 17.4143815041}},\n\t\t{\"yvmurs\", []float64{75.3002929688, 75.3057861328, 132.165527344, 132.176513672}},\n\t\t{\"tnjn6dgd8\", []float64{34.8641681671, 34.8642110825, 52.1459197998, 52.1459627151}},\n\t\t{\"gyee1hnu\", []float64{82.1125030518, 82.1126747131, -6.27490997314, -6.27456665039}},\n\t\t{\"gjwh9n02wfmc\", []float64{76.7615726776, 76.7615728453, -36.5179139748, -36.5179136395}},\n\t\t{\"n6jh51hrnfws\", []float64{-78.0401661247, -78.0401659571, 108.41922082, 108.419221155}},\n\t\t{\"shgszu46pudj\", []float64{27.5760518946, 27.5760520622, 5.26587635279, 5.26587668806}},\n\t\t{\"3c\", []float64{-39.375, -33.75, -101.25, -90.0}},\n\t\t{\"fy\", []float64{78.75, 84.375, -56.25, -45.0}},\n\t\t{\"s75d5tn3m\", []float64{17.254242897, 17.2542858124, 16.3344812393, 16.3345241547}},\n\t\t{\"2c1c9kkw8dk5\", []float64{-39.0868538059, -39.0868536383, -143.727924228, -143.727923892}},\n\t\t{\"5yugurey8e2\", []float64{-51.3297383487, -51.3297370076, -4.37837362289, -4.37837228179}},\n\t\t{\"rd\", []float64{-33.75, -28.125, 157.5, 168.75}},\n\t\t{\"um\", []float64{73.125, 78.75, 11.25, 22.5}},\n\t\t{\"bkgc\", []float64{71.89453125, 72.0703125, -163.4765625, -163.125}},\n\t\t{\"8cfxnrphhu0\", []float64{11.1133790016, 11.1133803427, -142.449899912, -142.449898571}},\n\t\t{\"tdm\", []float64{12.65625, 14.0625, 74.53125, 75.9375}},\n\t\t{\"y9usehucn02q\", []float64{55.6610321626, 55.6610323302, 118.966741897, 118.966742232}},\n\t\t{\"vk7kbfk0vf33\", []float64{69.7537115403, 69.7537117079, 60.859013088, 60.8590134233}},\n\t\t{\"ewyx82\", []float64{39.287109375, 39.2926025391, -13.3483886719, -13.3374023438}},\n\t\t{\"ev8c3\", []float64{31.1572265625, 31.201171875, -10.1513671875, -10.107421875}},\n\t\t{\"37p4fhuc\", []float64{-27.6153373718, -27.6151657104, -113.811836243, -113.81149292}},\n\t\t{\"0yvh2p9wbu\", []float64{-51.2418007851, -51.2417954206, -139.216657877, -139.216647148}},\n\t\t{\"81k\", []float64{7.03125, 8.4375, -174.375, -172.96875}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"5c67rs\", []float64{-82.3754882812, -82.3699951172, -7.75634765625, -7.74536132812}},\n\t\t{\"udjvtg\", []float64{57.2332763672, 57.2387695312, 30.8386230469, 30.849609375}},\n\t\t{\"8b56vfqrppu\", []float64{0.497001260519, 0.497002601624, -141.418113112, -141.418111771}},\n\t\t{\"xv50\", []float64{28.125, 28.30078125, 172.96875, 173.3203125}},\n\t\t{\"7ep\", []float64{-28.125, -26.71875, -12.65625, -11.25}},\n\t\t{\"bxzp4746\", []float64{89.8410415649, 89.8412132263, -147.554283142, -147.553939819}},\n\t\t{\"5d54r\", []float64{-78.3544921875, -78.310546875, -17.9736328125, -17.9296875}},\n\t\t{\"hhknsytff\", []float64{-64.9149942398, -64.9149513245, 5.8417224884, 5.84176540375}},\n\t\t{\"gvjjq75\", []float64{74.0643310547, 74.0657043457, -3.93997192383, -3.93859863281}},\n\t\t{\"6ryrt5\", []float64{-0.0714111328125, -0.06591796875, -69.7412109375, -69.7302246094}},\n\t\t{\"tykj1vq1gj\", []float64{36.0643225908, 36.0643279552, 84.460272789, 84.4602835178}},\n\t\t{\"20tdw84b1w\", []float64{-41.7480146885, -41.7480093241, -171.976139545, -171.976128817}},\n\t\t{\"r\", []float64{-45.0, 0.0, 135.0, 180.0}},\n\t\t{\"sbwdq1c4355\", []float64{3.21802318096, 3.21802452207, 43.1557171047, 43.1557184458}},\n\t\t{\"rwhkzg\", []float64{-10.3985595703, -10.3930664062, 163.817138672, 163.828125}},\n\t\t{\"wrzphwjw57\", []float64{44.8582237959, 44.8582291603, 111.299196482, 111.299207211}},\n\t\t{\"674\", []float64{-28.125, -26.71875, -75.9375, -74.53125}},\n\t\t{\"z8kb\", []float64{46.40625, 46.58203125, 164.1796875, 164.53125}},\n\t\t{\"pmudq9vs33\", []float64{-57.2503942251, -57.2503888607, 152.871376276, 152.871387005}},\n\t\t{\"j1br4n9\", []float64{-78.8900756836, -78.8887023926, 45.440826416, 45.442199707}},\n\t\t{\"ccc\", []float64{54.84375, 56.25, -99.84375, -98.4375}},\n\t\t{\"src\", []float64{43.59375, 45.0, 12.65625, 14.0625}},\n\t\t{\"cc51keq\", []float64{50.8625793457, 50.8639526367, -96.8252563477, -96.8238830566}},\n\t\t{\"pr\", []float64{-50.625, -45.0, 146.25, 157.5}},\n\t\t{\"tvd9\", []float64{31.11328125, 31.2890625, 82.265625, 82.6171875}},\n\t\t{\"489bdms44x\", []float64{-87.069016099, -87.0690107346, -64.9345850945, -64.9345743656}},\n\t\t{\"cmn23svsyv\", []float64{73.1958800554, 73.1958854198, -114.887176752, -114.887166023}},\n\t\t{\"dm9vug1194\", []float64{31.9649899006, 31.964995265, -76.0789060593, -76.0788953304}},\n\t\t{\"45f7\", []float64{-68.37890625, -68.203125, -86.8359375, -86.484375}},\n\t\t{\"mxuxkdtgy50\", []float64{-0.117443203926, -0.117441862822, 74.0340328217, 74.0340341628}},\n\t\t{\"tdwd7\", []float64{14.4580078125, 14.501953125, 76.7724609375, 76.81640625}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"jss4y\", []float64{-64.2041015625, -64.16015625, 73.388671875, 73.4326171875}},\n\t\t{\"tmcs\", []float64{33.046875, 33.22265625, 58.359375, 58.7109375}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"gc61ncnxuec\", []float64{52.2138749063, 52.2138762474, -8.13174828887, -8.13174694777}},\n\t\t{\"jwpc\", []float64{-56.07421875, -55.8984375, 78.3984375, 78.75}},\n\t\t{\"yq5gn\", []float64{79.27734375, 79.3212890625, 106.787109375, 106.831054688}},\n\t\t{\"uhtmex\", []float64{71.3177490234, 71.3232421875, 7.53662109375, 7.54760742188}},\n\t\t{\"n0hbvyc\", []float64{-89.8310852051, -89.8297119141, 96.9337463379, 96.9351196289}},\n\t\t{\"kp8q0gk0h\", []float64{-1.7399597168, -1.73991680145, 0.390186309814, 0.390229225159}},\n\t\t{\"yjzduj46p\", []float64{77.8549575806, 77.8550004959, 100.726046562, 100.726089478}},\n\t\t{\"8hhkjyy4v5s\", []float64{23.2406947017, 23.2406960428, -173.762292266, -173.762290925}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"wwu63\", []float64{38.3642578125, 38.408203125, 118.520507812, 118.564453125}},\n\t\t{\"z9rnym\", []float64{53.2452392578, 53.2507324219, 167.618408203, 167.629394531}},\n\t\t{\"78fnfc\", []float64{-39.5892333984, -39.5837402344, -19.5666503906, -19.5556640625}},\n\t\t{\"8dc1mqyer\", []float64{15.7261133194, 15.7261562347, -155.85381031, -155.853767395}},\n\t\t{\"b5\", []float64{61.875, 67.5, -180.0, -168.75}},\n\t\t{\"q3zq3gz6w7\", []float64{-34.0365725756, -34.0365672112, 111.532441378, 111.532452106}},\n\t\t{\"xx6\", []float64{40.78125, 42.1875, 160.3125, 161.71875}},\n\t\t{\"r3\", []float64{-39.375, -33.75, 146.25, 157.5}},\n\t\t{\"dytz0swq74e\", []float64{37.8187742829, 37.818775624, -48.1333740056, -48.1333726645}},\n\t\t{\"gpwu\", []float64{87.890625, 88.06640625, -35.5078125, -35.15625}},\n\t\t{\"9ywdf6cr2w7\", []float64{37.0622827113, 37.0622840524, -92.0087559521, -92.008754611}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"ues\", []float64{64.6875, 66.09375, 28.125, 29.53125}},\n\t\t{\"qggvs5q7pr\", []float64{-22.9210478067, -22.9210424423, 129.208112955, 129.208123684}},\n\t\t{\"fy9d8y2mv2\", []float64{82.0372724533, 82.0372778177, -54.1070973873, -54.1070866585}},\n\t\t{\"bxx92bb60s\", []float64{87.411711216, 87.4117165804, -146.919801235, -146.919790506}},\n\t\t{\"uugkmp4jkm0\", []float64{72.5052005053, 72.5052018464, 38.5429680347, 38.5429693758}},\n\t\t{\"7mmd13sd30\", []float64{-15.1085615158, -15.1085561514, -25.9544706345, -25.9544599056}},\n\t\t{\"5nn2\", []float64{-56.25, -56.07421875, -36.2109375, -35.859375}},\n\t\t{\"jf9sz2r\", []float64{-75.1011657715, -75.0997924805, 81.1875915527, 81.1889648438}},\n\t\t{\"3r\", []float64{-5.625, 0.0, -123.75, -112.5}},\n\t\t{\"yw\", []float64{78.75, 84.375, 112.5, 123.75}},\n\t\t{\"yt31y5hwc3c5\", []float64{74.8565152846, 74.8565154523, 114.17615667, 114.176157005}},\n\t\t{\"7vgv9xhmn\", []float64{-11.6501426697, -11.6500997543, -5.90455055237, -5.90450763702}},\n\t\t{\"f10tgm4q6\", []float64{51.6642808914, 51.6643238068, -89.1508769989, -89.1508340836}},\n\t\t{\"hepj84tfcj\", []float64{-72.143971324, -72.1439659595, 32.3516893387, 32.3517000675}},\n\t\t{\"zg\", []float64{61.875, 67.5, 168.75, 180.0}},\n\t\t{\"by12\", []float64{78.75, 78.92578125, -144.4921875, -144.140625}},\n\t\t{\"51\", []float64{-84.375, -78.75, -45.0, -33.75}},\n\t\t{\"w44s78ur\", []float64{12.0023918152, 12.0025634766, 93.6752700806, 93.6756134033}},\n\t\t{\"2tcr0\", []float64{-11.42578125, -11.3818359375, -155.7421875, -155.698242188}},\n\t\t{\"p0n\", []float64{-90.0, -88.59375, 143.4375, 144.84375}},\n\t\t{\"u1\", []float64{50.625, 56.25, 0.0, 11.25}},\n\t\t{\"nygu1mshqxku\", []float64{-51.2971434742, -51.2971433066, 129.084147625, 129.08414796}},\n\t\t{\"3khrs6gty5\", []float64{-21.1655312777, -21.1655259132, -117.581605911, -117.581595182}},\n\t\t{\"4n\", []float64{-56.25, -50.625, -90.0, -78.75}},\n\t\t{\"tj8pjxb5\", []float64{32.2110557556, 32.211227417, 45.2416992188, 45.2420425415}},\n\t\t{\"7nhpg3stdjgu\", []float64{-9.87847991288, -9.87847974524, -39.225907065, -39.2259067297}},\n\t\t{\"rhtcg24t2s50\", []float64{-19.3789601326, -19.378959965, 143.232218474, 143.232218809}},\n\t\t{\"5vx7c75x\", []float64{-58.3856391907, -58.3854675293, -0.99494934082, -0.994606018066}},\n\t\t{\"cs4kgc65z\", []float64{68.3424711227, 68.3425140381, -109.168095589, -109.168052673}},\n\t\t{\"k3sn2mb\", []float64{-35.4322814941, -35.4309082031, 16.8859863281, 16.8873596191}},\n\t\t{\"ud7s8v\", []float64{58.4747314453, 58.4802246094, 27.4548339844, 27.4658203125}},\n\t\t{\"8qqg2ue1mr6b\", []float64{35.7525117695, 35.7525119372, -159.220504649, -159.220504314}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"kwtg55d3me\", []float64{-7.89069950581, -7.89069414139, 30.7210993767, 30.7211101055}},\n\t\t{\"mqsedtkq3\", []float64{-7.79235363007, -7.79231071472, 62.6938676834, 62.6939105988}},\n\t\t{\"94j6tg\", []float64{11.7059326172, 11.7114257812, -127.364501953, -127.353515625}},\n\t\t{\"wd45s\", []float64{11.865234375, 11.9091796875, 115.48828125, 115.532226562}},\n\t\t{\"fwgxjq0n2\", []float64{84.233250618, 84.2332935333, -62.3474121094, -62.347369194}},\n\t\t{\"1k8fh1\", []float64{-64.3304443359, -64.3249511719, -122.51953125, -122.508544922}},\n\t\t{\"h\", []float64{-90.0, -45.0, 0.0, 45.0}},\n\t\t{\"f\", []float64{45.0, 90.0, -90.0, -45.0}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"qh\", []float64{-22.5, -16.875, 90.0, 101.25}},\n\t\t{\"24nszw5\", []float64{-32.8820800781, -32.8807067871, -170.525665283, -170.524291992}},\n\t\t{\"p8rfy\", []float64{-88.1103515625, -88.06640625, 168.662109375, 168.706054688}},\n\t\t{\"st23m\", []float64{29.7509765625, 29.794921875, 23.0712890625, 23.115234375}},\n\t\t{\"zgg3\", []float64{66.26953125, 66.4453125, 173.3203125, 173.671875}},\n\t\t{\"zgsgk7bcz8k\", []float64{65.2796901762, 65.2796915174, 175.617812276, 175.617813617}},\n\t\t{\"js0\", []float64{-67.5, -66.09375, 67.5, 68.90625}},\n\t\t{\"9zs3bh\", []float64{42.5170898438, 42.5225830078, -95.2734375, -95.2624511719}},\n\t\t{\"k8qd0d6mqfz\", []float64{-43.2289119065, -43.2289105654, 31.6659866273, 31.6659879684}},\n\t\t{\"xrj\", []float64{39.375, 40.78125, 153.28125, 154.6875}},\n\t\t{\"0mbxv2r4\", []float64{-56.2922286987, -56.2920570374, -167.806549072, -167.80620575}},\n\t\t{\"bdf8wz8g1\", []float64{60.5983543396, 60.5983972549, -153.686671257, -153.686628342}},\n\t\t{\"emgeh3mkyu\", []float64{32.8787970543, 32.8788024187, -28.6338579655, -28.6338472366}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"156p3nxfwq\", []float64{-70.4081690311, -70.4081636667, -132.132643461, -132.132632732}},\n\t\t{\"stduvpcnp4\", []float64{31.8160736561, 31.8160790205, 26.5885877609, 26.5885984898}},\n\t\t{\"shk8evrmmbgr\", []float64{24.0238861553, 24.023886323, 6.50312740356, 6.50312773883}},\n\t\t{\"7ynw\", []float64{-10.1953125, -10.01953125, -2.109375, -1.7578125}},\n\t\t{\"r4zgvkp\", []float64{-28.8500976562, -28.8487243652, 146.138763428, 146.140136719}},\n\t\t{\"5b9p\", []float64{-85.95703125, -85.78125, -9.84375, -9.4921875}},\n\t\t{\"gsp\", []float64{67.5, 68.90625, -12.65625, -11.25}},\n\t\t{\"ek4mmvr\", []float64{23.4516906738, 23.4530639648, -30.323638916, -30.322265625}},\n\t\t{\"hd2nu\", []float64{-76.1572265625, -76.11328125, 22.67578125, 22.7197265625}},\n\t\t{\"xzp0t\", []float64{39.462890625, 39.5068359375, 178.813476562, 178.857421875}},\n\t\t{\"fycjx9\", []float64{83.9410400391, 83.9465332031, -54.5141601562, -54.5031738281}},\n\t\t{\"x4ygy62x5\", []float64{16.1414909363, 16.1415338516, 144.767661095, 144.76770401}},\n\t\t{\"37\", []float64{-28.125, -22.5, -123.75, -112.5}},\n\t\t{\"m40\", []float64{-33.75, -32.34375, 45.0, 46.40625}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"b5\", []float64{61.875, 67.5, -180.0, -168.75}},\n\t\t{\"zwd\", []float64{81.5625, 82.96875, 160.3125, 161.71875}},\n\t\t{\"qgnb2g\", []float64{-28.0645751953, -28.0590820312, 133.275146484, 133.286132812}},\n\t\t{\"7w\", []float64{-11.25, -5.625, -22.5, -11.25}},\n\t\t{\"v6x\", []float64{59.0625, 60.46875, 66.09375, 67.5}},\n\t\t{\"018v8j6y\", []float64{-80.5658340454, -80.565662384, -178.94153595, -178.941192627}},\n\t\t{\"mm4p02h18xc\", []float64{-15.6442321837, -15.6442308426, 59.079002291, 59.0790036321}},\n\t\t{\"6cty1v7\", []float64{-35.4789733887, -35.4776000977, -48.0830383301, -48.0816650391}},\n\t\t{\"g6rzs\", []float64{58.974609375, 59.0185546875, -22.67578125, -22.6318359375}},\n\t\t{\"58qb\", []float64{-88.59375, -88.41796875, -13.0078125, -12.65625}},\n\t\t{\"n8v\", []float64{-85.78125, -84.375, 119.53125, 120.9375}},\n\t\t{\"h1nj4b1uv\", []float64{-83.4952783585, -83.4952354431, 8.56096744537, 8.56101036072}},\n\t\t{\"qt4ukphd\", []float64{-16.0891342163, -16.0889625549, 116.54914856, 116.549491882}},\n\t\t{\"n2\", []float64{-90.0, -84.375, 101.25, 112.5}},\n\t\t{\"50sux01hejpj\", []float64{-86.3956842385, -86.3956840709, -38.0111838877, -38.0111835524}},\n\t\t{\"4exy\", []float64{-69.2578125, -69.08203125, -56.6015625, -56.25}},\n\t\t{\"x9t8r\", []float64{8.4814453125, 8.525390625, 165.541992188, 165.5859375}},\n\t\t{\"785m2wrxgw2\", []float64{-44.0414522588, -44.0414509177, -17.8972649574, -17.8972636163}},\n\t\t{\"0exegdddqf\", []float64{-69.6391904354, -69.639185071, -146.7955935, -146.795582771}},\n\t\t{\"ynrw\", []float64{81.2109375, 81.38671875, 100.546875, 100.8984375}},\n\t\t{\"uqdmuzrz6\", []float64{82.6143121719, 82.6143550873, 14.6335315704, 14.6335744858}},\n\t\t{\"gchp3yu15c\", []float64{51.9366699457, 51.9366753101, -5.54244160652, -5.54243087769}},\n\t\t{\"415qzwpj2r\", []float64{-83.154578805, -83.1545734406, -85.0904738903, -85.0904631615}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"vzq07z4\", []float64{85.8636474609, 85.865020752, 87.3550415039, 87.3564147949}},\n\t\t{\"w43s\", []float64{13.359375, 13.53515625, 92.109375, 92.4609375}},\n\t\t{\"b7td\", []float64{65.0390625, 65.21484375, -161.015625, -160.6640625}},\n\t\t{\"2fpe\", []float64{-33.22265625, -33.046875, -135.703125, -135.3515625}},\n\t\t{\"nyt\", []float64{-53.4375, -52.03125, 130.78125, 132.1875}},\n\t\t{\"g\", []float64{45.0, 90.0, -45.0, 0.0}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"dkzk\", []float64{27.421875, 27.59765625, -68.5546875, -68.203125}},\n\t\t{\"r3925m207w\", []float64{-36.5335857868, -36.5335804224, 148.150784969, 148.150795698}},\n\t\t{\"0zy5gndwbzt3\", []float64{-45.710165631, -45.7101654634, -137.677191608, -137.677191272}},\n\t\t{\"46ex1dmu9\", []float64{-74.6938991547, -74.6938562393, -73.7542676926, -73.7542247772}},\n\t\t{\"jf74h9rrvnz8\", []float64{-76.9839544594, -76.9839542918, 83.1766849011, 83.1766852364}},\n\t\t{\"6vj1jtgv\", []float64{-16.6667747498, -16.6666030884, -48.9719009399, -48.9715576172}},\n\t\t{\"ntsx\", []float64{-57.83203125, -57.65625, 118.828125, 119.1796875}},\n\t\t{\"ehr3ejb\", []float64{24.2015075684, 24.2028808594, -34.6728515625, -34.6714782715}},\n\t\t{\"p7\", []float64{-73.125, -67.5, 146.25, 157.5}},\n\t\t{\"re7\", []float64{-26.71875, -25.3125, 161.71875, 163.125}},\n\t\t{\"66x6j7sc0t\", []float64{-30.5665129423, -30.5665075779, -68.3174300194, -68.3174192905}},\n\t\t{\"mywcjb2q\", []float64{-8.25931549072, -8.25914382935, 88.4952163696, 88.4955596924}},\n\t\t{\"f88jrw\", []float64{48.7683105469, 48.7738037109, -67.1704101562, -67.1594238281}},\n\t\t{\"bjty3ef9n3y6\", []float64{77.0569135621, 77.0569137298, -171.844434701, -171.844434366}},\n\t\t{\"jhz66rh4fj7s\", []float64{-62.8467891365, -62.8467889689, 55.2997731417, 55.299773477}},\n\t\t{\"u16r3nm\", []float64{53.3399963379, 53.3413696289, 3.21487426758, 3.21624755859}},\n\t\t{\"b\", []float64{45.0, 90.0, -180.0, -135.0}},\n\t\t{\"q5rwnj4\", []float64{-25.6365966797, -25.6352233887, 100.813293457, 100.814666748}},\n\t\t{\"dsqd4vc85c\", []float64{24.2894035578, 24.2894089222, -58.2363045216, -58.2362937927}},\n\t\t{\"bzpu7z2qxf9\", []float64{85.1630249619, 85.1630263031, -135.18609032, -135.186088979}},\n\t\t{\"e805cvqt\", []float64{0.688877105713, 0.68904876709, -22.4141693115, -22.4138259888}},\n\t\t{\"y9su01vg\", []float64{54.1507530212, 54.1509246826, 119.187583923, 119.187927246}},\n\t\t{\"41\", []float64{-84.375, -78.75, -90.0, -78.75}},\n\t\t{\"vu3f\", []float64{69.2578125, 69.43359375, 81.2109375, 81.5625}},\n\t\t{\"86\", []float64{11.25, 16.875, -168.75, -157.5}},\n\t\t{\"wpuksnn65\", []float64{44.4180679321, 44.4181108475, 96.1610555649, 96.1610984802}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"nrqk483fq0kp\", []float64{-48.5138629563, -48.5138627887, 110.151591897, 110.151592232}},\n\t\t{\"cg0mc\", []float64{62.8857421875, 62.9296875, -100.854492188, -100.810546875}},\n\t\t{\"myt75g\", []float64{-7.89367675781, -7.88818359375, 86.2976074219, 86.30859375}},\n\t\t{\"ugxe27b\", []float64{65.2793884277, 65.2807617188, 44.3078613281, 44.3092346191}},\n\t\t{\"wd845dtbhs8\", []float64{14.42781955, 14.4278208911, 112.661898136, 112.661899477}},\n\t\t{\"c8ef9h6mr\", []float64{48.2762002945, 48.2762432098, -107.179226875, -107.17918396}},\n\t\t{\"bx\", []float64{84.375, 90.0, -157.5, -146.25}},\n\t\t{\"qduv3\", []float64{-28.6083984375, -28.564453125, 119.223632812, 119.267578125}},\n\t\t{\"j86depxm\", []float64{-88.1122398376, -88.1120681763, 71.1574172974, 71.1577606201}},\n\t\t{\"4semjs5hr9p\", []float64{-63.7858861685, -63.7858848274, -62.6835371554, -62.6835358143}},\n\t\t{\"ee\", []float64{16.875, 22.5, -22.5, -11.25}},\n\t\t{\"jxv25m96n\", []float64{-46.3756942749, -46.3756513596, 75.0276088715, 75.0276517868}},\n\t\t{\"vx7gj0006xwe\", []float64{86.3086774014, 86.308677569, 72.993280068, 72.9932804033}},\n\t\t{\"6c\", []float64{-39.375, -33.75, -56.25, -45.0}},\n\t\t{\"ukstfgt78f\", []float64{71.3430798054, 71.3430851698, 17.7062165737, 17.7062273026}},\n\t\t{\"g1h0ye\", []float64{50.7733154297, 50.7788085938, -39.0893554688, -39.0783691406}},\n\t\t{\"5s3j5er\", []float64{-65.1969909668, -65.1956176758, -20.9303283691, -20.9289550781}},\n\t\t{\"6yrnnwq1\", []float64{-8.75455856323, -8.75438690186, -46.1123657227, -46.1120223999}},\n\t\t{\"20fjy\", []float64{-39.7705078125, -39.7265625, -176.923828125, -176.879882812}},\n\t\t{\"1tt417q\", []float64{-58.6930847168, -58.6917114258, -105.405578613, -105.404205322}},\n\t\t{\"hh68\", []float64{-66.09375, -65.91796875, 3.515625, 3.8671875}},\n\t\t{\"85s823r7j\", []float64{19.7388267517, 19.7388696671, -173.650717735, -173.65067482}},\n\t\t{\"3u6n\", []float64{-20.0390625, -19.86328125, -98.4375, -98.0859375}},\n\t\t{\"7w5y8\", []float64{-10.107421875, -10.0634765625, -17.2265625, -17.1826171875}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"zk8j\", []float64{71.19140625, 71.3671875, 146.25, 146.6015625}},\n\t\t{\"mgm1evw0\", []float64{-26.4248657227, -26.4246940613, 85.954284668, 85.9546279907}},\n\t\t{\"m7nzv\", []float64{-26.7626953125, -26.71875, 65.9619140625, 66.005859375}},\n\t\t{\"ev89re48\", []float64{31.1737060547, 31.1738777161, -10.2138519287, -10.213508606}},\n\t\t{\"dnc7mrxvb3\", []float64{38.5822302103, 38.5822355747, -88.0008208752, -88.0008101463}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"wbksmz281\", []float64{2.19314575195, 2.1931886673, 130.331540108, 130.331583023}},\n\t\t{\"0x3zqt\", []float64{-47.9168701172, -47.9113769531, -154.753417969, -154.742431641}},\n\t\t{\"h97q\", []float64{-81.9140625, -81.73828125, 27.0703125, 27.421875}},\n\t\t{\"ypjt38wtc\", []float64{85.3015851974, 85.3016281128, 97.8092622757, 97.809305191}},\n\t\t{\"d3ey054b7p\", []float64{9.50874745846, 9.50875282288, -73.4726572037, -73.4726464748}},\n\t\t{\"zpbkps3qd9r\", []float64{89.3213434517, 89.3213447928, 135.682985634, 135.682986975}},\n\t\t{\"sqxhhhs68mxy\", []float64{37.2908039019, 37.2908040695, 21.2753888592, 21.2753891945}},\n\t\t{\"293cyj4g6q\", []float64{-37.6330769062, -37.6330715418, -154.771517515, -154.771506786}},\n\t\t{\"w3\", []float64{5.625, 11.25, 101.25, 112.5}},\n\t\t{\"r\", []float64{-45.0, 0.0, 135.0, 180.0}},\n\t\t{\"q69s\", []float64{-30.234375, -30.05859375, 103.359375, 103.7109375}},\n\t\t{\"fy1qerq2ven\", []float64{79.9325484037, 79.9325497448, -54.3405380845, -54.3405367434}},\n\t\t{\"62\", []float64{-45.0, -39.375, -78.75, -67.5}},\n\t\t{\"yke1jyek0fg\", []float64{70.5246882141, 70.5246895552, 105.725934952, 105.725936294}},\n\t\t{\"2rcbq1z5c\", []float64{-1.35204792023, -1.35200500488, -166.015734673, -166.015691757}},\n\t\t{\"gue\", []float64{70.3125, 71.71875, -7.03125, -5.625}},\n\t\t{\"t8kqv\", []float64{2.5927734375, 2.63671875, 73.6962890625, 73.740234375}},\n\t\t{\"bgtecc1b\", []float64{65.3521728516, 65.3523445129, -138.436317444, -138.435974121}},\n\t\t{\"8s550\", []float64{23.02734375, 23.0712890625, -153.28125, -153.237304688}},\n\t\t{\"j655kpm1w0b\", []float64{-78.1386239827, -78.1386226416, 60.6516551971, 60.6516565382}},\n\t\t{\"980h\", []float64{0.703125, 0.87890625, -112.5, -112.1484375}},\n\t\t{\"ssywj\", []float64{27.7734375, 27.8173828125, 31.8603515625, 31.904296875}},\n\t\t{\"hrvu\", []float64{-45.703125, -45.52734375, 19.3359375, 19.6875}},\n\t\t{\"3ftuv\", []float64{-30.1025390625, -30.05859375, -92.9443359375, -92.900390625}},\n\t\t{\"zcphg93jux\", []float64{51.4678519964, 51.4678573608, 178.749125004, 178.749135733}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"vjxk\", []float64{76.640625, 76.81640625, 55.1953125, 55.546875}},\n\t\t{\"t6cgynpnh4\", []float64{16.161929369, 16.1619347334, 58.9843940735, 58.9844048023}},\n\t\t{\"jspc\", []float64{-67.32421875, -67.1484375, 78.3984375, 78.75}},\n\t\t{\"6w7k2v\", []float64{-9.06921386719, -9.06372070312, -62.8967285156, -62.8857421875}},\n\t\t{\"n6z4\", []float64{-74.1796875, -74.00390625, 111.09375, 111.4453125}},\n\t\t{\"y507hx0\", []float64{62.4407958984, 62.4421691895, 90.5493164062, 90.5506896973}},\n\t\t{\"z17p35rnsc6v\", []float64{53.3246401884, 53.324640356, 139.272515886, 139.272516221}},\n\t\t{\"p8uj5760\", []float64{-84.8844909668, -84.8843193054, 163.270568848, 163.27091217}},\n\t\t{\"8pcszdrxwp\", []float64{44.4423955679, 44.4424009323, -177.550477982, -177.550467253}},\n\t\t{\"mckjc7q4r\", []float64{-36.9397687912, -36.9397258759, 84.4384717941, 84.4385147095}},\n\t\t{\"p6sngm70\", []float64{-74.7221374512, -74.7219657898, 152.021942139, 152.022285461}},\n\t\t{\"59gcptbk\", []float64{-79.9481964111, -79.9480247498, -16.8966293335, -16.8962860107}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"fmnv3ug5m\", []float64{74.0745019913, 74.0745449066, -69.1765737534, -69.176530838}},\n\t\t{\"qc8kkbc0xev\", []float64{-35.8112038672, -35.8112025261, 124.312004596, 124.312005937}},\n\t\t{\"b4\", []float64{56.25, 61.875, -180.0, -168.75}},\n\t\t{\"vkxeh\", []float64{70.83984375, 70.8837890625, 66.97265625, 67.0166015625}},\n\t\t{\"399qcn2zv\", []float64{-35.3403139114, -35.3402709961, -110.696997643, -110.696954727}},\n\t\t{\"ybz3\", []float64{49.39453125, 49.5703125, 133.9453125, 134.296875}},\n\t\t{\"d5e25\", []float64{19.6875, 19.7314453125, -85.2978515625, -85.25390625}},\n\t\t{\"5n\", []float64{-56.25, -50.625, -45.0, -33.75}},\n\t\t{\"wdw\", []float64{14.0625, 15.46875, 120.9375, 122.34375}},\n\t\t{\"29q7\", []float64{-37.44140625, -37.265625, -148.7109375, -148.359375}},\n\t\t{\"tqe1y4trw\", []float64{36.885137558, 36.8851804733, 60.7398891449, 60.7399320602}},\n\t\t{\"zfwy172d\", []float64{60.135383606, 60.1355552673, 178.297805786, 178.298149109}},\n\t\t{\"tfd57f\", []float64{14.6447753906, 14.6502685547, 81.7272949219, 81.73828125}},\n\t\t{\"27f6s6h\", []float64{-23.4558105469, -23.4544372559, -165.393676758, -165.392303467}},\n\t\t{\"zk2q2ph7db\", []float64{70.0439357758, 70.0439411402, 146.607517004, 146.607527733}},\n\t\t{\"ptp\", []float64{-61.875, -60.46875, 167.34375, 168.75}},\n\t\t{\"7pcgp042wz\", []float64{-0.878782868385, -0.878777503967, -42.2280657291, -42.2280550003}},\n\t\t{\"9t\", []float64{28.125, 33.75, -112.5, -101.25}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"qd0pwby4y\", []float64{-32.4270486832, -32.4270057678, 112.805128098, 112.805171013}},\n\t\t{\"b\", []float64{45.0, 90.0, -180.0, -135.0}},\n\t\t{\"yet\", []float64{64.6875, 66.09375, 119.53125, 120.9375}},\n\t\t{\"v4pbsh7cp\", []float64{56.3614082336, 56.361451149, 56.0796689987, 56.0797119141}},\n\t\t{\"kvr7u7pm7jeu\", []float64{-14.7921594232, -14.7921592556, 44.1421702132, 44.1421705484}},\n\t\t{\"687jj\", []float64{-42.71484375, -42.6708984375, -63.0615234375, -63.017578125}},\n\t\t{\"4w29\", []float64{-54.66796875, -54.4921875, -66.796875, -66.4453125}},\n\t\t{\"6bz45dve\", []float64{-40.4140663147, -40.4138946533, -46.2448883057, -46.2445449829}},\n\t\t{\"gfysykk0nw29\", []float64{61.32709058, 61.3270907477, -1.82894401252, -1.82894367725}},\n\t\t{\"pdw4scw\", []float64{-75.4898071289, -75.4884338379, 166.15447998, 166.155853271}},\n\t\t{\"65f4\", []float64{-23.5546875, -23.37890625, -87.1875, -86.8359375}},\n\t\t{\"jc9g6tpd08j\", []float64{-80.9634017944, -80.9634004533, 81.3311286271, 81.3311299682}},\n\t\t{\"ckw1tvf18r09\", []float64{70.608052779, 70.6080529466, -115.057056472, -115.057056136}},\n\t\t{\"2cbtp\", []float64{-34.27734375, -34.2333984375, -145.239257812, -145.1953125}},\n\t\t{\"54myf\", []float64{-76.1572265625, -76.11328125, -36.826171875, -36.7822265625}},\n\t\t{\"kw\", []float64{-11.25, -5.625, 22.5, 33.75}},\n\t\t{\"mw\", []float64{-11.25, -5.625, 67.5, 78.75}},\n\t\t{\"2bjvee8sgek\", []float64{-44.0131442249, -44.0131428838, -138.009411693, -138.009410352}},\n\t\t{\"yj6r4eyxuv1\", []float64{75.783675313, 75.7836766541, 93.2830573618, 93.2830587029}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"pkec4ng\", []float64{-64.4746398926, -64.4732666016, 151.615447998, 151.616821289}},\n\t\t{\"uvpynzd0v\", []float64{74.2210149765, 74.2210578918, 44.9480295181, 44.9480724335}},\n\t\t{\"grxrc9by8g4\", []float64{88.5605496168, 88.5605509579, -23.4877046943, -23.4877033532}},\n\t\t{\"6z0\", []float64{-5.625, -4.21875, -56.25, -54.84375}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"6npe6\", []float64{-10.6787109375, -10.634765625, -79.365234375, -79.3212890625}},\n\t\t{\"04wzmgz23\", []float64{-74.6424436569, -74.6424007416, -170.245127678, -170.245084763}},\n\t\t{\"dmfzk9\", []float64{33.6236572266, 33.6291503906, -74.6850585938, -74.6740722656}},\n\t\t{\"eeu\", []float64{21.09375, 22.5, -16.875, -15.46875}},\n\t\t{\"84bd9k3t\", []float64{15.9324073792, 15.9325790405, -179.239883423, -179.2395401}},\n\t\t{\"7q6ywwxq2kc\", []float64{-8.664367944, -8.6643666029, -29.5871995389, -29.5871981978}},\n\t\t{\"bve6jyuwk\", []float64{76.327214241, 76.3272571564, -141.420650482, -141.420607567}},\n\t\t{\"h558\", []float64{-73.125, -72.94921875, 4.921875, 5.2734375}},\n\t\t{\"sk4f3d2h1vq\", []float64{22.9085822403, 22.9085835814, 15.1831886172, 15.1831899583}},\n\t\t{\"tffnvn\", []float64{16.6882324219, 16.6937255859, 81.7822265625, 81.7932128906}},\n\t\t{\"eppvmm886m\", []float64{40.3281337023, 40.3281390667, -33.8700664043, -33.8700556755}},\n\t\t{\"d2w\", []float64{2.8125, 4.21875, -70.3125, -68.90625}},\n\t\t{\"sd3\", []float64{12.65625, 14.0625, 23.90625, 25.3125}},\n\t\t{\"q73kk\", []float64{-25.9716796875, -25.927734375, 103.18359375, 103.227539062}},\n\t\t{\"wtz5yx\", []float64{33.0413818359, 33.046875, 122.629394531, 122.640380859}},\n\t\t{\"xw8z\", []float64{37.79296875, 37.96875, 158.5546875, 158.90625}},\n\t\t{\"dyhuchkt7qr\", []float64{34.6092416346, 34.6092429757, -49.5200385153, -49.5200371742}},\n\t\t{\"pyb1es47f\", []float64{-51.7449617386, -51.7449188232, 168.906984329, 168.907027245}},\n\t\t{\"v6jcgfj1nus\", []float64{56.5687993169, 56.568800658, 64.5078939199, 64.507895261}},\n\t\t{\"xxe\", []float64{42.1875, 43.59375, 161.71875, 163.125}},\n\t\t{\"7xdvcext7rq\", []float64{-1.78159162402, -1.78159028292, -18.5564473271, -18.556445986}},\n\t\t{\"1f35b7w\", []float64{-76.6653442383, -76.6639709473, -99.8245239258, -99.8231506348}},\n\t\t{\"he675b8pjek\", []float64{-71.187440604, -71.1874392629, 25.8290988207, 25.8291001618}},\n\t\t{\"pzj3\", []float64{-50.44921875, -50.2734375, 176.1328125, 176.484375}},\n\t\t{\"1s1m8j10zwq3\", []float64{-66.5055748634, -66.5055746958, -110.740483962, -110.740483627}},\n\t\t{\"sk4xsg4ufxm\", []float64{23.8356931508, 23.8356944919, 14.9782557786, 14.9782571197}},\n\t\t{\"r3m2ker83\", []float64{-37.906908989, -37.9068660736, 153.840909004, 153.84095192}},\n\t\t{\"p7bj\", []float64{-68.02734375, -67.8515625, 146.25, 146.6015625}},\n\t\t{\"4wws597u\", []float64{-52.7268218994, -52.726650238, -58.2004165649, -58.2000732422}},\n\t\t{\"u9rk\", []float64{52.734375, 52.91015625, 32.6953125, 33.046875}},\n\t\t{\"2mv856bp9uj4\", []float64{-12.6398345456, -12.6398343779, -160.872720927, -160.872720592}},\n\t\t{\"57th4543xnbs\", []float64{-69.5926011354, -69.5926009677, -26.6274683923, -26.627468057}},\n\t\t{\"8pct\", []float64{44.47265625, 44.6484375, -177.890625, -177.5390625}},\n\t\t{\"me0skjqbk8\", []float64{-27.3490476608, -27.3490422964, 68.3883690834, 68.3883798122}},\n\t\t{\"30s74\", []float64{-41.66015625, -41.6162109375, -128.935546875, -128.891601562}},\n\t\t{\"9r0h9fedn\", []float64{40.1800918579, 40.1801347733, -123.668031693, -123.667988777}},\n\t\t{\"9v6gvte7r\", []float64{30.2211999893, 30.2212429047, -97.136349678, -97.1363067627}},\n\t\t{\"j5d\", []float64{-70.3125, -68.90625, 47.8125, 49.21875}},\n\t\t{\"hf8ge\", []float64{-75.322265625, -75.2783203125, 34.9365234375, 34.98046875}},\n\t\t{\"hf2hmz\", []float64{-76.5582275391, -76.552734375, 34.0026855469, 34.013671875}},\n\t\t{\"5wc405\", []float64{-51.6632080078, -51.6577148438, -21.09375, -21.0827636719}},\n\t\t{\"x2dk49y7p8\", []float64{3.52575302124, 3.52575838566, 149.532830715, 149.532841444}},\n\t\t{\"350jjfux7dbk\", []float64{-27.2297275811, -27.2297274135, -134.740984105, -134.740983769}},\n\t\t{\"04wd\", []float64{-75.5859375, -75.41015625, -170.859375, -170.5078125}},\n\t\t{\"1zxmhstk\", []float64{-46.9081878662, -46.9080162048, -90.8497238159, -90.8493804932}},\n\t\t{\"b2sj26ddce3\", []float64{48.7495739758, 48.7495753169, -163.11051473, -163.110513389}},\n\t\t{\"vznd8mz363\", []float64{84.8462587595, 84.8462641239, 87.9116642475, 87.9116749763}},\n\t\t{\"f\", []float64{45.0, 90.0, -90.0, -45.0}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"kw\", []float64{-11.25, -5.625, 22.5, 33.75}},\n\t\t{\"1s0nq579\", []float64{-66.3833427429, -66.3831710815, -112.231521606, -112.231178284}},\n\t\t{\"mzky2scd\", []float64{-3.09368133545, -3.09350967407, 85.4537200928, 85.4540634155}},\n\t\t{\"kctpzwx2rxg\", []float64{-35.1644052565, -35.1644039154, 41.121122092, 41.1211234331}},\n\t\t{\"19\", []float64{-84.375, -78.75, -112.5, -101.25}},\n\t\t{\"pmg4wc8pn\", []float64{-57.2073554993, -57.2073125839, 150.765638351, 150.765681267}},\n\t\t{\"sxcn0yd0jck\", []float64{44.6841497719, 44.684151113, 23.9422076941, 23.9422090352}},\n\t\t{\"000dsj4g\", []float64{-89.5325660706, -89.5323944092, -179.1173172, -179.116973877}},\n\t\t{\"pgv0\", []float64{-68.90625, -68.73046875, 175.78125, 176.1328125}},\n\t\t{\"dhw0935d8\", []float64{25.4063129425, 25.4063558578, -81.5027618408, -81.5027189255}},\n\t\t{\"4gvz08kbk9\", []float64{-67.6743596792, -67.6743543148, -48.1353735924, -48.1353628635}},\n\t\t{\"djz4g1m\", []float64{32.8340148926, 32.8353881836, -80.0175476074, -80.0161743164}},\n\t\t{\"x4yhn1h5m1z\", []float64{16.1779354513, 16.1779367924, 143.706889004, 143.706890345}},\n\t\t{\"9t\", []float64{28.125, 33.75, -112.5, -101.25}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"4fgytpvp2v\", []float64{-73.3448284864, -73.344823122, -50.7499372959, -50.7499265671}},\n\t\t{\"jggxx\", []float64{-67.587890625, -67.5439453125, 83.9794921875, 84.0234375}},\n\t\t{\"mye8t1\", []float64{-8.34411621094, -8.33862304688, 83.8916015625, 83.9025878906}},\n\t\t{\"bssun7dstj\", []float64{71.0356503725, 71.0356557369, -150.542006493, -150.541995764}},\n\t\t{\"ehhv\", []float64{23.37890625, 23.5546875, -38.3203125, -37.96875}},\n\t\t{\"484tc\", []float64{-88.9892578125, -88.9453125, -63.9404296875, -63.896484375}},\n\t\t{\"d4tjtv\", []float64{15.0567626953, 15.0622558594, -82.7160644531, -82.705078125}},\n\t\t{\"2z\", []float64{-5.625, 0.0, -146.25, -135.0}},\n\t\t{\"t5ey32\", []float64{20.7861328125, 20.7916259766, 50.3283691406, 50.3393554688}},\n\t\t{\"um\", []float64{73.125, 78.75, 11.25, 22.5}},\n\t\t{\"pe\", []float64{-73.125, -67.5, 157.5, 168.75}},\n\t\t{\"2x05zw6z\", []float64{-4.93028640747, -4.93011474609, -157.166633606, -157.166290283}},\n\t\t{\"67\", []float64{-28.125, -22.5, -78.75, -67.5}},\n\t\t{\"dxfr3yndexbg\", []float64{44.9015942775, 44.9015944451, -64.249955602, -64.2499552667}},\n\t\t{\"y87\", []float64{46.40625, 47.8125, 116.71875, 118.125}},\n\t\t{\"2h29w\", []float64{-20.830078125, -20.7861328125, -179.033203125, -178.989257812}},\n\t\t{\"cv\", []float64{73.125, 78.75, -101.25, -90.0}},\n\t\t{\"jcx2m1mjy9e\", []float64{-81.5106931329, -81.5106917918, 89.1721884906, 89.1721898317}},\n\t\t{\"ryj9w\", []float64{-10.986328125, -10.9423828125, 176.748046875, 176.791992188}},\n\t\t{\"g5sftxrhf40\", []float64{65.1676046848, 65.1676060259, -38.0689144135, -38.0689130723}},\n\t\t{\"nhs95z98z\", []float64{-64.4703912735, -64.4703483582, 96.4952802658, 96.4953231812}},\n\t\t{\"s7n00se8u3n\", []float64{16.8998533487, 16.8998546898, 19.7144696116, 19.7144709527}},\n\t\t{\"4310r6m\", []float64{-84.3186950684, -84.3173217773, -77.0182800293, -77.0169067383}},\n\t\t{\"7kkmp\", []float64{-20.21484375, -20.1708984375, -27.4658203125, -27.421875}},\n\t\t{\"3dvnpf13665\", []float64{-28.4653508663, -28.4653495252, -105.126356632, -105.12635529}},\n\t\t{\"1cv2hegqd1s7\", []float64{-80.1345262863, -80.1345261186, -93.6648788676, -93.6648785323}},\n\t\t{\"uy0yttxwk65\", []float64{79.9238741398, 79.9238754809, 35.0568728149, 35.056874156}},\n\t\t{\"166cs2etxffy\", []float64{-77.0763716474, -77.0763714798, -119.690902121, -119.690901786}},\n\t\t{\"bm7n7\", []float64{75.6298828125, 75.673828125, -164.399414062, -164.35546875}},\n\t\t{\"5r7q\", []float64{-48.1640625, -47.98828125, -29.1796875, -28.828125}},\n\t\t{\"ymjqjv6\", []float64{74.2085266113, 74.2098999023, 108.888244629, 108.88961792}},\n\t\t{\"jx95kpvmv9\", []float64{-47.1976464987, -47.1976411343, 69.0894770622, 69.0894877911}},\n\t\t{\"f1m4m9\", []float64{52.4322509766, 52.4377441406, -82.7270507812, -82.7160644531}},\n\t\t{\"0cdr1wfep\", []float64{-80.2944374084, -80.2943944931, -143.016285896, -143.016242981}},\n\t\t{\"fe4q2v7\", []float64{63.0024719238, 63.0038452148, -64.2988586426, -64.2974853516}},\n\t\t{\"9pxfyb9p\", []float64{42.6748466492, 42.6750183105, -123.80355835, -123.803215027}},\n\t\t{\"8w088r\", []float64{33.8763427734, 33.8818359375, -156.785888672, -156.774902344}},\n\t\t{\"u569fj\", []float64{63.6163330078, 63.6218261719, 3.603515625, 3.61450195312}},\n\t\t{\"smvm0syj0kst\", []float64{33.2496320643, 33.2496322319, 18.6630416662, 18.6630420014}},\n\t\t{\"vx\", []float64{84.375, 90.0, 67.5, 78.75}},\n\t\t{\"zj\", []float64{73.125, 78.75, 135.0, 146.25}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"69n85w0\", []float64{-39.3420410156, -39.3406677246, -58.2055664062, -58.2041931152}},\n\t\t{\"ywmj\", []float64{81.03515625, 81.2109375, 119.53125, 119.8828125}},\n\t\t{\"2717c52t\", []float64{-27.4471092224, -27.446937561, -166.947555542, -166.947212219}},\n\t\t{\"t1h\", []float64{5.625, 7.03125, 50.625, 52.03125}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"5xphd2\", []float64{-49.833984375, -49.8284912109, -12.5573730469, -12.5463867188}},\n\t\t{\"xy8\", []float64{36.5625, 37.96875, 168.75, 170.15625}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"3pet007v7qb\", []float64{-1.93128302693, -1.93128168583, -130.072835684, -130.072834343}},\n\t\t{\"dyuz\", []float64{39.19921875, 39.375, -49.5703125, -49.21875}},\n\t\t{\"6r\", []float64{-5.625, 0.0, -78.75, -67.5}},\n\t\t{\"y8v06s5\", []float64{49.2846679688, 49.2860412598, 119.645233154, 119.646606445}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"mfuy2m\", []float64{-28.4051513672, -28.3996582031, 85.4406738281, 85.4516601562}},\n\t\t{\"dnp2u\", []float64{33.8818359375, 33.92578125, -79.62890625, -79.5849609375}},\n\t\t{\"g4u3ehx6\", []float64{60.757484436, 60.7576560974, -38.8816452026, -38.8813018799}},\n\t\t{\"q14h9tg88tx\", []float64{-38.5522833467, -38.5522820055, 92.8832553327, 92.8832566738}},\n\t\t{\"d4m6rn\", []float64{13.0847167969, 13.0902099609, -82.3095703125, -82.2985839844}},\n\t\t{\"64\", []float64{-33.75, -28.125, -90.0, -78.75}},\n\t\t{\"y2nzb\", []float64{46.3623046875, 46.40625, 110.7421875, 110.786132812}},\n\t\t{\"yhs5m8ytcgxb\", []float64{70.8889147639, 70.8889149316, 95.8757111058, 95.875711441}},\n\t\t{\"ytp9j8f0dv8\", []float64{73.305016458, 73.3050177991, 123.291438818, 123.291440159}},\n\t\t{\"pxcpm2h\", []float64{-45.1318359375, -45.1304626465, 159.142456055, 159.143829346}},\n\t\t{\"5zyf9\", []float64{-45.966796875, -45.9228515625, -1.7138671875, -1.669921875}},\n\t\t{\"wmz7v2\", []float64{33.0029296875, 33.0084228516, 111.676025391, 111.687011719}},\n\t\t{\"3fb7hkc8wus\", []float64{-28.9777037501, -28.977702409, -100.709314942, -100.709313601}},\n\t\t{\"ssstmsrv\", []float64{26.2595558167, 26.259727478, 29.0804672241, 29.0808105469}},\n\t\t{\"21\", []float64{-39.375, -33.75, -180.0, -168.75}},\n\t\t{\"w0du7r8257\", []float64{3.60078513622, 3.60079050064, 94.0104925632, 94.0105032921}},\n\t\t{\"fhx\", []float64{70.3125, 71.71875, -80.15625, -78.75}},\n\t\t{\"1xd\", []float64{-47.8125, -46.40625, -109.6875, -108.28125}},\n\t\t{\"s040p9v3shb2\", []float64{0.00989601016045, 0.00989617779851, 3.14947161824, 3.14947195351}},\n\t\t{\"gq\", []float64{78.75, 84.375, -33.75, -22.5}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"17b4wsgdq0q\", []float64{-68.4403167665, -68.4403154254, -123.459283412, -123.45928207}},\n\t\t{\"x3qs\", []float64{7.734375, 7.91015625, 155.390625, 155.7421875}},\n\t\t{\"rc\", []float64{-39.375, -33.75, 168.75, 180.0}},\n\t\t{\"sfxjv06b9\", []float64{15.0747013092, 15.0747442245, 43.8172960281, 43.8173389435}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"ngveyg\", []float64{-68.2305908203, -68.2250976562, 131.781005859, 131.791992188}},\n\t\t{\"pmxd5bd39qp\", []float64{-58.7079012394, -58.7078998983, 156.964822859, 156.9648242}},\n\t\t{\"xhw6\", []float64{25.6640625, 25.83984375, 143.7890625, 144.140625}},\n\t\t{\"bc6vx6\", []float64{53.0090332031, 53.0145263672, -142.064208984, -142.053222656}},\n\t\t{\"uuxy5sbu\", []float64{71.3939666748, 71.3941383362, 44.803276062, 44.8036193848}},\n\t\t{\"yu\", []float64{67.5, 73.125, 123.75, 135.0}},\n\t\t{\"610pf9c3ebh\", []float64{-38.0028247833, -38.0028234422, -89.888253808, -89.8882524669}},\n\t\t{\"9d\", []float64{11.25, 16.875, -112.5, -101.25}},\n\t\t{\"1s0tk4yp\", []float64{-66.5608406067, -66.5606689453, -111.612854004, -111.612510681}},\n\t\t{\"ss7yjqhs5su\", []float64{24.9946086109, 24.994609952, 28.0104857683, 28.0104871094}},\n\t\t{\"krqww1d0kp\", []float64{-3.06785166264, -3.06784629822, 20.6572151184, 20.6572258472}},\n\t\t{\"4yzg18c07qnm\", []float64{-51.4997104369, -51.4997102693, -45.2841233835, -45.2841230482}},\n\t\t{\"5wf241\", []float64{-52.0257568359, -52.0202636719, -19.248046875, -19.2370605469}},\n\t\t{\"gxrh60ce\", []float64{86.5329551697, 86.5331268311, -12.5662994385, -12.5659561157}},\n\t\t{\"5rzrgr9qf\", []float64{-45.0015878677, -45.0015449524, -23.4100627899, -23.4100198746}},\n\t\t{\"u3\", []float64{50.625, 56.25, 11.25, 22.5}},\n\t\t{\"8tm4uz7c\", []float64{30.0546455383, 30.0548171997, -150.254859924, -150.254516602}},\n\t\t{\"wz5f3\", []float64{39.7705078125, 39.814453125, 129.067382812, 129.111328125}},\n\t\t{\"ckz\", []float64{71.71875, 73.125, -113.90625, -112.5}},\n\t\t{\"h4s31\", []float64{-75.76171875, -75.7177734375, 6.0205078125, 6.064453125}},\n\t\t{\"hguje\", []float64{-67.939453125, -67.8955078125, 39.5068359375, 39.55078125}},\n\t\t{\"4rnfer\", []float64{-50.1470947266, -50.1416015625, -69.1149902344, -69.1040039062}},\n\t\t{\"gj\", []float64{73.125, 78.75, -45.0, -33.75}},\n\t\t{\"04\", []float64{-78.75, -73.125, -180.0, -168.75}},\n\t\t{\"kvj\", []float64{-16.875, -15.46875, 40.78125, 42.1875}},\n\t\t{\"7c3p\", []float64{-36.73828125, -36.5625, -9.84375, -9.4921875}},\n\t\t{\"rdw55\", []float64{-30.41015625, -30.3662109375, 166.069335938, 166.11328125}},\n\t\t{\"7spe18wk094b\", []float64{-21.969217658, -21.9692174904, -11.8785988167, -11.8785984814}},\n\t\t{\"uxumm\", []float64{89.5166015625, 89.560546875, 28.6962890625, 28.740234375}},\n\t\t{\"1n0sh0\", []float64{-55.546875, -55.5413818359, -134.12109375, -134.110107422}},\n\t\t{\"cphmy\", []float64{85.3857421875, 85.4296875, -128.759765625, -128.715820312}},\n\t\t{\"sd\", []float64{11.25, 16.875, 22.5, 33.75}},\n\t\t{\"h6jbb2gxyd0\", []float64{-78.6127030849, -78.6127017438, 19.3520092964, 19.3520106375}},\n\t\t{\"x3\", []float64{5.625, 11.25, 146.25, 157.5}},\n\t\t{\"yv289c0nbs\", []float64{74.625813961, 74.6258193254, 124.530050755, 124.530061483}},\n\t\t{\"g1fxbp5\", []float64{56.2445068359, 56.245880127, -41.480255127, -41.4788818359}},\n\t\t{\"bqdh\", []float64{82.265625, 82.44140625, -165.9375, -165.5859375}},\n\t\t{\"w558neznn3\", []float64{16.8966346979, 16.8966400623, 95.2174007893, 95.2174115181}},\n\t\t{\"up\", []float64{84.375, 90.0, 0.0, 11.25}},\n\t\t{\"pmy73pm2r\", []float64{-57.0450925827, -57.0450496674, 155.090517998, 155.090560913}},\n\t\t{\"jzerkufsjnh\", []float64{-46.5112745762, -46.5112732351, 83.5327059031, 83.5327072442}},\n\t\t{\"8eks\", []float64{18.984375, 19.16015625, -151.171875, -150.8203125}},\n\t\t{\"3rryyvfxc\", []float64{-2.99931049347, -2.99926757812, -112.551455498, -112.551412582}},\n\t\t{\"vn0cz\", []float64{79.0576171875, 79.1015625, 46.3623046875, 46.40625}},\n\t\t{\"skb5cnez\", []float64{27.4148368835, 27.4150085449, 11.2990951538, 11.2994384766}},\n\t\t{\"3p5cu0yju\", []float64{-5.31227588654, -5.31223297119, -129.542369843, -129.542326927}},\n\t\t{\"8yrwss1y2s\", []float64{36.3218951225, 36.3219004869, -135.502946377, -135.502935648}},\n\t\t{\"7x9kys4ydp\", []float64{-1.95441305637, -1.95440769196, -20.4526805878, -20.4526698589}},\n\t\t{\"u7gscks3\", []float64{66.9536018372, 66.9537734985, 16.2326431274, 16.2329864502}},\n\t\t{\"30c8pz7\", []float64{-40.7414245605, -40.7400512695, -132.545928955, -132.544555664}},\n\t\t{\"umertwj6wxuc\", []float64{77.2892892547, 77.2892894223, 16.0695068166, 16.0695071518}},\n\t\t{\"n6\", []float64{-78.75, -73.125, 101.25, 112.5}},\n\t\t{\"br8dqrhg058f\", []float64{87.6219940558, 87.6219942234, -167.765692659, -167.765692323}},\n\t\t{\"tp\", []float64{39.375, 45.0, 45.0, 56.25}},\n\t\t{\"33uy\", []float64{-34.1015625, -33.92578125, -117.0703125, -116.71875}},\n\t\t{\"u3ps1jnxg\", []float64{51.356921196, 51.3569641113, 21.8498754501, 21.8499183655}},\n\t\t{\"nqwmqymbyz\", []float64{-52.4801498652, -52.4801445007, 110.343879461, 110.34389019}},\n\t\t{\"wfugcgd\", []float64{16.1471557617, 16.1485290527, 130.509338379, 130.51071167}},\n\t\t{\"vp7g2eg\", []float64{86.3731384277, 86.3745117188, 50.2995300293, 50.3009033203}},\n\t\t{\"zz\", []float64{84.375, 90.0, 168.75, 180.0}},\n\t\t{\"859t0xmm6gwk\", []float64{20.6071523577, 20.6071525253, -177.861316167, -177.861315832}},\n\t\t{\"bvjnf49wp1hu\", []float64{74.3262923509, 74.3262925185, -139.128492661, -139.128492326}},\n\t\t{\"08ybs\", []float64{-85.693359375, -85.6494140625, -147.83203125, -147.788085938}},\n\t\t{\"dr908p7\", []float64{42.3152160645, 42.3165893555, -77.339630127, -77.3382568359}},\n\t\t{\"gufpbsu\", []float64{73.1071472168, 73.1085205078, -8.41003417969, -8.40866088867}},\n\t\t{\"623c388eg2t\", []float64{-43.3706304431, -43.370629102, -76.2223117054, -76.2223103642}},\n\t\t{\"8rc\", []float64{43.59375, 45.0, -167.34375, -165.9375}},\n\t\t{\"h4\", []float64{-78.75, -73.125, 0.0, 11.25}},\n\t\t{\"x84\", []float64{0.0, 1.40625, 160.3125, 161.71875}},\n\t\t{\"bbxgcx6nyzk\", []float64{48.5127027333, 48.5127040744, -135.282602906, -135.282601565}},\n\t\t{\"tqg06239nrht\", []float64{38.014278654, 38.0142788216, 60.5699611455, 60.5699614808}},\n\t\t{\"05fd\", []float64{-68.5546875, -68.37890625, -176.484375, -176.1328125}},\n\t\t{\"wuc\", []float64{26.71875, 28.125, 125.15625, 126.5625}},\n\t\t{\"m3hh\", []float64{-38.671875, -38.49609375, 61.875, 62.2265625}},\n\t\t{\"m2w4ru\", []float64{-41.7700195312, -41.7645263672, 65.0280761719, 65.0390625}},\n\t\t{\"4cz3k\", []float64{-79.9365234375, -79.892578125, -45.87890625, -45.8349609375}},\n\t\t{\"jqefz9v\", []float64{-52.9444885254, -52.9431152344, 61.8598937988, 61.8612670898}},\n\t\t{\"qqcnf60wjf\", []float64{-5.83269953728, -5.83269417286, 102.756060362, 102.756071091}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"n8qen549x9vr\", []float64{-88.0496587045, -88.0496585369, 121.908059008, 121.908059344}},\n\t\t{\"g01nwm2wyj\", []float64{46.1726027727, 46.1726081371, -43.3181476593, -43.3181369305}},\n\t\t{\"6xc\", []float64{-1.40625, 0.0, -66.09375, -64.6875}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"nmjjnhz67\", []float64{-60.9696149826, -60.9695720673, 108.555006981, 108.555049896}},\n\t\t{\"2gzwv1zggqg\", []float64{-22.7094335854, -22.7094322443, -135.472611934, -135.472610593}},\n\t\t{\"dt\", []float64{28.125, 33.75, -67.5, -56.25}},\n\t\t{\"v33ysf2y0s\", []float64{53.1872391701, 53.1872445345, 58.9207291603, 58.9207398891}},\n\t\t{\"fwy6tccjp3\", []float64{83.4186798334, 83.4186851978, -58.4565675259, -58.456556797}},\n\t\t{\"qug6fjx4ue8k\", []float64{-17.7671476454, -17.7671474777, 128.418009616, 128.418009952}},\n\t\t{\"8y59duq21\", []float64{34.0370178223, 34.0370607376, -141.198649406, -141.198606491}},\n\t\t{\"pxgfrpkhdnk\", []float64{-45.9701107442, -45.9701094031, 163.086639047, 163.086640388}},\n\t\t{\"91b\", []float64{9.84375, 11.25, -135.0, -133.59375}},\n\t\t{\"v83mnvugmh\", []float64{47.3173213005, 47.3173266649, 69.5611810684, 69.5611917973}},\n\t\t{\"k76cem9\", []float64{-26.4248657227, -26.4234924316, 15.2613830566, 15.2627563477}},\n\t\t{\"ydjq0mk\", []float64{57.3335266113, 57.3348999023, 119.899291992, 119.900665283}},\n\t\t{\"cdsuf\", []float64{59.8974609375, 59.94140625, -105.732421875, -105.688476562}},\n\t\t{\"tuyz3\", []float64{27.9931640625, 28.037109375, 88.2861328125, 88.330078125}},\n\t\t{\"r7r2skwfj00\", []float64{-26.605796814, -26.6057954729, 156.641564369, 156.64156571}},\n\t\t{\"8hx507p9f5x\", []float64{25.8566424251, 25.8566437662, -170.134868771, -170.13486743}},\n\t\t{\"cc\", []float64{50.625, 56.25, -101.25, -90.0}},\n\t\t{\"pkuuvxz1z3\", []float64{-62.4034112692, -62.4034059048, 153.181310892, 153.181321621}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"7vvz9gbf\", []float64{-11.316947937, -11.3167762756, -3.08612823486, -3.08578491211}},\n\t\t{\"54r\", []float64{-77.34375, -75.9375, -35.15625, -33.75}},\n\t\t{\"5r0mm0pwj1j\", []float64{-49.7011131048, -49.7011117637, -33.1681899726, -33.1681886315}},\n\t\t{\"mx\", []float64{-5.625, 0.0, 67.5, 78.75}},\n\t\t{\"rm39jwc\", []float64{-15.2558898926, -15.2545166016, 148.60244751, 148.603820801}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"4gqrkyvqc\", []float64{-70.4060983658, -70.4060554504, -47.2449445724, -47.2449016571}},\n\t\t{\"9s36btfr9\", []float64{24.4225215912, 24.4225645065, -110.717082024, -110.717039108}},\n\t\t{\"dxt5\", []float64{42.71484375, 42.890625, -60.46875, -60.1171875}},\n\t\t{\"21x8tven9\", []float64{-36.4432811737, -36.4432382584, -169.196276665, -169.196233749}},\n\t\t{\"7hh\", []float64{-22.5, -21.09375, -39.375, -37.96875}},\n\t\t{\"v7nm8q7s\", []float64{62.8768157959, 62.8769874573, 65.0548553467, 65.0551986694}},\n\t\t{\"e9qxh\", []float64{8.26171875, 8.3056640625, -13.18359375, -13.1396484375}},\n\t\t{\"273\", []float64{-26.71875, -25.3125, -167.34375, -165.9375}},\n\t\t{\"p99qpkcvb\", []float64{-80.4807329178, -80.4806900024, 159.578819275, 159.57886219}},\n\t\t{\"q2062f9csybw\", []float64{-44.5904645696, -44.590464402, 101.637129262, 101.637129597}},\n\t\t{\"btsq32\", []float64{77.0361328125, 77.0416259766, -151.468505859, -151.457519531}},\n\t\t{\"1cj5\", []float64{-83.84765625, -83.671875, -94.21875, -93.8671875}},\n\t\t{\"j71pemzwqxt9\", []float64{-71.7739416473, -71.7739414796, 57.8096582741, 57.8096586093}},\n\t\t{\"qnqcbndu0nf\", []float64{-9.49970439076, -9.49970304966, 99.4959667325, 99.4959680736}},\n\t\t{\"2v7cm6junqb\", []float64{-15.237314254, -15.2373129129, -140.737684965, -140.737683624}},\n\t\t{\"e7\", []float64{16.875, 22.5, -33.75, -22.5}},\n\t\t{\"s91p45\", []float64{6.87194824219, 6.87744140625, 23.994140625, 24.0051269531}},\n\t\t{\"xwk1de36\", []float64{35.438117981, 35.4382896423, 163.236579895, 163.236923218}},\n\t\t{\"gf2\", []float64{57.65625, 59.0625, -11.25, -9.84375}},\n\t\t{\"d1m\", []float64{7.03125, 8.4375, -82.96875, -81.5625}},\n\t\t{\"0b\", []float64{-90.0, -84.375, -146.25, -135.0}},\n\t\t{\"rw4\", []float64{-11.25, -9.84375, 160.3125, 161.71875}},\n\t\t{\"74pk9xsb\", []float64{-32.9177856445, -32.9176139832, -34.7322463989, -34.7319030762}},\n\t\t{\"4ghe0cn8s\", []float64{-72.5920772552, -72.5920343399, -49.8798179626, -49.8797750473}},\n\t\t{\"tw4\", []float64{33.75, 35.15625, 70.3125, 71.71875}},\n\t\t{\"gevx3\", []float64{67.3681640625, 67.412109375, -14.7216796875, -14.677734375}},\n\t\t{\"kw8w6yqc1ks\", []float64{-7.30433911085, -7.30433776975, 23.3333033323, 23.3333046734}},\n\t\t{\"vbdjwzpg7\", []float64{48.8183069229, 48.8183498383, 81.8699026108, 81.8699455261}},\n\t\t{\"tp\", []float64{39.375, 45.0, 45.0, 56.25}},\n\t\t{\"751q\", []float64{-27.0703125, -26.89453125, -43.2421875, -42.890625}},\n\t\t{\"yeurzu\", []float64{67.4780273438, 67.4835205078, 118.817138672, 118.828125}},\n\t\t{\"svx\", []float64{30.9375, 32.34375, 43.59375, 45.0}},\n\t\t{\"4yrg1w\", []float64{-54.2834472656, -54.2779541016, -45.2856445312, -45.2746582031}},\n\t\t{\"h6xmxenmzkc\", []float64{-74.9532110989, -74.9532097578, 21.7837978899, 21.7837992311}},\n\t\t{\"j5uzmqughwj1\", []float64{-67.5942097418, -67.5942095742, 51.9171233475, 51.9171236828}},\n\t\t{\"26trngxu\", []float64{-29.6871185303, -29.6869468689, -161.059913635, -161.059570312}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"h58jxs7g\", []float64{-69.3218421936, -69.3216705322, 0.334739685059, 0.335083007812}},\n\t\t{\"tg2kgbk3\", []float64{19.1177558899, 19.1179275513, 79.2721939087, 79.2725372314}},\n\t\t{\"x34hfu\", []float64{6.48193359375, 6.48742675781, 149.183349609, 149.194335938}},\n\t\t{\"774j\", []float64{-27.24609375, -27.0703125, -30.9375, -30.5859375}},\n\t\t{\"yf\", []float64{56.25, 61.875, 123.75, 135.0}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"e273k4gje9s\", []float64{1.64203494787, 1.64203628898, -28.9996308088, -28.9996294677}},\n\t\t{\"e\", []float64{0.0, 45.0, -45.0, 0.0}},\n\t\t{\"hx\", []float64{-50.625, -45.0, 22.5, 33.75}},\n\t\t{\"wz\", []float64{39.375, 45.0, 123.75, 135.0}},\n\t\t{\"w7p94b\", []float64{17.05078125, 17.0562744141, 111.917724609, 111.928710938}},\n\t\t{\"69sgzyb46pbs\", []float64{-35.8658129722, -35.8658128045, -60.4796498269, -60.4796494916}},\n\t\t{\"7f7m8k2u5\", []float64{-31.3529205322, -31.3528776169, -6.66754245758, -6.66749954224}},\n\t\t{\"gq4n7m\", []float64{79.8760986328, 79.8815917969, -30.7946777344, -30.7836914062}},\n\t\t{\"c1srg1pz8kt\", []float64{54.8066094518, 54.8066107929, -128.880941123, -128.880939782}},\n\t\t{\"b2h1dy\", []float64{45.2966308594, 45.3021240234, -163.004150391, -162.993164062}},\n\t\t{\"7170zy\", []float64{-37.8039550781, -37.7984619141, -40.4406738281, -40.4296875}},\n\t\t{\"p09\", []float64{-87.1875, -85.78125, 136.40625, 137.8125}},\n\t\t{\"sw970ux6\", []float64{37.114906311, 37.1150779724, 24.3007278442, 24.301071167}},\n\t\t{\"rc8hsdptqn\", []float64{-35.7595646381, -35.7595592737, 168.958311081, 168.95832181}},\n\t\t{\"qp3jujqc1\", []float64{-3.17899703979, -3.17895412445, 91.5913438797, 91.591386795}},\n\t\t{\"dn7n9krj2tgg\", []float64{36.3231066428, 36.3231068105, -85.7166788355, -85.7166785002}},\n\t\t{\"23e7vwt\", []float64{-35.8676147461, -35.8662414551, -163.931121826, -163.929748535}},\n\t\t{\"um\", []float64{73.125, 78.75, 11.25, 22.5}},\n\t\t{\"p8ws1\", []float64{-86.484375, -86.4404296875, 166.684570312, 166.728515625}},\n\t\t{\"vqt63tm6xx\", []float64{81.9873136282, 81.9873189926, 63.7062621117, 63.7062728405}},\n\t\t{\"b\", []float64{45.0, 90.0, -180.0, -135.0}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"bm20d\", []float64{74.619140625, 74.6630859375, -168.662109375, -168.618164062}},\n\t\t{\"p3t4k9c0\", []float64{-81.1573791504, -81.157207489, 153.480377197, 153.48072052}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"8rsnr3k0\", []float64{43.2929992676, 43.293170929, -162.80090332, -162.800559998}},\n\t\t{\"b739vykqw\", []float64{63.6243152618, 63.6243581772, -166.381845474, -166.381802559}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"13zrrq\", []float64{-78.8488769531, -78.8433837891, -113.236083984, -113.225097656}},\n\t\t{\"6yk9jt\", []float64{-9.64050292969, -9.63500976562, -49.6801757812, -49.6691894531}},\n\t\t{\"zmn0\", []float64{73.125, 73.30078125, 154.6875, 155.0390625}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"ps151cxkk8\", []float64{-66.9636869431, -66.9636815786, 158.993303776, 158.993314505}},\n\t\t{\"tn\", []float64{33.75, 39.375, 45.0, 56.25}},\n\t\t{\"u7\", []float64{61.875, 67.5, 11.25, 22.5}},\n\t\t{\"55yuz\", []float64{-68.0712890625, -68.02734375, -35.2001953125, -35.15625}},\n\t\t{\"x0p3grv8k\", []float64{0.350232124329, 0.350275039673, 145.345859528, 145.345902443}},\n\t\t{\"y5c6gjhshg\", []float64{66.6053169966, 66.605322361, 91.896032095, 91.8960428238}},\n\t\t{\"6fd2j\", []float64{-30.9375, -30.8935546875, -52.8662109375, -52.822265625}},\n\t\t{\"gegbfkt19cj6\", []float64{66.2505683675, 66.2505685352, -17.1207369491, -17.1207366139}},\n\t\t{\"k7\", []float64{-28.125, -22.5, 11.25, 22.5}},\n\t\t{\"y1ydbr\", []float64{55.3656005859, 55.37109375, 99.1516113281, 99.1625976562}},\n\t\t{\"byqsd\", []float64{80.947265625, 80.9912109375, -137.021484375, -136.977539062}},\n\t\t{\"mfcbxhf\", []float64{-29.4172668457, -29.4158935547, 81.5213012695, 81.5226745605}},\n\t\t{\"yxwd5901\", []float64{87.5447273254, 87.5448989868, 121.794433594, 121.794776917}},\n\t\t{\"24k7s7y0gqgj\", []float64{-31.7077504657, -31.7077502981, -173.828286678, -173.828286342}},\n\t\t{\"9031fbu\", []float64{1.71798706055, 1.71936035156, -133.467407227, -133.466033936}},\n\t\t{\"xqusuduvmc\", []float64{38.8197237253, 38.8197290897, 152.782648802, 152.782659531}},\n\t\t{\"5bxw\", []float64{-86.1328125, -85.95703125, -0.703125, -0.3515625}},\n\t\t{\"xw593h52f\", []float64{33.9918279648, 33.9918708801, 162.470369339, 162.470412254}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"9xxm2s\", []float64{43.1323242188, 43.1378173828, -102.282714844, -102.271728516}},\n\t\t{\"byf7t\", []float64{83.583984375, 83.6279296875, -142.866210938, -142.822265625}},\n\t\t{\"v6\", []float64{56.25, 61.875, 56.25, 67.5}},\n\t\t{\"yh\", []float64{67.5, 73.125, 90.0, 101.25}},\n\t\t{\"d6k43xp\", []float64{13.0902099609, 13.091583252, -73.0494689941, -73.0480957031}},\n\t\t{\"k4m5h\", []float64{-31.81640625, -31.7724609375, 7.20703125, 7.2509765625}},\n\t\t{\"r1t7\", []float64{-36.03515625, -35.859375, 142.3828125, 142.734375}},\n\t\t{\"cs4kvrjdkm7\", []float64{68.3738274872, 68.3738288283, -109.097485095, -109.097483754}},\n\t\t{\"unu\", []float64{82.96875, 84.375, 5.625, 7.03125}},\n\t\t{\"59xp\", []float64{-80.33203125, -80.15625, -12.65625, -12.3046875}},\n\t\t{\"542p4hbm\", []float64{-76.0863304138, -76.0861587524, -44.9117660522, -44.9114227295}},\n\t\t{\"5j\", []float64{-61.875, -56.25, -45.0, -33.75}},\n\t\t{\"v3\", []float64{50.625, 56.25, 56.25, 67.5}},\n\t\t{\"mstr13c0\", []float64{-18.4474182129, -18.4472465515, 74.9391174316, 74.9394607544}},\n\t\t{\"wcvtdg61swv8\", []float64{10.8286933601, 10.8286935277, 131.608171687, 131.608172022}},\n\t\t{\"3cm40m0w91z3\", []float64{-37.5885963254, -37.5885961577, -94.207024388, -94.2070240527}},\n\t\t{\"m9fup0\", []float64{-34.453125, -34.4476318359, 71.6748046875, 71.6857910156}},\n\t\t{\"jx\", []float64{-50.625, -45.0, 67.5, 78.75}},\n\t\t{\"7myut8cg\", []float64{-11.8605995178, -11.8604278564, -24.013710022, -24.0133666992}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"5dgu11uuf12g\", []float64{-73.8176893629, -73.8176891953, -17.1760072187, -17.1760068834}},\n\t\t{\"qp1ens1zqg\", []float64{-5.07442295551, -5.07441759109, 92.3977124691, 92.3977231979}},\n\t\t{\"vusxu8ewwbrz\", []float64{71.6786695831, 71.6786697507, 85.2809854969, 85.2809858322}},\n\t\t{\"8qjtgd1q\", []float64{34.7727584839, 34.7729301453, -160.860099792, -160.85975647}},\n\t\t{\"60dppvw\", []float64{-40.9268188477, -40.9254455566, -86.838684082, -86.837310791}},\n\t\t{\"tygxz83s2\", []float64{39.3331575394, 39.3332004547, 84.0035247803, 84.0035676956}},\n\t\t{\"e0qwrc\", []float64{2.51037597656, 2.51586914062, -35.5187988281, -35.5078125}},\n\t\t{\"5wyh2qbz\", []float64{-51.2458992004, -51.2457275391, -14.0504837036, -14.0501403809}},\n\t\t{\"0zqs\", []float64{-48.515625, -48.33984375, -137.109375, -136.7578125}},\n\t\t{\"n5ss\", []float64{-69.609375, -69.43359375, 96.328125, 96.6796875}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"v9q84x9n8ktx\", []float64{52.0735898428, 52.0735900104, 76.7518796772, 76.7518800125}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"2eh8wf4ktz\", []float64{-28.0253130198, -28.0253076553, -150.871907473, -150.871896744}},\n\t\t{\"cr96dg281x\", []float64{87.6448434591, 87.6448488235, -121.870586872, -121.870576143}},\n\t\t{\"56u\", []float64{-74.53125, -73.125, -28.125, -26.71875}},\n\t\t{\"628vt\", []float64{-41.220703125, -41.1767578125, -77.4755859375, -77.431640625}},\n\t\t{\"0heb\", []float64{-64.6875, -64.51171875, -174.7265625, -174.375}},\n\t\t{\"8q5skg4mk\", []float64{34.5144510269, 34.5144939423, -163.616123199, -163.616080284}},\n\t\t{\"7x0pw97495hw\", []float64{-4.2993279174, -4.29932774976, -22.2101866454, -22.2101863101}},\n\t\t{\"1j4vjyz1dqx\", []float64{-60.9587225318, -60.9587211907, -130.870407969, -130.870406628}},\n\t\t{\"u4q1szh\", []float64{57.9583740234, 57.9597473145, 8.65173339844, 8.65310668945}},\n\t\t{\"3v3dbm2uv2\", []float64{-14.9556970596, -14.9556916952, -99.1283833981, -99.1283726692}},\n\t\t{\"te3jtfwjnq\", []float64{19.2626702785, 19.262675643, 69.1674435139, 69.1674542427}},\n\t\t{\"sgvgx3ey6qe\", []float64{21.7183318734, 21.7183332145, 42.1597914398, 42.1597927809}},\n\t\t{\"2sw\", []float64{-19.6875, -18.28125, -149.0625, -147.65625}},\n\t\t{\"wzy2pqu8e\", []float64{43.6309146881, 43.6309576035, 132.863974571, 132.864017487}},\n\t\t{\"dk2849\", []float64{23.9117431641, 23.9172363281, -77.9370117188, -77.9260253906}},\n\t\t{\"0d65\", []float64{-76.81640625, -76.640625, -154.6875, -154.3359375}},\n\t\t{\"84\", []float64{11.25, 16.875, -180.0, -168.75}},\n\t\t{\"q17\", []float64{-37.96875, -36.5625, 94.21875, 95.625}},\n\t\t{\"x9wzer\", []float64{9.79431152344, 9.7998046875, 167.135009766, 167.145996094}},\n\t\t{\"7xk2s7425n6\", []float64{-4.1143463552, -4.1143450141, -16.3334485888, -16.3334472477}},\n\t\t{\"jv\", []float64{-61.875, -56.25, 78.75, 90.0}},\n\t\t{\"u5juvw\", []float64{62.7429199219, 62.7484130859, 8.32763671875, 8.33862304688}},\n\t\t{\"cnuczt0c\", []float64{83.3040046692, 83.3041763306, -127.989692688, -127.989349365}},\n\t\t{\"7f3jv330zuh\", []float64{-31.3259911537, -31.3259898126, -9.61132586002, -9.61132451892}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"70jqgb8uv\", []float64{-43.8099145889, -43.8098716736, -37.4511480331, -37.4511051178}},\n\t\t{\"u5t5d155\", []float64{65.3087425232, 65.3089141846, 7.12326049805, 7.1236038208}},\n\t\t{\"r9hy1gz8m4p\", []float64{-38.2996594906, -38.2996581495, 164.267115444, 164.267116785}},\n\t\t{\"qwmurb9920\", []float64{-9.09371852875, -9.09371316433, 120.928573608, 120.928584337}},\n\t\t{\"d2pt693fw4\", []float64{0.930157899857, 0.930163264275, -68.0906009674, -68.0905902386}},\n\t\t{\"zfgbx\", []float64{60.556640625, 60.6005859375, 174.331054688, 174.375}},\n\t\t{\"c7mtdqkfuuc\", []float64{64.2828767002, 64.2828780413, -115.910019726, -115.910018384}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"w89eqd\", []float64{3.39477539062, 3.40026855469, 114.895019531, 114.906005859}},\n\t\t{\"75mtem68vx\", []float64{-25.7229477167, -25.7229423523, -37.1191334724, -37.1191227436}},\n\t\t{\"12e9fwr\", []float64{-86.8455505371, -86.8441772461, -118.708648682, -118.707275391}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"nq\", []float64{-56.25, -50.625, 101.25, 112.5}},\n\t\t{\"svnx\", []float64{29.35546875, 29.53125, 42.890625, 43.2421875}},\n\t\t{\"0e1kd5pxdgt\", []float64{-72.316198647, -72.3161973059, -155.64387247, -155.643871129}},\n\t\t{\"pgk\", []float64{-71.71875, -70.3125, 174.375, 175.78125}},\n\t\t{\"vry9q\", []float64{88.8134765625, 88.857421875, 65.654296875, 65.6982421875}},\n\t\t{\"6f2x91u\", []float64{-31.0157775879, -31.0144042969, -55.4974365234, -55.4960632324}},\n\t\t{\"3hefw\", []float64{-19.248046875, -19.2041015625, -129.462890625, -129.418945312}},\n\t\t{\"g\", []float64{45.0, 90.0, -45.0, 0.0}},\n\t\t{\"m66erp\", []float64{-31.7340087891, -31.728515625, 60.0732421875, 60.0842285156}},\n\t\t{\"5nwny\", []float64{-52.2509765625, -52.20703125, -36.298828125, -36.2548828125}},\n\t\t{\"d5\", []float64{16.875, 22.5, -90.0, -78.75}},\n\t\t{\"wuphy8\", []float64{23.3349609375, 23.3404541016, 133.879394531, 133.890380859}},\n\t\t{\"b8u631kf\", []float64{49.6214675903, 49.6216392517, -151.472969055, -151.472625732}},\n\t\t{\"34d8j\", []float64{-30.9375, -30.8935546875, -131.264648438, -131.220703125}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"he\", []float64{-73.125, -67.5, 22.5, 33.75}},\n\t\t{\"yec1yepjd\", []float64{66.4187908173, 66.4188337326, 114.201593399, 114.201636314}},\n\t\t{\"h0p58hpz\", []float64{-89.3615913391, -89.3614196777, 9.85439300537, 9.85473632812}},\n\t\t{\"8u\", []float64{22.5, 28.125, -146.25, -135.0}},\n\t\t{\"hg9sxf0mu\", []float64{-69.509510994, -69.5094680786, 36.200466156, 36.2005090714}},\n\t\t{\"7zdg4c9868\", []float64{-2.27687358856, -2.27686822414, -7.25979566574, -7.2597849369}},\n\t\t{\"1h\", []float64{-67.5, -61.875, -135.0, -123.75}},\n\t\t{\"k7h239zr4097\", []float64{-28.0702368356, -28.070236668, 17.3025243357, 17.302524671}},\n\t\t{\"9h\", []float64{22.5, 28.125, -135.0, -123.75}},\n\t\t{\"fx\", []float64{84.375, 90.0, -67.5, -56.25}},\n\t\t{\"sf66h05\", []float64{13.0078125, 13.009185791, 37.093963623, 37.0953369141}},\n\t\t{\"4ge39\", []float64{-70.048828125, -70.0048828125, -51.6357421875, -51.591796875}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"ypd\", []float64{87.1875, 88.59375, 92.8125, 94.21875}},\n\t\t{\"gbzh2r\", []float64{50.0042724609, 50.009765625, -1.39526367188, -1.38427734375}},\n\t\t{\"h0\", []float64{-90.0, -84.375, 0.0, 11.25}},\n\t\t{\"nk7wkxwvku2\", []float64{-64.952994436, -64.9529930949, 106.379102468, 106.37910381}},\n\t\t{\"23qu\", []float64{-37.265625, -37.08984375, -159.2578125, -158.90625}},\n\t\t{\"n15b3\", []float64{-84.3310546875, -84.287109375, 95.3173828125, 95.361328125}},\n\t\t{\"8x9c6f1cjkn\", []float64{42.4184060097, 42.4184073508, -154.915576279, -154.915574938}},\n\t\t{\"5dr\", []float64{-77.34375, -75.9375, -12.65625, -11.25}},\n\t\t{\"022q4\", []float64{-87.5390625, -87.4951171875, -168.310546875, -168.266601562}},\n\t\t{\"52\", []float64{-90.0, -84.375, -33.75, -22.5}},\n\t\t{\"s0j8hb\", []float64{0.0, 0.0054931640625, 7.94311523438, 7.9541015625}},\n\t\t{\"58ygg9vn9nj\", []float64{-85.1113092899, -85.1113079488, -12.8470878303, -12.8470864892}},\n\t\t{\"ztzqs1\", []float64{78.4918212891, 78.4973144531, 167.87109375, 167.882080078}},\n\t\t{\"n5x\", []float64{-70.3125, -68.90625, 99.84375, 101.25}},\n\t\t{\"jh593g3fsf\", []float64{-67.261980772, -67.2619754076, 50.001386404, 50.0013971329}},\n\t\t{\"8vjg52364\", []float64{28.6540603638, 28.6541032791, -138.01943779, -138.019394875}},\n\t\t{\"dqeh1\", []float64{37.265625, 37.3095703125, -74.4873046875, -74.443359375}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"m1mjx\", []float64{-37.001953125, -36.9580078125, 52.3388671875, 52.3828125}},\n\t\t{\"w4fr\", []float64{16.69921875, 16.875, 93.1640625, 93.515625}},\n\t\t{\"k2v0kj\", []float64{-40.7098388672, -40.7043457031, 18.45703125, 18.4680175781}},\n\t\t{\"ytmdb30u\", []float64{75.0208282471, 75.0209999084, 120.246391296, 120.246734619}},\n\t\t{\"wzuhv\", []float64{44.4287109375, 44.47265625, 129.594726562, 129.638671875}},\n\t\t{\"84m2ue733hx1\", []float64{12.8061776049, 12.8061777726, -172.414918095, -172.41491776}},\n\t\t{\"02\", []float64{-90.0, -84.375, -168.75, -157.5}},\n\t\t{\"x8j5vd68s\", []float64{0.671625137329, 0.671668052673, 164.776554108, 164.776597023}},\n\t\t{\"7k\", []float64{-22.5, -16.875, -33.75, -22.5}},\n\t\t{\"4xtrffmbtxr\", []float64{-46.4377109706, -46.4377096295, -59.9881960452, -59.9881947041}},\n\t\t{\"k8x\", []float64{-42.1875, -40.78125, 32.34375, 33.75}},\n\t\t{\"yyxsh\", []float64{82.265625, 82.3095703125, 134.47265625, 134.516601562}},\n\t\t{\"f2rqpzxu\", []float64{47.502822876, 47.5029945374, -68.2034683228, -68.203125}},\n\t\t{\"d6h4jg\", []float64{11.6180419922, 11.6235351562, -72.8723144531, -72.861328125}},\n\t\t{\"e7d\", []float64{19.6875, 21.09375, -30.9375, -29.53125}},\n\t\t{\"bbs0tcr5wc9\", []float64{47.9078659415, 47.9078672826, -140.362410396, -140.362409055}},\n\t\t{\"7fuf\", []float64{-29.1796875, -29.00390625, -4.5703125, -4.21875}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"tt9p6y\", []float64{32.2448730469, 32.2503662109, 69.0270996094, 69.0380859375}},\n\t\t{\"xypsv42g6pr\", []float64{34.5979173481, 34.5979186893, 179.517726749, 179.51772809}},\n\t\t{\"p4gvh3sr97h\", []float64{-73.6428004503, -73.6427991092, 140.466100574, 140.466101915}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"6ecy4rk\", []float64{-22.8117370605, -22.8103637695, -64.9346923828, -64.9333190918}},\n\t\t{\"045xe38uj\", []float64{-77.4227142334, -77.4226713181, -174.934058189, -174.934015274}},\n\t\t{\"xyunrc48\", []float64{39.0728759766, 39.0730476379, 174.719009399, 174.719352722}},\n\t\t{\"enm03\", []float64{35.2001953125, 35.244140625, -37.9248046875, -37.880859375}},\n\t\t{\"n7r9k6ecbhm\", []float64{-71.4849673212, -71.4849659801, 111.988799125, 111.988800466}},\n\t\t{\"2bpxnf2\", []float64{-43.7571716309, -43.7557983398, -135.406494141, -135.40512085}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"vs65p7h9m\", []float64{69.4502878189, 69.4503307343, 70.6374979019, 70.6375408173}},\n\t\t{\"j6vt6yyjmms\", []float64{-73.5703888535, -73.5703875124, 64.1136950254, 64.1136963665}},\n\t\t{\"pp7z8s7\", []float64{-47.8770446777, -47.8756713867, 140.299530029, 140.30090332}},\n\t\t{\"skxsqyzdgn\", []float64{26.0971534252, 26.0971587896, 22.103934288, 22.1039450169}},\n\t\t{\"y1x7mdy\", []float64{54.0238952637, 54.0252685547, 100.445251465, 100.446624756}},\n\t\t{\"ck9n9vptne\", []float64{71.4834183455, 71.4834237099, -122.256267071, -122.256256342}},\n\t\t{\"fq7u12930mgt\", []float64{80.862324927, 80.8623250946, -73.4198988229, -73.4198984876}},\n\t\t{\"zs50j6\", []float64{67.5109863281, 67.5164794922, 161.949462891, 161.960449219}},\n\t\t{\"9jccmq0j\", []float64{32.5972938538, 32.5974655151, -132.308349609, -132.308006287}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"hhtn7\", []float64{-63.5888671875, -63.544921875, 7.1630859375, 7.20703125}},\n\t\t{\"480spdnxu\", []float64{-89.2845582962, -89.2845153809, -66.4581871033, -66.4581441879}},\n\t\t{\"ghz87yh\", []float64{71.7956542969, 71.7970275879, -34.2828369141, -34.281463623}},\n\t\t{\"cf\", []float64{56.25, 61.875, -101.25, -90.0}},\n\t\t{\"gd1n5pnc8p7\", []float64{57.3434360325, 57.3434373736, -20.9526403248, -20.9526389837}},\n\t\t{\"hef9dvw3q\", []float64{-68.6121511459, -68.6121082306, 26.1453151703, 26.1453580856}},\n\t\t{\"wbjwhe8rkyf0\", []float64{1.07519432902, 1.07519449666, 131.682678759, 131.682679094}},\n\t\t{\"pndb3wg\", []float64{-53.3564758301, -53.3551025391, 138.937225342, 138.938598633}},\n\t\t{\"bh2k52ewybs\", []float64{69.6132829785, 69.6132843196, -179.500513673, -179.500512332}},\n\t\t{\"t4r\", []float64{12.65625, 14.0625, 54.84375, 56.25}},\n\t\t{\"s7215\", []float64{18.45703125, 18.5009765625, 11.3818359375, 11.42578125}},\n\t\t{\"u8\", []float64{45.0, 50.625, 22.5, 33.75}},\n\t\t{\"f2\", []float64{45.0, 50.625, -78.75, -67.5}},\n\t\t{\"3hwb5r0etbt\", []float64{-19.6484443545, -19.6484430134, -125.36405012, -125.364048779}},\n\t\t{\"wendsfh63\", []float64{17.3258256912, 17.3258686066, 121.855244637, 121.855287552}},\n\t\t{\"90n\", []float64{0.0, 1.40625, -126.5625, -125.15625}},\n\t\t{\"e5mx13zs4t2\", []float64{19.5220465958, 19.5220479369, -37.2002863884, -37.2002850473}},\n\t\t{\"6kngstqxn\", []float64{-21.854724884, -21.8546819687, -69.0508747101, -69.0508317947}},\n\t\t{\"rgs\", []float64{-25.3125, -23.90625, 174.375, 175.78125}},\n\t\t{\"sbd1n1z90\", []float64{2.99806594849, 2.99810886383, 36.8364715576, 36.836514473}},\n\t\t{\"693c7m26y6\", []float64{-37.7197015285, -37.7196961641, -64.8956286907, -64.8956179619}},\n\t\t{\"ynu\", []float64{82.96875, 84.375, 95.625, 97.03125}},\n\t\t{\"68t74uch2444\", []float64{-41.6333230957, -41.6333229281, -59.9949619174, -59.9949615821}},\n\t\t{\"jcmv2zscc\", []float64{-82.0043992996, -82.0043563843, 86.875462532, 86.8755054474}},\n\t\t{\"3c\", []float64{-39.375, -33.75, -101.25, -90.0}},\n\t\t{\"r76ymc\", []float64{-25.6146240234, -25.6091308594, 150.369873047, 150.380859375}},\n\t\t{\"kj9vj024cd\", []float64{-13.1817376614, -13.1817322969, 2.68072843552, 2.68073916435}},\n\t\t{\"scr\", []float64{7.03125, 8.4375, 43.59375, 45.0}},\n\t\t{\"57g8wvk\", []float64{-68.7895202637, -68.7881469727, -28.5260009766, -28.5246276855}},\n\t\t{\"8qde6\", []float64{37.1337890625, 37.177734375, -165.146484375, -165.102539062}},\n\t\t{\"7ve260\", []float64{-14.0185546875, -14.0130615234, -6.591796875, -6.58081054688}},\n\t\t{\"trcxzhy6\", []float64{44.9824905396, 44.9826622009, 58.6755752563, 58.6759185791}},\n\t\t{\"syqw7fyhx\", []float64{36.2707614899, 36.2708044052, 43.0639600754, 43.0640029907}},\n\t\t{\"tb0\", []float64{0.0, 1.40625, 78.75, 80.15625}},\n\t\t{\"dttugd1xtj5p\", []float64{31.7847627215, 31.7847628891, -59.2579753697, -59.2579750344}},\n\t\t{\"v899zsbyh7f2\", []float64{48.1472598016, 48.1472599693, 69.9401802197, 69.940180555}},\n\t\t{\"8e\", []float64{16.875, 22.5, -157.5, -146.25}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"ny4017r8x\", []float64{-56.2320613861, -56.2320184708, 126.628031731, 126.628074646}},\n\t\t{\"8ued\", []float64{25.6640625, 25.83984375, -141.328125, -140.9765625}},\n\t\t{\"bqpsbj0h9p1t\", []float64{79.6132376231, 79.6132377908, -158.203080073, -158.203079738}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"b5m\", []float64{63.28125, 64.6875, -172.96875, -171.5625}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"w722c\", []float64{18.4130859375, 18.45703125, 101.645507812, 101.689453125}},\n\t\t{\"ey\", []float64{33.75, 39.375, -11.25, 0.0}},\n\t\t{\"6ndrutd\", []float64{-7.04498291016, -7.04360961914, -86.6354370117, -86.6340637207}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"47jtykh\", []float64{-72.0922851562, -72.0909118652, -70.7354736328, -70.7341003418}},\n\t\t{\"krb5qvkb\", []float64{-0.806121826172, -0.805950164795, 11.5531539917, 11.5534973145}},\n\t\t{\"mtpgnes7uugs\", []float64{-16.3277602941, -16.3277601264, 78.6901270598, 78.6901273951}},\n\t\t{\"6062z\", []float64{-43.4619140625, -43.41796875, -86.5283203125, -86.484375}},\n\t\t{\"241quukp\", []float64{-32.5389289856, -32.5387573242, -178.027954102, -178.027610779}},\n\t\t{\"g4m49\", []float64{58.095703125, 58.1396484375, -37.9248046875, -37.880859375}},\n\t\t{\"wwq\", []float64{35.15625, 36.5625, 120.9375, 122.34375}},\n\t\t{\"beujb1c\", []float64{67.1141052246, 67.1154785156, -151.873626709, -151.872253418}},\n\t\t{\"h4\", []float64{-78.75, -73.125, 0.0, 11.25}},\n\t\t{\"69xq1n9v\", []float64{-35.4712486267, -35.4710769653, -57.2583389282, -57.2579956055}},\n\t\t{\"cjpu\", []float64{73.828125, 74.00390625, -124.1015625, -123.75}},\n\t\t{\"pmks2611hw\", []float64{-59.7104895115, -59.7104841471, 152.590677738, 152.590688467}},\n\t\t{\"zd78cuj300z8\", []float64{57.8102342784, 57.8102344461, 162.505999133, 162.505999468}},\n\t\t{\"g9h460p7719\", []float64{51.0210737586, 51.0210750997, -16.777022928, -16.7770215869}},\n\t\t{\"crncwf7g0c\", []float64{84.6515518427, 84.6515572071, -113.955999613, -113.955988884}},\n\t\t{\"mp4\", []float64{-5.625, -4.21875, 47.8125, 49.21875}},\n\t\t{\"3ghgz3gsjgr\", []float64{-27.4555031955, -27.4555018544, -94.2466463149, -94.2466449738}},\n\t\t{\"m2g9\", []float64{-40.60546875, -40.4296875, 61.171875, 61.5234375}},\n\t\t{\"ngt309w\", []float64{-70.1284790039, -70.1271057129, 131.163024902, 131.164398193}},\n\t\t{\"hgn9u3537p\", []float64{-72.8116375208, -72.8116321564, 43.08198452, 43.0819952488}},\n\t\t{\"6spnhu\", []float64{-21.4233398438, -21.4178466797, -57.4475097656, -57.4365234375}},\n\t\t{\"z22v4r0d82\", []float64{47.3240375519, 47.3240429163, 147.404261827, 147.404272556}},\n\t\t{\"ytypn\", []float64{78.57421875, 78.6181640625, 121.201171875, 121.245117188}},\n\t\t{\"qstc\", []float64{-19.51171875, -19.3359375, 120.5859375, 120.9375}},\n\t\t{\"y8r48c5\", []float64{46.8511962891, 46.8525695801, 122.380828857, 122.382202148}},\n\t\t{\"uq2xybqqg\", []float64{81.5210866928, 81.5211296082, 12.2584676743, 12.2585105896}},\n\t\t{\"95xm8dtz5u42\", []float64{20.6692528725, 20.6692530401, -124.77465447, -124.774654135}},\n\t\t{\"x0uqxwj\", []float64{5.39428710938, 5.39566040039, 141.313018799, 141.31439209}},\n\t\t{\"0kve9v\", []float64{-62.6385498047, -62.6330566406, -160.938720703, -160.927734375}},\n\t\t{\"szwv1er87\", []float64{43.0843019485, 43.0843448639, 43.3185338974, 43.3185768127}},\n\t\t{\"88zscymedp\", []float64{5.08868157864, 5.08868694305, -146.868581772, -146.868571043}},\n\t\t{\"pd\", []float64{-78.75, -73.125, 157.5, 168.75}},\n\t\t{\"g6\", []float64{56.25, 61.875, -33.75, -22.5}},\n\t\t{\"3pg6r201vv51\", []float64{-1.01041479036, -1.01041462272, -130.110833198, -130.110832863}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"g02\", []float64{46.40625, 47.8125, -45.0, -43.59375}},\n\t\t{\"jx\", []float64{-50.625, -45.0, 67.5, 78.75}},\n\t\t{\"zksxs2nz2j3\", []float64{71.6321320832, 71.6321334243, 152.774163634, 152.774164975}},\n\t\t{\"erqyg8ff4d\", []float64{41.9722473621, 41.9722527266, -24.1001200676, -24.1001093388}},\n\t\t{\"n3wm1r3p4jev\", []float64{-80.6425363384, -80.6425361708, 110.095458291, 110.095458627}},\n\t\t{\"sqfbq13pjp9\", []float64{38.0208036304, 38.0208049715, 15.3824485838, 15.3824499249}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"y0\", []float64{45.0, 50.625, 90.0, 101.25}},\n\t\t{\"04q3tpcc68bq\", []float64{-77.0372864977, -77.03728633, -170.988700055, -170.988699719}},\n\t\t{\"81\", []float64{5.625, 11.25, -180.0, -168.75}},\n\t\t{\"nj\", []float64{-61.875, -56.25, 90.0, 101.25}},\n\t\t{\"b\", []float64{45.0, 90.0, -180.0, -135.0}},\n\t\t{\"8bxs0rfgp3n7\", []float64{3.55871787295, 3.55871804059, -135.688042603, -135.688042268}},\n\t\t{\"j0rm6\", []float64{-87.6708984375, -87.626953125, 55.283203125, 55.3271484375}},\n\t\t{\"86v9\", []float64{15.64453125, 15.8203125, -161.015625, -160.6640625}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"nmcsv\", []float64{-56.8212890625, -56.77734375, 103.579101562, 103.623046875}},\n\t\t{\"cqytkn1d56\", []float64{83.9249145985, 83.9249199629, -114.431394339, -114.43138361}},\n\t\t{\"jfpcxxud86r9\", []float64{-78.4433147125, -78.4433145449, 89.9842279404, 89.9842282757}},\n\t\t{\"zf8cm8fkrg\", []float64{59.2870920897, 59.2870974541, 170.049809217, 170.049819946}},\n\t\t{\"98x\", []float64{2.8125, 4.21875, -102.65625, -101.25}},\n\t\t{\"e5mbzuu\", []float64{18.4391784668, 18.4405517578, -36.5679931641, -36.566619873}},\n\t\t{\"03g5kqkcp\", []float64{-79.5504570007, -79.5504140854, -164.337658882, -164.337615967}},\n\t\t{\"b362q0b79x2\", []float64{52.0799548924, 52.0799562335, -165.321857929, -165.321856588}},\n\t\t{\"m3pq\", []float64{-38.3203125, -38.14453125, 66.4453125, 66.796875}},\n\t\t{\"6564m2w5b0\", []float64{-26.3198518753, -26.3198465109, -86.9485473633, -86.9485366344}},\n\t\t{\"m9puw71wrf\", []float64{-38.5664212704, -38.566415906, 78.6754882336, 78.6754989624}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"9t34u\", []float64{30.0146484375, 30.05859375, -110.91796875, -110.874023438}},\n\t\t{\"hewq3\", []float64{-69.2138671875, -69.169921875, 31.3330078125, 31.376953125}},\n\t\t{\"2m2uy1tgg0\", []float64{-14.6249055862, -14.6249002218, -167.423615456, -167.423604727}},\n\t\t{\"qu8dxgebd9wz\", []float64{-19.22872575, -19.2287255824, 124.798967354, 124.798967689}},\n\t\t{\"scwmj\", []float64{9.31640625, 9.3603515625, 42.7587890625, 42.802734375}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"zm02460g\", []float64{73.1365013123, 73.1366729736, 146.701469421, 146.701812744}},\n\t\t{\"cgtd84ky\", []float64{65.1403427124, 65.1405143738, -93.5091018677, -93.5087585449}},\n\t\t{\"eq\", []float64{33.75, 39.375, -33.75, -22.5}},\n\t\t{\"ht26dn8h\", []float64{-59.9929046631, -59.9927330017, 22.939453125, 22.9397964478}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"b\", []float64{45.0, 90.0, -180.0, -135.0}},\n\t\t{\"j70dzsu\", []float64{-72.6155090332, -72.6141357422, 57.2882080078, 57.2895812988}},\n\t\t{\"y241djby0dk\", []float64{45.2962996066, 45.2963009477, 104.151447415, 104.151448756}},\n\t\t{\"un9\", []float64{81.5625, 82.96875, 1.40625, 2.8125}},\n\t\t{\"2kzdj9\", []float64{-17.9241943359, -17.9187011719, -157.961425781, -157.950439453}},\n\t\t{\"m7b3gj70\", []float64{-23.5697937012, -23.5696220398, 56.7375183105, 56.7378616333}},\n\t\t{\"vvpmy3\", []float64{74.1412353516, 74.1467285156, 89.2199707031, 89.2309570312}},\n\t\t{\"ym0n0\", []float64{74.1796875, 74.2236328125, 101.25, 101.293945312}},\n\t\t{\"k6\", []float64{-33.75, -28.125, 11.25, 22.5}},\n\t\t{\"d96nj5sqmjfn\", []float64{8.10626830906, 8.10626847669, -64.4617196918, -64.4617193565}},\n\t\t{\"9yfrdwk\", []float64{39.3214416504, 39.3228149414, -97.9705810547, -97.9692077637}},\n\t\t{\"8vj712jd0kv\", []float64{28.6527125537, 28.6527138948, -138.804685324, -138.804683983}},\n\t\t{\"bk86yhd\", []float64{70.8206176758, 70.8219909668, -168.132019043, -168.130645752}},\n\t\t{\"6e3pb2s87q10\", []float64{-25.3536236286, -25.353623461, -66.0764430463, -66.0764427111}},\n\t\t{\"nxbn2n7yn\", []float64{-45.2722549438, -45.2722120285, 112.505407333, 112.505450249}},\n\t\t{\"cg4n\", []float64{62.9296875, 63.10546875, -98.4375, -98.0859375}},\n\t\t{\"4de6s37sk467\", []float64{-75.4904382862, -75.4904381186, -62.7379387245, -62.7379383892}},\n\t\t{\"pzegg\", []float64{-47.1533203125, -47.109375, 174.155273438, 174.19921875}},\n\t\t{\"xytyx62\", []float64{37.7174377441, 37.7188110352, 177.154541016, 177.155914307}},\n\t\t{\"2x4\", []float64{-5.625, -4.21875, -154.6875, -153.28125}},\n\t\t{\"j11m8j1eywcu\", []float64{-83.3800566941, -83.3800565265, 46.7601537332, 46.7601540685}},\n\t\t{\"k2xw57e\", []float64{-41.1135864258, -41.1122131348, 21.9438171387, 21.9451904297}},\n\t\t{\"2q74\", []float64{-9.4921875, -9.31640625, -164.53125, -164.1796875}},\n\t\t{\"9tp1960t8\", []float64{28.4006023407, 28.400645256, -102.600631714, -102.600588799}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"2b2mwb\", []float64{-42.626953125, -42.6214599609, -145.601806641, -145.590820312}},\n\t\t{\"b7ts80s\", []float64{65.481262207, 65.482635498, -161.010131836, -161.008758545}},\n\t\t{\"10x0\", []float64{-87.1875, -87.01171875, -125.15625, -124.8046875}},\n\t\t{\"c5p9s65qqch\", []float64{62.1507364511, 62.1507377923, -124.261599183, -124.261597842}},\n\t\t{\"n1j\", []float64{-84.375, -82.96875, 97.03125, 98.4375}},\n\t\t{\"gjy9e2g7hcvc\", []float64{77.6120662875, 77.6120664552, -35.7118779793, -35.7118776441}},\n\t\t{\"d0nngw26hnc0\", []float64{1.22123524547, 1.2212354131, -81.408175081, -81.4081747457}},\n\t\t{\"sqqh99ewn\", []float64{35.9565353394, 35.9565782547, 19.7584819794, 19.7585248947}},\n\t\t{\"27\", []float64{-28.125, -22.5, -168.75, -157.5}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"uv\", []float64{73.125, 78.75, 33.75, 45.0}},\n\t\t{\"nu6\", []float64{-66.09375, -64.6875, 126.5625, 127.96875}},\n\t\t{\"hjg9s97mjbfp\", []float64{-57.3848481663, -57.3848479986, 5.12434154749, 5.12434188277}},\n\t\t{\"cekd\", []float64{63.6328125, 63.80859375, -106.171875, -105.8203125}},\n\t\t{\"fx2s\", []float64{86.484375, 86.66015625, -66.796875, -66.4453125}},\n\t\t{\"zxx5n4wh37\", []float64{87.7293223143, 87.7293276787, 167.615715265, 167.615725994}},\n\t\t{\"k0redc\", []float64{-42.9730224609, -42.9675292969, 10.6677246094, 10.6787109375}},\n\t\t{\"xhj810f1r7\", []float64{22.504350543, 22.5043559074, 142.781378031, 142.78138876}},\n\t\t{\"9j\", []float64{28.125, 33.75, -135.0, -123.75}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"z59900erg\", []float64{64.8673582077, 64.867401123, 137.113966942, 137.114009857}},\n\t\t{\"6dnccesgx8ts\", []float64{-33.4225525707, -33.4225524031, -57.9350421578, -57.9350418225}},\n\t\t{\"jx\", []float64{-50.625, -45.0, 67.5, 78.75}},\n\t\t{\"312y4y56\", []float64{-36.8807601929, -36.8805885315, -133.819999695, -133.819656372}},\n\t\t{\"32vk3g9np\", []float64{-40.013923645, -40.0138807297, -116.288609505, -116.288566589}},\n\t\t{\"kn39\", []float64{-9.66796875, -9.4921875, 2.109375, 2.4609375}},\n\t\t{\"eed4ybdn4mpp\", []float64{20.1747029833, 20.174703151, -19.3880166113, -19.3880162761}},\n\t\t{\"st\", []float64{28.125, 33.75, 22.5, 33.75}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"0t8kw4xyhu5x\", []float64{-58.2566988654, -58.2566986978, -156.873914078, -156.873913743}},\n\t\t{\"dcy\", []float64{9.84375, 11.25, -47.8125, -46.40625}},\n\t\t{\"p3r\", []float64{-82.96875, -81.5625, 156.09375, 157.5}},\n\t\t{\"wdc0w9tbd6\", []float64{15.5649769306, 15.564982295, 114.199887514, 114.199898243}},\n\t\t{\"j3jc\", []float64{-84.19921875, -84.0234375, 64.3359375, 64.6875}},\n\t\t{\"k5cw3t\", []float64{-22.7801513672, -22.7746582031, 2.17529296875, 2.18627929688}},\n\t\t{\"pd6m\", []float64{-76.46484375, -76.2890625, 160.6640625, 161.015625}},\n\t\t{\"hnfqkjqrqnh\", []float64{-50.9025013447, -50.9025000036, 3.34868967533, 3.34869101644}},\n\t\t{\"6qkwd\", []float64{-8.701171875, -8.6572265625, -72.333984375, -72.2900390625}},\n\t\t{\"1q2x\", []float64{-53.61328125, -53.4375, -123.046875, -122.6953125}},\n\t\t{\"3nps1et4nde\", []float64{-10.527292192, -10.5272908509, -124.380057603, -124.380056262}},\n\t\t{\"gq2c8qnkx2\", []float64{80.4536533356, 80.4536587, -32.6754319668, -32.6754212379}},\n\t\t{\"q7d\", []float64{-25.3125, -23.90625, 104.0625, 105.46875}},\n\t\t{\"560\", []float64{-78.75, -77.34375, -33.75, -32.34375}},\n\t\t{\"j855ffmuxv4s\", []float64{-89.3276607245, -89.3276605569, 71.8478319794, 71.8478323147}},\n\t\t{\"sumfzt8z4\", []float64{24.4210624695, 24.4211053848, 42.1666431427, 42.166686058}},\n\t\t{\"bje9d2n0\", []float64{76.201171875, 76.2013435364, -174.971008301, -174.970664978}},\n\t\t{\"y943\", []float64{50.80078125, 50.9765625, 115.6640625, 116.015625}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"gmn\", []float64{73.125, 74.53125, -25.3125, -23.90625}},\n\t\t{\"1djwe5s\", []float64{-77.5881958008, -77.5868225098, -104.628295898, -104.626922607}},\n\t\t{\"tb5wv3\", []float64{1.19201660156, 1.19750976562, 83.9025878906, 83.9135742188}},\n\t\t{\"58h3qnfv9m\", []float64{-89.7422236204, -89.742218256, -16.2559354305, -16.2559247017}},\n\t\t{\"f0du0sguugkg\", []float64{48.5425508581, 48.5425510257, -86.1054797843, -86.105479449}},\n\t\t{\"h6zyvwk\", []float64{-73.3103942871, -73.3090209961, 22.3956298828, 22.3970031738}},\n\t\t{\"vg2nu\", []float64{64.4677734375, 64.51171875, 78.92578125, 78.9697265625}},\n\t\t{\"j7r7de\", []float64{-71.0870361328, -71.0815429688, 66.5551757812, 66.5661621094}},\n\t\t{\"hjshn\", []float64{-58.359375, -58.3154296875, 5.888671875, 5.9326171875}},\n\t\t{\"46khrs1t\", []float64{-76.5738487244, -76.573677063, -72.7933502197, -72.793006897}},\n\t\t{\"g5p5p\", []float64{62.40234375, 62.4462890625, -34.8486328125, -34.8046875}},\n\t\t{\"7pxgy2sf\", []float64{-2.15023040771, -2.15005874634, -33.8203811646, -33.8200378418}},\n\t\t{\"vrz\", []float64{88.59375, 90.0, 66.09375, 67.5}},\n\t\t{\"bry3ngyuq\", []float64{88.7908601761, 88.7909030914, -159.654779434, -159.654736519}},\n\t\t{\"9cu9t1g3cs\", []float64{10.1173567772, 10.1173621416, -94.6976208687, -94.6976101398}},\n\t\t{\"82\", []float64{0.0, 5.625, -168.75, -157.5}},\n\t\t{\"m9yzmzsmxx\", []float64{-33.8396555185, -33.8396501541, 77.2510313988, 77.2510421276}},\n\t\t{\"4wv4wwxb2dr2\", []float64{-51.5560363233, -51.5560361557, -60.1724312827, -60.1724309474}},\n\t\t{\"j7ufy8\", []float64{-68.4228515625, -68.4173583984, 63.2153320312, 63.2263183594}},\n\t\t{\"hzrjzg9xjj\", []float64{-48.1875532866, -48.1875479221, 43.9366006851, 43.936611414}},\n\t\t{\"z5eh3ghtm7\", []float64{65.4519671202, 65.4519724846, 139.302059412, 139.302070141}},\n\t\t{\"6sjhet2cts\", []float64{-21.6798663139, -21.6798609495, -60.3136754036, -60.3136646748}},\n\t\t{\"vjyn6v\", []float64{78.4698486328, 78.4753417969, 53.5583496094, 53.5693359375}},\n\t\t{\"hysxqy94\", []float64{-52.1270370483, -52.126865387, 40.3761291504, 40.3764724731}},\n\t\t{\"8yd4qxm0d\", []float64{36.9979190826, 36.997961998, -143.144903183, -143.144860268}},\n\t\t{\"w43b42f\", []float64{12.660369873, 12.6617431641, 92.5625610352, 92.5639343262}},\n\t\t{\"7suc291x\", []float64{-18.0548286438, -18.0546569824, -15.7962799072, -15.7959365845}},\n\t\t{\"pq\", []float64{-56.25, -50.625, 146.25, 157.5}},\n\t\t{\"frd\", []float64{87.1875, 88.59375, -75.9375, -74.53125}},\n\t\t{\"r2\", []float64{-45.0, -39.375, 146.25, 157.5}},\n\t\t{\"5rs1x50fe5m\", []float64{-47.531902045, -47.5319007039, -27.8162173927, -27.8162160516}},\n\t\t{\"zjv30682s21k\", []float64{77.5333506614, 77.533350829, 142.394326217, 142.394326553}},\n\t\t{\"zbbtmw\", []float64{50.1745605469, 50.1800537109, 169.694824219, 169.705810547}},\n\t\t{\"e56k\", []float64{18.984375, 19.16015625, -41.8359375, -41.484375}},\n\t\t{\"v7dzp\", []float64{65.91796875, 65.9619140625, 60.4248046875, 60.46875}},\n\t\t{\"n2z8qt2hnzss\", []float64{-85.707738027, -85.7077378593, 112.082815245, 112.08281558}},\n\t\t{\"zbp8bt07bb5\", []float64{45.159945488, 45.1599468291, 179.319227189, 179.31922853}},\n\t\t{\"s551t\", []float64{17.138671875, 17.1826171875, 4.4384765625, 4.482421875}},\n\t\t{\"5zp7trjp\", []float64{-49.9701118469, -49.9699401855, -0.817108154297, -0.816764831543}},\n\t\t{\"81n2z7dz\", []float64{5.77726364136, 5.77743530273, -170.888557434, -170.888214111}},\n\t\t{\"dw\", []float64{33.75, 39.375, -67.5, -56.25}},\n\t\t{\"vvw35\", []float64{76.11328125, 76.1572265625, 87.6708984375, 87.71484375}},\n\t\t{\"zhtuc3b8bg5n\", []float64{71.1572198197, 71.1572199874, 143.141591996, 143.141592331}},\n\t\t{\"042w4qgx\", []float64{-76.2507820129, -76.2506103516, -179.193191528, -179.192848206}},\n\t\t{\"9sntb7\", []float64{23.5272216797, 23.5327148438, -103.348388672, -103.337402344}},\n\t\t{\"wt6x5nxk\", []float64{30.7981109619, 30.7982826233, 116.157417297, 116.15776062}},\n\t\t{\"spzwehn\", []float64{44.7583007812, 44.7596740723, 10.6869506836, 10.6883239746}},\n\t\t{\"hv70sbbpw64\", []float64{-60.3754413128, -60.3754399717, 38.1777611375, 38.1777624786}},\n\t\t{\"wxt\", []float64{42.1875, 43.59375, 119.53125, 120.9375}},\n\t\t{\"7dvqsskj8\", []float64{-28.3643817902, -28.3643388748, -14.9139404297, -14.9138975143}},\n\t\t{\"wm22\", []float64{29.53125, 29.70703125, 101.6015625, 101.953125}},\n\t\t{\"5t4kf\", []float64{-61.0400390625, -60.99609375, -19.248046875, -19.2041015625}},\n\t\t{\"5sy67z47fz1\", []float64{-62.846608758, -62.8466074169, -13.542933315, -13.5429319739}},\n\t\t{\"c36nw2msv0t\", []float64{53.1760194898, 53.1760208309, -120.655067414, -120.655066073}},\n\t\t{\"311v\", []float64{-38.49609375, -38.3203125, -132.5390625, -132.1875}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"xcykeys\", []float64{10.6704711914, 10.6718444824, 177.709350586, 177.710723877}},\n\t\t{\"gtmvbbv\", []float64{75.5461120605, 75.5474853516, -14.3742370605, -14.3728637695}},\n\t\t{\"5n1\", []float64{-56.25, -54.84375, -43.59375, -42.1875}},\n\t\t{\"y08uf\", []float64{48.6474609375, 48.69140625, 91.142578125, 91.1865234375}},\n\t\t{\"ds4\", []float64{22.5, 23.90625, -64.6875, -63.28125}},\n\t\t{\"1t49n9\", []float64{-61.6937255859, -61.6882324219, -108.698730469, -108.687744141}},\n\t\t{\"v81d55x758\", []float64{45.3713035583, 45.3713089228, 69.7513175011, 69.7513282299}},\n\t\t{\"39j80vxmjh\", []float64{-39.3439078331, -39.3439024687, -104.722495079, -104.72248435}},\n\t\t{\"x3vd66k2vj46\", []float64{10.251773335, 10.2517735027, 154.089306034, 154.089306369}},\n\t\t{\"vg\", []float64{61.875, 67.5, 78.75, 90.0}},\n\t\t{\"06q04jnnuyz\", []float64{-77.3150892556, -77.3150879145, -160.216156393, -160.216155052}},\n\t\t{\"cvws\", []float64{76.640625, 76.81640625, -92.109375, -91.7578125}},\n\t\t{\"9d5j84gmgbv\", []float64{12.2328941524, 12.2328954935, -108.276619166, -108.276617825}},\n\t\t{\"mjg\", []float64{-12.65625, -11.25, 49.21875, 50.625}},\n\t\t{\"k7gspu\", []float64{-23.1811523438, -23.1756591797, 16.5124511719, 16.5234375}},\n\t\t{\"13nrmggfx\", []float64{-83.0795574188, -83.0795145035, -114.702801704, -114.702758789}},\n\t\t{\"ypj759\", []float64{84.9078369141, 84.9133300781, 97.5366210938, 97.5476074219}},\n\t\t{\"hhy3z0zjn\", []float64{-62.9686546326, -62.9686117172, 9.10655021667, 9.10659313202}},\n\t\t{\"b0xhjv\", []float64{48.5430908203, 48.5485839844, -169.903564453, -169.892578125}},\n\t\t{\"xucn7t7t11\", []float64{27.8470855951, 27.8470909595, 170.314908028, 170.314918756}},\n\t\t{\"f0yqwpw6\", []float64{50.4028701782, 50.4030418396, -80.9386825562, -80.9383392334}},\n\t\t{\"hcud164f7z\", []float64{-79.7932773829, -79.7932720184, 40.1369941235, 40.1370048523}},\n\t\t{\"k7b09t\", []float64{-23.7908935547, -23.7854003906, 11.3159179688, 11.3269042969}},\n\t\t{\"gzttr69\", []float64{88.1240844727, 88.1254577637, -3.19564819336, -3.19427490234}},\n\t\t{\"z1\", []float64{50.625, 56.25, 135.0, 146.25}},\n\t\t{\"mt\", []float64{-16.875, -11.25, 67.5, 78.75}},\n\t\t{\"vgpm3pe\", []float64{62.839050293, 62.840423584, 88.9933776855, 88.9947509766}},\n\t\t{\"xc2xk\", []float64{8.3056640625, 8.349609375, 169.62890625, 169.672851562}},\n\t\t{\"7gpegjrrn\", []float64{-27.4357795715, -27.4357366562, -0.561075210571, -0.561032295227}},\n\t\t{\"5m00s41bg21m\", []float64{-61.7759934627, -61.775993295, -33.5716743395, -33.5716740042}},\n\t\t{\"ybz9un\", []float64{49.5593261719, 49.5648193359, 134.47265625, 134.483642578}},\n\t\t{\"5cxhpu5jy6y\", []float64{-80.8364005387, -80.8363991976, -1.06127768755, -1.06127634645}},\n\t\t{\"0gk7412\", []float64{-71.1845397949, -71.1831665039, -140.185546875, -140.184173584}},\n\t\t{\"zj2\", []float64{74.53125, 75.9375, 135.0, 136.40625}},\n\t\t{\"6jj5vt6zv5\", []float64{-16.1856347322, -16.1856293678, -82.7230596542, -82.7230489254}},\n\t\t{\"mdp7x0cy8x\", []float64{-33.1294924021, -33.1294870377, 78.0053544044, 78.0053651333}},\n\t\t{\"515\", []float64{-84.375, -82.96875, -40.78125, -39.375}},\n\t\t{\"rcrpxqxje\", []float64{-36.613740921, -36.6136980057, 178.922095299, 178.922138214}},\n\t\t{\"ydd5n3qr6tu\", []float64{59.5979855955, 59.5979869366, 115.595853925, 115.595855266}},\n\t\t{\"hd\", []float64{-78.75, -73.125, 22.5, 33.75}},\n\t\t{\"rj4uc\", []float64{-16.0400390625, -15.99609375, 138.911132812, 138.955078125}},\n\t\t{\"0r3x\", []float64{-47.98828125, -47.8125, -166.640625, -166.2890625}},\n\t\t{\"490rp8g4y\", []float64{-83.1399393082, -83.1398963928, -66.8144702911, -66.8144273758}},\n\t\t{\"v19nuj\", []float64{54.6514892578, 54.6569824219, 46.58203125, 46.5930175781}},\n\t\t{\"wrg3hx7\", []float64{43.8093566895, 43.8107299805, 106.022186279, 106.02355957}},\n\t\t{\"ntzjry4\", []float64{-56.7004394531, -56.6990661621, 122.687072754, 122.688446045}},\n\t\t{\"3s5c85t2d\", []float64{-22.2170162201, -22.2169733047, -107.219266891, -107.219223976}},\n\t\t{\"wyy\", []float64{37.96875, 39.375, 132.1875, 133.59375}},\n\t\t{\"6wk17\", []float64{-9.6240234375, -9.580078125, -61.7431640625, -61.69921875}},\n\t\t{\"7m5wqccz7meg\", []float64{-15.7654795982, -15.7654794306, -28.5289463773, -28.5289460421}},\n\t\t{\"sk8n5z1qd2\", []float64{26.4067554474, 26.4067608118, 11.4166080952, 11.416618824}},\n\t\t{\"kw8hyjjj384\", []float64{-7.57417201996, -7.57417067885, 22.7706053853, 22.7706067264}},\n\t\t{\"14rw6bt3vx9v\", []float64{-76.2420291267, -76.2420289591, -124.324827231, -124.324826896}},\n\t\t{\"9cdu\", []float64{9.140625, 9.31640625, -97.3828125, -97.03125}},\n\t\t{\"0cptj3e6crgm\", []float64{-83.4873395227, -83.4873393551, -135.467890911, -135.467890576}},\n\t\t{\"0m\", []float64{-61.875, -56.25, -168.75, -157.5}},\n\t\t{\"mx\", []float64{-5.625, 0.0, 67.5, 78.75}},\n\t\t{\"p1sfuc\", []float64{-81.0736083984, -81.0681152344, 141.888427734, 141.899414062}},\n\t\t{\"03nduncm\", []float64{-83.8536643982, -83.8534927368, -159.431877136, -159.431533813}},\n\t\t{\"0m0j\", []float64{-60.99609375, -60.8203125, -168.75, -168.3984375}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"fn\", []float64{78.75, 84.375, -90.0, -78.75}},\n\t\t{\"srddrb0\", []float64{42.5830078125, 42.5843811035, 15.1062011719, 15.1075744629}},\n\t\t{\"wpbf1dnj\", []float64{43.957157135, 43.9573287964, 91.1288452148, 91.1291885376}},\n\t\t{\"nv8k1f98\", []float64{-58.3456420898, -58.3454704285, 124.180526733, 124.180870056}},\n\t\t{\"7upj4ct1\", []float64{-21.6126823425, -21.6125106812, -1.27853393555, -1.27819061279}},\n\t\t{\"k312nzxpwm\", []float64{-39.3324869871, -39.3324816227, 13.3143246174, 13.3143353462}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"qure08zn2n\", []float64{-20.5611813068, -20.5611759424, 134.328460693, 134.328471422}},\n\t\t{\"m1\", []float64{-39.375, -33.75, 45.0, 56.25}},\n\t\t{\"q7wbtv20e\", []float64{-25.195684433, -25.1956415176, 110.995001793, 110.995044708}},\n\t\t{\"fsqx0ywr3s\", []float64{70.1736903191, 70.1736956835, -58.3177685738, -58.3177578449}},\n\t\t{\"rn9k7k2927vd\", []float64{-7.66684871167, -7.66684854403, 136.901339516, 136.901339851}},\n\t\t{\"8ypg\", []float64{34.27734375, 34.453125, -135.3515625, -135.0}},\n\t\t{\"42ru\", []float64{-87.890625, -87.71484375, -67.8515625, -67.5}},\n\t\t{\"z3efhtnprhw\", []float64{53.8177970052, 53.8177983463, 151.729739606, 151.729740947}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"fu\", []float64{67.5, 73.125, -56.25, -45.0}},\n\t\t{\"uz9\", []float64{87.1875, 88.59375, 35.15625, 36.5625}},\n\t\t{\"bqdnsr4y4\", []float64{82.7445602417, 82.744603157, -165.746870041, -165.746827126}},\n\t\t{\"rshpnngvd\", []float64{-21.231508255, -21.2314653397, 163.393907547, 163.393950462}},\n\t\t{\"4egtrvjfe\", []float64{-67.9555034637, -67.9554605484, -62.2295236588, -62.2294807434}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"r2m2yc\", []float64{-43.4564208984, -43.4509277344, 153.929443359, 153.940429688}},\n\t\t{\"v9vps2z7\", []float64{56.1667442322, 56.1669158936, 74.727973938, 74.7283172607}},\n\t\t{\"dh\", []float64{22.5, 28.125, -90.0, -78.75}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"7\", []float64{-45.0, 0.0, -45.0, 0.0}},\n\t\t{\"rrtp\", []float64{-1.58203125, -1.40625, 153.28125, 153.6328125}},\n\t\t{\"57wdet\", []float64{-69.8455810547, -69.8400878906, -24.4555664062, -24.4445800781}},\n\t\t{\"sxm3rvsc1x0y\", []float64{41.031399183, 41.0313993506, 30.229977183, 30.2299775183}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"fmwp70q5\", []float64{77.2138023376, 77.213973999, -70.1724243164, -70.1720809937}},\n\t\t{\"vq3whey\", []float64{81.2315368652, 81.2329101562, 58.5653686523, 58.5667419434}},\n\t\t{\"vm55829xumn\", []float64{73.7443381548, 73.7443394959, 60.4819867015, 60.4819880426}},\n\t\t{\"pc\", []float64{-84.375, -78.75, 168.75, 180.0}},\n\t\t{\"76j\", []float64{-33.75, -32.34375, -26.71875, -25.3125}},\n\t\t{\"du5md\", []float64{23.466796875, 23.5107421875, -51.591796875, -51.5478515625}},\n\t\t{\"0b800r9\", []float64{-87.1463012695, -87.1449279785, -146.237640381, -146.23626709}},\n\t\t{\"r96suj\", []float64{-37.1063232422, -37.1008300781, 161.19140625, 161.202392578}},\n\t\t{\"dqn86\", []float64{33.7939453125, 33.837890625, -69.521484375, -69.4775390625}},\n\t\t{\"62jjysq7fun\", []float64{-43.9652466774, -43.9652453363, -71.4243963361, -71.424394995}},\n\t\t{\"s623s4p3v\", []float64{12.9312086105, 12.9312515259, 11.7875146866, 11.7875576019}},\n\t\t{\"j9w5\", []float64{-81.03515625, -80.859375, 75.9375, 76.2890625}},\n\t\t{\"cku2qh5ee64\", []float64{71.7852795124, 71.7852808535, -117.504816949, -117.504815608}},\n\t\t{\"ypmy864vvgs\", []float64{86.9358202815, 86.9358216226, 98.1009525061, 98.1009538472}},\n\t\t{\"kwe\", []float64{-8.4375, -7.03125, 26.71875, 28.125}},\n\t\t{\"gmq7083dvewj\", []float64{75.0604587235, 75.0604588911, -24.9366608262, -24.9366604909}},\n\t\t{\"9er\", []float64{18.28125, 19.6875, -102.65625, -101.25}},\n\t\t{\"5p89tmjs9j5\", []float64{-47.5205630064, -47.5205616653, -44.0585620701, -44.058560729}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"ewy\", []float64{37.96875, 39.375, -14.0625, -12.65625}},\n\t\t{\"jtgef\", []float64{-56.9970703125, -56.953125, 72.509765625, 72.5537109375}},\n\t\t{\"9yjjw\", []float64{34.716796875, 34.7607421875, -93.955078125, -93.9111328125}},\n\t\t{\"926\", []float64{1.40625, 2.8125, -120.9375, -119.53125}},\n\t\t{\"bz1\", []float64{84.375, 85.78125, -144.84375, -143.4375}},\n\t\t{\"yjjpq0ecnve\", []float64{74.4023618102, 74.4023631513, 97.3003654182, 97.3003667593}},\n\t\t{\"w5e\", []float64{19.6875, 21.09375, 94.21875, 95.625}},\n\t\t{\"hqcn9wtcr\", []float64{-50.8527517319, -50.8527088165, 12.7303647995, 12.7304077148}},\n\t\t{\"qfh6xphngs\", []float64{-33.2709145546, -33.2709091902, 130.039823055, 130.039833784}},\n\t\t{\"1he586fypp\", []float64{-64.0560919046, -64.0560865402, -130.766186714, -130.766175985}},\n\t\t{\"4cc5sh9n3s\", []float64{-79.5152020454, -79.515196681, -54.666531086, -54.6665203571}},\n\t\t{\"9y5wfm\", []float64{34.9639892578, 34.9694824219, -96.2292480469, -96.2182617188}},\n\t\t{\"c97809\", []float64{52.0367431641, 52.0422363281, -107.556152344, -107.545166016}},\n\t\t{\"k9g2nkbm3j5h\", []float64{-35.1292287558, -35.1292285882, 27.3453609645, 27.3453612998}},\n\t\t{\"thdwugw196t\", []float64{26.5185204148, 26.5185217559, 48.7326653302, 48.7326666713}},\n\t\t{\"34nm41n89c8v\", []float64{-32.8655058704, -32.8655057028, -126.114044376, -126.11404404}},\n\t\t{\"buf7qgu\", []float64{72.3106384277, 72.3120117188, -142.783813477, -142.782440186}},\n\t\t{\"mhvh0u7f4\", []float64{-17.55443573, -17.5543928146, 52.0694446564, 52.0694875717}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"f0vdwj1bu\", []float64{49.6857976913, 49.6858406067, -81.9993782043, -81.999335289}},\n\t\t{\"kcke59\", []float64{-37.4359130859, -37.4304199219, 40.2319335938, 40.2429199219}},\n\t\t{\"9rws4p0\", []float64{42.9290771484, 42.9304504395, -114.521484375, -114.520111084}},\n\t\t{\"fhj1u03epu\", []float64{67.8095269203, 67.8095322847, -82.7905762196, -82.7905654907}},\n\t\t{\"13296d9gwq1\", []float64{-82.734657526, -82.7346561849, -122.934338897, -122.934337556}},\n\t\t{\"4j\", []float64{-61.875, -56.25, -90.0, -78.75}},\n\t\t{\"gk5u1y2\", []float64{68.2374572754, 68.2388305664, -28.3996582031, -28.3982849121}},\n\t\t{\"9v6yrwx00\", []float64{30.6655883789, 30.6656312943, -97.0436096191, -97.0435667038}},\n\t\t{\"mc92\", []float64{-36.5625, -36.38671875, 80.5078125, 80.859375}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"vtzr1we0jh5\", []float64{78.6099457741, 78.6099471152, 77.7655689418, 77.7655702829}},\n\t\t{\"ytmmrjr08p\", []float64{75.4830640554, 75.4830694199, 120.200042725, 120.200053453}},\n\t\t{\"y7q525c0mgkz\", []float64{63.8731999509, 63.8732001185, 109.689126424, 109.68912676}},\n\t\t{\"s5nc\", []float64{17.05078125, 17.2265625, 9.4921875, 9.84375}},\n\t\t{\"wk2\", []float64{23.90625, 25.3125, 101.25, 102.65625}},\n\t\t{\"f4beky4z04y\", []float64{61.0742144287, 61.0742157698, -89.0843501687, -89.0843488276}},\n\t\t{\"ywdu5yj95\", []float64{82.2987556458, 82.2987985611, 116.539664268, 116.539707184}},\n\t\t{\"n3\", []float64{-84.375, -78.75, 101.25, 112.5}},\n\t\t{\"0334vnb6\", []float64{-82.4479293823, -82.4477577209, -167.123680115, -167.123336792}},\n\t\t{\"xg65\", []float64{18.80859375, 18.984375, 171.5625, 171.9140625}},\n\t\t{\"0ebmse71br\", []float64{-67.9212623835, -67.921257019, -156.946552992, -156.946542263}},\n\t\t{\"ycwd9fc\", []float64{53.8920593262, 53.8934326172, 132.968902588, 132.970275879}},\n\t\t{\"0z2gsvd0tfzy\", []float64{-48.573201634, -48.5732014664, -144.983568527, -144.983568192}},\n\t\t{\"e041\", []float64{0.17578125, 0.3515625, -42.1875, -41.8359375}},\n\t\t{\"ntzdfpdcphj7\", []float64{-57.1314592101, -57.1314590424, 123.138849624, 123.138849959}},\n\t\t{\"jx1bfyyhgds1\", []float64{-50.4552562349, -50.4552560672, 70.0901824236, 70.0901827589}},\n\t\t{\"dhzuvhgspbew\", []float64{27.5804938003, 27.580493968, -78.8766921312, -78.8766917959}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"teyjc173t\", []float64{22.1116161346, 22.11165905, 75.986123085, 75.9861660004}},\n\t\t{\"bg57uz4rxw5\", []float64{62.5739514828, 62.5739528239, -141.467531472, -141.467530131}},\n\t\t{\"52dtfpdc\", []float64{-86.1353874207, -86.1352157593, -30.1427078247, -30.142364502}},\n\t\t{\"vx1j39e\", []float64{85.3060913086, 85.3074645996, 68.9762878418, 68.9776611328}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"psmpz5\", []float64{-64.7149658203, -64.7094726562, 164.838867188, 164.849853516}},\n\t\t{\"4xr95eeee\", []float64{-49.023141861, -49.0230989456, -56.7943811417, -56.7943382263}},\n\t\t{\"5j\", []float64{-61.875, -56.25, -45.0, -33.75}},\n\t\t{\"kpb\", []float64{-1.40625, 0.0, 0.0, 1.40625}},\n\t\t{\"dsub48epk\", []float64{26.722741127, 26.7227840424, -60.7061576843, -60.706114769}},\n\t\t{\"2urtnwtdw17\", []float64{-20.1787023246, -20.1787009835, -135.409665853, -135.409664512}},\n\t\t{\"e6s30gwjxm\", []float64{14.2584782839, 14.2584836483, -27.7319276333, -27.7319169044}},\n\t\t{\"qtx\", []float64{-14.0625, -12.65625, 122.34375, 123.75}},\n\t\t{\"qj0qvndweq3k\", []float64{-15.651620999, -15.6516208313, 90.5748634413, 90.5748637766}},\n\t\t{\"ffetyh28uyj\", []float64{60.0967490673, 60.0967504084, -51.0635559261, -51.063554585}},\n\t\t{\"z56t8nwqq7\", []float64{64.2848414183, 64.2848467827, 138.52447629, 138.524487019}},\n\t\t{\"7h\", []float64{-22.5, -16.875, -45.0, -33.75}},\n\t\t{\"9tuuw1pkyh\", []float64{33.1410956383, 33.1411010027, -105.546426773, -105.546416044}},\n\t\t{\"2m\", []float64{-16.875, -11.25, -168.75, -157.5}},\n\t\t{\"h7qt\", []float64{-70.83984375, -70.6640625, 20.390625, 20.7421875}},\n\t\t{\"t832ztb6psn\", []float64{1.57003641129, 1.57003775239, 69.5880755782, 69.5880769193}},\n\t\t{\"wk\", []float64{22.5, 28.125, 101.25, 112.5}},\n\t\t{\"ndjbb8w3n\", []float64{-78.6152458191, -78.6152029037, 120.616750717, 120.616793633}},\n\t\t{\"14pf3eqg4zd5\", []float64{-78.3360836841, -78.3360835165, -124.026254117, -124.026253782}},\n\t\t{\"9j\", []float64{28.125, 33.75, -135.0, -123.75}},\n\t\t{\"fr6ng34\", []float64{86.9732666016, 86.9746398926, -75.7919311523, -75.7905578613}},\n\t\t{\"p3ggurx2c\", []float64{-79.455742836, -79.4556999207, 151.720204353, 151.720247269}},\n\t\t{\"1h1pg1myn06\", []float64{-66.1297975481, -66.129796207, -133.453757465, -133.453756124}},\n\t\t{\"cqsue\", []float64{82.353515625, 82.3974609375, -116.938476562, -116.89453125}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"s8jkw\", []float64{0.791015625, 0.8349609375, 30.146484375, 30.1904296875}},\n\t\t{\"67\", []float64{-28.125, -22.5, -78.75, -67.5}},\n\t\t{\"ywe4mn\", []float64{81.9909667969, 81.9964599609, 116.938476562, 116.949462891}},\n\t\t{\"0f5te71q9g\", []float64{-77.7655917406, -77.7655863762, -141.183511019, -141.18350029}},\n\t\t{\"v9s6tw70swwv\", []float64{53.911406938, 53.9114071056, 73.7225837633, 73.7225840986}},\n\t\t{\"0jbutv\", []float64{-56.8377685547, -56.8322753906, -178.692626953, -178.681640625}},\n\t\t{\"bn271bp\", []float64{80.68359375, 80.684967041, -179.561920166, -179.560546875}},\n\t\t{\"1vvyth\", []float64{-56.4916992188, -56.4862060547, -92.9443359375, -92.9333496094}},\n\t\t{\"7ruk94vup\", []float64{-0.59944152832, -0.599398612976, -27.7212953568, -27.7212524414}},\n\t\t{\"3hf\", []float64{-18.28125, -16.875, -132.1875, -130.78125}},\n\t\t{\"741rwgds3m4k\", []float64{-32.4116574973, -32.4116573296, -42.9420667514, -42.9420664161}},\n\t\t{\"2pye\", []float64{-0.87890625, -0.703125, -170.859375, -170.5078125}},\n\t\t{\"2\", []float64{-45.0, 0.0, -180.0, -135.0}},\n\t\t{\"e7\", []float64{16.875, 22.5, -33.75, -22.5}},\n\t\t{\"f\", []float64{45.0, 90.0, -90.0, -45.0}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"4c5p5ke\", []float64{-83.1198120117, -83.1184387207, -51.8843078613, -51.8829345703}},\n\t\t{\"h7q\", []float64{-71.71875, -70.3125, 19.6875, 21.09375}},\n\t\t{\"4fjp8\", []float64{-77.431640625, -77.3876953125, -49.21875, -49.1748046875}},\n\t\t{\"p2cbvvvdt8\", []float64{-85.6173992157, -85.6173938513, 148.971412182, 148.971422911}},\n\t\t{\"xxjtqz46qmm\", []float64{40.3367181122, 40.3367194533, 165.534370691, 165.534372032}},\n\t\t{\"w1e\", []float64{8.4375, 9.84375, 94.21875, 95.625}},\n\t\t{\"fxpg4v3e\", []float64{84.9316978455, 84.9318695068, -56.4786529541, -56.4783096313}},\n\t\t{\"3be6u\", []float64{-41.7041015625, -41.66015625, -96.50390625, -96.4599609375}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"seqqvkuphz\", []float64{19.4951051474, 19.4951105118, 31.5254724026, 31.5254831314}},\n\t\t{\"txy2t7xx\", []float64{43.7020683289, 43.7022399902, 76.5300750732, 76.530418396}},\n\t\t{\"s2hc2d2s\", []float64{0.232772827148, 0.232944488525, 17.9523468018, 17.9526901245}},\n\t\t{\"8zr0n4f62k\", []float64{40.7967638969, 40.7967692614, -136.139477491, -136.139466763}},\n\t\t{\"th1vxpfxnp9\", []float64{23.5106107593, 23.5106121004, 47.7722467482, 47.7722480893}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"33\", []float64{-39.375, -33.75, -123.75, -112.5}},\n\t\t{\"gu\", []float64{67.5, 73.125, -11.25, 0.0}},\n\t\t{\"9vq49\", []float64{29.970703125, 30.0146484375, -92.7685546875, -92.724609375}},\n\t\t{\"tm\", []float64{28.125, 33.75, 56.25, 67.5}},\n\t\t{\"dpzw0p\", []float64{44.6868896484, 44.6923828125, -79.453125, -79.4421386719}},\n\t\t{\"gwg12\", []float64{83.1884765625, 83.232421875, -18.28125, -18.2373046875}},\n\t\t{\"b8vphv0m5k\", []float64{50.4775643349, 50.4775696993, -150.259526968, -150.259516239}},\n\t\t{\"pgpffhw1\", []float64{-72.6167106628, -72.6165390015, 179.744567871, 179.744911194}},\n\t\t{\"3r3w\", []float64{-3.1640625, -2.98828125, -121.640625, -121.2890625}},\n\t\t{\"u1d\", []float64{53.4375, 54.84375, 2.8125, 4.21875}},\n\t\t{\"mznb8v5xu6\", []float64{-5.50830245018, -5.50829708576, 88.2801353931, 88.280146122}},\n\t\t{\"8mb57vjrex4\", []float64{32.9438298941, 32.9438312352, -168.577842414, -168.577841073}},\n\t\t{\"zm\", []float64{73.125, 78.75, 146.25, 157.5}},\n\t\t{\"c9ef6tm74sg\", []float64{53.8623873889, 53.86238873, -107.109378129, -107.109376788}},\n\t\t{\"spww\", []float64{43.2421875, 43.41796875, 9.140625, 9.4921875}},\n\t\t{\"snp97n\", []float64{34.0026855469, 34.0081787109, 10.6787109375, 10.6896972656}},\n\t\t{\"zp9r6emsk8xx\", []float64{88.4805002622, 88.4805004299, 136.875432059, 136.875432394}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"18zsh9bg\", []float64{-85.0679969788, -85.0678253174, -101.754341125, -101.753997803}},\n\t\t{\"v28\", []float64{47.8125, 49.21875, 56.25, 57.65625}},\n\t\t{\"4e\", []float64{-73.125, -67.5, -67.5, -56.25}},\n\t\t{\"evn0wp56\", []float64{28.2516860962, 28.2518577576, -2.5443649292, -2.54402160645}},\n\t\t{\"uyf9v\", []float64{83.2763671875, 83.3203125, 37.4853515625, 37.529296875}},\n\t\t{\"d7\", []float64{16.875, 22.5, -78.75, -67.5}},\n\t\t{\"05\", []float64{-73.125, -67.5, -180.0, -168.75}},\n\t\t{\"ujj8\", []float64{73.125, 73.30078125, 7.734375, 8.0859375}},\n\t\t{\"wcb7n8\", []float64{10.37109375, 10.3765869141, 124.387207031, 124.398193359}},\n\t\t{\"r35s2y4e2\", []float64{-38.5944128036, -38.5943698883, 151.208267212, 151.208310127}},\n\t\t{\"k\", []float64{-45.0, 0.0, 0.0, 45.0}},\n\t\t{\"8tm3h7b1f\", []float64{29.7279310226, 29.727973938, -149.930334091, -149.930291176}},\n\t\t{\"3xecw9gsguw3\", []float64{-2.53837538883, -2.53837522119, -106.935942136, -106.9359418}},\n\t\t{\"hqs10v\", []float64{-53.2342529297, -53.2287597656, 16.9079589844, 16.9189453125}},\n\t\t{\"b21g\", []float64{45.52734375, 45.703125, -166.2890625, -165.9375}},\n\t\t{\"vphhpnjt5b\", []float64{85.1119422913, 85.1119476557, 50.9403312206, 50.9403419495}},\n\t\t{\"kbd\", []float64{-42.1875, -40.78125, 36.5625, 37.96875}},\n\t\t{\"2c\", []float64{-39.375, -33.75, -146.25, -135.0}},\n\t\t{\"07ur\", []float64{-67.67578125, -67.5, -162.7734375, -162.421875}},\n\t\t{\"8e5ky1\", []float64{17.7154541016, 17.7209472656, -152.666015625, -152.655029297}},\n\t\t{\"k2w84t\", []float64{-42.1600341797, -42.1545410156, 20.5004882812, 20.5114746094}},\n\t\t{\"p9t4ncex81m\", []float64{-81.2014035881, -81.201402247, 164.832694083, 164.832695425}},\n\t\t{\"q67rduzsu6uz\", []float64{-30.9984667785, -30.9984666109, 105.951650552, 105.951650888}},\n\t\t{\"udwkypp0v\", []float64{59.936041832, 59.9360847473, 31.5625619888, 31.5626049042}},\n\t\t{\"pjsu1q9qg\", []float64{-58.3225107193, -58.322467804, 141.7364645, 141.736507416}},\n\t\t{\"2kj2w9b021b\", []float64{-22.4024440348, -22.4024426937, -161.081542969, -161.081541628}},\n\t\t{\"5k0\", []float64{-67.5, -66.09375, -33.75, -32.34375}},\n\t\t{\"t626vs8j\", []float64{13.1652259827, 13.165397644, 56.8432617188, 56.8436050415}},\n\t\t{\"hd0z4zr73\", []float64{-77.4791479111, -77.4791049957, 23.6855363846, 23.6855792999}},\n\t\t{\"79gjppfekhhk\", []float64{-34.2341917008, -34.2341915332, -17.9700222239, -17.9700218886}},\n\t\t{\"u9u\", []float64{54.84375, 56.25, 28.125, 29.53125}},\n\t\t{\"5zbfmj3n30\", []float64{-45.9808301926, -45.9808248281, -9.97416973114, -9.9741590023}},\n\t\t{\"1w1nt3g4t9pp\", []float64{-55.0973731466, -55.0973729789, -110.858671814, -110.858671479}},\n\t\t{\"f6bh910940\", []float64{61.2654304504, 61.2654358149, -78.7052822113, -78.7052714825}},\n\t\t{\"r65q38x\", []float64{-32.6486206055, -32.6472473145, 150.895843506, 150.897216797}},\n\t\t{\"xq2\", []float64{35.15625, 36.5625, 146.25, 147.65625}},\n\t\t{\"q87xbntvdv8d\", []float64{-42.1947657689, -42.1947656013, 117.429890111, 117.429890446}},\n\t\t{\"w1zhgmbpw\", []float64{10.7115840912, 10.7116270065, 99.9868297577, 99.986872673}},\n\t\t{\"5n\", []float64{-56.25, -50.625, -45.0, -33.75}},\n\t\t{\"9dz\", []float64{15.46875, 16.875, -102.65625, -101.25}},\n\t\t{\"n8r794hh15gv\", []float64{-87.9668216966, -87.966821529, 122.744798921, 122.744799256}},\n\t\t{\"px78re9\", []float64{-49.1555786133, -49.1542053223, 162.752838135, 162.754211426}},\n\t\t{\"3pps\", []float64{-4.921875, -4.74609375, -124.453125, -124.1015625}},\n\t\t{\"3s6um\", []float64{-20.3466796875, -20.302734375, -108.413085938, -108.369140625}},\n\t\t{\"9dj7zre6t7\", []float64{11.9508236647, 11.9508290291, -104.793895483, -104.793884754}},\n\t\t{\"4v1b\", []float64{-61.875, -61.69921875, -53.7890625, -53.4375}},\n\t\t{\"1k35z\", []float64{-65.4345703125, -65.390625, -122.036132812, -121.9921875}},\n\t\t{\"7z9n57\", []float64{-1.74133300781, -1.73583984375, -9.70092773438, -9.68994140625}},\n\t\t{\"3gzg\", []float64{-23.37890625, -23.203125, -90.3515625, -90.0}},\n\t\t{\"hy\", []float64{-56.25, -50.625, 33.75, 45.0}},\n\t\t{\"2rj6t\", []float64{-5.185546875, -5.1416015625, -161.147460938, -161.103515625}},\n\t\t{\"h\", []float64{-90.0, -45.0, 0.0, 45.0}},\n\t\t{\"dp44nc1t\", []float64{39.7329139709, 39.7330856323, -86.8888092041, -86.8884658813}},\n\t\t{\"0x1\", []float64{-50.625, -49.21875, -156.09375, -154.6875}},\n\t\t{\"dmwxxf\", []float64{32.2668457031, 32.2723388672, -69.2687988281, -69.2578125}},\n\t\t{\"khy29\", []float64{-18.193359375, -18.1494140625, 8.8330078125, 8.876953125}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"phn4\", []float64{-67.1484375, -66.97265625, 143.4375, 143.7890625}},\n\t\t{\"qzhvp\", []float64{-4.74609375, -4.7021484375, 130.737304688, 130.78125}},\n\t\t{\"3n\", []float64{-11.25, -5.625, -135.0, -123.75}},\n\t\t{\"0nx\", []float64{-53.4375, -52.03125, -170.15625, -168.75}},\n\t\t{\"19uwx04h21\", []float64{-79.0129369497, -79.0129315853, -105.86151123, -105.861500502}},\n\t\t{\"7ur1q\", []float64{-20.8740234375, -20.830078125, -1.142578125, -1.0986328125}},\n\t\t{\"8yn6q9vmm\", []float64{34.1560220718, 34.1560649872, -137.167868614, -137.167825699}},\n\t\t{\"m4zk\", []float64{-28.828125, -28.65234375, 55.1953125, 55.546875}},\n\t\t{\"9bgzpypspd0\", []float64{5.48287510872, 5.48287644982, -95.6253647804, -95.6253634393}},\n\t\t{\"y1s\", []float64{53.4375, 54.84375, 95.625, 97.03125}},\n\t\t{\"qsyp207nvy\", []float64{-17.0042717457, -17.0042663813, 120.941866636, 120.941877365}},\n\t\t{\"rfb\", []float64{-29.53125, -28.125, 168.75, 170.15625}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"5exm5p63\", []float64{-69.3935966492, -69.3934249878, -12.1697616577, -12.169418335}},\n\t\t{\"cnv22fdkruw\", []float64{83.0271819234, 83.0271832645, -127.58079797, -127.580796629}},\n\t\t{\"n7vg\", []float64{-68.37890625, -68.203125, 109.3359375, 109.6875}},\n\t\t{\"whvgd2h3sz9\", []float64{27.3342821002, 27.3342834413, 98.1908561289, 98.19085747}},\n\t\t{\"shbfuzk8vr\", []float64{27.2421401739, 27.2421455383, 1.2698328495, 1.26984357834}},\n\t\t{\"44vmk\", []float64{-73.6083984375, -73.564453125, -82.44140625, -82.3974609375}},\n\t\t{\"uhd1mfq\", []float64{70.5445861816, 70.5459594727, 3.07342529297, 3.07479858398}},\n\t\t{\"7bz\", []float64{-40.78125, -39.375, -1.40625, 0.0}},\n\t\t{\"h5b2wkdqpz\", []float64{-68.7925726175, -68.7925672531, 0.629643201828, 0.629653930664}},\n\t\t{\"h1\", []float64{-84.375, -78.75, 0.0, 11.25}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"408bm1\", []float64{-87.1380615234, -87.1325683594, -88.7255859375, -88.7145996094}},\n\t\t{\"ggysyy5e2be6\", []float64{66.9622308388, 66.9622310065, -1.80790107697, -1.8079007417}},\n\t\t{\"w4u7dn8m9ndw\", []float64{16.1206699535, 16.1206701212, 96.0648427159, 96.0648430511}},\n\t\t{\"yq\", []float64{78.75, 84.375, 101.25, 112.5}},\n\t\t{\"2nwuht4w\", []float64{-7.70587921143, -7.70570755005, -170.306625366, -170.306282043}},\n\t\t{\"v5gqe\", []float64{67.236328125, 67.2802734375, 49.7021484375, 49.74609375}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"cmehghzjm1\", []float64{76.7994600534, 76.7994654179, -119.389586449, -119.38957572}},\n\t\t{\"u207q361d\", []float64{45.5784130096, 45.578455925, 11.8790531158, 11.8790960312}},\n\t\t{\"n4pgq345pp\", []float64{-78.1726652384, -78.172659874, 101.176142693, 101.176153421}},\n\t\t{\"b2mn8\", []float64{47.548828125, 47.5927734375, -161.71875, -161.674804688}},\n\t\t{\"qbe6mp0g8et3\", []float64{-41.7529202811, -41.7529201135, 128.541097529, 128.541097865}},\n\t\t{\"sr04m\", []float64{39.7705078125, 39.814453125, 11.4697265625, 11.513671875}},\n\t\t{\"hfr0y7u988jx\", []float64{-77.1910560317, -77.1910558641, 43.8746168464, 43.8746171817}},\n\t\t{\"jrgxg5qkg7p\", []float64{-45.0252610445, -45.0252597034, 61.3124428689, 61.3124442101}},\n\t\t{\"gryut\", []float64{89.384765625, 89.4287109375, -24.0380859375, -23.994140625}},\n\t\t{\"14d5b766ppxg\", []float64{-75.2600834705, -75.2600833029, -132.173112966, -132.173112631}},\n\t\t{\"0ede2rgwt2\", []float64{-69.6975231171, -69.6975177526, -153.968356848, -153.968346119}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"dj\", []float64{28.125, 33.75, -90.0, -78.75}},\n\t\t{\"xf\", []float64{11.25, 16.875, 168.75, 180.0}},\n\t\t{\"szf3p5k9rmx\", []float64{43.7876281142, 43.7876294553, 37.228180021, 37.2281813622}},\n\t\t{\"b9fqkhfem7\", []float64{55.9690493345, 55.9690546989, -154.156497717, -154.156486988}},\n\t\t{\"t7zw8x5c\", []float64{22.2749519348, 22.2751235962, 66.8239974976, 66.8243408203}},\n\t\t{\"f87dmh\", []float64{46.8237304688, 46.8292236328, -62.3583984375, -62.3474121094}},\n\t\t{\"yrd1swq12\", []float64{87.4857187271, 87.4857616425, 104.268493652, 104.268536568}},\n\t\t{\"s2\", []float64{0.0, 5.625, 11.25, 22.5}},\n\t\t{\"q9dhkgwy4kum\", []float64{-35.7951473258, -35.7951471582, 115.530612208, 115.530612543}},\n\t\t{\"7dr\", []float64{-32.34375, -30.9375, -12.65625, -11.25}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"2rpk2ey43dw\", []float64{-4.85693067312, -4.85692933202, -158.524402678, -158.524401337}},\n\t\t{\"3wfeg\", []float64{-6.3720703125, -6.328125, -108.852539062, -108.80859375}},\n\t\t{\"ke5k4j\", []float64{-27.3944091797, -27.3889160156, 27.158203125, 27.1691894531}},\n\t\t{\"z0xq\", []float64{48.8671875, 49.04296875, 145.1953125, 145.546875}},\n\t\t{\"w1sy\", []float64{9.4921875, 9.66796875, 96.6796875, 97.03125}},\n\t\t{\"eqm14\", []float64{35.33203125, 35.3759765625, -26.630859375, -26.5869140625}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"hjp9d9f\", []float64{-61.6017150879, -61.6003417969, 10.6594848633, 10.6608581543}},\n\t\t{\"p92v0\", []float64{-82.08984375, -82.0458984375, 158.5546875, 158.598632812}},\n\t\t{\"36m7g02m\", []float64{-31.6823387146, -31.6821670532, -116.23500824, -116.234664917}},\n\t\t{\"5g70e57zjrf\", []float64{-71.6117633879, -71.6117620468, -6.89403623343, -6.89403489232}},\n\t\t{\"65rkyfq4se\", []float64{-25.8709841967, -25.8709788322, -79.4996237755, -79.4996130466}},\n\t\t{\"eev1\", []float64{21.26953125, 21.4453125, -15.46875, -15.1171875}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"m75926t\", []float64{-27.8915405273, -27.8901672363, 61.1897277832, 61.1911010742}},\n\t\t{\"1kjyeb\", []float64{-66.357421875, -66.3519287109, -115.499267578, -115.48828125}},\n\t\t{\"fb8rk2yfwmrp\", []float64{49.0914924257, 49.0914925933, -55.7021225989, -55.7021222636}},\n\t\t{\"y2qhd0j8x\", []float64{47.1973514557, 47.197394371, 109.783244133, 109.783287048}},\n\t\t{\"m2\", []float64{-45.0, -39.375, 56.25, 67.5}},\n\t\t{\"0543np5pgd23\", []float64{-72.9094239883, -72.9094238207, -176.567995213, -176.567994878}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"d4h5zdhe5gy\", []float64{11.9207011163, 11.9207024574, -84.0390613675, -84.0390600264}},\n\t\t{\"9rcd\", []float64{43.9453125, 44.12109375, -121.640625, -121.2890625}},\n\t\t{\"ne9nrh75tq3\", []float64{-69.1898868978, -69.1898855567, 114.218213707, 114.218215048}},\n\t\t{\"7wk7\", []float64{-9.31640625, -9.140625, -16.5234375, -16.171875}},\n\t\t{\"995f97e\", []float64{6.08367919922, 6.08505249023, -107.167510986, -107.166137695}},\n\t\t{\"60kmung\", []float64{-42.5459289551, -42.5445556641, -83.843536377, -83.8421630859}},\n\t\t{\"845\", []float64{11.25, 12.65625, -175.78125, -174.375}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"jehdxn0\", []float64{-72.6525878906, -72.6512145996, 74.1357421875, 74.1371154785}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"1d\", []float64{-78.75, -73.125, -112.5, -101.25}},\n\t\t{\"rbjy\", []float64{-43.9453125, -43.76953125, 176.8359375, 177.1875}},\n\t\t{\"r8qgzf4r9uy\", []float64{-42.9222710431, -42.922269702, 167.335936725, 167.335938066}},\n\t\t{\"k5p\", []float64{-28.125, -26.71875, 9.84375, 11.25}},\n\t\t{\"f4z7\", []float64{60.99609375, 61.171875, -79.8046875, -79.453125}},\n\t\t{\"7rp35b\", []float64{-5.44921875, -5.44372558594, -23.3898925781, -23.37890625}},\n\t\t{\"zn71yyn0pbc\", []float64{80.4968301952, 80.4968315363, 139.52395454, 139.523955882}},\n\t\t{\"ppj7\", []float64{-50.09765625, -49.921875, 142.3828125, 142.734375}},\n\t\t{\"mqv3q\", []float64{-6.8115234375, -6.767578125, 63.896484375, 63.9404296875}},\n\t\t{\"tsdtmfq\", []float64{26.2477111816, 26.2490844727, 71.276550293, 71.277923584}},\n\t\t{\"72ey8b14uynx\", []float64{-41.0444164462, -41.0444162786, -28.4420176595, -28.4420173243}},\n\t\t{\"7qrgb\", []float64{-9.1845703125, -9.140625, -22.8515625, -22.8076171875}},\n\t\t{\"w7zmkdpezcm\", []float64{22.0282383263, 22.0282396674, 111.653705388, 111.653706729}},\n\t\t{\"kqwr1dh9jdbc\", []float64{-7.19585834071, -7.19585817307, 20.1113973185, 20.1113976538}},\n\t\t{\"kv9jx\", []float64{-13.095703125, -13.0517578125, 35.4638671875, 35.5078125}},\n\t\t{\"09\", []float64{-84.375, -78.75, -157.5, -146.25}},\n\t\t{\"f8ztmmp0\", []float64{50.1690673828, 50.1692390442, -56.7127990723, -56.7124557495}},\n\t\t{\"k5dj8cuwbxjg\", []float64{-24.3348933198, -24.3348931521, 2.85166796297, 2.85166829824}},\n\t\t{\"xd72j5qndwhn\", []float64{12.6752517745, 12.6752519421, 162.298391461, 162.298391797}},\n\t\t{\"esp42d\", []float64{22.9064941406, 22.9119873047, -12.6342773438, -12.6232910156}},\n\t\t{\"5sbfys\", []float64{-62.7758789062, -62.7703857422, -21.1596679688, -21.1486816406}},\n\t\t{\"8wsz02n\", []float64{37.79296875, 37.794342041, -150.801086426, -150.799713135}},\n\t\t{\"zeghw8\", []float64{66.884765625, 66.8902587891, 162.004394531, 162.015380859}},\n\t\t{\"u0xg7ug\", []float64{48.4098815918, 48.4112548828, 11.0673522949, 11.0687255859}},\n\t\t{\"0jb11\", []float64{-57.48046875, -57.4365234375, -179.956054688, -179.912109375}},\n\t\t{\"xv8cwtybm\", []float64{31.2328004837, 31.232843399, 170.099816322, 170.099859238}},\n\t\t{\"ef0cwqt7\", []float64{11.5498924255, 11.5500640869, -9.91344451904, -9.91310119629}},\n\t\t{\"hrh5k\", []float64{-50.0537109375, -50.009765625, 17.05078125, 17.0947265625}},\n\t\t{\"pnpdsx4eb\", []float64{-55.7714509964, -55.7714080811, 145.748062134, 145.748105049}},\n\t\t{\"8g2sx4gn\", []float64{19.0884017944, 19.0885734558, -145.235137939, -145.234794617}},\n\t\t{\"tsue3yr4z\", []float64{27.3248434067, 27.324886322, 73.9149427414, 73.9149856567}},\n\t\t{\"k4vq\", []float64{-28.4765625, -28.30078125, 7.3828125, 7.734375}},\n\t\t{\"mr1f1d430h\", []float64{-5.26225805283, -5.26225268841, 58.7799453735, 58.7799561024}},\n\t\t{\"dtuqkjybm\", []float64{33.4740114212, 33.4740543365, -61.3381719589, -61.3381290436}},\n\t\t{\"p00zpfbj5350\", []float64{-88.7535613775, -88.7535612099, 136.39540717, 136.395407505}},\n\t\t{\"n16jy8wrg38\", []float64{-81.9539228082, -81.9539214671, 93.106867075, 93.1068684161}},\n\t\t{\"3ckf9t6\", []float64{-37.5004577637, -37.4990844727, -94.5016479492, -94.5002746582}},\n\t\t{\"vvch78h7q7\", []float64{78.0913943052, 78.0913996696, 80.3161633015, 80.3161740303}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"qkr8dj44\", []float64{-20.9780502319, -20.9778785706, 111.887512207, 111.88785553}},\n\t\t{\"s5dw7s\", []float64{20.8081054688, 20.8135986328, 3.66943359375, 3.68041992188}},\n\t\t{\"tpt\", []float64{42.1875, 43.59375, 52.03125, 53.4375}},\n\t\t{\"6vqn07ep\", []float64{-14.3936347961, -14.3934631348, -47.7973937988, -47.7970504761}},\n\t\t{\"7zbup2\", []float64{-0.703125, -0.697631835938, -9.87670898438, -9.86572265625}},\n\t\t{\"xd0j0wrn39f8\", []float64{12.1643207967, 12.1643209644, 157.531653419, 157.531653754}},\n\t\t{\"254kywz4\", []float64{-27.2526168823, -27.2524452209, -176.540679932, -176.540336609}},\n\t\t{\"6pkmr1875rp\", []float64{-3.28710615635, -3.28710481524, -83.7153281271, -83.715326786}},\n\t\t{\"69bmhmbw0de\", []float64{-34.2447146773, -34.2447133362, -66.9609577954, -66.9609564543}},\n\t\t{\"47jd\", []float64{-72.7734375, -72.59765625, -71.015625, -70.6640625}},\n\t\t{\"mw3ngtnj\", []float64{-8.6289024353, -8.62873077393, 69.0682983398, 69.0686416626}},\n\t\t{\"v\", []float64{45.0, 90.0, 45.0, 90.0}},\n\t\t{\"4uyq1\", []float64{-62.2265625, -62.1826171875, -47.4169921875, -47.373046875}},\n\t\t{\"9748v3e\", []float64{17.0150756836, 17.0164489746, -119.999542236, -119.998168945}},\n\t\t{\"sjy7\", []float64{32.87109375, 33.046875, 8.7890625, 9.140625}},\n\t\t{\"nc1jb2kb\", []float64{-83.3628845215, -83.3627128601, 125.17375946, 125.174102783}},\n\t\t{\"ffryw\", []float64{58.798828125, 58.8427734375, -45.087890625, -45.0439453125}},\n\t\t{\"3qfr7scg5s\", []float64{-5.7302069664, -5.73020160198, -120.429575443, -120.429564714}},\n\t\t{\"1x\", []float64{-50.625, -45.0, -112.5, -101.25}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"n4wk\", []float64{-75.234375, -75.05859375, 98.7890625, 99.140625}},\n\t\t{\"bw2d2\", []float64{80.5517578125, 80.595703125, -156.796875, -156.752929688}},\n\t\t{\"ztgehr9mpt\", []float64{77.9131776094, 77.9131829739, 162.610681057, 162.610691786}},\n\t\t{\"bnkb\", []float64{80.15625, 80.33203125, -173.3203125, -172.96875}},\n\t\t{\"q0fmcn\", []float64{-39.7375488281, -39.7320556641, 93.2080078125, 93.2189941406}},\n\t\t{\"e0e1sxt9gwm\", []float64{3.11770454049, 3.1177058816, -40.5757860839, -40.5757847428}},\n\t\t{\"9qc\", []float64{37.96875, 39.375, -122.34375, -120.9375}},\n\t\t{\"0cybm9snr\", []float64{-80.1029920578, -80.1029491425, -136.51031971, -136.510276794}},\n\t\t{\"fp\", []float64{84.375, 90.0, -90.0, -78.75}},\n\t\t{\"7u69k7\", []float64{-20.8575439453, -20.8520507812, -7.54760742188, -7.53662109375}},\n\t\t{\"guh3mbvnwv7y\", []float64{67.7249914035, 67.7249915712, -5.01359079033, -5.01359045506}},\n\t\t{\"vgw4wgnrd58e\", []float64{65.1447393559, 65.1447395235, 87.4928004295, 87.4928007647}},\n\t\t{\"rzk732w\", []float64{-3.64471435547, -3.64334106445, 174.789733887, 174.791107178}},\n\t\t{\"kf\", []float64{-33.75, -28.125, 33.75, 45.0}},\n\t\t{\"rcfr28t0\", []float64{-33.8790893555, -33.8789176941, 171.942901611, 171.943244934}},\n\t\t{\"5bqnms\", []float64{-87.4731445312, -87.4676513672, -2.57080078125, -2.55981445312}},\n\t\t{\"fs84w\", []float64{70.751953125, 70.7958984375, -67.236328125, -67.1923828125}},\n\t\t{\"mcjrsmx\", []float64{-38.0264282227, -38.0250549316, 86.3291931152, 86.3305664062}},\n\t\t{\"u84\", []float64{45.0, 46.40625, 25.3125, 26.71875}},\n\t\t{\"gkv4g14m\", []float64{72.2084999084, 72.2086715698, -26.5838241577, -26.583480835}},\n\t\t{\"27dhxu\", []float64{-24.4995117188, -24.4940185547, -165.596923828, -165.5859375}},\n\t\t{\"0v\", []float64{-61.875, -56.25, -146.25, -135.0}},\n\t\t{\"bpurn\", []float64{89.82421875, 89.8681640625, -173.759765625, -173.715820312}},\n\t\t{\"p5\", []float64{-73.125, -67.5, 135.0, 146.25}},\n\t\t{\"f3ffsuh\", []float64{55.3051757812, 55.3065490723, -74.6685791016, -74.6672058105}},\n\t\t{\"j0zbr0tb\", []float64{-85.7345581055, -85.7343864441, 56.2139511108, 56.2142944336}},\n\t\t{\"vyz\", []float64{82.96875, 84.375, 88.59375, 90.0}},\n\t\t{\"082p96b5ey\", []float64{-87.2596514225, -87.2596460581, -157.444907427, -157.444896698}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"g030qs\", []float64{46.4721679688, 46.4776611328, -43.3081054688, -43.2971191406}},\n\t\t{\"54\", []float64{-78.75, -73.125, -45.0, -33.75}},\n\t\t{\"fp5rcptn2gc\", []float64{85.7795964181, 85.7795977592, -85.3788422048, -85.3788408637}},\n\t\t{\"dk8z85s6516h\", []float64{26.650436148, 26.6504363157, -77.6893445849, -77.6893442497}},\n\t\t{\"3v1ebh18qujh\", []float64{-16.1937826127, -16.193782445, -99.1382686794, -99.1382683441}},\n\t\t{\"un50j3xf9\", []float64{78.7586688995, 78.7587118149, 4.46014881134, 4.46019172668}},\n\t\t{\"4b8y3gh\", []float64{-86.0723876953, -86.0710144043, -55.1129150391, -55.111541748}},\n\t\t{\"efdgh\", []float64{14.58984375, 14.6337890625, -7.20703125, -7.1630859375}},\n\t\t{\"1xxuk2\", []float64{-47.0654296875, -47.0599365234, -101.414794922, -101.403808594}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"1z6\", []float64{-49.21875, -47.8125, -98.4375, -97.03125}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"mxkjr1jdu5ru\", []float64{-3.28991509974, -3.2899149321, 73.440352343, 73.4403526783}},\n\t\t{\"x\", []float64{0.0, 45.0, 135.0, 180.0}},\n\t\t{\"3kpcn3sm\", []float64{-22.315120697, -22.3149490356, -112.57106781, -112.570724487}},\n\t\t{\"buk3t0ctrmt0\", []float64{69.1749724746, 69.1749726422, -140.051333159, -140.051332824}},\n\t\t{\"pp\", []float64{-50.625, -45.0, 135.0, 146.25}},\n\t\t{\"4h\", []float64{-67.5, -61.875, -90.0, -78.75}},\n\t\t{\"fjw1kcg4\", []float64{76.1671829224, 76.1673545837, -81.3496398926, -81.3492965698}},\n\t\t{\"877wsvjfz5\", []float64{19.4517821074, 19.4517874718, -163.611187935, -163.611177206}},\n\t\t{\"ru3\", []float64{-21.09375, -19.6875, 170.15625, 171.5625}},\n\t\t{\"yr\", []float64{84.375, 90.0, 101.25, 112.5}},\n\t\t{\"cu5x6cxq\", []float64{68.7836837769, 68.7838554382, -96.1973190308, -96.196975708}},\n\t\t{\"w04vuf4bdzjm\", []float64{1.02185273543, 1.02185290307, 94.0798293427, 94.0798296779}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"4zdcmmp2p4\", []float64{-47.5652968884, -47.5652915239, -52.1418428421, -52.1418321133}},\n\t\t{\"eft02s1hu\", []float64{14.1292333603, 14.1292762756, -4.19523239136, -4.19518947601}},\n\t\t{\"zk4v9qdeg5q1\", []float64{68.5031637736, 68.5031639412, 150.175689161, 150.175689496}},\n\t\t{\"8xr\", []float64{40.78125, 42.1875, -147.65625, -146.25}},\n\t\t{\"3pxyrt\", []float64{-1.68640136719, -1.68090820312, -123.771972656, -123.760986328}},\n\t\t{\"cmh39xszs8\", []float64{73.4311580658, 73.4311634302, -117.70080328, -117.700792551}},\n\t\t{\"xrm9d0wb48\", []float64{41.047668457, 41.0476738214, 154.081642628, 154.081653357}},\n\t\t{\"d4bh0k\", []float64{16.1938476562, 16.1993408203, -89.9890136719, -89.9780273438}},\n\t\t{\"hk8\", []float64{-64.6875, -63.28125, 11.25, 12.65625}},\n\t\t{\"9hxqk54m0wn\", []float64{26.4285027981, 26.4285041392, -124.625786841, -124.6257855}},\n\t\t{\"mygnv0\", []float64{-5.8447265625, -5.83923339844, 83.1884765625, 83.1994628906}},\n\t\t{\"yrjmvs\", []float64{85.4077148438, 85.4132080078, 108.874511719, 108.885498047}},\n\t\t{\"52csyemvf12\", []float64{-84.9274425209, -84.9274411798, -31.3469982147, -31.3469968736}},\n\t\t{\"4jrvjj\", []float64{-59.5623779297, -59.5568847656, -78.8818359375, -78.8708496094}},\n\t\t{\"ys1\", []float64{67.5, 68.90625, 113.90625, 115.3125}},\n\t\t{\"unf91\", []float64{83.14453125, 83.1884765625, 3.5595703125, 3.603515625}},\n\t\t{\"h5che0vnt\", []float64{-68.109998703, -68.1099557877, 1.5451669693, 1.54520988464}},\n\t\t{\"ugrk3\", []float64{64.0283203125, 64.072265625, 43.9892578125, 44.033203125}},\n\t\t{\"9ush8c\", []float64{26.1090087891, 26.1145019531, -95.5920410156, -95.5810546875}},\n\t\t{\"q92pzb\", []float64{-36.6064453125, -36.6009521484, 112.840576172, 112.8515625}},\n\t\t{\"0e\", []float64{-73.125, -67.5, -157.5, -146.25}},\n\t\t{\"dbt1mchu\", []float64{3.03840637207, 3.03857803345, -48.9595413208, -48.959197998}},\n\t\t{\"98xv2m\", []float64{3.76281738281, 3.76831054688, -101.590576172, -101.579589844}},\n\t\t{\"rqd8u195kgu\", []float64{-8.29684630036, -8.29684495926, 149.942988753, 149.942990094}},\n\t\t{\"504wk58ccv\", []float64{-88.8818138838, -88.8818085194, -41.3074886799, -41.307477951}},\n\t\t{\"0dzjhbn\", []float64{-73.65234375, -73.650970459, -147.43927002, -147.437896729}},\n\t\t{\"sgcn\", []float64{22.1484375, 22.32421875, 35.15625, 35.5078125}},\n\t\t{\"46k78jw0x65w\", []float64{-76.6982056573, -76.6982054897, -72.7648819238, -72.7648815885}},\n\t\t{\"6w2cbxx3nf9\", []float64{-9.49474900961, -9.4947476685, -66.4130924642, -66.4130911231}},\n\t\t{\"zxmf4\", []float64{86.1328125, 86.1767578125, 165.673828125, 165.717773438}},\n\t\t{\"unf\", []float64{82.96875, 84.375, 2.8125, 4.21875}},\n\t\t{\"m4p\", []float64{-33.75, -32.34375, 54.84375, 56.25}},\n\t\t{\"dsc1rqss2w\", []float64{26.9749438763, 26.9749492407, -65.7689452171, -65.7689344883}},\n\t\t{\"cxp\", []float64{84.375, 85.78125, -102.65625, -101.25}},\n\t\t{\"zmh\", []float64{73.125, 74.53125, 151.875, 153.28125}},\n\t\t{\"tynvnjc8hdb\", []float64{34.6605066955, 34.6605080366, 88.5081124306, 88.5081137717}},\n\t\t{\"uk8hb\", []float64{71.1474609375, 71.19140625, 11.25, 11.2939453125}},\n\t\t{\"34d\", []float64{-30.9375, -29.53125, -132.1875, -130.78125}},\n\t\t{\"ts39vet4rzw5\", []float64{24.2335202359, 24.2335204035, 69.8582813144, 69.8582816496}},\n\t\t{\"3rt1fx5\", []float64{-2.46643066406, -2.46505737305, -116.604766846, -116.603393555}},\n\t\t{\"ujn8yhfpg\", []float64{73.2842588425, 73.2843017578, 9.40717220306, 9.40721511841}},\n\t\t{\"pdbvhzj\", []float64{-73.6138916016, -73.6125183105, 158.770294189, 158.77166748}},\n\t\t{\"q35\", []float64{-39.375, -37.96875, 105.46875, 106.875}},\n\t\t{\"szh5424hc\", []float64{39.9031591415, 39.9032020569, 39.4766664505, 39.4767093658}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"tt1wjkr44e\", []float64{29.2033928633, 29.2033982277, 69.8498082161, 69.8498189449}},\n\t\t{\"1u3hdkn\", []float64{-65.2807617188, -65.2793884277, -99.7366333008, -99.7352600098}},\n\t\t{\"jc9\", []float64{-81.5625, -80.15625, 80.15625, 81.5625}},\n\t\t{\"627pp\", []float64{-42.36328125, -42.3193359375, -74.2236328125, -74.1796875}},\n\t\t{\"g46wqb4z\", []float64{58.7560844421, 58.7562561035, -41.1839675903, -41.1836242676}},\n\t\t{\"2407674\", []float64{-33.1622314453, -33.1608581543, -179.546813965, -179.545440674}},\n\t\t{\"3vbsrcxu\", []float64{-11.9002532959, -11.9000816345, -100.195655823, -100.1953125}},\n\t\t{\"u0mr9fpy\", []float64{47.7366256714, 47.7367973328, 7.47035980225, 7.470703125}},\n\t\t{\"p1s1\", []float64{-81.38671875, -81.2109375, 140.625, 140.9765625}},\n\t\t{\"ce7y6s1ugjpu\", []float64{64.4026983529, 64.4026985206, -107.11415682, -107.114156485}},\n\t\t{\"tujn\", []float64{23.5546875, 23.73046875, 85.78125, 86.1328125}},\n\t\t{\"fes\", []float64{64.6875, 66.09375, -61.875, -60.46875}},\n\t\t{\"28te871t29y\", []float64{-41.5548755229, -41.5548741817, -149.752549231, -149.75254789}},\n\t\t{\"2z9j0591\", []float64{-1.9141960144, -1.91402435303, -144.842376709, -144.842033386}},\n\t\t{\"e\", []float64{0.0, 45.0, -45.0, 0.0}},\n\t\t{\"90\", []float64{0.0, 5.625, -135.0, -123.75}},\n\t\t{\"jbfm12r\", []float64{-84.900970459, -84.899597168, 81.9786071777, 81.9799804688}},\n\t\t{\"y0ws\", []float64{48.515625, 48.69140625, 99.140625, 99.4921875}},\n\t\t{\"m2\", []float64{-45.0, -39.375, 56.25, 67.5}},\n\t\t{\"gpspv95sz\", []float64{88.5561132431, 88.5561561584, -39.1281938553, -39.1281509399}},\n\t\t{\"7k8u95cyjdx6\", []float64{-18.8748412952, -18.8748411275, -32.6487181708, -32.6487178355}},\n\t\t{\"c1fe0r\", []float64{55.4095458984, 55.4150390625, -131.473388672, -131.462402344}},\n\t\t{\"668wjecj2d\", []float64{-29.8613011837, -29.8612958193, -77.8037810326, -77.8037703037}},\n\t\t{\"dnq3\", []float64{35.33203125, 35.5078125, -81.2109375, -80.859375}},\n\t\t{\"m3sxdxnvmrr\", []float64{-35.2047483623, -35.2047470212, 62.6974926889, 62.69749403}},\n\t\t{\"zz3qpfvqzu\", []float64{86.8522238731, 86.8522292376, 170.855931044, 170.855941772}},\n\t\t{\"98mjjx8bu\", []float64{2.3264837265, 2.32652664185, -105.225849152, -105.225806236}},\n\t\t{\"pkmusy0e4j35\", []float64{-65.2692317404, -65.2692315727, 154.545451552, 154.545451887}},\n\t\t{\"j3f9dtm5r5n\", []float64{-79.8631650209, -79.8631636798, 59.8826631904, 59.8826645315}},\n\t\t{\"67up3c0uh9jn\", []float64{-22.6256497577, -22.62564959, -73.0468659103, -73.046865575}},\n\t\t{\"6q0fd9wn2\", []float64{-10.8012342453, -10.80119133, -77.5772094727, -77.5771665573}},\n\t\t{\"t82e5zrs\", []float64{1.97410583496, 1.97427749634, 68.3782196045, 68.3785629272}},\n\t\t{\"0hstxh\", []float64{-63.6987304688, -63.6932373047, -173.364257812, -173.353271484}},\n\t\t{\"qe1egcuetqe\", []float64{-27.4555715919, -27.4555702507, 114.78057906, 114.780580401}},\n\t\t{\"yhp25wc4v\", []float64{67.5375509262, 67.5375938416, 100.350708961, 100.350751877}},\n\t\t{\"z6uvby2nrt4k\", []float64{61.5149248391, 61.5149250068, 152.962971367, 152.962971702}},\n\t\t{\"29sd0863cx\", []float64{-36.2092262506, -36.2092208862, -151.146748066, -151.146737337}},\n\t\t{\"kvnx614\", []float64{-15.5950927734, -15.5937194824, 42.981262207, 42.982635498}},\n\t\t{\"mu1srk07\", []float64{-21.7304420471, -21.7302703857, 81.1783218384, 81.1786651611}},\n\t\t{\"5bz5bmq\", []float64{-85.0932312012, -85.0918579102, -1.38702392578, -1.38565063477}},\n\t\t{\"fu4yx9fr8gtk\", []float64{68.6534980685, 68.6534982361, -52.0500935242, -52.0500931889}},\n\t\t{\"3hyhj92rn\", []float64{-17.5700569153, -17.5700139999, -126.320199966, -126.320157051}},\n\t\t{\"345nw\", []float64{-32.607421875, -32.5634765625, -130.517578125, -130.473632812}},\n\t\t{\"q5f2p327mhy\", []float64{-23.8988001645, -23.8987988234, 93.4832319617, 93.4832333028}},\n\t\t{\"0wmufb9\", []float64{-54.0060424805, -54.0046691895, -149.2918396, -149.290466309}},\n\t\t{\"r\", []float64{-45.0, 0.0, 135.0, 180.0}},\n\t\t{\"07d2sde\", []float64{-70.2108764648, -70.2095031738, -165.384063721, -165.38269043}},\n\t\t{\"d0r2\", []float64{1.40625, 1.58203125, -79.8046875, -79.453125}},\n\t\t{\"znegsexfs23h\", []float64{82.1973916143, 82.197391782, 140.482018143, 140.482018478}},\n\t\t{\"sfr69qxg\", []float64{13.1319236755, 13.1320953369, 44.010887146, 44.0112304688}},\n\t\t{\"tr44b8brc\", []float64{39.8638486862, 39.8638916016, 59.0848588943, 59.0849018097}},\n\t\t{\"tbnqctecsf\", []float64{1.21700406075, 1.21700942516, 87.6103341579, 87.6103448868}},\n\t\t{\"jpfy538qu\", []float64{-45.3421640396, -45.3421211243, 49.0105247498, 49.0105676651}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"gskrg0z5e\", []float64{70.2732753754, 70.2733182907, -16.3818597794, -16.381816864}},\n\t\t{\"6cz\", []float64{-35.15625, -33.75, -46.40625, -45.0}},\n\t\t{\"u67hm7b47423\", []float64{58.4243181534, 58.424318321, 15.6995919719, 15.6995923072}},\n\t\t{\"j154zhnnkyt3\", []float64{-83.8685209863, -83.8685208187, 49.5348178223, 49.5348181576}},\n\t\t{\"muqdpev4smv\", []float64{-20.7211281359, -20.7211267948, 88.2272703946, 88.2272717357}},\n\t\t{\"47h3upynmsru\", []float64{-72.7737144381, -72.7737142704, -72.589170076, -72.5891697407}},\n\t\t{\"g6j200\", []float64{56.25, 56.2554931641, -26.3671875, -26.3562011719}},\n\t\t{\"tw\", []float64{33.75, 39.375, 67.5, 78.75}},\n\t\t{\"c0pjhr520q\", []float64{45.9173905849, 45.9173959494, -124.965008497, -124.964997768}},\n\t\t{\"8nx\", []float64{36.5625, 37.96875, -170.15625, -168.75}},\n\t\t{\"47b2wvtns\", []float64{-68.7870311737, -68.7869882584, -78.0947685242, -78.0947256088}},\n\t\t{\"vrsbq\", []float64{87.2314453125, 87.275390625, 63.193359375, 63.2373046875}},\n\t\t{\"sz\", []float64{39.375, 45.0, 33.75, 45.0}},\n\t\t{\"xe61b0bnw\", []float64{18.5941028595, 18.5941457748, 160.312757492, 160.312800407}},\n\t\t{\"dky6qedz3w\", []float64{27.1347606182, 27.1347659826, -69.6714520454, -69.6714413166}},\n\t\t{\"vmvkqx2hb\", []float64{78.1314611435, 78.1315040588, 63.9184570312, 63.9184999466}},\n\t\t{\"t96m49xgr1y\", []float64{7.9189632833, 7.9189646244, 70.7848772407, 70.7848785818}},\n\t\t{\"brw2urrqcfex\", []float64{87.3603346758, 87.3603348434, -159.764133766, -159.764133431}},\n\t\t{\"z7m8\", []float64{63.28125, 63.45703125, 153.984375, 154.3359375}},\n\t\t{\"wm6w7f38\", []float64{30.6422424316, 30.642414093, 104.932479858, 104.932823181}},\n\t\t{\"rxj23rtt4y\", []float64{-5.53896546364, -5.53896009922, 164.945415258, 164.945425987}},\n\t\t{\"sfr9xsyzfn\", []float64{12.9473769665, 12.9473823309, 44.6358203888, 44.6358311176}},\n\t\t{\"9ubf9uq02e\", []float64{27.1816080809, 27.1816134453, -100.110146999, -100.110136271}},\n\t\t{\"kj25zp1gb6j\", []float64{-14.7704637051, -14.770462364, 0.310037881136, 0.31003922224}},\n\t\t{\"x4f\", []float64{15.46875, 16.875, 137.8125, 139.21875}},\n\t\t{\"xnn27kkf4c\", []float64{33.8176399469, 33.8176453114, 143.938525915, 143.938536644}},\n\t\t{\"61bhs9byn4\", []float64{-34.3545806408, -34.3545752764, -89.8009586334, -89.8009479046}},\n\t\t{\"rv2sve92mngr\", []float64{-14.6144826896, -14.614482522, 169.696759768, 169.696760103}},\n\t\t{\"zkvq2w\", []float64{72.8503417969, 72.8558349609, 153.654785156, 153.665771484}},\n\t\t{\"qprmp68h7kcd\", []float64{-3.32535546273, -3.32535529509, 100.514057502, 100.514057837}},\n\t\t{\"77pmzubu\", []float64{-27.0874786377, -27.0873069763, -23.2130813599, -23.2127380371}},\n\t\t{\"q73t2sumh3b\", []float64{-25.7689382136, -25.7689368725, 103.387366533, 103.387367874}},\n\t\t{\"3kxch9c\", []float64{-19.5021057129, -19.5007324219, -112.652435303, -112.651062012}},\n\t\t{\"t\", []float64{0.0, 45.0, 45.0, 90.0}},\n\t\t{\"3um1y618chw\", []float64{-20.7749935985, -20.7749922574, -93.9419808984, -93.9419795573}},\n\t\t{\"45nj7sxww\", []float64{-72.1763134003, -72.1762704849, -81.3981342316, -81.3980913162}},\n\t\t{\"rnkyjdv404\", []float64{-8.77360224724, -8.77359688282, 141.928253174, 141.928263903}},\n\t\t{\"p3\", []float64{-84.375, -78.75, 146.25, 157.5}},\n\t\t{\"sxbz\", []float64{44.82421875, 45.0, 23.5546875, 23.90625}},\n\t\t{\"xuj2k\", []float64{22.5439453125, 22.587890625, 176.30859375, 176.352539062}},\n\t\t{\"yhp9\", []float64{67.67578125, 67.8515625, 100.546875, 100.8984375}},\n\t\t{\"1yq4\", []float64{-54.4921875, -54.31640625, -92.8125, -92.4609375}},\n\t\t{\"u4m2jkw\", []float64{57.6809692383, 57.6823425293, 7.62176513672, 7.62313842773}},\n\t\t{\"xb9\", []float64{2.8125, 4.21875, 170.15625, 171.5625}},\n\t\t{\"ebf4e478jp\", []float64{4.67060029507, 4.67060565948, -8.30064296722, -8.30063223839}},\n\t\t{\"y7venx9\", []float64{66.6622924805, 66.6636657715, 109.271392822, 109.272766113}},\n\t\t{\"8qu\", []float64{37.96875, 39.375, -163.125, -161.71875}},\n\t\t{\"jw2jbzms66\", []float64{-53.7924420834, -53.7924367189, 67.5406086445, 67.5406193733}},\n\t\t{\"n\", []float64{-90.0, -45.0, 90.0, 135.0}},\n\t\t{\"jbx\", []float64{-87.1875, -85.78125, 88.59375, 90.0}},\n\t\t{\"3v4n\", []float64{-15.8203125, -15.64453125, -98.4375, -98.0859375}},\n\t\t{\"0z1theg\", []float64{-49.7254943848, -49.7241210938, -143.938751221, -143.93737793}},\n\t\t{\"zbz00jf21m\", []float64{49.2503625154, 49.2503678799, 178.596893549, 178.596904278}},\n\t\t{\"dfpq2eg2\", []float64{12.3692321777, 12.3694038391, -46.0282516479, -46.0279083252}},\n\t\t{\"z2j5bc1ph562\", []float64{45.6658919156, 45.6658920832, 153.315756954, 153.31575729}},\n\t\t{\"3p3g\", []float64{-3.69140625, -3.515625, -132.5390625, -132.1875}},\n\t\t{\"4rfgeu3\", []float64{-45.7676696777, -45.7662963867, -74.7166442871, -74.7152709961}},\n\t\t{\"nykq\", []float64{-53.7890625, -53.61328125, 129.7265625, 130.078125}},\n\t\t{\"h\", []float64{-90.0, -45.0, 0.0, 45.0}},\n\t\t{\"85\", []float64{16.875, 22.5, -180.0, -168.75}},\n\t\t{\"bdsdxr\", []float64{59.5404052734, 59.5458984375, -150.853271484, -150.842285156}},\n\t\t{\"wsyt3duqg2\", []float64{27.657866478, 27.6578718424, 121.71251893, 121.712529659}},\n\t\t{\"90\", []float64{0.0, 5.625, -135.0, -123.75}},\n\t\t{\"butw\", []float64{71.3671875, 71.54296875, -138.515625, -138.1640625}},\n\t\t{\"ddhpjv6b7tqh\", []float64{12.5093796104, 12.5093797781, -61.6183796525, -61.6183793172}},\n\t\t{\"18ueqgd\", []float64{-85.1907348633, -85.1893615723, -105.872497559, -105.871124268}},\n\t\t{\"v2g8jh1\", []float64{49.2407226562, 49.2420959473, 61.3929748535, 61.3943481445}},\n\t\t{\"84umeh3gmupk\", []float64{16.45947285, 16.4594730176, -173.888941817, -173.888941482}},\n\t\t{\"s4g900\", []float64{15.64453125, 15.6500244141, 4.921875, 4.93286132812}},\n\t\t{\"0b313fz2\", []float64{-88.3589172363, -88.358745575, -144.756889343, -144.756546021}},\n\t\t{\"4q\", []float64{-56.25, -50.625, -78.75, -67.5}},\n\t\t{\"d61\", []float64{11.25, 12.65625, -77.34375, -75.9375}},\n\t\t{\"w5q5298pq\", []float64{18.8620233536, 18.8620662689, 98.4597301483, 98.4597730637}},\n\t\t{\"ushgx399\", []float64{68.1236457825, 68.1238174438, 29.5003509521, 29.5006942749}},\n\t\t{\"73ngt\", []float64{-38.759765625, -38.7158203125, -24.0380859375, -23.994140625}},\n\t\t{\"2f4smcem\", []float64{-32.9938316345, -32.9936599731, -142.477226257, -142.476882935}},\n\t\t{\"0\", []float64{-90.0, -45.0, -180.0, -135.0}},\n\t\t{\"8\", []float64{0.0, 45.0, -180.0, -135.0}},\n\t\t{\"5u14weqgz\", []float64{-67.0420503616, -67.0420074463, -9.54853534698, -9.54849243164}},\n\t\t{\"xxhuu8y4xb\", []float64{40.214509964, 40.2145153284, 164.386013746, 164.386024475}},\n\t\t{\"272xeqmj\", []float64{-25.3652000427, -25.3650283813, -167.897186279, -167.896842957}},\n\t\t{\"2trrhunrd1\", []float64{-14.215015769, -14.2150104046, -147.087278366, -147.087267637}},\n\t\t{\"e4\", []float64{11.25, 16.875, -45.0, -33.75}},\n\t\t{\"p5duz8tp\", []float64{-69.4735908508, -69.4734191895, 139.203643799, 139.203987122}},\n\t\t{\"5qprz78e\", []float64{-54.8679542542, -54.8677825928, -23.2353973389, -23.2350540161}},\n\t\t{\"ch5yq40qtu3\", []float64{68.6107577384, 68.6107590795, -129.462299198, -129.462297857}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"7qmv9c5\", []float64{-8.87145996094, -8.87008666992, -25.5830383301, -25.5816650391}},\n\t\t{\"c\", []float64{45.0, 90.0, -135.0, -90.0}},\n\t\t{\"hp8s7\", []float64{-47.0654296875, -47.021484375, 0.8349609375, 0.87890625}},\n\t\t{\"9e4y04d17\", []float64{17.9436349869, 17.9436779022, -108.629937172, -108.629894257}},\n\t\t{\"39nh\", []float64{-38.671875, -38.49609375, -104.0625, -103.7109375}},\n\t\t{\"6\", []float64{-45.0, 0.0, -90.0, -45.0}},\n\t\t{\"pjpxe1pvyzhm\", []float64{-60.5501220189, -60.5501218513, 145.689649321, 145.689649656}},\n\t\t{\"drx\", []float64{42.1875, 43.59375, -68.90625, -67.5}},\n\t\t{\"zu1c5qg0s\", []float64{67.7129459381, 67.7129888535, 171.3580513, 171.358094215}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"t8d\", []float64{2.8125, 4.21875, 70.3125, 71.71875}},\n\t\t{\"d47w70rwe23h\", []float64{13.7573739141, 13.7573740818, -84.9358485639, -84.9358482286}},\n\t\t{\"3t617\", []float64{-15.2490234375, -15.205078125, -109.555664062, -109.51171875}},\n\t\t{\"qnkq1pz\", []float64{-8.74649047852, -8.7451171875, 96.0301208496, 96.0314941406}},\n\t\t{\"fu\", []float64{67.5, 73.125, -56.25, -45.0}},\n\t\t{\"7vs\", []float64{-14.0625, -12.65625, -5.625, -4.21875}},\n\t\t{\"bztqz0h\", []float64{88.3740234375, 88.3753967285, -138.554077148, -138.552703857}},\n\t\t{\"b8j\", []float64{45.0, 46.40625, -150.46875, -149.0625}},\n\t\t{\"cetkxmq73\", []float64{65.5079126358, 65.5079555511, -104.789958, -104.789915085}},\n\t\t{\"p91\", []float64{-84.375, -82.96875, 158.90625, 160.3125}},\n\t\t{\"z4g7bn4w38\", []float64{61.1619615555, 61.1619669199, 139.573810101, 139.573820829}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"g2eej64jbwj\", []float64{48.3518493176, 48.3518506587, -28.5946373641, -28.594636023}},\n\t\t{\"fwshzb30mj\", []float64{82.398903966, 82.3989093304, -61.5328359604, -61.5328252316}},\n\t\t{\"fv2mqt\", []float64{75.4815673828, 75.4870605469, -55.6127929688, -55.6018066406}},\n\t\t{\"bzr6m3zdun5\", []float64{86.1868751049, 86.186876446, -135.813499242, -135.813497901}},\n\t\t{\"et5rq77j8b\", []float64{29.4182109833, 29.4182163477, -17.6508772373, -17.6508665085}},\n\t\t{\"1c\", []float64{-84.375, -78.75, -101.25, -90.0}},\n\t\t{\"y1hyumh10jq\", []float64{51.8391890824, 51.8391904235, 96.8719562888, 96.8719576299}},\n\t\t{\"qd42djnqxq\", []float64{-33.6334955692, -33.6334902048, 115.76084733, 115.760858059}},\n\t\t{\"hsd9s\", []float64{-64.423828125, -64.3798828125, 26.19140625, 26.2353515625}},\n\t\t{\"8289gq947\", []float64{3.156208992, 3.15625190735, -167.902550697, -167.902507782}},\n\t\t{\"em37sw72zq4\", []float64{30.1809775829, 30.180978924, -31.7896565795, -31.7896552384}},\n\t\t{\"zms25\", []float64{75.9375, 75.9814453125, 152.358398438, 152.40234375}},\n\t\t{\"h25d54\", []float64{-89.6374511719, -89.6319580078, 16.3037109375, 16.3146972656}},\n\t\t{\"6qc7y2t4bb\", []float64{-6.36885166168, -6.36884629726, -76.7106306553, -76.7106199265}},\n\t\t{\"06vt5z8j\", []float64{-73.6102867126, -73.6101150513, -160.850830078, -160.850486755}},\n\t\t{\"37q3\", []float64{-26.54296875, -26.3671875, -114.9609375, -114.609375}},\n\t\t{\"sey9wu\", []float64{21.3793945312, 21.3848876953, 31.9372558594, 31.9482421875}},\n\t\t{\"qk0jrj\", []float64{-21.5496826172, -21.5441894531, 101.557617188, 101.568603516}},\n\t\t{\"8x6jjpm0\", []float64{41.6999816895, 41.7001533508, -154.460906982, -154.46056366}},\n\t\t{\"5j1etu\", []float64{-61.2377929688, -61.2322998047, -42.6379394531, -42.626953125}},\n\t\t{\"r6b\", []float64{-29.53125, -28.125, 146.25, 147.65625}},\n\t\t{\"ddu3vyj07\", []float64{15.8093690872, 15.8094120026, -61.263756752, -61.2637138367}},\n\t\t{\"m9fm5q2d91\", []float64{-34.2425769567, -34.2425715923, 70.8076143265, 70.8076250553}},\n\t\t{\"0pxdx\", []float64{-47.373046875, -47.3291015625, -169.145507812, -169.1015625}},\n\t\t{\"w\", []float64{0.0, 45.0, 90.0, 135.0}},\n\t\t{\"q1e\", []float64{-36.5625, -35.15625, 94.21875, 95.625}},\n\t\t{\"h3vxhm8tu\", []float64{-78.8945817947, -78.8945388794, 19.172000885, 19.1720438004}},\n\t\t{\"bcxsz\", []float64{54.2724609375, 54.31640625, -135.395507812, -135.3515625}},\n\t\t{\"crjh\", []float64{85.078125, 85.25390625, -116.71875, -116.3671875}},\n\t\t{\"bdqejqqgwj\", []float64{58.2185536623, 58.2185590267, -148.119134903, -148.119124174}},\n\t\t{\"x7zhc480u7\", []float64{21.9425886869, 21.9425940514, 156.137877703, 156.137888432}},\n\t\t{\"xhr7c9nd6js9\", []float64{24.5713387616, 24.5713389292, 145.270248726, 145.270249061}},\n\t\t{\"f25r3r\", []float64{46.3128662109, 46.318359375, -74.1247558594, -74.1137695312}},\n\t\t{\"b4v1e8zek\", []float64{60.7370996475, 60.7371425629, -172.804470062, -172.804427147}},\n\t\t{\"95cwh1k\", []float64{22.1553039551, 22.1566772461, -132.709350586, -132.707977295}},\n\t\t{\"kh1r\", []float64{-21.26953125, -21.09375, 1.7578125, 2.109375}},\n\t\t{\"7p\", []float64{-5.625, 0.0, -45.0, -33.75}},\n\t\t{\"mgsvj\", []float64{-24.43359375, -24.3896484375, 85.6494140625, 85.693359375}},\n\t\t{\"k70\", []float64{-28.125, -26.71875, 11.25, 12.65625}},\n\t\t{\"pxjr5g\", []float64{-49.3780517578, -49.3725585938, 165.047607422, 165.05859375}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"81pv\", []float64{6.50390625, 6.6796875, -169.1015625, -168.75}},\n\t\t{\"jjg\", []float64{-57.65625, -56.25, 49.21875, 50.625}},\n\t\t{\"732kjtvw\", []float64{-37.2330093384, -37.232837677, -33.1491851807, -33.1488418579}},\n\t\t{\"kuc2\", []float64{-18.28125, -18.10546875, 35.5078125, 35.859375}},\n\t\t{\"wn91fmw18yp\", []float64{36.9006192684, 36.9006206095, 91.5134082735, 91.5134096146}},\n\t\t{\"5wdnzyz5r\", []float64{-52.2133398056, -52.2132968903, -19.3370103836, -19.3369674683}},\n\t\t{\"m682wkeu80\", []float64{-30.8241176605, -30.8241122961, 56.8813705444, 56.8813812733}},\n\t\t{\"r18jv9k\", []float64{-35.5448913574, -35.5435180664, 135.247192383, 135.248565674}},\n\t\t{\"zr079yhvttr\", []float64{85.0241656601, 85.0241670012, 146.685235351, 146.685236692}},\n\t\t{\"r4umz4vhvm\", []float64{-28.5045593977, -28.5045540333, 141.291271448, 141.291282177}},\n\t\t{\"58gdwpzc3zs\", []float64{-85.2989700437, -85.2989687026, -17.3037296534, -17.3037283123}},\n\t\t{\"64frgqpt0yj\", []float64{-28.1350958347, -28.1350944936, -86.6827766597, -86.6827753186}},\n\t\t{\"8n18eckkw5\", []float64{33.8455456495, 33.8455510139, -177.719736099, -177.71972537}},\n\t\t{\"mz326c81b1\", []float64{-4.16625916958, -4.16625380516, 80.6286621094, 80.6286728382}},\n\t\t{\"hx\", []float64{-50.625, -45.0, 22.5, 33.75}},\n\t\t{\"ush2juq\", []float64{67.5233459473, 67.5247192383, 28.737487793, 28.738861084}},\n\t\t{\"bp6h7m8fe\", []float64{86.5589618683, 86.5590047836, -177.04351902, -177.043476105}},\n\t\t{\"111\", []float64{-84.375, -82.96875, -133.59375, -132.1875}},\n\t\t{\"m9hwzz\", []float64{-38.1500244141, -38.14453125, 74.1687011719, 74.1796875}},\n\t\t{\"100u6e92zuk\", []float64{-89.2335520685, -89.2335507274, -133.833394647, -133.833393306}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"9wbmdmhu\", []float64{38.9636993408, 38.9638710022, -112.043037415, -112.042694092}},\n\t\t{\"9p1jg8k\", []float64{40.3871154785, 40.3884887695, -133.434448242, -133.433074951}},\n\t\t{\"vqf6kw\", []float64{83.3972167969, 83.4027099609, 59.6118164062, 59.6228027344}},\n\t\t{\"gw\", []float64{78.75, 84.375, -22.5, -11.25}},\n\t\t{\"h49v9\", []float64{-74.970703125, -74.9267578125, 2.5048828125, 2.548828125}},\n\t\t{\"23cmz\", []float64{-34.1455078125, -34.1015625, -166.684570312, -166.640625}},\n\t\t{\"71\", []float64{-39.375, -33.75, -45.0, -33.75}},\n\t\t{\"5x2kvmbu\", []float64{-48.3515167236, -48.3513450623, -21.9166946411, -21.9163513184}},\n\t\t{\"1nywfjs8e\", []float64{-50.8144283295, -50.8143854141, -125.765175819, -125.765132904}},\n\t\t{\"7u4vm3b9qy\", []float64{-21.5672886372, -21.5672832727, -7.15112328529, -7.15111255646}},\n\t\t{\"rx4n750gn\", []float64{-4.50937271118, -4.50932979584, 160.445623398, 160.445666313}},\n\t\t{\"9\", []float64{0.0, 45.0, -135.0, -90.0}},\n\t\t{\"nxfyqng3\", []float64{-45.2703666687, -45.2701950073, 116.635322571, 116.635665894}},\n\t\t{\"tgnt\", []float64{17.75390625, 17.9296875, 87.890625, 88.2421875}},\n\t\t{\"qe2k5jtbm2c\", []float64{-25.985365659, -25.9853643179, 112.991521508, 112.991522849}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"1jhq\", []float64{-60.8203125, -60.64453125, -129.0234375, -128.671875}},\n\t\t{\"p874n4ubjm\", []float64{-88.2270544767, -88.2270491123, 161.989170313, 161.989181042}},\n\t\t{\"91h2qc\", []float64{5.67443847656, 5.67993164062, -128.726806641, -128.715820312}},\n\t\t{\"mzp8s0p\", []float64{-5.537109375, -5.53573608398, 89.4822692871, 89.4836425781}},\n\t\t{\"ptp0rem\", []float64{-61.8132019043, -61.8118286133, 167.680206299, 167.68157959}},\n\t\t{\"14\", []float64{-78.75, -73.125, -135.0, -123.75}},\n\t\t{\"s4dq\", []float64{15.1171875, 15.29296875, 3.1640625, 3.515625}},\n\t\t{\"uvs7\", []float64{76.46484375, 76.640625, 39.7265625, 40.078125}},\n\t\t{\"wh9xq3mqh\", []float64{26.5948104858, 26.5948534012, 92.3914146423, 92.3914575577}},\n\t\t{\"kz\", []float64{-5.625, 0.0, 33.75, 45.0}},\n\t\t{\"s8t4hkb3\", []float64{3.19032669067, 3.19049835205, 29.7183609009, 29.7187042236}},\n\t\t{\"3ry9w\", []float64{-1.142578125, -1.0986328125, -114.345703125, -114.301757812}},\n\t\t{\"mf1wt5\", []float64{-32.5909423828, -32.5854492188, 81.0791015625, 81.0900878906}},\n\t\t{\"e\", []float64{0.0, 45.0, -45.0, 0.0}},\n\t\t{\"mh\", []float64{-22.5, -16.875, 45.0, 56.25}},\n\t\t{\"75y3665k\", []float64{-23.6748504639, -23.6746788025, -36.1075973511, -36.1072540283}},\n\t\t{\"sts\", []float64{30.9375, 32.34375, 28.125, 29.53125}},\n\t\t{\"6fdmxb\", []float64{-29.970703125, -29.9652099609, -52.7453613281, -52.734375}},\n\t\t{\"xcvf9qbx13jx\", []float64{10.3214901499, 10.3214903176, 176.891616806, 176.891617142}},\n\t\t{\"n1r6pkxu86t\", []float64{-82.5916823745, -82.5916810334, 100.524576455, 100.524577796}},\n\t\t{\"1m2p\", []float64{-59.23828125, -59.0625, -123.75, -123.3984375}},\n\t\t{\"fz\", []float64{84.375, 90.0, -56.25, -45.0}},\n\t\t{\"hgw\", []float64{-70.3125, -68.90625, 42.1875, 43.59375}},\n\t\t{\"ssktp8e5hsub\", []float64{24.7884432971, 24.7884434648, 29.1620342061, 29.1620345414}},\n\t\t{\"8wbw4\", []float64{39.0234375, 39.0673828125, -156.708984375, -156.665039062}},\n\t\t{\"wbcdsqtpnkh9\", []float64{4.69513194636, 4.69513211399, 126.053283289, 126.053283624}},\n\t\t{\"f0md\", []float64{46.7578125, 46.93359375, -82.265625, -81.9140625}},\n\t\t{\"hngnmbt2pe4\", []float64{-50.9298545122, -50.9298531711, 4.478969872, 4.4789712131}},\n\t\t{\"gbkn8cgjewxc\", []float64{47.559420336, 47.5594205037, -5.58776054531, -5.58776021004}},\n\t\t{\"u\", []float64{45.0, 90.0, 0.0, 45.0}},\n\t\t{\"b5\", []float64{61.875, 67.5, -180.0, -168.75}},\n\t\t{\"w1r042nrqyk\", []float64{7.0325280726, 7.0325294137, 99.951505065, 99.9515064061}},\n\t\t{\"tv6r1uuh1h\", []float64{30.7885193825, 30.7885247469, 81.9965028763, 81.9965136051}},\n\t\t{\"r1hw8pqpr3\", []float64{-38.1913465261, -38.1913411617, 141.336675882, 141.336686611}},\n\t\t{\"j2bcyrxdgj\", []float64{-85.4319351912, -85.4319298267, 57.5897741318, 57.5897848606}},\n\t\t{\"m\", []float64{-45.0, 0.0, 45.0, 90.0}},\n\t\t{\"qtxvgfvmrbf\", []float64{-13.0357463658, -13.0357450247, 123.570777476, 123.570778817}},\n\t\t{\"jp\", []float64{-50.625, -45.0, 45.0, 56.25}},\n\t\t{\"f76\", []float64{63.28125, 64.6875, -75.9375, -74.53125}},\n\t\t{\"vz9\", []float64{87.1875, 88.59375, 80.15625, 81.5625}},\n\t\t{\"wm\", []float64{28.125, 33.75, 101.25, 112.5}},\n\t\t{\"c0wn5\", []float64{48.8671875, 48.9111328125, -126.430664062, -126.38671875}},\n\t\t{\"7pn7whg\", []float64{-4.9836730957, -4.98229980469, -35.943145752, -35.9417724609}},\n\t\t{\"s\", []float64{0.0, 45.0, 0.0, 45.0}},\n\t\t{\"txwr3\", []float64{43.4619140625, 43.505859375, 76.3330078125, 76.376953125}},\n\t\t{\"zc0\", []float64{50.625, 52.03125, 168.75, 170.15625}},\n\t\t{\"sq7pru6gr\", []float64{36.4545679092, 36.4546108246, 15.8134031296, 15.8134460449}},\n\t\t{\"nu\", []float64{-67.5, -61.875, 123.75, 135.0}},\n\t\t{\"7dkt6vr\", []float64{-31.3920593262, -31.3906860352, -16.0414123535, -16.0400390625}},\n\t\t{\"xm2uwefdyf1\", []float64{30.3433477879, 30.343349129, 147.594056278, 147.59405762}},\n\t\t{\"mgmnc0\", []float64{-25.5322265625, -25.5267333984, 85.8251953125, 85.8361816406}},\n\t\t{\"jj1shq\", []float64{-61.1389160156, -61.1334228516, 47.2961425781, 47.3071289062}},\n\t\t{\"3\", []float64{-45.0, 0.0, -135.0, -90.0}},\n\t\t{\"0p4y3p8tgr\", []float64{-49.4841438532, -49.4841384888, -176.088041067, -176.088030338}},\n\t\t{\"gu\", []float64{67.5, 73.125, -11.25, 0.0}},\n\t\t{\"e94\", []float64{5.625, 7.03125, -19.6875, -18.28125}},\n\t\t{\"u7khr\", []float64{64.0283203125, 64.072265625, 17.1826171875, 17.2265625}},\n\t\t{\"k1k\", []float64{-37.96875, -36.5625, 5.625, 7.03125}},\n\t\t{\"wks48m7f\", []float64{25.7811355591, 25.7813072205, 106.891136169, 106.891479492}},\n\t\t{\"z91w3\", []float64{51.7236328125, 51.767578125, 159.653320312, 159.697265625}},\n\t\t{\"c2d6xmbp1\", []float64{48.284740448, 48.2847833633, -120.267291069, -120.267248154}},\n\t\t{\"s9yur\", []float64{10.5908203125, 10.634765625, 32.2998046875, 32.34375}},\n\t\t{\"7u09b46\", []float64{-22.1800231934, -22.1786499023, -10.544128418, -10.542755127}},\n\t\t{\"8sndxb\", []float64{22.939453125, 22.9449462891, -148.018798828, -148.0078125}},\n\t\t{\"j2g761bhsex\", []float64{-85.1995566487, -85.1995553076, 60.9084056318, 60.9084069729}},\n\t\t{\"5wg\", []float64{-52.03125, -50.625, -18.28125, -16.875}},\n\t\t{\"fzn\", []float64{84.375, 85.78125, -47.8125, -46.40625}},\n\t\t{\"ugdpz8p4mu\", []float64{66.0502123833, 66.0502177477, 36.9019496441, 36.9019603729}},\n\t\t{\"nx\", []float64{-50.625, -45.0, 112.5, 123.75}},\n\t\t{\"d\", []float64{0.0, 45.0, -90.0, -45.0}},\n\t\t{\"crr9e8mz59se\", []float64{86.0475053452, 86.0475055128, -113.041263744, -113.041263409}},\n\t\t{\"dgmpsg\", []float64{19.6160888672, 19.6215820312, -49.0100097656, -48.9990234375}},\n\t\t{\"jcfk52m03\", []float64{-79.4517087936, -79.4516658783, 82.063794136, 82.0638370514}},\n\t\t{\"d8trpccuk\", []float64{4.05331134796, 4.05335426331, -59.7740364075, -59.7739934921}},\n\t\t{\"93g9665\", []float64{10.0744628906, 10.0758361816, -118.725128174, -118.723754883}},\n\t\t{\"sqt7cuf8d0f\", []float64{37.2478620708, 37.2478634119, 18.7132385373, 18.7132398784}},\n\t\t{\"f9\", []float64{50.625, 56.25, -67.5, -56.25}},\n\t\t{\"k90\", []float64{-39.375, -37.96875, 22.5, 23.90625}},\n\t\t{\"k8xdhcv\", []float64{-41.8263244629, -41.8249511719, 33.2624816895, 33.2638549805}},\n\t\t{\"4989w4r926t7\", []float64{-81.2862400152, -81.2862398475, -66.5228856727, -66.5228853375}},\n\t\t{\"c3\", []float64{50.625, 56.25, -123.75, -112.5}},\n\t\t{\"bd908pg0\", []float64{59.1929626465, 59.1931343079, -156.089630127, -156.089286804}},\n\t\t{\"bq\", []float64{78.75, 84.375, -168.75, -157.5}},\n\t\t{\"chcdt\", []float64{72.158203125, 72.2021484375, -132.670898438, -132.626953125}},\n\t\t{\"hff8vsrzhy39\", []float64{-74.3748327903, -74.3748326227, 37.5181730837, 37.5181734189}},\n\t\t{\"9gef7g6ezj\", []float64{20.101531148, 20.1015365124, -95.8080339432, -95.8080232143}},\n\t\t{\"yc0u2dp\", []float64{51.3830566406, 51.3844299316, 124.836273193, 124.837646484}},\n\t\t{\"w0b41f7\", []float64{4.58267211914, 4.58404541016, 90.0810241699, 90.0823974609}},\n\t\t{\"8cdmwjc\", []float64{9.43588256836, 9.43725585938, -142.820892334, -142.819519043}},\n\t\t{\"p4ngqtjm2\", []float64{-78.150343895, -78.1503009796, 144.785041809, 144.785084724}},\n\t\t{\"5\", []float64{-90.0, -45.0, -45.0, 0.0}},\n\t\t{\"3qtzqdknx\", []float64{-7.14961051941, -7.14956760406, -115.372624397, -115.372581482}},\n\t\t{\"gzjhuv9xmkg5\", []float64{85.2414438687, 85.2414440364, -4.00772050023, -4.00772016495}},\n\t\t{\"8g8t7eh4y0fh\", []float64{20.6273078173, 20.627307985, -145.387313068, -145.387312733}},\n\t\t{\"39w\", []float64{-36.5625, -35.15625, -104.0625, -102.65625}},\n\t\t{\"z34rj8\", []float64{51.85546875, 51.8609619141, 149.655761719, 149.666748047}},\n\t\t{\"c9p0zv2\", []float64{50.7856750488, 50.7870483398, -102.315673828, -102.314300537}},\n\t\t{\"vh871y\", []float64{70.8728027344, 70.8782958984, 45.4284667969, 45.439453125}},\n\t\t{\"7ggt8b4yfw\", []float64{-22.9382622242, -22.9382568598, -6.29128217697, -6.29127144814}},\n\t\t{\"3qz3d324z\", []float64{-6.76023960114, -6.76019668579, -113.455510139, -113.455467224}},\n\t\t{\"4sm2463w0w\", []float64{-66.0803282261, -66.0803228617, -60.0162291527, -60.0162184238}},\n\t\t{\"26ewbu0gw3\", []float64{-29.728397727, -29.7283923626, -163.793867826, -163.793857098}},\n\t\t{\"bre7js2w9z\", []float64{87.7393430471, 87.7393484116, -163.937226534, -163.937215805}},\n\t\t{\"sy5ug08bn\", []float64{34.5877075195, 34.5877504349, 39.1565608978, 39.1566038132}},\n\t\t{\"p4r\", []float64{-77.34375, -75.9375, 144.84375, 146.25}},\n\t\t{\"qb\", []float64{-45.0, -39.375, 123.75, 135.0}},\n\t\t{\"f4hj\", []float64{57.12890625, 57.3046875, -84.375, -84.0234375}},\n\t\t{\"5r0f5t7d5\", []float64{-50.2442550659, -50.2442121506, -32.5365686417, -32.5365257263}},\n\t\t{\"2j7n4r6r\", []float64{-14.3730354309, -14.3728637695, -175.679283142, -175.678939819}},\n\t\t{\"wu92egv2wsy\", []float64{25.4211013019, 25.421102643, 125.680104196, 125.680105537}},\n\t\t{\"vgtwey347bg\", []float64{65.8648006618, 65.8648020029, 86.6507081687, 86.6507095098}},\n\t\t{\"q2meny7pbd18\", []float64{-43.0307328701, -43.0307327025, 109.285149202, 109.285149537}},\n\t\t{\"rpe\", []float64{-2.8125, -1.40625, 139.21875, 140.625}},\n\t\t{\"m69\", []float64{-30.9375, -29.53125, 57.65625, 59.0625}},\n\t\t{\"w1zwd8\", []float64{10.986328125, 10.9918212891, 100.656738281, 100.667724609}},\n\t\t{\"fzpf\", []float64{84.7265625, 84.90234375, -45.3515625, -45.0}},\n\t\t{\"t3w\", []float64{8.4375, 9.84375, 64.6875, 66.09375}},\n\t\t{\"zb11\", []float64{45.17578125, 45.3515625, 170.15625, 170.5078125}},\n\t\t{\"r2prkmxpgtww\", []float64{-43.6940126494, -43.6940124817, 156.641852036, 156.641852371}},\n\t\t{\"zr34g1zcj\", []float64{86.274433136, 86.2744760513, 147.79894352, 147.798986435}},\n\t\t{\"19mgdk8\", []float64{-82.3287963867, -82.3274230957, -104.315185547, -104.313812256}},\n\t\t{\"mkp\", []float64{-22.5, -21.09375, 66.09375, 67.5}},\n\t\t{\"934qy86ssc\", []float64{6.81367456913, 6.81367993355, -120.296655893, -120.296645164}},\n\t\t{\"byydj4mrm8\", []float64{83.3339166641, 83.3339220285, -136.882202625, -136.882191896}},\n\t\t{\"j\", []float64{-90.0, -45.0, 45.0, 90.0}},\n\t\t{\"9cjzqv5n6\", []float64{6.92795276642, 6.92799568176, -92.8632259369, -92.8631830215}},\n\t\t{\"vkg1wg6s\", []float64{72.0009613037, 72.0011329651, 60.7688140869, 60.7691574097}},\n\t\t{\"ynp42e9x\", []float64{79.1659355164, 79.1661071777, 99.8677825928, 99.8681259155}},\n\t\t{\"uv9zwddtwn\", []float64{77.2705686092, 77.2705739737, 36.5002727509, 36.5002834797}},\n\t\t{\"t17zkzszpm\", []float64{8.3480912447, 8.34809660912, 50.4890120029, 50.4890227318}},\n\t\t{\"tuw779c04ukm\", []float64{25.8934257366, 25.8934259042, 87.6943681017, 87.6943684369}},\n\t\t{\"37rm598\", []float64{-25.8316040039, -25.8302307129, -113.400878906, -113.399505615}},\n\t\t{\"ymf18\", []float64{77.607421875, 77.6513671875, 104.0625, 104.106445312}},\n\t\t{\"gd\", []float64{56.25, 61.875, -22.5, -11.25}},\n\t\t{\"smz\", []float64{32.34375, 33.75, 21.09375, 22.5}},\n\t\t{\"p\", []float64{-90.0, -45.0, 135.0, 180.0}},\n\t\t{\"muzkh95w2u42\", []float64{-17.5715374947, -17.571537327, 89.1479081288, 89.1479084641}},\n\t\t{\"hh53c48eg\", []float64{-67.1780061722, -67.1779632568, 4.61507320404, 4.61511611938}},\n\t\t{\"739ewv\", []float64{-35.9197998047, -35.9143066406, -31.3439941406, -31.3330078125}},\n\t\t{\"cw883\", []float64{81.6064453125, 81.650390625, -111.752929688, -111.708984375}},\n\t\t{\"41xu1w37yf\", []float64{-80.8243882656, -80.8243829012, -79.0336382389, -79.0336275101}},\n\t\t{\"0y750v2k\", []float64{-54.2868804932, -54.2867088318, -141.997947693, -141.99760437}},\n\t\t{\"gqbgw\", []float64{83.583984375, 83.6279296875, -32.431640625, -32.3876953125}},\n\t\t{\"pej\", []float64{-73.125, -71.71875, 164.53125, 165.9375}},\n\t\t{\"r05t\", []float64{-44.12109375, -43.9453125, 139.921875, 140.2734375}},\n\t\t{\"qfuew7wk4f\", []float64{-28.8960921764, -28.896086812, 130.361484289, 130.361495018}},\n\t\t{\"nu357yhp72y\", []float64{-65.4882533848, -65.4882520437, 125.326685607, 125.326686949}},\n\t\t{\"8qt7rnggz\", []float64{37.1715116501, 37.1715545654, -161.054120064, -161.054077148}},\n\t\t{\"dhjq2nz\", []float64{23.6357116699, 23.6370849609, -82.6075744629, -82.6062011719}},\n\t\t{\"5s4\", []float64{-67.5, -66.09375, -19.6875, -18.28125}},\n\t\t{\"ge8nq842\", []float64{65.7861328125, 65.7863044739, -22.211265564, -22.2109222412}},\n\t\t{\"71\", []float64{-39.375, -33.75, -45.0, -33.75}},\n\t\t{\"sz59kteks87q\", []float64{39.625713788, 39.6257139556, 38.8742895797, 38.874289915}},\n\t\t{\"ur2bh\", []float64{85.78125, 85.8251953125, 12.48046875, 12.5244140625}},\n\t\t{\"w140u2f\", []float64{5.76095581055, 5.76232910156, 93.0020141602, 93.0033874512}},\n\t\t{\"fpd3zkt6whz4\", []float64{87.5202913955, 87.5202915631, -86.5098573267, -86.5098569915}},\n\t\t{\"zmej764h8kb8\", []float64{76.8721358478, 76.8721360154, 150.614330247, 150.614330582}},\n\t\t{\"k4p6z\", []float64{-33.2666015625, -33.22265625, 10.5029296875, 10.546875}},\n\t\t{\"f8\", []float64{45.0, 50.625, -67.5, -56.25}},\n\t\t{\"utsy6pv17m\", []float64{77.0789462328, 77.0789515972, 29.2745840549, 29.2745947838}},\n\t\t{\"6z5\", []float64{-5.625, -4.21875, -52.03125, -50.625}},\n\t\t{\"mjdc1\", []float64{-13.88671875, -13.8427734375, 48.9111328125, 48.955078125}},\n\t\t{\"gjks4c2\", []float64{75.2412414551, 75.2426147461, -38.5510253906, -38.5496520996}},\n\t\t{\"fvkvvrh42ju\", []float64{75.5808614194, 75.5808627605, -49.3341010809, -49.3340997398}},\n\t\t{\"yp63x30p7u7\", []float64{86.0516823828, 86.0516837239, 93.4828309715, 93.4828323126}},\n\t\t{\"6rw\", []float64{-2.8125, -1.40625, -70.3125, -68.90625}},\n\t\t{\"28vsqm8sb6\", []float64{-40.0031411648, -40.0031358004, -149.490269423, -149.490258694}},\n\t\t{\"be72g2zcek5\", []float64{63.4174847603, 63.4174861014, -152.776078731, -152.77607739}},\n\t\t{\"xry6fn699f\", []float64{44.1117489338, 44.1117542982, 155.130461454, 155.130472183}},\n\t\t{\"3sf7bw01y4\", []float64{-17.5888001919, -17.5887948275, -109.313707352, -109.313696623}},\n\t\t{\"k729yrqr77\", []float64{-26.3700467348, -26.3700413704, 12.2365057468, 12.2365164757}},\n\t\t{\"e\", []float64{0.0, 45.0, -45.0, 0.0}},\n\t\t{\"63x6dd\", []float64{-36.1120605469, -36.1065673828, -68.4448242188, -68.4338378906}},\n\t\t{\"z\", []float64{45.0, 90.0, 135.0, 180.0}},\n\t\t{\"mzyj\", []float64{-0.52734375, -0.3515625, 87.1875, 87.5390625}},\n\t\t{\"3j6r\", []float64{-14.23828125, -14.0625, -131.8359375, -131.484375}},\n\t\t{\"u3q\", []float64{52.03125, 53.4375, 19.6875, 21.09375}},\n\t\t{\"nueu7mbnbn4\", []float64{-63.9076530933, -63.9076517522, 129.166262448, 129.166263789}},\n\t\t{\"cyq2pkr\", []float64{80.1795959473, 80.1809692383, -92.1327209473, -92.1313476562}},\n\t\t{\"gptzzke9hvh\", []float64{88.5747224092, 88.5747237504, -36.5904432535, -36.5904419124}},\n\t\t{\"khd\", []float64{-19.6875, -18.28125, 2.8125, 4.21875}},\n\t\t{\"ghm92hg6p5\", []float64{69.1524285078, 69.1524338722, -37.2608613968, -37.260850668}},\n\t\t{\"n9e20w0\", []float64{-81.5295410156, -81.5281677246, 117.092285156, 117.093658447}},\n\t\t{\"826dzs0k\", []float64{1.91230773926, 1.91247940063, -164.904441833, -164.904098511}},\n\t\t{\"0d5f2\", []float64{-78.3544921875, -78.310546875, -152.2265625, -152.182617188}},\n\t\t{\"70zsyg3\", []float64{-39.9284362793, -39.9270629883, -34.1551208496, -34.1537475586}},\n\t\t{\"zykh8gwy\", []float64{80.9675216675, 80.9676933289, 174.417228699, 174.417572021}},\n\t\t{\"4spd3s\", []float64{-67.0825195312, -67.0770263672, -56.8872070312, -56.8762207031}},\n\t\t{\"r9p6f2t\", []float64{-38.8888549805, -38.8874816895, 167.801055908, 167.802429199}},\n\t\t{\"6q3merq7w\", []float64{-8.83652687073, -8.83648395538, -76.8405246735, -76.8404817581}},\n\t\t{\"qx1zr32\", []float64{-4.34371948242, -4.34234619141, 115.279541016, 115.280914307}},\n\t\t{\"3zfnk\", []float64{-0.3076171875, -0.263671875, -98.26171875, -98.2177734375}},\n\t\t{\"kd2e3\", []float64{-31.7724609375, -31.728515625, 23.2470703125, 23.291015625}},\n\t\t{\"stkhr6\", []float64{30.2893066406, 30.2947998047, 28.4436035156, 28.4545898438}},\n\t\t{\"nh4pzbm\", []float64{-66.1363220215, -66.1349487305, 93.159942627, 93.161315918}},\n\t\t{\"zt8tf\", []float64{76.9482421875, 76.9921875, 158.291015625, 158.334960938}},\n\t\t{\"gd37\", []float64{58.18359375, 58.359375, -20.7421875, -20.390625}},\n\t\t{\"gnx5b45dd\", []float64{82.2330951691, 82.2331380844, -35.1513576508, -35.1513147354}},\n\t\t{\"2qm7\", []float64{-9.31640625, -9.140625, -161.3671875, -161.015625}},\n\t\t{\"4g0zf0kq9cpp\", []float64{-71.7601996846, -71.760199517, -55.1015008986, -55.1015005633}},\n\t\t{\"977tgt\", []float64{19.3194580078, 19.3249511719, -118.674316406, -118.663330078}},\n\t\t{\"md0k\", []float64{-33.046875, -32.87109375, 67.8515625, 68.203125}},\n\t\t{\"v1q3nvfb3h\", []float64{52.2386813164, 52.2386866808, 54.089512825, 54.0895235538}},\n\t\t{\"z96pt6u96w\", []float64{53.3649623394, 53.3649677038, 160.549499989, 160.549510717}},\n\t\t{\"pu\", []float64{-67.5, -61.875, 168.75, 180.0}},\n\t\t{\"6uydh\", []float64{-17.9296875, -17.8857421875, -46.93359375, -46.8896484375}},\n\t\t{\"nx5mt4sk\", []float64{-49.6437835693, -49.643611908, 117.295875549, 117.296218872}},\n\t\t{\"nk8jt\", []float64{-63.720703125, -63.6767578125, 101.469726562, 101.513671875}},\n\t\t{\"kec1015b\", []float64{-23.7249755859, -23.7248039246, 23.9113998413, 23.9117431641}},\n\t\t{\"fk388b\", []float64{68.994140625, 68.9996337891, -76.6076660156, -76.5966796875}},\n\t\t{\"nsb\", []float64{-63.28125, -61.875, 112.5, 113.90625}},\n\t\t{\"ndbws\", []float64{-73.388671875, -73.3447265625, 113.37890625, 113.422851562}},\n\t\t{\"fs5\", []float64{67.5, 68.90625, -63.28125, -61.875}},\n\t\t{\"h6x0kbec6p15\", []float64{-75.8905554749, -75.8905553073, 21.3077272475, 21.3077275828}},\n\t\t{\"hy78\", []float64{-54.84375, -54.66796875, 38.671875, 39.0234375}},\n\t\t{\"vpun9j4ce9\", []float64{89.7640568018, 89.7640621662, 50.6728720665, 50.6728827953}},\n\t\t{\"6tyevvqrj665\", []float64{-11.9670169987, -11.967016831, -58.0978783965, -58.0978780612}},\n\t\t{\"d3ktekr48n\", []float64{8.02185416222, 8.02185952663, -72.2694396973, -72.2694289684}},\n\t\t{\"heyfm2up5\", []float64{-68.5054206848, -68.5053777695, 32.2285223007, 32.2285652161}},\n\t\t{\"mn3f7qjr22v\", []float64{-9.41403463483, -9.41403329372, 47.6109869778, 47.6109883189}},\n\t\t{\"1wngqx9b\", []float64{-55.637512207, -55.6373405457, -102.719764709, -102.719421387}},\n\t\t{\"xc9jv49jej\", []float64{9.46294605732, 9.46295142174, 170.3774786, 170.377489328}},\n\t\t{\"27\", []float64{-28.125, -22.5, -168.75, -157.5}},\n\t\t{\"6yhqnw\", []float64{-10.1623535156, -10.1568603516, -49.9877929688, -49.9768066406}},\n\t\t{\"rmhhu3qd\", []float64{-16.0328292847, -16.0326576233, 152.07069397, 152.071037292}},\n\t\t{\"y00b8mhby\", []float64{45.1154851913, 45.1155281067, 91.0724544525, 91.0724973679}},\n\t\t{\"yq5tr\", []float64{79.6728515625, 79.716796875, 106.479492188, 106.5234375}},\n\t\t{\"cuvxw\", []float64{73.037109375, 73.0810546875, -93.251953125, -93.2080078125}},\n\t\t{\"exvb4bcn\", []float64{43.5988998413, 43.5990715027, -14.2918395996, -14.2914962769}},\n\t\t{\"uhdpsyx75n0\", []float64{71.667112112, 71.6671134531, 3.03132534027, 3.03132668138}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"qvwbzgw7q\", []float64{-13.9108800888, -13.9108371735, 133.591604233, 133.591647148}},\n\t\t{\"u0rqp0bres\", []float64{47.466366291, 47.4663716555, 10.503423214, 10.5034339428}},\n\t\t{\"43k2u0vbnrbd\", []float64{-82.8327522799, -82.8327521123, -72.5894909352, -72.5894905999}},\n\t\t{\"7r7kwz7xxr\", []float64{-3.38658392429, -3.38657855988, -28.8779389858, -28.877928257}},\n\t\t{\"fu2f\", []float64{69.2578125, 69.43359375, -55.1953125, -54.84375}},\n\t\t{\"9tkjsvzxw6\", []float64{30.5309307575, 30.5309361219, -106.655691862, -106.655681133}},\n\t\t{\"y9x0z\", []float64{53.5693359375, 53.61328125, 122.651367188, 122.6953125}},\n\t\t{\"y4rk4b25g\", []float64{58.3613920212, 58.3614349365, 100.316290855, 100.316333771}},\n\t\t{\"sxmdmu9xcn\", []float64{41.202839613, 41.2028449774, 30.4891633987, 30.4891741276}},\n\t\t{\"fb0\", []float64{45.0, 46.40625, -56.25, -54.84375}},\n\t\t{\"7ffkxt5\", []float64{-28.7127685547, -28.7113952637, -7.7522277832, -7.75085449219}},\n\t\t{\"n1x5jn4dr\", []float64{-81.0018110275, -81.0017681122, 100.067210197, 100.067253113}},\n\t\t{\"uxcdctgmj\", []float64{89.1095924377, 89.1096353531, 24.6799707413, 24.6800136566}},\n\t\t{\"h7\", []float64{-73.125, -67.5, 11.25, 22.5}},\n\t\t{\"b\", []float64{45.0, 90.0, -180.0, -135.0}},\n\t\t{\"us1n4udk\", []float64{68.5800933838, 68.5802650452, 24.0301895142, 24.0305328369}},\n\t\t{\"zjtptsj2vn\", []float64{77.2779929638, 77.2779983282, 142.280373573, 142.280384302}},\n\t\t{\"x6utsqz\", []float64{16.4726257324, 16.4739990234, 152.774505615, 152.775878906}},\n\t\t{\"cs9dn901rqn\", []float64{70.6698024273, 70.6698037684, -110.104661286, -110.104659945}},\n\t\t{\"sjbn\", []float64{33.3984375, 33.57421875, 0.0, 0.3515625}},\n\t\t{\"0fwxjyc3g9q\", []float64{-74.6696452796, -74.6696439385, -136.854814589, -136.854813248}},\n\t\t{\"fk\", []float64{67.5, 73.125, -78.75, -67.5}},\n\t\t{\"75hq9jh\", []float64{-26.9549560547, -26.9535827637, -38.9739990234, -38.9726257324}},\n\t\t{\"kr3hg1q1\", []float64{-3.37675094604, -3.37657928467, 12.7963256836, 12.7966690063}},\n\t\t{\"hfq4d2wu\", []float64{-76.9008636475, -76.9006919861, 42.2956466675, 42.2959899902}},\n\t\t{\"rg6ygh\", []float64{-25.5102539062, -25.5047607422, 172.749023438, 172.760009766}},\n\t\t{\"995pvrg\", []float64{7.02987670898, 7.03125, -108.046417236, -108.045043945}},\n\t\t{\"s5ys\", []float64{21.796875, 21.97265625, 9.140625, 9.4921875}},\n\t\t{\"289ucubzj6\", []float64{-41.3252341747, -41.3252288103, -154.960902929, -154.9608922}},\n\t\t{\"4\", []float64{-90.0, -45.0, -90.0, -45.0}},\n\t\t{\"7g0e3gp7cz\", []float64{-27.5365501642, -27.5365447998, -10.4599392414, -10.4599285126}},\n\t\t{\"9suuudg\", []float64{27.5688171387, 27.5701904297, -105.618438721, -105.61706543}},\n\t\t{\"8vdt3j8zb0\", []float64{31.8918943405, 31.8918997049, -142.689399719, -142.68938899}},\n\t\t{\"cf\", []float64{56.25, 61.875, -101.25, -90.0}},\n\t\t{\"jnp33f5pr9\", []float64{-56.0180372, -56.0180318356, 55.276658535, 55.2766692638}},\n\t\t{\"czgmgyb\", []float64{89.6415710449, 89.6429443359, -96.5148925781, -96.5135192871}},\n\t\t{\"c1kk\", []float64{52.734375, 52.91015625, -129.0234375, -128.671875}},\n\t\t{\"kfm4hfe8cp4s\", []float64{-31.9782876223, -31.9782874547, 40.994843021, 40.9948433563}},\n\t\t{\"9mnws4hc8h\", []float64{29.2788434029, 29.2788487673, -114.427070618, -114.427059889}},\n\t\t{\"t0j7chwg\", []float64{0.684413909912, 0.684585571289, 52.4360275269, 52.4363708496}},\n\t\t{\"y\", []float64{45.0, 90.0, 90.0, 135.0}},\n\t\t{\"suj\", []float64{22.5, 23.90625, 40.78125, 42.1875}},\n\t}\n\n\tfor _, test := range tests {\n\t\tlat, lon := DecodeGeoHash(test.hash)\n\n\t\tif !compareLatitude(test.box, lat) {\n\t\t\tt.Errorf(\"expected lat %f, got %f, hash %s\", (test.box[0]+test.box[1])/2, lat, test.hash)\n\t\t}\n\t\tif !compareLogitude(test.box, lon) {\n\t\t\tt.Errorf(\"expected lon %f, got %f, hash %s\", (test.box[2]+test.box[3])/2, lon, test.hash)\n\t\t}\n\t}\n}\n\nfunc compareLatitude(box []float64, v float64) bool {\n\tavg := (box[0] + box[1]) / 2\n\n\treturn compareGeo(avg, v) == 0\n}\n\nfunc compareLogitude(box []float64, v float64) bool {\n\tavg := (box[2] + box[3]) / 2\n\n\treturn compareGeo(avg, v) == 0\n}\n"
  },
  {
    "path": "go.mod",
    "content": "module github.com/blevesearch/bleve/v2\n\ngo 1.24.0\n\nrequire (\n\tgithub.com/RoaringBitmap/roaring/v2 v2.14.5\n\tgithub.com/bits-and-blooms/bitset v1.24.2\n\tgithub.com/blevesearch/bleve_index_api v1.3.4\n\tgithub.com/blevesearch/geo v0.2.5\n\tgithub.com/blevesearch/go-faiss v1.0.28\n\tgithub.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475\n\tgithub.com/blevesearch/go-porterstemmer v1.0.3\n\tgithub.com/blevesearch/goleveldb v1.0.1\n\tgithub.com/blevesearch/gtreap v0.1.1\n\tgithub.com/blevesearch/scorch_segment_api/v2 v2.4.3\n\tgithub.com/blevesearch/segment v0.9.1\n\tgithub.com/blevesearch/snowball v0.6.1\n\tgithub.com/blevesearch/snowballstem v0.9.0\n\tgithub.com/blevesearch/stempel v0.2.0\n\tgithub.com/blevesearch/upsidedown_store_api v1.0.2\n\tgithub.com/blevesearch/vellum v1.2.0\n\tgithub.com/blevesearch/zapx/v11 v11.4.3\n\tgithub.com/blevesearch/zapx/v12 v12.4.3\n\tgithub.com/blevesearch/zapx/v13 v13.4.3\n\tgithub.com/blevesearch/zapx/v14 v14.4.3\n\tgithub.com/blevesearch/zapx/v15 v15.4.3\n\tgithub.com/blevesearch/zapx/v16 v16.3.1\n\tgithub.com/blevesearch/zapx/v17 v17.0.4\n\tgithub.com/couchbase/moss v0.2.0\n\tgithub.com/spf13/cobra v1.10.2\n\tgo.etcd.io/bbolt v1.4.0\n\tgolang.org/x/text v0.22.0\n\tgoogle.golang.org/protobuf v1.36.6\n)\n\nrequire (\n\tgithub.com/blevesearch/mmap-go v1.2.0 // indirect\n\tgithub.com/couchbase/ghistogram v0.1.0 // indirect\n\tgithub.com/golang/snappy v1.0.0 // indirect\n\tgithub.com/inconshreveable/mousetrap v1.1.0 // indirect\n\tgithub.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect\n\tgithub.com/mschoch/smat v0.2.0 // indirect\n\tgithub.com/spf13/pflag v1.0.9 // indirect\n\tgolang.org/x/sys v0.40.0 // indirect\n)\n"
  },
  {
    "path": "index/scorch/README.md",
    "content": "# scorch\n\n## Definitions\n\nBatch\n\n- A collection of Documents to mutate in the index.\n\nDocument\n\n- Has a unique identifier (arbitrary bytes).\n- Is comprised of a list of fields.\n\nField\n\n- Has a name (string).\n- Has a type (text, number, date, geopoint).\n- Has a value (depending on type).\n- Can be indexed, stored, or both.\n- If indexed, can be analyzed.\n-m If indexed, can optionally store term vectors.\n\n## Scope\n\nScorch *MUST* implement the bleve.index API without requiring any changes to this API.\n\nScorch *MAY* introduce new interfaces, which can be discovered to allow use of new capabilities not in the current API.\n\n## Implementation\n\nThe scorch implementation starts with the concept of a segmented index.\n\nA segment is simply a slice, subset, or portion of the entire index.  A segmented index is one which is composed of one or more segments.  Although segments are created in a particular order, knowing this ordering is not required to achieve correct semantics when querying.  Because there is no ordering, this means that when searching an index, you can (and should) search all the segments concurrently.\n\n### Internal Wrapper\n\nIn order to accommodate the existing APIs while also improving the implementation, the scorch implementation includes some wrapper functionality that must be described.\n\n#### \\_id field\n\nIn scorch, field 0 is prearranged to be named \\_id.  All documents have a value for this field, which is the documents external identifier.  In this version the field *MUST* be both indexed AND stored.  The scorch wrapper adds this field, as it will not be present in the Document from the calling bleve code.\n\nNOTE: If a document already contains a field \\_id, it will be replaced.  If this is problematic, the caller must ensure such a scenario does not happen.\n\n### Proposed Structures\n\n```go\ntype Segment interface {\n\n  Dictionary(field string) TermDictionary\n\n}\n\ntype TermDictionary interface {\n\n  PostingsList(term string, excluding PostingsList) PostingsList\n\n}\n\ntype PostingsList interface {\n\n  Next() Posting\n\n  And(other PostingsList) PostingsList\n  Or(other PostingsList) PostingsList\n\n}\n\ntype Posting interface {\n  Number() uint64\n\n  Frequency() uint64\n  Norm() float64\n\n  Locations() Locations\n}\n\ntype Locations interface {\n  Start() uint64\n  End() uint64\n  Pos() uint64\n  ArrayPositions() ...\n}\n\ntype DeletedDocs {\n\n}\n\ntype SegmentSnapshot struct {\n  segment Segment\n  deleted PostingsList\n}\n\ntype IndexSnapshot struct {\n  segment []SegmentSnapshot\n}\n```\n\n**What about errors?**\n**What about memory mgmnt or context?**\n**Postings List separate iterator to separate stateful from stateless**\n\n### Mutating the Index\n\nThe bleve.index API has methods for directly making individual mutations (Update/Delete/SetInternal/DeleteInternal), however for this first implementation, we assume that all of these calls can simply be turned into a Batch of size 1.  This may be highly inefficient, but it will be correct.  This decision is made based on the fact that Couchbase FTS always uses Batches.\n\nNOTE: As a side-effect of this decision, it should be clear that performance tuning may depend on the batch size, which may in-turn require changes in FTS.\n\nFrom this point forward, only Batch mutations will be discussed.\n\nSequence of Operations:\n\n1. For each document in the batch, search through all existing segments.  The goal is to build up a per-segment bitset which tells us which documents in that segment are obsoleted by the addition of the new segment we're currently building.  NOTE: we're not ready for this change to take effect yet, so rather than this operation mutating anything, they simply return bitsets, which we can apply later.  Logically, this is something like:\n\n    ```go\n      foreach segment {\n        dict := segment.Dictionary(\"\\_id\")\n        postings := empty postings list\n        foreach docID {\n          postings = postings.Or(dict.PostingsList(docID, nil))\n        }\n      }\n    ```\n\n    NOTE: it is illustrated above as nested for loops, but some or all of these could be concurrently.  The end result is that for each segment, we have (possibly empty) bitset.\n\n2. Also concurrent with 1, the documents in the batch are analyzed.  This analysis proceeds using the existing analyzer pool.\n\n3. (after 2 completes) Analyzed documents are fed into a function which builds a new Segment representing this information.\n\n4. We now have everything we need to update the state of the system to include this new snapshot.\n    - Acquire a lock\n    - Create a new IndexSnapshot\n    - For each SegmentSnapshot in the IndexSnapshot, take the deleted PostingsList and OR it with the new postings list for this Segment.  Construct a new SegmentSnapshot for the segment using this new deleted PostingsList.  Append this SegmentSnapshot to the IndexSnapshot.\n    - Create a new SegmentSnapshot wrapping our new segment with nil deleted docs.\n    - Append the new SegmentSnapshot to the IndexSnapshot\n    - Release the lock\n\nAn ASCII art example:\n\n```text\n  0 - Empty Index\n\n  No segments\n\n  IndexSnapshot\n    segments []\n    deleted []\n\n\n  1 - Index Batch [ A B C ]\n\n  segment       0\n  numbers   [ 1 2 3 ]\n  \\_id      [ A B C ]\n\n  IndexSnapshot\n    segments [ 0 ]\n    deleted [ nil ]\n\n\n  2 - Index Batch [ B' ]\n\n  segment       0           1\n  numbers   [ 1 2 3 ]     [ 1 ]\n  \\_id      [ A B C ]     [ B ]\n\n  Compute bitset segment-0-deleted-by-1:\n            [ 0 1 0 ]\n\n  OR it with previous (nil) (call it 0-1)\n            [ 0 1 0 ]\n\n  IndexSnapshot\n    segments [  0    1 ]\n    deleted  [ 0-1 nil ]\n\n  3 - Index Batch [ C' ]\n\n    segment       0           1      2\n    numbers   [ 1 2 3 ]     [ 1 ]  [ 1 ]\n    \\_id      [ A B C ]     [ B ]  [ C ]\n\n    Compute bitset segment-0-deleted-by-2:\n              [ 0 0 1 ]\n\n    OR it with previous ([ 0 1 0 ]) (call it 0-12)\n              [ 0 1 1 ]\n\n  Compute bitset segment-1-deleted-by-2:\n              [ 0 ]\n\n  OR it with previous (nil)\n              still just nil\n\n\n    IndexSnapshot\n      segments [  0    1    2 ]\n      deleted  [ 0-12 nil  nil ]\n  ```\n\n**is there opportunity to stop early when doc is found in one segment**\n**also, more efficient way to find bits for long lists of ids?**\n\n### Searching\n\nIn the bleve.index API all searching starts by getting an IndexReader, which represents a snapshot of the index at a point in time.\n\nAs described in the section above, our index implementation maintains a pointer to the current IndexSnapshot.  When a caller gets an IndexReader, they get a copy of this pointer, and can use it as long as they like.  The IndexSnapshot contains SegmentSnapshots, which only contain pointers to immutable segments.  The deleted posting lists associated with a segment change over time, but the particular deleted posting list in YOUR snapshot is immutable.  This gives a stable view of the data.\n\n#### Term Search\n\nTerm search is the only searching primitive exposed in today's bleve.index API.  This ultimately could limit our ability to take advantage of the indexing improvements, but it also means it will be easier to get a first version of this working.\n\nA term search for term T in field F will look something like this:\n\n```go\n  searchResultPostings = empty\n  foreach segment {\n    dict := segment.Dictionary(F)\n    segmentResultPostings = dict.PostingsList(T, segmentSnapshotDeleted)\n    // make segmentLocal numbers into global numbers, and flip bits in searchResultPostings\n  }\n```\n\nThe searchResultPostings will be a new implementation of the TermFieldReader interface.\n\nAs a reminder this interface is:\n\n```go\n// TermFieldReader is the interface exposing the enumeration of documents\n// containing a given term in a given field. Documents are returned in byte\n// lexicographic order over their identifiers.\ntype TermFieldReader interface {\n  // Next returns the next document containing the term in this field, or nil\n  // when it reaches the end of the enumeration.  The preAlloced TermFieldDoc\n  // is optional, and when non-nil, will be used instead of allocating memory.\n  Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error)\n\n  // Advance resets the enumeration at specified document or its immediate\n  // follower.\n  Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error)\n\n  // Count returns the number of documents contains the term in this field.\n  Count() uint64\n  Close() error\n}\n```\n\nAt first glance this appears problematic, we have no way to return documents in order of their identifiers.  But it turns out the wording of this perhaps too strong, or a bit ambiguous.  Originally, this referred to the external identifiers, but with the introduction of a distinction between internal/external identifiers, returning them in order of their internal identifiers is also acceptable.  **ASIDE**: the reason for this is that most callers just use Next() and literally don't care what the order is, they could be in any order and it would be fine.  There is only one search that cares and that is the ConjunctionSearcher, which relies on Next/Advance having very specific semantics.  Later in this document we will have a proposal to split into multiple interfaces:\n\n- The weakest interface, only supports Next() no ordering at all.\n- Ordered, supporting Advance()\n- And/Or'able capable of internally efficiently doing these ops with like interfaces (if not capable then can always fall back to external walking)\n\nBut, the good news is that we don't even have to do that for our first implementation.  As long as the global numbers we use for internal identifiers are consistent within this IndexSnapshot, then Next() will be ordered by ascending document number, and Advance() will still work correctly.\n\nNOTE: there is another place where we rely on the ordering of these hits, and that is in the \"\\_id\" sort order.  Previously this was the natural order, and a NOOP for the collector, now it must be implemented by actually sorting on the \"\\_id\" field.  We probably should introduce at least a marker interface to detect this.\n\nAn ASCII art example:\n\n```text\nLet's start with the IndexSnapshot we ended with earlier:\n\n3 - Index Batch [ C' ]\n\n  segment       0           1      2\n  numbers   [ 1 2 3 ]     [ 1 ]  [ 1 ]\n  \\_id      [ A B C ]     [ B ]  [ C ]\n\n  Compute bitset segment-0-deleted-by-2:\n            [ 0 0 1 ]\n\n  OR it with previous ([ 0 1 0 ]) (call it 0-12)\n            [ 0 1 1 ]\n\nCompute bitset segment-1-deleted-by-2:\n            [ 0 0 0 ]\n\nOR it with previous (nil)\n            still just nil\n\n\n  IndexSnapshot\n    segments [  0    1    2 ]\n    deleted  [ 0-12 nil  nil ]\n\nNow let's search for the term 'cat' in the field 'desc' and let's assume that Document C (both versions) would match it.\n\nConcurrently:\n\n  - Segment 0\n   - Get Term Dictionary For Field 'desc'\n   - From it get Postings List for term 'cat' EXCLUDING 0-12\n   - raw segment matches [ 0 0 1 ] but excluding [ 0 1 1 ] gives [ 0 0 0 ]\n  - Segment 1\n   - Get Term Dictionary For Field 'desc'\n   - From it get Postings List for term 'cat' excluding nil\n   - [ 0 ]\n  - Segment 2\n   - Get Term Dictionary For Field 'desc'\n   - From it get Postings List for term 'cat' excluding nil\n   - [ 1 ]\n\nMap local bitsets into global number space (global meaning cross-segment but still unique to this snapshot)\n\nIndexSnapshot already should have mapping something like:\n0 - Offset 0\n1 - Offset 3 (because segment 0 had 3 docs)\n2 - Offset 4 (because segment 1 had 1 doc)\n\nThis maps to search result bitset:\n\n[ 0 0 0 0 1]\n\nCaller would call Next() and get doc number 5 (assuming 1 based indexing for now)\n\nCaller could then ask to get term locations, stored fields, external doc ID for document number 5.  Internally in the IndexSnapshot, we can now convert that back, and realize doc number 5 comes from segment 2, 5-4=1 so we're looking for doc number 1 in segment 2.  That happens to be C...\n\n```\n\n#### Future improvements\n\nIn the future, interfaces to detect these non-serially operating TermFieldReaders could expose their own And() and Or() up to the higher level Conjunction/Disjunction searchers.  Doing this alone offers some win, but also means there would be greater burden on the Searcher code rewriting logical expressions for maximum performance.\n\nAnother related topic is that of peak memory usage.  With serially operating TermFieldReaders it was necessary to start them all at the same time and operate in unison.  However, with these non-serially operating TermFieldReaders we have the option of doing a few at a time, consolidating them, dispoting the intermediaries, and then doing a few more.  For very complex queries with many clauses this could reduce peak memory usage.\n\n### Memory Tracking\n\nAll segments must be able to produce two statistics, an estimate of their explicit memory usage, and their actual size on disk (if any).  For in-memory segments, disk usage could be zero, and the memory usage represents the entire information content.  For mmap-based disk segments, the memory could be as low as the size of tracking structure itself (say just a few pointers).\n\nThis would allow the implementation to throttle or block incoming mutations when a threshold memory usage has (or would be) exceeded.\n\n### Persistence\n\nObviously, we want to support (but maybe not require) asynchronous persistence of segments.  My expectation is that segments are initially built in memory.  At some point they are persisted to disk.  This poses some interesting challenges.\n\nAt runtime, the state of an index (it's IndexSnapshot) is not only the contents of the segments, but also the bitmasks of deleted documents.  These bitmasks indirectly encode an ordering in which the segments were added.  The reason is that the bitmasks encode which items have been obsoleted by other (subsequent or more future) segments.  In the runtime implementation we compute bitmask deltas and then merge them at the same time we bring the new segment in.  One idea is that we could take a similar approach on disk.  When we persist a segment, we persist the bitmask deltas of segments known to exist at that time, and eventually these can get merged up into a base segment deleted bitmask.\n\nThis also relates to the topic rollback, addressed next...\n\n### Rollback\n\nOne desirable property in the Couchbase ecosystem is the ability to rollback to some previous (though typically not long ago) state.  One idea for keeping this property in this design is to protect some of the most recent segments from merging.  Then, if necessary, they could be \"undone\" to reveal previous states of the system.  In these scenarios \"undone\" has to properly undo the deleted bitmasks on the other segments.  Again, the current thinking is that rather than \"undo\" anything, it could be work that was deferred in the first place, thus making it easier to logically undo.\n\nAnother possibly related approach would be to tie this into our existing snapshot mechanism.  Perhaps simulating a slow reader (holding onto index snapshots) for some period of time, can be the mechanism to achieve the desired end goal.\n\n### Internal Storage\n\nThe bleve.index API has support for \"internal storage\".  The ability to store information under a separate name space.\n\nThis is not used for high volume storage, so it is tempting to think we could just put a small k/v store alongside the rest of the index.  But, the reality is that this storage is used to maintain key information related to the rollback scenario.  Because of this, its crucial that ordering and overwriting of key/value pairs correspond with actual segment persistence in the index.  Based on this, I believe its important to put the internal key/value pairs inside the segments themselves.  But, this also means that they must follow a similar \"deleted\" bitmask approach to obsolete values in older segments.  But, this also seems to substantially increase the complexity of the solution because of the separate name space, it would appear to require its own bitmask.  Further keys aren't numeric, which then implies yet another mapping from internal key to number, etc.\n\nMore thought is required here.\n\n### Merging\n\nThe segmented index approach requires merging to prevent the number of segments from growing too large.\n\nRecent experience with LSMs has taught us that having the correct merge strategy can make a huge difference in the overall performance of the system.  In particular, a simple merge strategy which merges segments too aggressively can lead to high write amplification and unnecessarily rendering cached data useless.\n\nA few simple principles have been identified.\n\n- Roughly we merge multiple smaller segments into a single larger one.\n- The larger a segment gets the less likely we should be to ever merge it.\n- Segments with large numbers of deleted/obsoleted items are good candidates as the merge will result in a space savings.\n- Segments with all items deleted/obsoleted can be dropped.\n\nMerging of a segment should be able to proceed even if that segment is held by an ongoing snapshot, it should only delay the removal of it.\n"
  },
  {
    "path": "index/scorch/builder.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n\tbolt \"go.etcd.io/bbolt\"\n)\n\nconst DefaultBuilderBatchSize = 1000\nconst DefaultBuilderMergeMax = 10\n\ntype Builder struct {\n\tm         sync.Mutex\n\tsegCount  uint64\n\tpath      string\n\tbuildPath string\n\tsegPaths  []string\n\tbatchSize int\n\tmergeMax  int\n\tbatch     *index.Batch\n\tinternal  map[string][]byte\n\tsegPlugin SegmentPlugin\n}\n\nfunc NewBuilder(config map[string]interface{}) (*Builder, error) {\n\tpath, ok := config[\"path\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify path\")\n\t}\n\n\tbuildPathPrefix, _ := config[\"buildPathPrefix\"].(string)\n\tbuildPath, err := os.MkdirTemp(buildPathPrefix, \"scorch-offline-build\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trv := &Builder{\n\t\tpath:      path,\n\t\tbuildPath: buildPath,\n\t\tmergeMax:  DefaultBuilderMergeMax,\n\t\tbatchSize: DefaultBuilderBatchSize,\n\t\tbatch:     index.NewBatch(),\n\t\tsegPlugin: defaultSegmentPlugin,\n\t}\n\n\terr = rv.parseConfig(config)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error parsing builder config: %v\", err)\n\t}\n\n\treturn rv, nil\n}\n\nfunc (o *Builder) parseConfig(config map[string]interface{}) (err error) {\n\tif v, ok := config[\"mergeMax\"]; ok {\n\t\tvar t int\n\t\tif t, err = parseToInteger(v); err != nil {\n\t\t\treturn fmt.Errorf(\"mergeMax parse err: %v\", err)\n\t\t}\n\t\tif t > 0 {\n\t\t\to.mergeMax = t\n\t\t}\n\t}\n\n\tif v, ok := config[\"batchSize\"]; ok {\n\t\tvar t int\n\t\tif t, err = parseToInteger(v); err != nil {\n\t\t\treturn fmt.Errorf(\"batchSize parse err: %v\", err)\n\t\t}\n\t\tif t > 0 {\n\t\t\to.batchSize = t\n\t\t}\n\t}\n\n\tif v, ok := config[\"internal\"]; ok {\n\t\tif vinternal, ok := v.(map[string][]byte); ok {\n\t\t\to.internal = vinternal\n\t\t}\n\t}\n\n\tforcedSegmentType, forcedSegmentVersion, err := configForceSegmentTypeVersion(config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif forcedSegmentType != \"\" && forcedSegmentVersion != 0 {\n\t\tsegPlugin, err := chooseSegmentPlugin(forcedSegmentType,\n\t\t\tuint32(forcedSegmentVersion))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\to.segPlugin = segPlugin\n\t}\n\n\treturn nil\n}\n\n// Index will place the document into the index.\n// It is invalid to index the same document multiple times.\nfunc (o *Builder) Index(doc index.Document) error {\n\to.m.Lock()\n\tdefer o.m.Unlock()\n\n\to.batch.Update(doc)\n\n\treturn o.maybeFlushBatchLOCKED(o.batchSize)\n}\n\nfunc (o *Builder) maybeFlushBatchLOCKED(moreThan int) error {\n\tif len(o.batch.IndexOps) >= moreThan {\n\t\tdefer o.batch.Reset()\n\t\treturn o.executeBatchLOCKED(o.batch)\n\t}\n\treturn nil\n}\n\nfunc (o *Builder) executeBatchLOCKED(batch *index.Batch) (err error) {\n\tanalysisResults := make([]index.Document, 0, len(batch.IndexOps))\n\tfor _, doc := range batch.IndexOps {\n\t\tif doc != nil {\n\t\t\t// insert _id field\n\t\t\tdoc.AddIDField()\n\t\t\t// perform analysis directly\n\t\t\tanalyze(doc, nil)\n\t\t\tanalysisResults = append(analysisResults, doc)\n\t\t}\n\t}\n\n\tseg, _, err := o.segPlugin.New(analysisResults)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error building segment base: %v\", err)\n\t}\n\n\tfilename := zapFileName(o.segCount)\n\to.segCount++\n\tpath := o.buildPath + string(os.PathSeparator) + filename\n\n\tif segUnpersisted, ok := seg.(segment.UnpersistedSegment); ok {\n\t\terr = segUnpersisted.Persist(path)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error persisting segment base to %s: %v\", path, err)\n\t\t}\n\n\t\to.segPaths = append(o.segPaths, path)\n\t\treturn nil\n\t}\n\n\treturn fmt.Errorf(\"new segment does not implement unpersisted: %T\", seg)\n}\n\nfunc (o *Builder) doMerge() error {\n\t// as long as we have more than 1 segment, keep merging\n\tfor len(o.segPaths) > 1 {\n\n\t\t// merge the next <mergeMax> number of segments into one new one\n\t\t// or, if there are fewer than <mergeMax> remaining, merge them all\n\t\tmergeCount := o.mergeMax\n\t\tif mergeCount > len(o.segPaths) {\n\t\t\tmergeCount = len(o.segPaths)\n\t\t}\n\n\t\tmergePaths := o.segPaths[0:mergeCount]\n\t\to.segPaths = o.segPaths[mergeCount:]\n\n\t\t// open each of the segments to be merged\n\t\tmergeSegs := make([]segment.Segment, 0, mergeCount)\n\n\t\t// closeOpenedSegs attempts to close all opened\n\t\t// segments even if an error occurs, in which case\n\t\t// the first error is returned\n\t\tcloseOpenedSegs := func() error {\n\t\t\tvar err error\n\t\t\tfor _, seg := range mergeSegs {\n\t\t\t\tclErr := seg.Close()\n\t\t\t\tif clErr != nil && err == nil {\n\t\t\t\t\terr = clErr\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\n\t\tfor _, mergePath := range mergePaths {\n\t\t\tseg, err := o.segPlugin.Open(mergePath)\n\t\t\tif err != nil {\n\t\t\t\t_ = closeOpenedSegs()\n\t\t\t\treturn fmt.Errorf(\"error opening segment (%s) for merge: %v\", mergePath, err)\n\t\t\t}\n\t\t\tmergeSegs = append(mergeSegs, seg)\n\t\t}\n\n\t\t// do the merge\n\t\tmergedSegPath := o.buildPath + string(os.PathSeparator) + zapFileName(o.segCount)\n\t\tdrops := make([]*roaring.Bitmap, mergeCount)\n\t\t_, _, err := o.segPlugin.Merge(mergeSegs, drops, mergedSegPath, nil, nil)\n\t\tif err != nil {\n\t\t\t_ = closeOpenedSegs()\n\t\t\treturn fmt.Errorf(\"error merging segments (%v): %v\", mergePaths, err)\n\t\t}\n\t\to.segCount++\n\t\to.segPaths = append(o.segPaths, mergedSegPath)\n\n\t\t// close segments opened for merge\n\t\terr = closeOpenedSegs()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error closing opened segments: %v\", err)\n\t\t}\n\n\t\t// remove merged segments\n\t\tfor _, mergePath := range mergePaths {\n\t\t\terr = os.RemoveAll(mergePath)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"error removing segment %s after merge: %v\", mergePath, err)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (o *Builder) Close() error {\n\to.m.Lock()\n\tdefer o.m.Unlock()\n\n\t// see if there is a partial batch\n\terr := o.maybeFlushBatchLOCKED(1)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error flushing batch before close: %v\", err)\n\t}\n\n\t// perform all the merging\n\terr = o.doMerge()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error while merging: %v\", err)\n\t}\n\n\t// ensure the store path exists\n\terr = os.MkdirAll(o.path, 0700)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// move final segment into place\n\t// segment id 2 is chosen to match the behavior of a scorch\n\t// index which indexes a single batch of data\n\tfinalSegPath := o.path + string(os.PathSeparator) + zapFileName(2)\n\terr = os.Rename(o.segPaths[0], finalSegPath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error moving final segment into place: %v\", err)\n\t}\n\n\t// remove the buildPath, as it is no longer needed\n\terr = os.RemoveAll(o.buildPath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error removing build path: %v\", err)\n\t}\n\n\t// prepare wrapping\n\tseg, err := o.segPlugin.Open(finalSegPath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error opening final segment\")\n\t}\n\n\t// create a segment snapshot for this segment\n\tss := &SegmentSnapshot{\n\t\tsegment: seg,\n\t}\n\tis := &IndexSnapshot{\n\t\tepoch:    3, // chosen to match scorch behavior when indexing a single batch\n\t\tsegment:  []*SegmentSnapshot{ss},\n\t\tcreator:  \"scorch-builder\",\n\t\tinternal: o.internal,\n\t}\n\n\t// create the root bolt\n\trootBoltPath := o.path + string(os.PathSeparator) + \"root.bolt\"\n\trootBolt, err := bolt.Open(rootBoltPath, 0600, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// start a write transaction\n\ttx, err := rootBolt.Begin(true)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// fill the root bolt with this fake index snapshot\n\t_, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin, nil, nil)\n\tif err != nil {\n\t\t_ = tx.Rollback()\n\t\t_ = rootBolt.Close()\n\t\treturn fmt.Errorf(\"error preparing bolt snapshot in root.bolt: %v\", err)\n\t}\n\n\t// commit bolt data\n\terr = tx.Commit()\n\tif err != nil {\n\t\t_ = rootBolt.Close()\n\t\treturn fmt.Errorf(\"error committing bolt tx in root.bolt: %v\", err)\n\t}\n\n\t// close bolt\n\terr = rootBolt.Close()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error closing root.bolt: %v\", err)\n\t}\n\n\t// close final segment\n\terr = seg.Close()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error closing final segment: %v\", err)\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "index/scorch/builder_test.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestBuilder(t *testing.T) {\n\ttmpDir, err := os.MkdirTemp(\"\", \"scorch-builder-test\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = os.RemoveAll(tmpDir)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error cleaning up test index: %v\", err)\n\t\t}\n\t}()\n\toptions := map[string]interface{}{\n\t\t\"path\":      tmpDir,\n\t\t\"batchSize\": 2,\n\t\t\"mergeMax\":  2,\n\t}\n\tb, err := NewBuilder(options)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tdoc := document.NewDocument(fmt.Sprintf(\"%d\", i))\n\t\tdoc.AddField(document.NewTextField(\"name\", nil, []byte(\"hello\")))\n\t\terr = b.Index(doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = b.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcheckIndex(t, tmpDir, []byte(\"hello\"), \"name\", 10)\n}\n\nfunc checkIndex(t *testing.T, path string, term []byte, field string, expectCount int) {\n\tcfg := make(map[string]interface{})\n\tcfg[\"path\"] = path\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error closing index: %v\", err)\n\t\t}\n\t}()\n\n\tr, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatalf(\"error accessing index reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr = r.Close()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error closing reader: %v\", err)\n\t\t}\n\t}()\n\n\t// check the count, expect 10 docs\n\tcount, err := r.DocCount()\n\tif err != nil {\n\t\tt.Errorf(\"error accessing index doc count: %v\", err)\n\t} else if count != uint64(expectCount) {\n\t\tt.Errorf(\"expected %d docs, got %d\", expectCount, count)\n\t}\n\n\t// run a search for hello\n\ttfr, err := r.TermFieldReader(context.TODO(), term, field, false, false, false)\n\tif err != nil {\n\t\tt.Errorf(\"error accessing term field reader: %v\", err)\n\t} else {\n\t\tvar rows int\n\t\ttfd, err := tfr.Next(nil)\n\t\tfor err == nil && tfd != nil {\n\t\t\trows++\n\t\t\ttfd, err = tfr.Next(nil)\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Errorf(\"error calling next on term field reader: %v\", err)\n\t\t}\n\t\tif rows != expectCount {\n\t\t\tt.Errorf(\"expected %d rows for term hello, field name, got %d\", expectCount, rows)\n\t\t}\n\t}\n}\n\nfunc TestBuilderFlushFinalBatch(t *testing.T) {\n\ttmpDir, err := os.MkdirTemp(\"\", \"scorch-builder-test\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = os.RemoveAll(tmpDir)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error cleaning up test index: %v\", err)\n\t\t}\n\t}()\n\toptions := map[string]interface{}{\n\t\t\"path\":      tmpDir,\n\t\t\"batchSize\": 2,\n\t\t\"mergeMax\":  2,\n\t}\n\tb, err := NewBuilder(options)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := 0; i < 9; i++ {\n\t\tdoc := document.NewDocument(fmt.Sprintf(\"%d\", i))\n\t\tdoc.AddField(document.NewTextField(\"name\", nil, []byte(\"hello\")))\n\t\terr = b.Index(doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = b.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcheckIndex(t, tmpDir, []byte(\"hello\"), \"name\", 9)\n}\n"
  },
  {
    "path": "index/scorch/empty.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport segment \"github.com/blevesearch/scorch_segment_api/v2\"\n\ntype emptyPostingsIterator struct{}\n\nfunc (e *emptyPostingsIterator) Next() (segment.Posting, error) {\n\treturn nil, nil\n}\n\nfunc (e *emptyPostingsIterator) Advance(uint64) (segment.Posting, error) {\n\treturn nil, nil\n}\n\nfunc (e *emptyPostingsIterator) Size() int {\n\treturn 0\n}\n\nfunc (e *emptyPostingsIterator) BytesRead() uint64 {\n\treturn 0\n}\n\nfunc (e *emptyPostingsIterator) ResetBytesRead(uint64) {}\n\nfunc (e *emptyPostingsIterator) BytesWritten() uint64 { return 0 }\n\nvar anEmptyPostingsIterator = &emptyPostingsIterator{}\n"
  },
  {
    "path": "index/scorch/event.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport \"time\"\n\n// RegistryAsyncErrorCallbacks should be treated as read-only after\n// process init()'ialization.\nvar RegistryAsyncErrorCallbacks = map[string]func(error, string){}\n\n// RegistryEventCallbacks should be treated as read-only after\n// process init()'ialization.\n// In the event of not having a callback, these return true.\nvar RegistryEventCallbacks = map[string]func(Event) bool{}\n\n// Event represents the information provided in an OnEvent() callback.\ntype Event struct {\n\tKind     EventKind\n\tScorch   *Scorch\n\tDuration time.Duration\n}\n\n// EventKind represents an event code for OnEvent() callbacks.\ntype EventKind int\n\nconst (\n\t// EventKindCloseStart is fired when a Scorch.Close() has begun.\n\tEventKindCloseStart EventKind = iota\n\n\t// EventKindClose is fired when a scorch index has been fully closed.\n\tEventKindClose\n\n\t// EventKindMergerProgress is fired when the merger has completed a\n\t// round of merge processing.\n\tEventKindMergerProgress\n\n\t// EventKindPersisterProgress is fired when the persister has completed\n\t// a round of persistence processing.\n\tEventKindPersisterProgress\n\n\t// EventKindBatchIntroductionStart is fired when Batch() is invoked which\n\t// introduces a new segment.\n\tEventKindBatchIntroductionStart\n\n\t// EventKindBatchIntroduction is fired when Batch() completes.\n\tEventKindBatchIntroduction\n\n\t// EventKindMergeTaskIntroductionStart is fired when the merger is about to\n\t// start the introduction of merged segment from a single merge task.\n\tEventKindMergeTaskIntroductionStart\n\n\t// EventKindMergeTaskIntroduction is fired when the merger has completed\n\t// the introduction of merged segment from a single merge task.\n\tEventKindMergeTaskIntroduction\n\n\t// EventKindPreMergeCheck is fired before the merge begins to check if\n\t// the caller should proceed with the merge.\n\tEventKindPreMergeCheck\n\n\t// EventKindIndexStart is fired when Index() is invoked which\n\t// creates a new Document object from an interface using the index mapping.\n\tEventKindIndexStart\n\n\t// EventKindPurgerCheck is fired before the purge code is invoked and decides\n\t// whether to execute or not. For unit test purposes\n\tEventKindPurgerCheck\n)\n"
  },
  {
    "path": "index/scorch/event_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestEventBatchIntroductionStart(t *testing.T) {\n\ttestConfig := CreateConfig(\"TestEventBatchIntroductionStart\")\n\terr := InitTest(testConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(testConfig)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar count int\n\tRegistryEventCallbacks[\"test\"] = func(e Event) bool {\n\t\tif e.Kind == EventKindBatchIntroductionStart {\n\t\t\tcount++\n\t\t}\n\t\treturn true\n\t}\n\n\tourConfig := make(map[string]interface{}, len(testConfig))\n\tfor k, v := range testConfig {\n\t\tourConfig[k] = v\n\t}\n\tourConfig[\"eventCallbackName\"] = \"test\"\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, ourConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tif count != 1 {\n\t\tt.Fatalf(\"expected to see 1 batch introduction event event, saw %d\", count)\n\t}\n}\n"
  },
  {
    "path": "index/scorch/field_dict_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestIndexFieldDict(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexFieldDict\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\tcerr := idx.Close()\n\t\tif cerr != nil {\n\t\t\tt.Fatal(cerr)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextFieldWithAnalyzer(\"name\", []uint64{}, []byte(\"test test test\"), testAnalyzer))\n\tdoc.AddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"eat more rice\"), index.IndexField|index.IncludeTermVectors, testAnalyzer))\n\tdoc.AddField(document.NewTextFieldCustom(\"prefix\", []uint64{}, []byte(\"bob cat cats catting dog doggy zoo\"), index.IndexField|index.IncludeTermVectors, testAnalyzer))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdict, err := indexReader.FieldDict(\"name\")\n\tif err != nil {\n\t\tt.Errorf(\"error creating reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := dict.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermCount := 0\n\tcurr, err := dict.Next()\n\tfor err == nil && curr != nil {\n\t\ttermCount++\n\t\tif curr.Term != \"test\" {\n\t\t\tt.Errorf(\"expected term to be 'test', got '%s'\", curr.Term)\n\t\t}\n\t\tcurr, err = dict.Next()\n\t}\n\tif termCount != 1 {\n\t\tt.Errorf(\"expected 1 term for this field, got %d\", termCount)\n\t}\n\n\tdict2, err := indexReader.FieldDict(\"desc\")\n\tif err != nil {\n\t\tt.Fatalf(\"error creating reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := dict2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermCount = 0\n\tterms := make([]string, 0)\n\tcurr, err = dict2.Next()\n\tfor err == nil && curr != nil {\n\t\ttermCount++\n\t\tterms = append(terms, curr.Term)\n\t\tcurr, err = dict2.Next()\n\t}\n\tif termCount != 3 {\n\t\tt.Errorf(\"expected 3 term for this field, got %d\", termCount)\n\t}\n\texpectedTerms := []string{\"eat\", \"more\", \"rice\"}\n\tif !reflect.DeepEqual(expectedTerms, terms) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedTerms, terms)\n\t}\n\t// test start and end range\n\tdict3, err := indexReader.FieldDictRange(\"desc\", []byte(\"fun\"), []byte(\"nice\"))\n\tif err != nil {\n\t\tt.Errorf(\"error creating reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := dict3.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermCount = 0\n\tterms = make([]string, 0)\n\tcurr, err = dict3.Next()\n\tfor err == nil && curr != nil {\n\t\ttermCount++\n\t\tterms = append(terms, curr.Term)\n\t\tcurr, err = dict3.Next()\n\t}\n\tif termCount != 1 {\n\t\tt.Errorf(\"expected 1 term for this field, got %d\", termCount)\n\t}\n\texpectedTerms = []string{\"more\"}\n\tif !reflect.DeepEqual(expectedTerms, terms) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedTerms, terms)\n\t}\n\n\t// test use case for prefix\n\tdict4, err := indexReader.FieldDictPrefix(\"prefix\", []byte(\"cat\"))\n\tif err != nil {\n\t\tt.Errorf(\"error creating reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := dict4.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermCount = 0\n\tterms = make([]string, 0)\n\tcurr, err = dict4.Next()\n\tfor err == nil && curr != nil {\n\t\ttermCount++\n\t\tterms = append(terms, curr.Term)\n\t\tcurr, err = dict4.Next()\n\t}\n\tif termCount != 3 {\n\t\tt.Errorf(\"expected 3 term for this field, got %d\", termCount)\n\t}\n\texpectedTerms = []string{\"cat\", \"cats\", \"catting\"}\n\tif !reflect.DeepEqual(expectedTerms, terms) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedTerms, terms)\n\t}\n}\n"
  },
  {
    "path": "index/scorch/int.go",
    "content": "// Copyright 2014 The Cockroach Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n// implied. See the License for the specific language governing\n// permissions and limitations under the License.\n\n// This code originated from:\n// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go\n\n// Modified to not use pkg/errors\n\npackage scorch\n\nimport \"fmt\"\n\nconst (\n\t// intMin is chosen such that the range of int tags does not overlap the\n\t// ascii character set that is frequently used in testing.\n\tintMin      = 0x80 // 128\n\tintMaxWidth = 8\n\tintZero     = intMin + intMaxWidth           // 136\n\tintSmall    = intMax - intZero - intMaxWidth // 109\n\t// intMax is the maximum int tag value.\n\tintMax = 0xfd // 253\n)\n\n// encodeUvarintAscending encodes the uint64 value using a variable length\n// (length-prefixed) representation. The length is encoded as a single\n// byte indicating the number of encoded bytes (-8) to follow. See\n// EncodeVarintAscending for rationale. The encoded bytes are appended to the\n// supplied buffer and the final buffer is returned.\nfunc encodeUvarintAscending(b []byte, v uint64) []byte {\n\tswitch {\n\tcase v <= intSmall:\n\t\treturn append(b, intZero+byte(v))\n\tcase v <= 0xff:\n\t\treturn append(b, intMax-7, byte(v))\n\tcase v <= 0xffff:\n\t\treturn append(b, intMax-6, byte(v>>8), byte(v))\n\tcase v <= 0xffffff:\n\t\treturn append(b, intMax-5, byte(v>>16), byte(v>>8), byte(v))\n\tcase v <= 0xffffffff:\n\t\treturn append(b, intMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))\n\tcase v <= 0xffffffffff:\n\t\treturn append(b, intMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8),\n\t\t\tbyte(v))\n\tcase v <= 0xffffffffffff:\n\t\treturn append(b, intMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16),\n\t\t\tbyte(v>>8), byte(v))\n\tcase v <= 0xffffffffffffff:\n\t\treturn append(b, intMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24),\n\t\t\tbyte(v>>16), byte(v>>8), byte(v))\n\tdefault:\n\t\treturn append(b, intMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32),\n\t\t\tbyte(v>>24), byte(v>>16), byte(v>>8), byte(v))\n\t}\n}\n\n// decodeUvarintAscending decodes a varint encoded uint64 from the input\n// buffer. The remainder of the input buffer and the decoded uint64\n// are returned.\nfunc decodeUvarintAscending(b []byte) ([]byte, uint64, error) {\n\tif len(b) == 0 {\n\t\treturn nil, 0, fmt.Errorf(\"insufficient bytes to decode uvarint value\")\n\t}\n\tlength := int(b[0]) - intZero\n\tb = b[1:] // skip length byte\n\tif length <= intSmall {\n\t\treturn b, uint64(length), nil\n\t}\n\tlength -= intSmall\n\tif length < 0 || length > 8 {\n\t\treturn nil, 0, fmt.Errorf(\"invalid uvarint length of %d\", length)\n\t} else if len(b) < length {\n\t\treturn nil, 0, fmt.Errorf(\"insufficient bytes to decode uvarint value: %q\", b)\n\t}\n\tvar v uint64\n\t// It is faster to range over the elements in a slice than to index\n\t// into the slice on each loop iteration.\n\tfor _, t := range b[:length] {\n\t\tv = (v << 8) | uint64(t)\n\t}\n\treturn b[length:], v, nil\n}\n"
  },
  {
    "path": "index/scorch/int_test.go",
    "content": "// Copyright 2014 The Cockroach Authors.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n// implied. See the License for the specific language governing\n// permissions and limitations under the License.\n\n// This code originated from:\n// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding_test.go\n\n// Modified to only test the parts we borrowed\n\npackage scorch\n\nimport (\n\t\"bytes\"\n\t\"math\"\n\t\"testing\"\n)\n\ntype testCaseUint64 struct {\n\tvalue  uint64\n\texpEnc []byte\n}\n\nfunc TestEncodeDecodeUvarint(t *testing.T) {\n\ttestBasicEncodeDecodeUint64(encodeUvarintAscending, decodeUvarintAscending, false, t)\n\ttestCases := []testCaseUint64{\n\t\t{0, []byte{0x88}},\n\t\t{1, []byte{0x89}},\n\t\t{109, []byte{0xf5}},\n\t\t{110, []byte{0xf6, 0x6e}},\n\t\t{1 << 8, []byte{0xf7, 0x01, 0x00}},\n\t\t{math.MaxUint64, []byte{0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}},\n\t}\n\ttestCustomEncodeUint64(testCases, encodeUvarintAscending, t)\n}\n\nfunc testBasicEncodeDecodeUint64(\n\tencFunc func([]byte, uint64) []byte,\n\tdecFunc func([]byte) ([]byte, uint64, error),\n\tdescending bool, t *testing.T,\n) {\n\ttestCases := []uint64{\n\t\t0, 1,\n\t\t1<<8 - 1, 1 << 8,\n\t\t1<<16 - 1, 1 << 16,\n\t\t1<<24 - 1, 1 << 24,\n\t\t1<<32 - 1, 1 << 32,\n\t\t1<<40 - 1, 1 << 40,\n\t\t1<<48 - 1, 1 << 48,\n\t\t1<<56 - 1, 1 << 56,\n\t\tmath.MaxUint64 - 1, math.MaxUint64,\n\t}\n\n\tvar lastEnc []byte\n\tfor i, v := range testCases {\n\t\tenc := encFunc(nil, v)\n\t\tif i > 0 {\n\t\t\tif (descending && bytes.Compare(enc, lastEnc) >= 0) ||\n\t\t\t\t(!descending && bytes.Compare(enc, lastEnc) < 0) {\n\t\t\t\tt.Errorf(\"ordered constraint violated for %d: [% x] vs. [% x]\", v, enc, lastEnc)\n\t\t\t}\n\t\t}\n\t\tb, decode, err := decFunc(enc)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\tif len(b) != 0 {\n\t\t\tt.Errorf(\"leftover bytes: [% x]\", b)\n\t\t}\n\t\tif decode != v {\n\t\t\tt.Errorf(\"decode yielded different value than input: %d vs. %d\", decode, v)\n\t\t}\n\t\tlastEnc = enc\n\t}\n}\n\nfunc testCustomEncodeUint64(\n\ttestCases []testCaseUint64, encFunc func([]byte, uint64) []byte, t *testing.T,\n) {\n\tfor _, test := range testCases {\n\t\tenc := encFunc(nil, test.value)\n\t\tif !bytes.Equal(enc, test.expEnc) {\n\t\t\tt.Errorf(\"expected [% x]; got [% x] (value: %d)\", test.expEnc, enc, test.value)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "index/scorch/introducer.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"sync/atomic\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\nconst introducer = \"introducer\"\n\ntype segmentIntroduction struct {\n\tid        uint64\n\tdata      segment.Segment\n\tobsoletes map[uint64]*roaring.Bitmap\n\tids       []string\n\tinternal  map[string][]byte\n\tstats     *fieldStats\n\n\tapplied           chan error\n\tpersisted         chan error\n\tpersistedCallback index.BatchCallback\n}\n\ntype persistIntroduction struct {\n\tpersisted map[uint64]segment.Segment\n\tapplied   notificationChan\n}\n\ntype epochWatcher struct {\n\tepoch    uint64\n\tnotifyCh notificationChan\n}\n\nfunc (s *Scorch) introducerLoop() {\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\ts.fireAsyncError(NewScorchError(\n\t\t\t\tintroducer,\n\t\t\t\tfmt.Sprintf(\"panic: %v, path: %s\", r, s.path),\n\t\t\t\tErrAsyncPanic,\n\t\t\t))\n\t\t}\n\n\t\ts.asyncTasks.Done()\n\t}()\n\n\tvar epochWatchers []*epochWatcher\nOUTER:\n\tfor {\n\t\tatomic.AddUint64(&s.stats.TotIntroduceLoop, 1)\n\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\tbreak OUTER\n\n\t\tcase epochWatcher := <-s.introducerNotifier:\n\t\t\tepochWatchers = append(epochWatchers, epochWatcher)\n\n\t\tcase nextMerge := <-s.merges:\n\t\t\ts.introduceMerge(nextMerge)\n\n\t\tcase next := <-s.introductions:\n\t\t\terr := s.introduceSegment(next)\n\t\t\tif err != nil {\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\n\t\tcase persist := <-s.persists:\n\t\t\ts.introducePersist(persist)\n\n\t\t}\n\n\t\tvar epochCurr uint64\n\t\ts.rootLock.RLock()\n\t\tif s.root != nil {\n\t\t\tepochCurr = s.root.epoch\n\t\t}\n\t\ts.rootLock.RUnlock()\n\t\tvar epochWatchersNext []*epochWatcher\n\t\tfor _, w := range epochWatchers {\n\t\t\tif w.epoch < epochCurr {\n\t\t\t\tclose(w.notifyCh)\n\t\t\t} else {\n\t\t\t\tepochWatchersNext = append(epochWatchersNext, w)\n\t\t\t}\n\t\t}\n\t\tepochWatchers = epochWatchersNext\n\t}\n}\n\nfunc (s *Scorch) introduceSegment(next *segmentIntroduction) error {\n\tatomic.AddUint64(&s.stats.TotIntroduceSegmentBeg, 1)\n\tdefer atomic.AddUint64(&s.stats.TotIntroduceSegmentEnd, 1)\n\n\ts.rootLock.RLock()\n\troot := s.root\n\troot.AddRef()\n\ts.rootLock.RUnlock()\n\n\tdefer func() { _ = root.DecRef() }()\n\n\tnsegs := len(root.segment)\n\n\t// prepare new index snapshot\n\tnewSnapshot := &IndexSnapshot{\n\t\tparent:   s,\n\t\tsegment:  make([]*SegmentSnapshot, 0, nsegs+1),\n\t\toffsets:  make([]uint64, 0, nsegs+1),\n\t\tinternal: make(map[string][]byte, len(root.internal)),\n\t\trefs:     1,\n\t\tcreator:  \"introduceSegment\",\n\t}\n\n\t// iterate through current segments\n\tvar running uint64\n\tvar docsToPersistCount, memSegments, fileSegments uint64\n\tvar droppedSegmentFiles []string\n\tfor i := range root.segment {\n\t\t// see if optimistic work included this segment\n\t\tdelta, ok := next.obsoletes[root.segment[i].id]\n\t\tif !ok {\n\t\t\tvar err error\n\t\t\tdelta, err = root.segment[i].segment.DocNumbers(next.ids)\n\t\t\tif err != nil {\n\t\t\t\tnext.applied <- fmt.Errorf(\"error computing doc numbers: %v\", err)\n\t\t\t\tclose(next.applied)\n\t\t\t\t_ = newSnapshot.DecRef()\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tnewss := &SegmentSnapshot{\n\t\t\tid:         root.segment[i].id,\n\t\t\tsegment:    root.segment[i].segment,\n\t\t\tstats:      root.segment[i].stats,\n\t\t\tcachedDocs: root.segment[i].cachedDocs,\n\t\t\tcachedMeta: root.segment[i].cachedMeta,\n\t\t\tcreator:    root.segment[i].creator,\n\t\t}\n\n\t\t// apply new obsoletions\n\t\tif root.segment[i].deleted == nil {\n\t\t\tnewss.deleted = delta\n\t\t} else {\n\t\t\tif delta.IsEmpty() {\n\t\t\t\tnewss.deleted = root.segment[i].deleted\n\t\t\t} else {\n\t\t\t\tnewss.deleted = roaring.Or(root.segment[i].deleted, delta)\n\t\t\t}\n\t\t}\n\t\tif newss.deleted.IsEmpty() {\n\t\t\tnewss.deleted = nil\n\t\t}\n\n\t\t// update the deleted bitmap to include any nested/sub-documents as well\n\t\t// if the segment supports that\n\t\tif ns, ok := newss.segment.(segment.NestedSegment); ok {\n\t\t\tnewss.deleted = ns.AddNestedDocuments(newss.deleted)\n\t\t}\n\t\t// check for live size before copying\n\t\tif newss.LiveSize() > 0 {\n\t\t\tnewSnapshot.segment = append(newSnapshot.segment, newss)\n\t\t\troot.segment[i].segment.AddRef()\n\t\t\tnewSnapshot.offsets = append(newSnapshot.offsets, running)\n\t\t\trunning += newss.segment.Count()\n\t\t} else if seg, ok := newss.segment.(segment.PersistedSegment); ok {\n\t\t\tdroppedSegmentFiles = append(droppedSegmentFiles,\n\t\t\t\tfilepath.Base(seg.Path()))\n\t\t}\n\n\t\tif isMemorySegment(root.segment[i]) {\n\t\t\tdocsToPersistCount += root.segment[i].Count()\n\t\t\tmemSegments++\n\t\t} else {\n\t\t\tfileSegments++\n\t\t}\n\t}\n\n\tatomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount)\n\tatomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments)\n\tatomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments)\n\n\t// append new segment, if any, to end of the new index snapshot\n\tif next.data != nil {\n\t\tnewSegmentSnapshot := &SegmentSnapshot{\n\t\t\tid:         next.id,\n\t\t\tsegment:    next.data, // take ownership of next.data's ref-count\n\t\t\tstats:      next.stats,\n\t\t\tcachedDocs: &cachedDocs{cache: nil},\n\t\t\tcachedMeta: &cachedMeta{meta: nil},\n\t\t\tcreator:    \"introduceSegment\",\n\t\t}\n\t\tnewSnapshot.segment = append(newSnapshot.segment, newSegmentSnapshot)\n\t\tnewSnapshot.offsets = append(newSnapshot.offsets, running)\n\n\t\t// increment numItemsIntroduced which tracks the number of items\n\t\t// queued for persistence.\n\t\tatomic.AddUint64(&s.stats.TotIntroducedItems, newSegmentSnapshot.Count())\n\t\tatomic.AddUint64(&s.stats.TotIntroducedSegmentsBatch, 1)\n\t}\n\t// copy old values\n\tfor key, oldVal := range root.internal {\n\t\tnewSnapshot.internal[key] = oldVal\n\t}\n\t// set new values and apply deletes\n\tfor key, newVal := range next.internal {\n\t\tif newVal != nil {\n\t\t\tnewSnapshot.internal[key] = newVal\n\t\t} else {\n\t\t\tdelete(newSnapshot.internal, key)\n\t\t}\n\t}\n\n\tnewSnapshot.updateSize()\n\ts.rootLock.Lock()\n\tif next.persisted != nil {\n\t\ts.rootPersisted = append(s.rootPersisted, next.persisted)\n\t}\n\tif next.persistedCallback != nil {\n\t\ts.persistedCallbacks = append(s.persistedCallbacks, next.persistedCallback)\n\t}\n\t// swap in new index snapshot\n\tnewSnapshot.epoch = s.nextSnapshotEpoch\n\ts.nextSnapshotEpoch++\n\trootPrev := s.root\n\ts.root = newSnapshot\n\tatomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch)\n\t// release lock\n\ts.rootLock.Unlock()\n\n\tif rootPrev != nil {\n\t\t_ = rootPrev.DecRef()\n\t}\n\n\t// update the removal eligibility for those segment files\n\t// that are not a part of the latest root.\n\tfor _, filename := range droppedSegmentFiles {\n\t\ts.unmarkIneligibleForRemoval(filename)\n\t}\n\n\tclose(next.applied)\n\n\treturn nil\n}\n\nfunc (s *Scorch) introducePersist(persist *persistIntroduction) {\n\tatomic.AddUint64(&s.stats.TotIntroducePersistBeg, 1)\n\tdefer atomic.AddUint64(&s.stats.TotIntroducePersistEnd, 1)\n\n\ts.rootLock.Lock()\n\troot := s.root\n\troot.AddRef()\n\tnextSnapshotEpoch := s.nextSnapshotEpoch\n\ts.nextSnapshotEpoch++\n\ts.rootLock.Unlock()\n\n\tdefer func() { _ = root.DecRef() }()\n\n\tnewIndexSnapshot := &IndexSnapshot{\n\t\tparent:   s,\n\t\tepoch:    nextSnapshotEpoch,\n\t\tsegment:  make([]*SegmentSnapshot, len(root.segment)),\n\t\toffsets:  make([]uint64, len(root.offsets)),\n\t\tinternal: make(map[string][]byte, len(root.internal)),\n\t\trefs:     1,\n\t\tcreator:  \"introducePersist\",\n\t}\n\n\tvar docsToPersistCount, memSegments, fileSegments uint64\n\tfor i, segmentSnapshot := range root.segment {\n\t\t// see if this segment has been replaced\n\t\tif replacement, ok := persist.persisted[segmentSnapshot.id]; ok {\n\t\t\tnewSegmentSnapshot := &SegmentSnapshot{\n\t\t\t\tid:         segmentSnapshot.id,\n\t\t\t\tsegment:    replacement,\n\t\t\t\tdeleted:    segmentSnapshot.deleted,\n\t\t\t\tstats:      segmentSnapshot.stats,\n\t\t\t\tcachedDocs: segmentSnapshot.cachedDocs,\n\t\t\t\tcachedMeta: segmentSnapshot.cachedMeta,\n\t\t\t\tcreator:    \"introducePersist\",\n\t\t\t\tmmaped:     1,\n\t\t\t}\n\t\t\tnewIndexSnapshot.segment[i] = newSegmentSnapshot\n\t\t\tdelete(persist.persisted, segmentSnapshot.id)\n\n\t\t\t// update items persisted in case of a new segment snapshot\n\t\t\tatomic.AddUint64(&s.stats.TotPersistedItems, newSegmentSnapshot.Count())\n\t\t\tatomic.AddUint64(&s.stats.TotPersistedSegments, 1)\n\t\t\tfileSegments++\n\t\t} else {\n\t\t\tnewIndexSnapshot.segment[i] = root.segment[i]\n\t\t\tnewIndexSnapshot.segment[i].segment.AddRef()\n\n\t\t\tif isMemorySegment(root.segment[i]) {\n\t\t\t\tdocsToPersistCount += root.segment[i].Count()\n\t\t\t\tmemSegments++\n\t\t\t} else {\n\t\t\t\tfileSegments++\n\t\t\t}\n\t\t}\n\t\tnewIndexSnapshot.offsets[i] = root.offsets[i]\n\t}\n\n\tfor k, v := range root.internal {\n\t\tnewIndexSnapshot.internal[k] = v\n\t}\n\n\tatomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount)\n\tatomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments)\n\tatomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments)\n\tnewIndexSnapshot.updateSize()\n\ts.rootLock.Lock()\n\trootPrev := s.root\n\ts.root = newIndexSnapshot\n\tatomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch)\n\ts.rootLock.Unlock()\n\n\tif rootPrev != nil {\n\t\t_ = rootPrev.DecRef()\n\t}\n\n\tclose(persist.applied)\n}\n\n// The introducer should definitely handle the segmentMerge.notify\n// channel before exiting the introduceMerge.\nfunc (s *Scorch) introduceMerge(nextMerge *segmentMerge) {\n\tatomic.AddUint64(&s.stats.TotIntroduceMergeBeg, 1)\n\tdefer atomic.AddUint64(&s.stats.TotIntroduceMergeEnd, 1)\n\n\ts.rootLock.RLock()\n\troot := s.root\n\troot.AddRef()\n\ts.rootLock.RUnlock()\n\n\tdefer func() { _ = root.DecRef() }()\n\n\tnewSnapshot := &IndexSnapshot{\n\t\tparent:   s,\n\t\tinternal: root.internal,\n\t\trefs:     1,\n\t\tcreator:  \"introduceMerge\",\n\t}\n\n\tvar running, docsToPersistCount, memSegments, fileSegments uint64\n\tvar droppedSegmentFiles []string\n\tnewSegmentDeleted := make([]*roaring.Bitmap, len(nextMerge.new))\n\tfor i := range newSegmentDeleted {\n\t\t// create a bitmaps to track the obsoletes per newly merged segments\n\t\tnewSegmentDeleted[i] = roaring.NewBitmap()\n\t}\n\n\t// iterate through current segments\n\tfor i := range root.segment {\n\t\tsegmentID := root.segment[i].id\n\t\tif segSnapAtMerge, ok := nextMerge.mergedSegHistory[segmentID]; ok {\n\t\t\t// this segment is going away, see if anything else was deleted since we started the merge\n\t\t\tif segSnapAtMerge != nil && root.segment[i].deleted != nil {\n\t\t\t\t// assume all these deletes are new\n\t\t\t\tdeletedSince := root.segment[i].deleted\n\t\t\t\t// if we already knew about some of them, remove\n\t\t\t\tif segSnapAtMerge.oldSegment.deleted != nil {\n\t\t\t\t\tdeletedSince = roaring.AndNot(root.segment[i].deleted, segSnapAtMerge.oldSegment.deleted)\n\t\t\t\t}\n\t\t\t\tdeletedSinceItr := deletedSince.Iterator()\n\t\t\t\tfor deletedSinceItr.HasNext() {\n\t\t\t\t\toldDocNum := deletedSinceItr.Next()\n\t\t\t\t\tnewDocNum := segSnapAtMerge.oldNewDocIDs[oldDocNum]\n\t\t\t\t\tnewSegmentDeleted[segSnapAtMerge.workerID].Add(uint32(newDocNum))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// clean up the old segment map to figure out the\n\t\t\t// obsolete segments wrt root in meantime, whatever\n\t\t\t// segments left behind in old map after processing\n\t\t\t// the root segments would be the obsolete segment set\n\t\t\tdelete(nextMerge.mergedSegHistory, segmentID)\n\t\t} else if root.segment[i].LiveSize() > 0 {\n\t\t\t// this segment is staying\n\t\t\tnewSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{\n\t\t\t\tid:         root.segment[i].id,\n\t\t\t\tsegment:    root.segment[i].segment,\n\t\t\t\tdeleted:    root.segment[i].deleted,\n\t\t\t\tstats:      root.segment[i].stats,\n\t\t\t\tcachedDocs: root.segment[i].cachedDocs,\n\t\t\t\tcachedMeta: root.segment[i].cachedMeta,\n\t\t\t\tcreator:    root.segment[i].creator,\n\t\t\t})\n\t\t\troot.segment[i].segment.AddRef()\n\t\t\tnewSnapshot.offsets = append(newSnapshot.offsets, running)\n\t\t\trunning += root.segment[i].segment.Count()\n\n\t\t\tif isMemorySegment(root.segment[i]) {\n\t\t\t\tdocsToPersistCount += root.segment[i].Count()\n\t\t\t\tmemSegments++\n\t\t\t} else {\n\t\t\t\tfileSegments++\n\t\t\t}\n\t\t} else if root.segment[i].LiveSize() == 0 {\n\t\t\tif seg, ok := root.segment[i].segment.(segment.PersistedSegment); ok {\n\t\t\t\tdroppedSegmentFiles = append(droppedSegmentFiles,\n\t\t\t\t\tfilepath.Base(seg.Path()))\n\t\t\t}\n\t\t}\n\t}\n\t// before the newMerge introduction, need to clean the newly\n\t// merged segment wrt the current root segments, hence\n\t// applying the obsolete segment contents to newly merged segment\n\tfor _, ss := range nextMerge.mergedSegHistory {\n\t\tobsoleted := ss.oldSegment.DocNumbersLive()\n\t\tif obsoleted != nil {\n\t\t\tobsoletedIter := obsoleted.Iterator()\n\t\t\tfor obsoletedIter.HasNext() {\n\t\t\t\toldDocNum := obsoletedIter.Next()\n\t\t\t\tnewDocNum := ss.oldNewDocIDs[oldDocNum]\n\t\t\t\tnewSegmentDeleted[ss.workerID].Add(uint32(newDocNum))\n\t\t\t}\n\t\t}\n\t}\n\n\tskipped := true\n\t// make the newly merged segments part of the newSnapshot being constructed\n\tfor i, newMergedSegment := range nextMerge.new {\n\t\t// checking if this newly merged segment is worth keeping based on\n\t\t// obsoleted doc count since the merge intro started\n\t\tif newMergedSegment != nil &&\n\t\t\tnewMergedSegment.Count() > newSegmentDeleted[i].GetCardinality() {\n\t\t\tstats := newFieldStats()\n\t\t\tif fsr, ok := newMergedSegment.(segment.FieldStatsReporter); ok {\n\t\t\t\tfsr.UpdateFieldStats(stats)\n\t\t\t}\n\n\t\t\t// put the merged segment at the end of newSnapshot\n\t\t\tnewSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{\n\t\t\t\tid:         nextMerge.id[i],\n\t\t\t\tsegment:    newMergedSegment, // take ownership for nextMerge.new's ref-count\n\t\t\t\tdeleted:    newSegmentDeleted[i],\n\t\t\t\tstats:      stats,\n\t\t\t\tcachedDocs: &cachedDocs{cache: nil},\n\t\t\t\tcachedMeta: &cachedMeta{meta: nil},\n\t\t\t\tcreator:    \"introduceMerge\",\n\t\t\t\tmmaped:     nextMerge.mmaped,\n\t\t\t})\n\t\t\tnewSnapshot.offsets = append(newSnapshot.offsets, running)\n\t\t\trunning += newMergedSegment.Count()\n\n\t\t\tswitch newMergedSegment.(type) {\n\t\t\tcase segment.PersistedSegment:\n\t\t\t\tfileSegments++\n\t\t\tdefault:\n\t\t\t\tdocsToPersistCount += newMergedSegment.Count() - newSegmentDeleted[i].GetCardinality()\n\t\t\t\tmemSegments++\n\t\t\t}\n\t\t\tskipped = false\n\t\t}\n\t}\n\n\tif skipped {\n\t\tatomic.AddUint64(&s.stats.TotFileMergeIntroductionsObsoleted, 1)\n\t} else {\n\t\tatomic.AddUint64(&s.stats.TotIntroducedSegmentsMerge, uint64(len(nextMerge.new)))\n\t}\n\n\tatomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount)\n\tatomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments)\n\tatomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments)\n\n\tnewSnapshot.AddRef() // 1 ref for the nextMerge.notify response\n\n\tnewSnapshot.updateSize()\n\ts.rootLock.Lock()\n\t// swap in new index snapshot\n\tnewSnapshot.epoch = s.nextSnapshotEpoch\n\ts.nextSnapshotEpoch++\n\trootPrev := s.root\n\ts.root = newSnapshot\n\tatomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch)\n\t// release lock\n\ts.rootLock.Unlock()\n\n\tif rootPrev != nil {\n\t\t_ = rootPrev.DecRef()\n\t}\n\n\t// update the removal eligibility for those segment files\n\t// that are not a part of the latest root.\n\tfor _, filename := range droppedSegmentFiles {\n\t\ts.unmarkIneligibleForRemoval(filename)\n\t}\n\n\t// notify requester that we incorporated this\n\tnextMerge.notifyCh <- &mergeTaskIntroStatus{\n\t\tindexSnapshot: newSnapshot,\n\t\tskipped:       skipped}\n\tclose(nextMerge.notifyCh)\n}\n\nfunc isMemorySegment(s *SegmentSnapshot) bool {\n\tswitch s.segment.(type) {\n\tcase segment.PersistedSegment:\n\t\treturn false\n\tdefault:\n\t\treturn true\n\t}\n}\n"
  },
  {
    "path": "index/scorch/merge.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch/mergeplan\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\nconst merger = \"merger\"\n\nfunc (s *Scorch) mergerLoop() {\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\ts.fireAsyncError(NewScorchError(\n\t\t\t\tmerger,\n\t\t\t\tfmt.Sprintf(\"panic: %v, path: %s\", r, s.path),\n\t\t\t\tErrAsyncPanic,\n\t\t\t))\n\t\t}\n\n\t\ts.asyncTasks.Done()\n\t}()\n\n\tvar lastEpochMergePlanned uint64\n\tvar ctrlMsg *mergerCtrl\n\tmergePlannerOptions, err := s.parseMergePlannerOptions()\n\tif err != nil {\n\t\ts.fireAsyncError(NewScorchError(\n\t\t\tmerger,\n\t\t\tfmt.Sprintf(\"mergerPlannerOptions json parsing err: %v\", err),\n\t\t\tErrOptionsParse,\n\t\t))\n\t\treturn\n\t}\n\tctrlMsgDflt := &mergerCtrl{ctx: context.Background(),\n\t\toptions: mergePlannerOptions,\n\t\tdoneCh:  nil}\n\nOUTER:\n\tfor {\n\t\tatomic.AddUint64(&s.stats.TotFileMergeLoopBeg, 1)\n\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\tbreak OUTER\n\n\t\tdefault:\n\t\t\t// check to see if there is a new snapshot to persist\n\t\t\ts.rootLock.Lock()\n\t\t\tourSnapshot := s.root\n\t\t\tourSnapshot.AddRef()\n\t\t\tatomic.StoreUint64(&s.iStats.mergeSnapshotSize, uint64(ourSnapshot.Size()))\n\t\t\tatomic.StoreUint64(&s.iStats.mergeEpoch, ourSnapshot.epoch)\n\t\t\ts.rootLock.Unlock()\n\n\t\t\tif ctrlMsg == nil && ourSnapshot.epoch != lastEpochMergePlanned {\n\t\t\t\tctrlMsg = ctrlMsgDflt\n\t\t\t}\n\t\t\tif ctrlMsg != nil {\n\t\t\t\tcontinueMerge := s.fireEvent(EventKindPreMergeCheck, 0)\n\t\t\t\t// The default, if there's no handler, is to continue the merge.\n\t\t\t\tif !continueMerge {\n\t\t\t\t\t// If it's decided that this merge can't take place now,\n\t\t\t\t\t// begin the merge process all over again.\n\t\t\t\t\t// Retry instead of blocking/waiting here since a long wait\n\t\t\t\t\t// can result in more segments introduced i.e. s.root will\n\t\t\t\t\t// be updated.\n\n\t\t\t\t\t// decrement the ref count since its no longer needed in this\n\t\t\t\t\t// iteration\n\t\t\t\t\t_ = ourSnapshot.DecRef()\n\t\t\t\t\tcontinue OUTER\n\t\t\t\t}\n\n\t\t\t\tstartTime := time.Now()\n\n\t\t\t\t// lets get started\n\t\t\t\terr := s.planMergeAtSnapshot(ctrlMsg.ctx, ctrlMsg.options,\n\t\t\t\t\tourSnapshot)\n\t\t\t\tif err != nil {\n\t\t\t\t\tatomic.StoreUint64(&s.iStats.mergeEpoch, 0)\n\t\t\t\t\tif err == segment.ErrClosed {\n\t\t\t\t\t\t// index has been closed\n\t\t\t\t\t\t_ = ourSnapshot.DecRef()\n\n\t\t\t\t\t\t// continue the workloop on a user triggered cancel\n\t\t\t\t\t\tif ctrlMsg.doneCh != nil {\n\t\t\t\t\t\t\tclose(ctrlMsg.doneCh)\n\t\t\t\t\t\t\tctrlMsg = nil\n\t\t\t\t\t\t\tcontinue OUTER\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// exit the workloop on index closure\n\t\t\t\t\t\tctrlMsg = nil\n\t\t\t\t\t\tbreak OUTER\n\t\t\t\t\t}\n\n\t\t\t\t\ts.fireAsyncError(NewScorchError(\n\t\t\t\t\t\tmerger,\n\t\t\t\t\t\tfmt.Sprintf(\"merging err: %v\", err),\n\t\t\t\t\t\tErrPersist,\n\t\t\t\t\t))\n\t\t\t\t\t_ = ourSnapshot.DecRef()\n\t\t\t\t\tatomic.AddUint64(&s.stats.TotFileMergeLoopErr, 1)\n\t\t\t\t\tcontinue OUTER\n\t\t\t\t}\n\n\t\t\t\tif ctrlMsg.doneCh != nil {\n\t\t\t\t\tclose(ctrlMsg.doneCh)\n\t\t\t\t}\n\t\t\t\tctrlMsg = nil\n\n\t\t\t\tlastEpochMergePlanned = ourSnapshot.epoch\n\n\t\t\t\tatomic.StoreUint64(&s.stats.LastMergedEpoch, ourSnapshot.epoch)\n\n\t\t\t\ts.fireEvent(EventKindMergerProgress, time.Since(startTime))\n\t\t\t}\n\t\t\t_ = ourSnapshot.DecRef()\n\n\t\t\t// tell the persister we're waiting for changes\n\t\t\t// first make a epochWatcher chan\n\t\t\tew := &epochWatcher{\n\t\t\t\tepoch:    lastEpochMergePlanned,\n\t\t\t\tnotifyCh: make(notificationChan, 1),\n\t\t\t}\n\n\t\t\t// give it to the persister\n\t\t\tselect {\n\t\t\tcase <-s.closeCh:\n\t\t\t\tbreak OUTER\n\t\t\tcase s.persisterNotifier <- ew:\n\t\t\tcase ctrlMsg = <-s.forceMergeRequestCh:\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\n\t\t\t// now wait for persister (but also detect close)\n\t\t\tselect {\n\t\t\tcase <-s.closeCh:\n\t\t\t\tbreak OUTER\n\t\t\tcase <-ew.notifyCh:\n\t\t\tcase ctrlMsg = <-s.forceMergeRequestCh:\n\t\t\t}\n\t\t}\n\n\t\tatomic.AddUint64(&s.stats.TotFileMergeLoopEnd, 1)\n\t}\n}\n\ntype mergerCtrl struct {\n\tctx     context.Context\n\toptions *mergeplan.MergePlanOptions\n\tdoneCh  chan struct{}\n}\n\n// ForceMerge helps users trigger a merge operation on\n// an online scorch index.\nfunc (s *Scorch) ForceMerge(ctx context.Context,\n\tmo *mergeplan.MergePlanOptions) error {\n\t// check whether force merge is already under processing\n\ts.rootLock.Lock()\n\tif s.stats.TotFileMergeForceOpsStarted >\n\t\ts.stats.TotFileMergeForceOpsCompleted {\n\t\ts.rootLock.Unlock()\n\t\treturn fmt.Errorf(\"force merge already in progress\")\n\t}\n\n\ts.stats.TotFileMergeForceOpsStarted++\n\ts.rootLock.Unlock()\n\n\tif mo != nil {\n\t\terr := mergeplan.ValidateMergePlannerOptions(mo)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\t// assume the default single segment merge policy\n\t\tmo = &mergeplan.SingleSegmentMergePlanOptions\n\t}\n\tmsg := &mergerCtrl{options: mo,\n\t\tdoneCh: make(chan struct{}),\n\t\tctx:    ctx,\n\t}\n\n\t// request the merger perform a force merge\n\tselect {\n\tcase s.forceMergeRequestCh <- msg:\n\tcase <-s.closeCh:\n\t\treturn nil\n\t}\n\n\t// wait for the force merge operation completion\n\tselect {\n\tcase <-msg.doneCh:\n\t\tatomic.AddUint64(&s.stats.TotFileMergeForceOpsCompleted, 1)\n\tcase <-s.closeCh:\n\t}\n\n\treturn nil\n}\n\nfunc (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions,\n\terror) {\n\tmergePlannerOptions := mergeplan.DefaultMergePlanOptions\n\n\tpo, err := s.parsePersisterOptions()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// by default use the MaxSizeInMemoryMergePerWorker from the persister option\n\t// as the FloorSegmentFileSize for the merge planner which would be the\n\t// first tier size in the planning. If the value is 0, then we don't use the\n\t// file size in the planning.\n\tmergePlannerOptions.FloorSegmentFileSize = int64(po.MaxSizeInMemoryMergePerWorker)\n\n\tif v, ok := s.config[\"scorchMergePlanOptions\"]; ok {\n\t\tb, err := util.MarshalJSON(v)\n\t\tif err != nil {\n\t\t\treturn &mergePlannerOptions, err\n\t\t}\n\n\t\terr = util.UnmarshalJSON(b, &mergePlannerOptions)\n\t\tif err != nil {\n\t\t\treturn &mergePlannerOptions, err\n\t\t}\n\n\t\terr = mergeplan.ValidateMergePlannerOptions(&mergePlannerOptions)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn &mergePlannerOptions, nil\n}\n\ntype closeChWrapper struct {\n\tch1      chan struct{}\n\tctx      context.Context\n\tcloseCh  chan struct{}\n\tcancelCh chan struct{}\n}\n\nfunc newCloseChWrapper(ch1 chan struct{},\n\tctx context.Context) *closeChWrapper {\n\treturn &closeChWrapper{\n\t\tch1:      ch1,\n\t\tctx:      ctx,\n\t\tcloseCh:  make(chan struct{}),\n\t\tcancelCh: make(chan struct{}),\n\t}\n}\n\nfunc (w *closeChWrapper) close() {\n\tclose(w.closeCh)\n}\n\nfunc (w *closeChWrapper) listen() {\n\tselect {\n\tcase <-w.ch1:\n\t\tclose(w.cancelCh)\n\tcase <-w.ctx.Done():\n\t\tclose(w.cancelCh)\n\tcase <-w.closeCh:\n\t}\n}\n\nfunc (s *Scorch) planMergeAtSnapshot(ctx context.Context,\n\toptions *mergeplan.MergePlanOptions, ourSnapshot *IndexSnapshot) error {\n\t// build list of persisted segments in this snapshot\n\tvar onlyPersistedSnapshots []mergeplan.Segment\n\tfor _, segmentSnapshot := range ourSnapshot.segment {\n\t\tif _, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok {\n\t\t\tonlyPersistedSnapshots = append(onlyPersistedSnapshots, segmentSnapshot)\n\t\t}\n\t}\n\n\tatomic.AddUint64(&s.stats.TotFileMergePlan, 1)\n\n\t// give this list to the planner\n\tresultMergePlan, err := mergeplan.Plan(onlyPersistedSnapshots, options)\n\tif err != nil {\n\t\tatomic.AddUint64(&s.stats.TotFileMergePlanErr, 1)\n\t\treturn fmt.Errorf(\"merge planning err: %v\", err)\n\t}\n\tif resultMergePlan == nil {\n\t\t// nothing to do\n\t\tatomic.AddUint64(&s.stats.TotFileMergePlanNone, 1)\n\t\treturn nil\n\t}\n\tatomic.AddUint64(&s.stats.TotFileMergePlanOk, 1)\n\n\tatomic.AddUint64(&s.stats.TotFileMergePlanTasks, uint64(len(resultMergePlan.Tasks)))\n\n\t// process tasks in serial for now\n\tvar filenames []string\n\n\tcw := newCloseChWrapper(s.closeCh, ctx)\n\tdefer cw.close()\n\n\tgo cw.listen()\n\n\tfor _, task := range resultMergePlan.Tasks {\n\t\tif len(task.Segments) == 0 {\n\t\t\tatomic.AddUint64(&s.stats.TotFileMergePlanTasksSegmentsEmpty, 1)\n\t\t\tcontinue\n\t\t}\n\n\t\tatomic.AddUint64(&s.stats.TotFileMergePlanTasksSegments, uint64(len(task.Segments)))\n\n\t\toldMap := make(map[uint64]*SegmentSnapshot, len(task.Segments))\n\t\tnewSegmentID := atomic.AddUint64(&s.nextSegmentID, 1)\n\t\tsegmentsToMerge := make([]segment.Segment, 0, len(task.Segments))\n\t\tdocsToDrop := make([]*roaring.Bitmap, 0, len(task.Segments))\n\t\tmergedSegHistory := make(map[uint64]*mergedSegmentHistory, len(task.Segments))\n\n\t\tfor _, planSegment := range task.Segments {\n\t\t\tif segSnapshot, ok := planSegment.(*SegmentSnapshot); ok {\n\t\t\t\toldMap[segSnapshot.id] = segSnapshot\n\t\t\t\tmergedSegHistory[segSnapshot.id] = &mergedSegmentHistory{\n\t\t\t\t\tworkerID:   0,\n\t\t\t\t\toldSegment: segSnapshot,\n\t\t\t\t}\n\t\t\t\tif persistedSeg, ok := segSnapshot.segment.(segment.PersistedSegment); ok {\n\t\t\t\t\tif segSnapshot.LiveSize() == 0 {\n\t\t\t\t\t\tatomic.AddUint64(&s.stats.TotFileMergeSegmentsEmpty, 1)\n\t\t\t\t\t\toldMap[segSnapshot.id] = nil\n\t\t\t\t\t\tdelete(mergedSegHistory, segSnapshot.id)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tsegmentsToMerge = append(segmentsToMerge, segSnapshot.segment)\n\t\t\t\t\t\tdocsToDrop = append(docsToDrop, segSnapshot.deleted)\n\t\t\t\t\t}\n\t\t\t\t\t// track the files getting merged for unsetting the\n\t\t\t\t\t// removal ineligibility. This helps to unflip files\n\t\t\t\t\t// even with fast merger, slow persister work flows.\n\t\t\t\t\tpath := persistedSeg.Path()\n\t\t\t\t\tfilenames = append(filenames,\n\t\t\t\t\t\tstrings.TrimPrefix(path, s.path+string(os.PathSeparator)))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tvar seg segment.Segment\n\t\tvar filename string\n\t\tif len(segmentsToMerge) > 0 {\n\t\t\tfilename = zapFileName(newSegmentID)\n\t\t\ts.markIneligibleForRemoval(filename)\n\t\t\tpath := s.path + string(os.PathSeparator) + filename\n\n\t\t\tfileMergeZapStartTime := time.Now()\n\n\t\t\tatomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1)\n\t\t\tprevBytesReadTotal := cumulateBytesRead(segmentsToMerge)\n\t\t\tnewDocNums, _, err := s.segPlugin.MergeUsing(segmentsToMerge, docsToDrop, path,\n\t\t\t\tcw.cancelCh, s, s.segmentConfig)\n\t\t\tatomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1)\n\n\t\t\tfileMergeZapTime := uint64(time.Since(fileMergeZapStartTime))\n\t\t\tatomic.AddUint64(&s.stats.TotFileMergeZapTime, fileMergeZapTime)\n\t\t\tif atomic.LoadUint64(&s.stats.MaxFileMergeZapTime) < fileMergeZapTime {\n\t\t\t\tatomic.StoreUint64(&s.stats.MaxFileMergeZapTime, fileMergeZapTime)\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\ts.unmarkIneligibleForRemoval(filename)\n\t\t\t\tatomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1)\n\t\t\t\tif err == segment.ErrClosed {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"merging failed: %v\", err)\n\t\t\t}\n\n\t\t\tseg, err = s.segPlugin.OpenUsing(path, s.segmentConfig)\n\t\t\tif err != nil {\n\t\t\t\ts.unmarkIneligibleForRemoval(filename)\n\t\t\t\tatomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1)\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\ttotalBytesRead := seg.BytesRead() + prevBytesReadTotal\n\t\t\tseg.ResetBytesRead(totalBytesRead)\n\n\t\t\tfor i, segNewDocNums := range newDocNums {\n\t\t\t\tif mergedSegHistory[task.Segments[i].Id()] != nil {\n\t\t\t\t\tmergedSegHistory[task.Segments[i].Id()].oldNewDocIDs = segNewDocNums\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tatomic.AddUint64(&s.stats.TotFileMergeSegments, uint64(len(segmentsToMerge)))\n\t\t}\n\n\t\tsm := &segmentMerge{\n\t\t\tid:               []uint64{newSegmentID},\n\t\t\tmergedSegHistory: mergedSegHistory,\n\t\t\tnew:              []segment.Segment{seg},\n\t\t\tnewCount:         seg.Count(),\n\t\t\tnotifyCh:         make(chan *mergeTaskIntroStatus),\n\t\t\tmmaped:           1,\n\t\t}\n\n\t\ts.fireEvent(EventKindMergeTaskIntroductionStart, 0)\n\n\t\t// give it to the introducer\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\t_ = seg.Close()\n\t\t\treturn segment.ErrClosed\n\t\tcase s.merges <- sm:\n\t\t\tatomic.AddUint64(&s.stats.TotFileMergeIntroductions, 1)\n\t\t}\n\n\t\tintroStartTime := time.Now()\n\t\t// it is safe to blockingly wait for the merge introduction\n\t\t// here as the introducer is bound to handle the notify channel.\n\t\tintroStatus := <-sm.notifyCh\n\t\tintroTime := uint64(time.Since(introStartTime))\n\t\tatomic.AddUint64(&s.stats.TotFileMergeZapIntroductionTime, introTime)\n\t\tif atomic.LoadUint64(&s.stats.MaxFileMergeZapIntroductionTime) < introTime {\n\t\t\tatomic.StoreUint64(&s.stats.MaxFileMergeZapIntroductionTime, introTime)\n\t\t}\n\t\tatomic.AddUint64(&s.stats.TotFileMergeIntroductionsDone, 1)\n\t\tif introStatus != nil && introStatus.indexSnapshot != nil {\n\t\t\t_ = introStatus.indexSnapshot.DecRef()\n\t\t\tif introStatus.skipped {\n\t\t\t\t// close the segment on skipping introduction.\n\t\t\t\ts.unmarkIneligibleForRemoval(filename)\n\t\t\t\t_ = seg.Close()\n\t\t\t}\n\t\t}\n\n\t\tatomic.AddUint64(&s.stats.TotFileMergePlanTasksDone, 1)\n\n\t\ts.fireEvent(EventKindMergeTaskIntroduction, 0)\n\t}\n\n\t// once all the newly merged segment introductions are done,\n\t// its safe to unflip the removal ineligibility for the replaced\n\t// older segments\n\tfor _, f := range filenames {\n\t\ts.unmarkIneligibleForRemoval(f)\n\t}\n\n\treturn nil\n}\n\ntype mergeTaskIntroStatus struct {\n\tindexSnapshot *IndexSnapshot\n\tskipped       bool\n}\n\n// this is important when it comes to introducing multiple merged segments in a\n// single introducer channel push. That way there is a check to ensure that the\n// file count doesn't explode during the index's lifetime.\ntype mergedSegmentHistory struct {\n\tworkerID     uint64\n\toldNewDocIDs []uint64\n\toldSegment   *SegmentSnapshot\n}\n\ntype segmentMerge struct {\n\tid               []uint64\n\tnew              []segment.Segment\n\tmergedSegHistory map[uint64]*mergedSegmentHistory\n\tnotifyCh         chan *mergeTaskIntroStatus\n\tmmaped           uint32\n\tnewCount         uint64\n}\n\nfunc cumulateBytesRead(sbs []segment.Segment) uint64 {\n\tvar rv uint64\n\tfor _, seg := range sbs {\n\t\trv += seg.BytesRead()\n\t}\n\treturn rv\n}\n\nfunc closeNewMergedSegments(segs []segment.Segment) error {\n\tfor _, seg := range segs {\n\t\tif seg != nil {\n\t\t\t_ = seg.DecRef()\n\t\t}\n\t}\n\treturn nil\n}\n\n// mergeAndPersistInMemorySegments takes an IndexSnapshot and a list of in-memory segments,\n// which are merged and persisted to disk concurrently. These are then introduced as\n// the new root snapshot in one-shot.\nfunc (s *Scorch) mergeAndPersistInMemorySegments(snapshot *IndexSnapshot,\n\tflushableObjs []*flushable) (*IndexSnapshot, []uint64, error) {\n\tatomic.AddUint64(&s.stats.TotMemMergeBeg, 1)\n\n\tmemMergeZapStartTime := time.Now()\n\n\tatomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1)\n\n\tvar wg sync.WaitGroup\n\t// we're tracking the merged segments and their doc number per worker\n\t// to be able to introduce them all at once, so the first dimension of the\n\t// slices here correspond to workerID\n\tnewDocIDsSet := make([][][]uint64, len(flushableObjs))\n\tnewMergedSegments := make([]segment.Segment, len(flushableObjs))\n\tnewMergedSegmentIDs := make([]uint64, len(flushableObjs))\n\tnumFlushes := len(flushableObjs)\n\tvar numSegments, newMergedCount uint64\n\tvar em sync.Mutex\n\tvar errs []error\n\n\t// deploy the workers to merge and flush the batches of segments concurrently\n\t// and create a new file segment\n\tfor i := 0; i < numFlushes; i++ {\n\t\twg.Add(1)\n\t\tgo func(segsBatch []segment.Segment, dropsBatch []*roaring.Bitmap, id int) {\n\t\t\tdefer wg.Done()\n\t\t\tnewSegmentID := atomic.AddUint64(&s.nextSegmentID, 1)\n\t\t\tfilename := zapFileName(newSegmentID)\n\t\t\tpath := s.path + string(os.PathSeparator) + filename\n\n\t\t\t// the newly merged segment is already flushed out to disk, just needs\n\t\t\t// to be opened using mmap.\n\t\t\tnewDocIDs, _, err :=\n\t\t\t\ts.segPlugin.MergeUsing(segsBatch, dropsBatch, path, s.closeCh, s, s.segmentConfig)\n\t\t\tif err != nil {\n\t\t\t\tem.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tem.Unlock()\n\t\t\t\tatomic.AddUint64(&s.stats.TotMemMergeErr, 1)\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// to prevent accidental cleanup of this newly created file, mark it\n\t\t\t// as ineligible for removal. this will be flipped back when the bolt\n\t\t\t// is updated - which is valid, since the snapshot updated in bolt is\n\t\t\t// cleaned up only if its zero ref'd (MB-66163 for more details)\n\t\t\ts.markIneligibleForRemoval(filename)\n\t\t\tnewMergedSegmentIDs[id] = newSegmentID\n\t\t\tnewDocIDsSet[id] = newDocIDs\n\t\t\tnewMergedSegments[id], err = s.segPlugin.OpenUsing(path, s.segmentConfig)\n\t\t\tif err != nil {\n\t\t\t\tem.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tem.Unlock()\n\t\t\t\tatomic.AddUint64(&s.stats.TotMemMergeErr, 1)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tatomic.AddUint64(&newMergedCount, newMergedSegments[id].Count())\n\t\t\tatomic.AddUint64(&numSegments, uint64(len(segsBatch)))\n\t\t}(flushableObjs[i].segments, flushableObjs[i].drops, i)\n\t}\n\twg.Wait()\n\n\tif errs != nil {\n\t\t// close the new merged segments\n\t\t_ = closeNewMergedSegments(newMergedSegments)\n\t\tvar errf error\n\t\tfor _, err := range errs {\n\t\t\tif err == segment.ErrClosed {\n\t\t\t\t// the index snapshot was closed which will be handled gracefully\n\t\t\t\t// by retrying the whole merge+flush operation in a later iteration\n\t\t\t\t// so its safe to early exit the same error.\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t\terrf = fmt.Errorf(\"%w; %v\", errf, err)\n\t\t}\n\t\treturn nil, nil, errf\n\t}\n\n\tatomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1)\n\n\tmemMergeZapTime := uint64(time.Since(memMergeZapStartTime))\n\tatomic.AddUint64(&s.stats.TotMemMergeZapTime, memMergeZapTime)\n\tif atomic.LoadUint64(&s.stats.MaxMemMergeZapTime) < memMergeZapTime {\n\t\tatomic.StoreUint64(&s.stats.MaxMemMergeZapTime, memMergeZapTime)\n\t}\n\n\t// update the segmentMerge task with the newly merged + flushed segments which\n\t// are to be introduced atomically.\n\tsm := &segmentMerge{\n\t\tid:               newMergedSegmentIDs,\n\t\tnew:              newMergedSegments,\n\t\tmergedSegHistory: make(map[uint64]*mergedSegmentHistory, numSegments),\n\t\tnotifyCh:         make(chan *mergeTaskIntroStatus),\n\t\tnewCount:         newMergedCount,\n\t}\n\n\t// create a history map which maps the old in-memory segments with the specific\n\t// persister worker (also the specific file segment its going to be part of)\n\t// which flushed it out. This map will be used on the introducer side to out-ref\n\t// the in-memory segments and also track the new tombstones if present.\n\tfor i, flushable := range flushableObjs {\n\t\tfor j, idx := range flushable.sbIdxs {\n\t\t\tss := snapshot.segment[idx]\n\t\t\t// oldSegmentSnapshot.id -> {workerID, oldSegmentSnapshot, docIDs}\n\t\t\tsm.mergedSegHistory[ss.id] = &mergedSegmentHistory{\n\t\t\t\tworkerID:     uint64(i),\n\t\t\t\toldNewDocIDs: newDocIDsSet[i][j],\n\t\t\t\toldSegment:   ss,\n\t\t\t}\n\t\t}\n\t}\n\n\tselect { // send to introducer\n\tcase <-s.closeCh:\n\t\t_ = closeNewMergedSegments(newMergedSegments)\n\t\treturn nil, nil, segment.ErrClosed\n\tcase s.merges <- sm:\n\t}\n\n\t// blockingly wait for the introduction to complete\n\tvar newSnapshot *IndexSnapshot\n\tintroStatus := <-sm.notifyCh\n\tif introStatus != nil && introStatus.indexSnapshot != nil {\n\t\tnewSnapshot = introStatus.indexSnapshot\n\t\tatomic.AddUint64(&s.stats.TotMemMergeSegments, uint64(numSegments))\n\t\tatomic.AddUint64(&s.stats.TotMemMergeDone, 1)\n\t\tif introStatus.skipped {\n\t\t\t// close the segment on skipping introduction.\n\t\t\t_ = newSnapshot.DecRef()\n\t\t\t_ = closeNewMergedSegments(newMergedSegments)\n\t\t\tnewSnapshot = nil\n\t\t}\n\t}\n\n\treturn newSnapshot, newMergedSegmentIDs, nil\n}\n\nfunc (s *Scorch) ReportBytesWritten(bytesWritten uint64) {\n\tatomic.AddUint64(&s.stats.TotFileMergeWrittenBytes, bytesWritten)\n}\n"
  },
  {
    "path": "index/scorch/merge_test.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestObsoleteSegmentMergeIntroduction(t *testing.T) {\n\ttestConfig := CreateConfig(\"TestObsoleteSegmentMergeIntroduction\")\n\terr := InitTest(testConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(testConfig)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar introComplete, mergeIntroStart, mergeIntroComplete sync.WaitGroup\n\tintroComplete.Add(1)\n\tmergeIntroStart.Add(1)\n\tmergeIntroComplete.Add(1)\n\tvar segIntroCompleted int\n\tRegistryEventCallbacks[\"test\"] = func(e Event) bool {\n\t\tswitch e.Kind {\n\t\tcase EventKindBatchIntroduction:\n\t\t\tsegIntroCompleted++\n\t\t\tif segIntroCompleted == 3 {\n\t\t\t\t// all 3 segments introduced\n\t\t\t\tintroComplete.Done()\n\t\t\t}\n\t\tcase EventKindMergeTaskIntroductionStart:\n\t\t\t// signal the start of merge task introduction so that\n\t\t\t// we can introduce a new batch which obsoletes the\n\t\t\t// merged segment's contents.\n\t\t\tmergeIntroStart.Done()\n\t\t\t// hold the merge task introduction until the merged segment contents\n\t\t\t// are obsoleted with the next batch/segment introduction.\n\t\t\tintroComplete.Wait()\n\t\tcase EventKindMergeTaskIntroduction:\n\t\t\t// signal the completion of the merge task introduction.\n\t\t\tmergeIntroComplete.Done()\n\n\t\t}\n\n\t\treturn true\n\t}\n\n\tourConfig := make(map[string]interface{}, len(testConfig))\n\tfor k, v := range testConfig {\n\t\tourConfig[k] = v\n\t}\n\tourConfig[\"eventCallbackName\"] = \"test\"\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, ourConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// first introduce two documents over two batches.\n\tbatch := index.NewBatch()\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test3\")))\n\tbatch.Update(doc)\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tbatch.Reset()\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test2updated\")))\n\tbatch.Update(doc)\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\t// wait until the merger trying to introduce the new merged segment.\n\tmergeIntroStart.Wait()\n\n\t// execute another batch which obsoletes the contents of the new merged\n\t// segment awaiting introduction.\n\tbatch.Reset()\n\tbatch.Delete(\"1\")\n\tbatch.Delete(\"2\")\n\tdoc = document.NewDocument(\"3\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test3updated\")))\n\tbatch.Update(doc)\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\t// wait until the merge task introduction complete.\n\tmergeIntroComplete.Wait()\n\n\tidxr, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tnumSegments := len(idxr.(*IndexSnapshot).segment)\n\tif numSegments != 1 {\n\t\tt.Errorf(\"expected one segment at the root, got: %d\", numSegments)\n\t}\n\n\tskipIntroCount := atomic.LoadUint64(&idxr.(*IndexSnapshot).parent.stats.TotFileMergeIntroductionsObsoleted)\n\tif skipIntroCount != 1 {\n\t\tt.Errorf(\"expected one obsolete merge segment skipping the introduction, got: %d\", skipIntroCount)\n\t}\n\n\tdocCount, err := idxr.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docCount != 1 {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", 1, docCount)\n\t}\n\n\terr = idxr.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n"
  },
  {
    "path": "index/scorch/mergeplan/merge_plan.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package mergeplan provides a segment merge planning approach that's\n// inspired by Lucene's TieredMergePolicy.java and descriptions like\n// http://blog.mikemccandless.com/2011/02/visualizing-lucenes-segment-merges.html\npackage mergeplan\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"sort\"\n\t\"strings\"\n)\n\n// A Segment represents the information that the planner needs to\n// calculate segment merging.\ntype Segment interface {\n\t// Unique id of the segment -- used for sorting.\n\tId() uint64\n\n\t// Full segment size (the size before any logical deletions).\n\tFullSize() int64\n\n\t// Size of the live data of the segment; i.e., FullSize() minus\n\t// any logical deletions.\n\tLiveSize() int64\n\n\tHasVector() bool\n\n\t// Size of the persisted segment file.\n\tFileSize() int64\n}\n\n// Plan() will functionally compute a merge plan.  A segment will be\n// assigned to at most a single MergeTask in the output MergePlan.  A\n// segment not assigned to any MergeTask means the segment should\n// remain unmerged.\nfunc Plan(segments []Segment, o *MergePlanOptions) (*MergePlan, error) {\n\treturn plan(segments, o)\n}\n\n// A MergePlan is the result of the Plan() API.\n//\n// The planner doesn’t know how or whether these tasks are executed --\n// that’s up to a separate merge execution system, which might execute\n// these tasks concurrently or not, and which might execute all the\n// tasks or not.\ntype MergePlan struct {\n\tTasks []*MergeTask\n}\n\n// A MergeTask represents several segments that should be merged\n// together into a single segment.\ntype MergeTask struct {\n\tSegments []Segment\n}\n\n// The MergePlanOptions is designed to be reusable between planning calls.\ntype MergePlanOptions struct {\n\t// Max # segments per logarithmic tier, or max width of any\n\t// logarithmic “step”.  Smaller values mean more merging but fewer\n\t// segments.  Should be >= SegmentsPerMergeTask, else you'll have\n\t// too much merging.\n\tMaxSegmentsPerTier int\n\n\t// Max size of any segment produced after merging.  Actual\n\t// merging, however, may produce segment sizes different than the\n\t// planner’s predicted sizes.\n\tMaxSegmentSize int64\n\n\t// Max size (in bytes) of the persisted segment file that contains the\n\t// vectors.  This is used to prevent merging of segments that\n\t// contain vectors that are too large.\n\tMaxSegmentFileSize int64\n\n\t// The growth factor for each tier in a staircase of idealized\n\t// segments computed by CalcBudget().\n\tTierGrowth float64\n\n\t// The number of segments in any resulting MergeTask.  e.g.,\n\t// len(result.Tasks[ * ].Segments) == SegmentsPerMergeTask.\n\tSegmentsPerMergeTask int\n\n\t// Small segments are rounded up to this size, i.e., treated as\n\t// equal (floor) size for consideration.  This is to prevent lots\n\t// of tiny segments from resulting in a long tail in the index.\n\tFloorSegmentSize int64\n\n\t// Small segments' file size are rounded up to this size to prevent lot\n\t// of tiny segments causing a long tail in the index.\n\tFloorSegmentFileSize int64\n\n\t// Controls how aggressively merges that reclaim more deletions\n\t// are favored.  Higher values will more aggressively target\n\t// merges that reclaim deletions, but be careful not to go so high\n\t// that way too much merging takes place; a value of 3.0 is\n\t// probably nearly too high.  A value of 0.0 means deletions don't\n\t// impact merge selection.\n\tReclaimDeletesWeight float64\n\n\t// Optional, defaults to mergeplan.CalcBudget().\n\tCalcBudget func(totalSize int64, firstTierSize int64,\n\t\to *MergePlanOptions) (budgetNumSegments int)\n\n\t// Optional, defaults to mergeplan.ScoreSegments().\n\tScoreSegments func(segments []Segment, o *MergePlanOptions) float64\n\n\t// Optional.\n\tLogger func(string)\n}\n\n// Returns the higher of the input or FloorSegmentSize.\nfunc (o *MergePlanOptions) RaiseToFloorSegmentSize(s int64) int64 {\n\tif s > o.FloorSegmentSize {\n\t\treturn s\n\t}\n\treturn o.FloorSegmentSize\n}\n\nfunc (o *MergePlanOptions) RaiseToFloorSegmentFileSize(s int64) int64 {\n\tif s > o.FloorSegmentFileSize {\n\t\treturn s\n\t}\n\treturn o.FloorSegmentFileSize\n}\n\n// MaxSegmentSizeLimit represents the maximum size of a segment,\n// this limit comes with hit-1 optimisation/max encoding limit uint31.\nconst MaxSegmentSizeLimit = 1<<31 - 1\n\n// ErrMaxSegmentSizeTooLarge is returned when the size of the segment\n// exceeds the MaxSegmentSizeLimit\nvar ErrMaxSegmentSizeTooLarge = errors.New(\"MaxSegmentSize exceeds the size limit\")\n\n// DefaultMergePlanOptions suggests the default options.\nvar DefaultMergePlanOptions = MergePlanOptions{\n\tMaxSegmentsPerTier:   10,\n\tMaxSegmentSize:       5000000,\n\tMaxSegmentFileSize:   4000000000, // 4GB\n\tTierGrowth:           10.0,\n\tSegmentsPerMergeTask: 10,\n\tFloorSegmentSize:     2000,\n\tReclaimDeletesWeight: 2.0,\n}\n\n// SingleSegmentMergePlanOptions helps in creating a\n// single segment index.\nvar SingleSegmentMergePlanOptions = MergePlanOptions{\n\tMaxSegmentsPerTier:   1,\n\tMaxSegmentSize:       1 << 30,\n\tMaxSegmentFileSize:   1 << 40,\n\tTierGrowth:           1.0,\n\tSegmentsPerMergeTask: 10,\n\tFloorSegmentSize:     1 << 30,\n\tReclaimDeletesWeight: 2.0,\n\tFloorSegmentFileSize: 1 << 40,\n}\n\n// -------------------------------------------\n\nfunc plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) {\n\tif len(segmentsIn) <= 1 {\n\t\treturn nil, nil\n\t}\n\n\tif o == nil {\n\t\to = &DefaultMergePlanOptions\n\t}\n\n\tsegments := append([]Segment(nil), segmentsIn...) // Copy.\n\n\tsort.Sort(byLiveSizeDescending(segments))\n\n\tvar minLiveSize int64 = math.MaxInt64\n\n\tvar eligibles []Segment\n\tvar eligiblesLiveSize int64\n\tvar eligiblesFileSize int64\n\tvar minFileSize int64 = math.MaxInt64\n\n\tfor _, segment := range segments {\n\t\tif minLiveSize > segment.LiveSize() {\n\t\t\tminLiveSize = segment.LiveSize()\n\t\t}\n\n\t\tif minFileSize > segment.FileSize() {\n\t\t\tminFileSize = segment.FileSize()\n\t\t}\n\n\t\tisEligible := segment.LiveSize() < o.MaxSegmentSize/2\n\t\t// An eligible segment (based on #documents) may be too large\n\t\t// and thus need a stricter check based on the file size.\n\t\t// This is particularly important for segments that contain\n\t\t// vectors.\n\t\tif isEligible && segment.HasVector() && o.MaxSegmentFileSize > 0 {\n\t\t\tisEligible = segment.FileSize() < o.MaxSegmentFileSize/2\n\t\t}\n\n\t\t// Only small-enough segments are eligible.\n\t\tif isEligible {\n\t\t\teligibles = append(eligibles, segment)\n\t\t\teligiblesLiveSize += segment.LiveSize()\n\t\t\teligiblesFileSize += segment.FileSize()\n\t\t}\n\t}\n\n\tcalcBudget := o.CalcBudget\n\tif calcBudget == nil {\n\t\tcalcBudget = CalcBudget\n\t}\n\n\tvar budgetNumSegments int\n\tif o.FloorSegmentFileSize > 0 {\n\t\tminFileSize = o.RaiseToFloorSegmentFileSize(minFileSize)\n\t\tbudgetNumSegments = calcBudget(eligiblesFileSize, minFileSize, o)\n\n\t} else {\n\t\tminLiveSize = o.RaiseToFloorSegmentSize(minLiveSize)\n\t\tbudgetNumSegments = calcBudget(eligiblesLiveSize, minLiveSize, o)\n\t}\n\n\tscoreSegments := o.ScoreSegments\n\tif scoreSegments == nil {\n\t\tscoreSegments = ScoreSegments\n\t}\n\n\trv := &MergePlan{}\n\n\tvar empties []Segment\n\tfor _, eligible := range eligibles {\n\t\tif eligible.LiveSize() <= 0 {\n\t\t\tempties = append(empties, eligible)\n\t\t}\n\t}\n\tif len(empties) > 0 {\n\t\trv.Tasks = append(rv.Tasks, &MergeTask{Segments: empties})\n\t\teligibles = removeSegments(eligibles, empties)\n\t}\n\n\t// While we’re over budget, keep looping, which might produce\n\t// another MergeTask.\n\tfor len(eligibles) > 0 && (len(eligibles)+len(rv.Tasks)) > budgetNumSegments {\n\t\t// Track a current best roster as we examine and score\n\t\t// potential rosters of merges.\n\t\tvar bestRoster []Segment\n\t\tvar bestRosterScore float64 // Lower score is better.\n\n\t\tfor startIdx := 0; startIdx < len(eligibles); startIdx++ {\n\t\t\tvar roster []Segment\n\t\t\tvar rosterLiveSize int64\n\t\t\tvar rosterFileSize int64 // useful for segments with vectors\n\n\t\t\tfor idx := startIdx; idx < len(eligibles) && len(roster) < o.SegmentsPerMergeTask; idx++ {\n\t\t\t\teligible := eligibles[idx]\n\n\t\t\t\tif rosterLiveSize+eligible.LiveSize() >= o.MaxSegmentSize {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tif eligible.HasVector() {\n\t\t\t\t\tefs := eligible.FileSize()\n\t\t\t\t\tif rosterFileSize+efs >= o.MaxSegmentFileSize {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\trosterFileSize += efs\n\t\t\t\t}\n\n\t\t\t\troster = append(roster, eligible)\n\t\t\t\trosterLiveSize += eligible.LiveSize()\n\t\t\t}\n\n\t\t\tif len(roster) > 0 {\n\t\t\t\trosterScore := scoreSegments(roster, o)\n\n\t\t\t\tif len(bestRoster) == 0 || rosterScore < bestRosterScore {\n\t\t\t\t\tbestRoster = roster\n\t\t\t\t\tbestRosterScore = rosterScore\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif len(bestRoster) == 0 {\n\t\t\treturn rv, nil\n\t\t}\n\t\t// create tasks with valid merges - i.e. there should be at least 2 non-empty segments\n\t\tif len(bestRoster) > 1 {\n\t\t\trv.Tasks = append(rv.Tasks, &MergeTask{Segments: bestRoster})\n\t\t}\n\n\t\teligibles = removeSegments(eligibles, bestRoster)\n\t}\n\n\treturn rv, nil\n}\n\n// Compute the number of segments that would be needed to cover the\n// totalSize, by climbing up a logarithmically growing staircase of\n// segment tiers.\nfunc CalcBudget(totalSize int64, firstTierSize int64, o *MergePlanOptions) (\n\tbudgetNumSegments int) {\n\ttierSize := firstTierSize\n\tif tierSize < 1 {\n\t\ttierSize = 1\n\t}\n\n\tmaxSegmentsPerTier := o.MaxSegmentsPerTier\n\tif maxSegmentsPerTier < 1 {\n\t\tmaxSegmentsPerTier = 1\n\t}\n\n\ttierGrowth := o.TierGrowth\n\tif tierGrowth < 1.0 {\n\t\ttierGrowth = 1.0\n\t}\n\n\tfor totalSize > 0 {\n\t\tsegmentsInTier := float64(totalSize) / float64(tierSize)\n\t\tif segmentsInTier < float64(maxSegmentsPerTier) {\n\t\t\tbudgetNumSegments += int(math.Ceil(segmentsInTier))\n\t\t\tbreak\n\t\t}\n\n\t\tbudgetNumSegments += maxSegmentsPerTier\n\t\ttotalSize -= int64(maxSegmentsPerTier) * tierSize\n\t\ttierSize = int64(float64(tierSize) * tierGrowth)\n\t}\n\n\treturn budgetNumSegments\n}\n\n// Of note, removeSegments() keeps the ordering of the results stable.\nfunc removeSegments(segments []Segment, toRemove []Segment) []Segment {\n\trv := make([]Segment, 0, len(segments)-len(toRemove))\nOUTER:\n\tfor _, segment := range segments {\n\t\tfor _, r := range toRemove {\n\t\t\tif segment == r {\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\t\t}\n\t\trv = append(rv, segment)\n\t}\n\treturn rv\n}\n\n// Smaller result score is better.\nfunc ScoreSegments(segments []Segment, o *MergePlanOptions) float64 {\n\tvar totBeforeSize int64\n\tvar totAfterSize int64\n\tvar totAfterSizeFloored int64\n\n\tfor _, segment := range segments {\n\t\ttotBeforeSize += segment.FullSize()\n\t\ttotAfterSize += segment.LiveSize()\n\t\ttotAfterSizeFloored += o.RaiseToFloorSegmentSize(segment.LiveSize())\n\t}\n\n\tif totBeforeSize <= 0 || totAfterSize <= 0 || totAfterSizeFloored <= 0 {\n\t\treturn 0\n\t}\n\n\t// Roughly guess the \"balance\" of the segments -- whether the\n\t// segments are about the same size.\n\tbalance :=\n\t\tfloat64(o.RaiseToFloorSegmentSize(segments[0].LiveSize())) /\n\t\t\tfloat64(totAfterSizeFloored)\n\n\t// Gently favor smaller merges over bigger ones.  We don't want to\n\t// make the exponent too large else we end up with poor merges of\n\t// small segments in order to avoid the large merges.\n\tscore := balance * math.Pow(float64(totAfterSize), 0.05)\n\n\t// Strongly favor merges that reclaim deletes.\n\tnonDelRatio := float64(totAfterSize) / float64(totBeforeSize)\n\n\tscore *= math.Pow(nonDelRatio, o.ReclaimDeletesWeight)\n\n\treturn score\n}\n\n// ------------------------------------------\n\n// ToBarChart returns an ASCII rendering of the segments and the plan.\n// The barMax is the max width of the bars in the bar chart.\nfunc ToBarChart(prefix string, barMax int, segments []Segment, plan *MergePlan) string {\n\trv := make([]string, 0, len(segments))\n\n\tvar maxFullSize int64\n\tfor _, segment := range segments {\n\t\tif maxFullSize < segment.FullSize() {\n\t\t\tmaxFullSize = segment.FullSize()\n\t\t}\n\t}\n\tif maxFullSize < 0 {\n\t\tmaxFullSize = 1\n\t}\n\n\tfor _, segment := range segments {\n\t\tbarFull := int(segment.FullSize())\n\t\tbarLive := int(segment.LiveSize())\n\n\t\tif maxFullSize > int64(barMax) {\n\t\t\tbarFull = int(float64(barMax) * float64(barFull) / float64(maxFullSize))\n\t\t\tbarLive = int(float64(barMax) * float64(barLive) / float64(maxFullSize))\n\t\t}\n\n\t\tbarKind := \" \"\n\t\tbarChar := \".\"\n\n\t\tif plan != nil {\n\t\tTASK_LOOP:\n\t\t\tfor taski, task := range plan.Tasks {\n\t\t\t\tfor _, taskSegment := range task.Segments {\n\t\t\t\t\tif taskSegment == segment {\n\t\t\t\t\t\tbarKind = \"*\"\n\t\t\t\t\t\tbarChar = fmt.Sprintf(\"%d\", taski)\n\t\t\t\t\t\tbreak TASK_LOOP\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tbar :=\n\t\t\tstrings.Repeat(barChar, barLive)[0:barLive] +\n\t\t\t\tstrings.Repeat(\"x\", barFull-barLive)[0:barFull-barLive]\n\n\t\trv = append(rv, fmt.Sprintf(\"%s %5d: %5d /%5d - %s %s\", prefix,\n\t\t\tsegment.Id(),\n\t\t\tsegment.LiveSize(),\n\t\t\tsegment.FullSize(),\n\t\t\tbarKind, bar))\n\t}\n\n\treturn strings.Join(rv, \"\\n\")\n}\n\n// ValidateMergePlannerOptions validates the merge planner options\nfunc ValidateMergePlannerOptions(options *MergePlanOptions) error {\n\tif options.MaxSegmentSize > MaxSegmentSizeLimit {\n\t\treturn ErrMaxSegmentSizeTooLarge\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "index/scorch/mergeplan/merge_plan_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mergeplan\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n)\n\n// Implements the Segment interface for testing,\ntype segment struct {\n\tMyId       uint64\n\tMyFullSize int64\n\tMyLiveSize int64\n\n\tMyHasVector bool\n\tMyFileSize  int64\n}\n\nfunc (s *segment) Id() uint64      { return s.MyId }\nfunc (s *segment) FullSize() int64 { return s.MyFullSize }\nfunc (s *segment) LiveSize() int64 { return s.MyLiveSize }\nfunc (s *segment) HasVector() bool { return s.MyHasVector }\nfunc (s *segment) FileSize() int64 { return s.MyFileSize }\n\nfunc makeLinearSegments(n int) (rv []Segment) {\n\tfor i := 0; i < n; i++ {\n\t\trv = append(rv, &segment{\n\t\t\tMyId:       uint64(i),\n\t\t\tMyFullSize: int64(i),\n\t\t\tMyLiveSize: int64(i),\n\t\t})\n\t}\n\treturn rv\n}\n\n// ----------------------------------------\n\nfunc TestSimplePlan(t *testing.T) {\n\tsegs := makeLinearSegments(10)\n\n\ttests := []struct {\n\t\tDesc       string\n\t\tSegments   []Segment\n\t\tOptions    *MergePlanOptions\n\t\tExpectPlan *MergePlan\n\t\tExpectErr  error\n\t}{\n\t\t{\n\t\t\t\"nil segments\",\n\t\t\tnil, nil, nil, nil,\n\t\t},\n\t\t{\n\t\t\t\"empty segments\",\n\t\t\t[]Segment{},\n\t\t\tnil, nil, nil,\n\t\t},\n\t\t{\n\t\t\t\"1 segment\",\n\t\t\t[]Segment{segs[1]},\n\t\t\tnil,\n\t\t\tnil,\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"2 segments\",\n\t\t\t[]Segment{\n\t\t\t\tsegs[1],\n\t\t\t\tsegs[2],\n\t\t\t},\n\t\t\tnil,\n\t\t\t&MergePlan{\n\t\t\t\tTasks: []*MergeTask{\n\t\t\t\t\t{\n\t\t\t\t\t\tSegments: []Segment{\n\t\t\t\t\t\t\tsegs[2],\n\t\t\t\t\t\t\tsegs[1],\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"3 segments\",\n\t\t\t[]Segment{\n\t\t\t\tsegs[1],\n\t\t\t\tsegs[2],\n\t\t\t\tsegs[9],\n\t\t\t},\n\t\t\tnil,\n\t\t\t&MergePlan{\n\t\t\t\tTasks: []*MergeTask{\n\t\t\t\t\t{\n\t\t\t\t\t\tSegments: []Segment{\n\t\t\t\t\t\t\tsegs[9],\n\t\t\t\t\t\t\tsegs[2],\n\t\t\t\t\t\t\tsegs[1],\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t\"many segments\",\n\t\t\t[]Segment{\n\t\t\t\tsegs[1],\n\t\t\t\tsegs[2],\n\t\t\t\tsegs[3],\n\t\t\t\tsegs[4],\n\t\t\t\tsegs[5],\n\t\t\t\tsegs[6],\n\t\t\t},\n\t\t\t&MergePlanOptions{\n\t\t\t\tMaxSegmentsPerTier:   1,\n\t\t\t\tMaxSegmentSize:       1000,\n\t\t\t\tTierGrowth:           2.0,\n\t\t\t\tSegmentsPerMergeTask: 2,\n\t\t\t\tFloorSegmentSize:     1,\n\t\t\t},\n\t\t\t&MergePlan{\n\t\t\t\tTasks: []*MergeTask{\n\t\t\t\t\t{\n\t\t\t\t\t\tSegments: []Segment{\n\t\t\t\t\t\t\tsegs[6],\n\t\t\t\t\t\t\tsegs[5],\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tnil,\n\t\t},\n\t}\n\n\tfor testi, test := range tests {\n\t\tplan, err := Plan(test.Segments, test.Options)\n\n\t\tif err != test.ExpectErr {\n\t\t\ttestj, _ := json.Marshal(&test)\n\n\t\t\tt.Errorf(\"testi: %d, test: %s, got err: %v\", testi, testj, err)\n\t\t}\n\n\t\tif !reflect.DeepEqual(plan, test.ExpectPlan) {\n\t\t\ttestj, _ := json.Marshal(&test)\n\n\t\t\tplanj, _ := json.Marshal(&plan)\n\n\t\t\tt.Errorf(\"testi: %d, test: %s, got plan: %s\",\n\t\t\t\ttesti, testj, planj)\n\t\t}\n\t}\n}\n\n// ----------------------------------------\n\nfunc TestSort(t *testing.T) {\n\tsegs := makeLinearSegments(10)\n\n\tsort.Sort(byLiveSizeDescending(segs))\n\n\tfor i := 1; i < len(segs); i++ {\n\t\tif segs[i].LiveSize() >= segs[i-1].LiveSize() {\n\t\t\tt.Errorf(\"not descending\")\n\t\t}\n\t}\n}\n\n// ----------------------------------------\n\nfunc TestCalcBudget(t *testing.T) {\n\ttests := []struct {\n\t\ttotalSize     int64\n\t\tfirstTierSize int64\n\t\to             MergePlanOptions\n\t\texpect        int\n\t}{\n\t\t{0, 0, MergePlanOptions{}, 0},\n\t\t{1, 0, MergePlanOptions{}, 1},\n\t\t{9, 0, MergePlanOptions{}, 9},\n\t\t{\n\t\t\t1, 1,\n\t\t\tMergePlanOptions{\n\t\t\t\tMaxSegmentsPerTier:   1,\n\t\t\t\tMaxSegmentSize:       1000,\n\t\t\t\tTierGrowth:           2.0,\n\t\t\t\tSegmentsPerMergeTask: 2,\n\t\t\t\tFloorSegmentSize:     1,\n\t\t\t},\n\t\t\t1,\n\t\t},\n\t\t{\n\t\t\t21, 1,\n\t\t\tMergePlanOptions{\n\t\t\t\tMaxSegmentsPerTier:   1,\n\t\t\t\tMaxSegmentSize:       1000,\n\t\t\t\tTierGrowth:           2.0,\n\t\t\t\tSegmentsPerMergeTask: 2,\n\t\t\t\tFloorSegmentSize:     1,\n\t\t\t},\n\t\t\t5,\n\t\t},\n\t\t{\n\t\t\t21, 1,\n\t\t\tMergePlanOptions{\n\t\t\t\tMaxSegmentsPerTier:   2,\n\t\t\t\tMaxSegmentSize:       1000,\n\t\t\t\tTierGrowth:           2.0,\n\t\t\t\tSegmentsPerMergeTask: 2,\n\t\t\t\tFloorSegmentSize:     1,\n\t\t\t},\n\t\t\t7,\n\t\t},\n\t\t{\n\t\t\t1000, 2000, DefaultMergePlanOptions,\n\t\t\t1,\n\t\t},\n\t\t{\n\t\t\t5000, 2000, DefaultMergePlanOptions,\n\t\t\t3,\n\t\t},\n\t\t{\n\t\t\t10000, 2000, DefaultMergePlanOptions,\n\t\t\t5,\n\t\t},\n\t\t{\n\t\t\t30000, 2000, DefaultMergePlanOptions,\n\t\t\t11,\n\t\t},\n\t\t{\n\t\t\t1000000, 2000, DefaultMergePlanOptions,\n\t\t\t24,\n\t\t},\n\t\t{\n\t\t\t1000000000, 2000, DefaultMergePlanOptions,\n\t\t\t54,\n\t\t},\n\t}\n\n\tfor testi, test := range tests {\n\t\tres := CalcBudget(test.totalSize, test.firstTierSize, &test.o)\n\t\tif res != test.expect {\n\t\t\tt.Errorf(\"testi: %d, test: %#v, res: %v\",\n\t\t\t\ttesti, test, res)\n\t\t}\n\t}\n}\n\nfunc TestCalcBudgetForSingleSegmentMergePolicy(t *testing.T) {\n\tmpolicy := MergePlanOptions{\n\t\tMaxSegmentsPerTier:   1,\n\t\tMaxSegmentSize:       1 << 30, // ~ 1 Billion\n\t\tSegmentsPerMergeTask: 10,\n\t\tFloorSegmentSize:     1 << 30,\n\t}\n\n\ttests := []struct {\n\t\ttotalSize     int64\n\t\tfirstTierSize int64\n\t\to             MergePlanOptions\n\t\texpect        int\n\t}{\n\t\t{0, mpolicy.RaiseToFloorSegmentSize(0), mpolicy, 0},\n\t\t{1, mpolicy.RaiseToFloorSegmentSize(1), mpolicy, 1},\n\t\t{9, mpolicy.RaiseToFloorSegmentSize(0), mpolicy, 1},\n\t\t{1, mpolicy.RaiseToFloorSegmentSize(1), mpolicy, 1},\n\t\t{21, mpolicy.RaiseToFloorSegmentSize(21), mpolicy, 1},\n\t\t{21, mpolicy.RaiseToFloorSegmentSize(21), mpolicy, 1},\n\t\t{1000, mpolicy.RaiseToFloorSegmentSize(2000), mpolicy, 1},\n\t\t{5000, mpolicy.RaiseToFloorSegmentSize(5000), mpolicy, 1},\n\t\t{10000, mpolicy.RaiseToFloorSegmentSize(10000), mpolicy, 1},\n\t\t{30000, mpolicy.RaiseToFloorSegmentSize(30000), mpolicy, 1},\n\t\t{1000000, mpolicy.RaiseToFloorSegmentSize(1000000), mpolicy, 1},\n\t\t{1000000000, 1 << 30, mpolicy, 1},\n\t\t{1013423541, 1 << 30, mpolicy, 1},\n\t\t{98765442, 1 << 30, mpolicy, 1},\n\t}\n\n\tfor testi, test := range tests {\n\t\tres := CalcBudget(test.totalSize, test.firstTierSize, &test.o)\n\t\tif res != test.expect {\n\t\t\tt.Errorf(\"testi: %d, test: %#v, res: %v\",\n\t\t\t\ttesti, test, res)\n\t\t}\n\t}\n}\n\n// ----------------------------------------\n\nfunc TestInsert1SameSizedSegmentBetweenMerges(t *testing.T) {\n\to := &MergePlanOptions{\n\t\tMaxSegmentSize:       1000,\n\t\tMaxSegmentsPerTier:   3,\n\t\tTierGrowth:           3.0,\n\t\tSegmentsPerMergeTask: 3,\n\t}\n\n\tspec := testCyclesSpec{\n\t\tdescrip: \"i1sssbm\",\n\t\tverbose: os.Getenv(\"VERBOSE\") == \"i1sssbm\" || os.Getenv(\"VERBOSE\") == \"y\",\n\t\tn:       200,\n\t\to:       o,\n\t\tbeforePlan: func(spec *testCyclesSpec) {\n\t\t\tspec.segments = append(spec.segments, &segment{\n\t\t\t\tMyId:       spec.nextSegmentId,\n\t\t\t\tMyFullSize: 1,\n\t\t\t\tMyLiveSize: 1,\n\t\t\t})\n\t\t\tspec.nextSegmentId++\n\t\t},\n\t}\n\n\tspec.runCycles(t)\n}\n\nfunc TestInsertManySameSizedSegmentsBetweenMerges(t *testing.T) {\n\to := &MergePlanOptions{\n\t\tMaxSegmentSize:       1000,\n\t\tMaxSegmentsPerTier:   3,\n\t\tTierGrowth:           3.0,\n\t\tSegmentsPerMergeTask: 3,\n\t}\n\n\tspec := testCyclesSpec{\n\t\tdescrip: \"imsssbm\",\n\t\tverbose: os.Getenv(\"VERBOSE\") == \"imsssbm\" || os.Getenv(\"VERBOSE\") == \"y\",\n\t\tn:       20,\n\t\to:       o,\n\t\tbeforePlan: func(spec *testCyclesSpec) {\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tspec.segments = append(spec.segments, &segment{\n\t\t\t\t\tMyId:       spec.nextSegmentId,\n\t\t\t\t\tMyFullSize: 1,\n\t\t\t\t\tMyLiveSize: 1,\n\t\t\t\t})\n\t\t\t\tspec.nextSegmentId++\n\t\t\t}\n\t\t},\n\t}\n\n\tspec.runCycles(t)\n}\n\nfunc TestInsertManySameSizedSegmentsWithDeletionsBetweenMerges(t *testing.T) {\n\to := &MergePlanOptions{\n\t\tMaxSegmentSize:       1000,\n\t\tMaxSegmentsPerTier:   3,\n\t\tTierGrowth:           3.0,\n\t\tSegmentsPerMergeTask: 3,\n\t}\n\n\tspec := testCyclesSpec{\n\t\tdescrip: \"imssswdbm\",\n\t\tverbose: os.Getenv(\"VERBOSE\") == \"imssswdbm\" || os.Getenv(\"VERBOSE\") == \"y\",\n\t\tn:       20,\n\t\to:       o,\n\t\tbeforePlan: func(spec *testCyclesSpec) {\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\t// Deletions are a shrinking of the live size.\n\t\t\t\tfor i, seg := range spec.segments {\n\t\t\t\t\tif (spec.cycle+i)%5 == 0 {\n\t\t\t\t\t\ts := seg.(*segment)\n\t\t\t\t\t\tif s.MyLiveSize > 0 {\n\t\t\t\t\t\t\ts.MyLiveSize -= 1\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tspec.segments = append(spec.segments, &segment{\n\t\t\t\t\tMyId:       spec.nextSegmentId,\n\t\t\t\t\tMyFullSize: 1,\n\t\t\t\t\tMyLiveSize: 1,\n\t\t\t\t})\n\t\t\t\tspec.nextSegmentId++\n\t\t\t}\n\t\t},\n\t}\n\n\tspec.runCycles(t)\n}\n\nfunc TestInsertManyDifferentSizedSegmentsBetweenMerges(t *testing.T) {\n\to := &MergePlanOptions{\n\t\tMaxSegmentSize:       1000,\n\t\tMaxSegmentsPerTier:   3,\n\t\tTierGrowth:           3.0,\n\t\tSegmentsPerMergeTask: 3,\n\t}\n\n\tspec := testCyclesSpec{\n\t\tdescrip: \"imdssbm\",\n\t\tverbose: os.Getenv(\"VERBOSE\") == \"imdssbm\" || os.Getenv(\"VERBOSE\") == \"y\",\n\t\tn:       20,\n\t\to:       o,\n\t\tbeforePlan: func(spec *testCyclesSpec) {\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tspec.segments = append(spec.segments, &segment{\n\t\t\t\t\tMyId:       spec.nextSegmentId,\n\t\t\t\t\tMyFullSize: int64(1 + (i % 5)),\n\t\t\t\t\tMyLiveSize: int64(1 + (i % 5)),\n\t\t\t\t})\n\t\t\t\tspec.nextSegmentId++\n\t\t\t}\n\t\t},\n\t}\n\n\tspec.runCycles(t)\n}\n\nfunc TestManySameSizedSegmentsWithDeletesBetweenMerges(t *testing.T) {\n\to := &MergePlanOptions{\n\t\tMaxSegmentSize:       1000,\n\t\tMaxSegmentsPerTier:   3,\n\t\tTierGrowth:           3.0,\n\t\tSegmentsPerMergeTask: 3,\n\t}\n\n\tvar numPlansWithTasks int\n\n\tspec := testCyclesSpec{\n\t\tdescrip: \"mssswdbm\",\n\t\tverbose: os.Getenv(\"VERBOSE\") == \"mssswdbm\" || os.Getenv(\"VERBOSE\") == \"y\",\n\t\tn:       20,\n\t\to:       o,\n\t\tbeforePlan: func(spec *testCyclesSpec) {\n\t\t\t// Deletions are a shrinking of the live size.\n\t\t\tfor i, seg := range spec.segments {\n\t\t\t\tif (spec.cycle+i)%5 == 0 {\n\t\t\t\t\ts := seg.(*segment)\n\t\t\t\t\tif s.MyLiveSize > 0 {\n\t\t\t\t\t\ts.MyLiveSize -= 1\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tspec.segments = append(spec.segments, &segment{\n\t\t\t\t\tMyId:       spec.nextSegmentId,\n\t\t\t\t\tMyFullSize: 1,\n\t\t\t\t\tMyLiveSize: 1,\n\t\t\t\t})\n\t\t\t\tspec.nextSegmentId++\n\t\t\t}\n\t\t},\n\t\tafterPlan: func(spec *testCyclesSpec, plan *MergePlan) {\n\t\t\tif plan != nil && len(plan.Tasks) > 0 {\n\t\t\t\tnumPlansWithTasks++\n\t\t\t}\n\t\t},\n\t}\n\n\tspec.runCycles(t)\n\n\tif numPlansWithTasks <= 0 {\n\t\tt.Errorf(\"expected some plans with tasks\")\n\t}\n}\n\nfunc TestValidateMergePlannerOptions(t *testing.T) {\n\to := &MergePlanOptions{\n\t\tMaxSegmentSize:       1 << 32,\n\t\tMaxSegmentsPerTier:   3,\n\t\tTierGrowth:           3.0,\n\t\tSegmentsPerMergeTask: 3,\n\t}\n\terr := ValidateMergePlannerOptions(o)\n\tif err != ErrMaxSegmentSizeTooLarge {\n\t\tt.Error(\"Validation expected to fail as the MaxSegmentSize exceeds limit\")\n\t}\n}\n\nfunc TestPlanMaxSegmentSizeLimit(t *testing.T) {\n\to := &MergePlanOptions{\n\t\tMaxSegmentSize:       20,\n\t\tMaxSegmentsPerTier:   5,\n\t\tTierGrowth:           3.0,\n\t\tSegmentsPerMergeTask: 5,\n\t\tFloorSegmentSize:     5,\n\t}\n\tsegments := makeLinearSegments(20)\n\n\ts := rand.NewSource(time.Now().UnixNano())\n\tr := rand.New(s)\n\n\tmax := 20\n\tmin := 5\n\trandomInRange := func() int64 {\n\t\treturn int64(r.Intn(max-min) + min)\n\t}\n\tfor i := 1; i < 20; i++ {\n\t\to.MaxSegmentSize = randomInRange()\n\t\tplans, err := Plan(segments, o)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Plan failed, err: %v\", err)\n\t\t}\n\t\tif len(plans.Tasks) == 0 {\n\t\t\tt.Errorf(\"expected some plans with tasks\")\n\t\t}\n\n\t\tfor _, task := range plans.Tasks {\n\t\t\tvar totalLiveSize int64\n\t\t\tfor _, segs := range task.Segments {\n\t\t\t\ttotalLiveSize += segs.LiveSize()\n\t\t\t}\n\t\t\tif totalLiveSize >= o.MaxSegmentSize {\n\t\t\t\tt.Errorf(\"merged segments size: %d exceeding the MaxSegmentSize\"+\n\t\t\t\t\t\"limit: %d\", totalLiveSize, o.MaxSegmentSize)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// ----------------------------------------\n\ntype testCyclesSpec struct {\n\tdescrip string\n\tverbose bool\n\n\tn int // Number of cycles to run.\n\to *MergePlanOptions\n\n\tbeforePlan func(*testCyclesSpec)\n\tafterPlan  func(*testCyclesSpec, *MergePlan)\n\n\tcycle         int\n\tsegments      []Segment\n\tnextSegmentId uint64\n}\n\nfunc (spec *testCyclesSpec) runCycles(t *testing.T) {\n\tnumPlansWithTasks := 0\n\n\tfor spec.cycle < spec.n {\n\t\tif spec.verbose {\n\t\t\temit(spec.descrip, spec.cycle, 0, spec.segments, nil)\n\t\t}\n\n\t\tif spec.beforePlan != nil {\n\t\t\tspec.beforePlan(spec)\n\t\t}\n\n\t\tif spec.verbose {\n\t\t\temit(spec.descrip, spec.cycle, 1, spec.segments, nil)\n\t\t}\n\n\t\tplan, err := Plan(spec.segments, spec.o)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"expected no err, got: %v\", err)\n\t\t}\n\n\t\tif spec.afterPlan != nil {\n\t\t\tspec.afterPlan(spec, plan)\n\t\t}\n\n\t\tif spec.verbose {\n\t\t\temit(spec.descrip, spec.cycle, 2, spec.segments, plan)\n\t\t}\n\n\t\tif plan != nil {\n\t\t\tif len(plan.Tasks) > 0 {\n\t\t\t\tnumPlansWithTasks++\n\t\t\t}\n\n\t\t\tfor _, task := range plan.Tasks {\n\t\t\t\tspec.segments = removeSegments(spec.segments, task.Segments)\n\n\t\t\t\tvar totLiveSize int64\n\t\t\t\tfor _, segment := range task.Segments {\n\t\t\t\t\ttotLiveSize += segment.LiveSize()\n\t\t\t\t}\n\n\t\t\t\tif totLiveSize > 0 {\n\t\t\t\t\tspec.segments = append(spec.segments, &segment{\n\t\t\t\t\t\tMyId:       spec.nextSegmentId,\n\t\t\t\t\t\tMyFullSize: totLiveSize,\n\t\t\t\t\t\tMyLiveSize: totLiveSize,\n\t\t\t\t\t})\n\t\t\t\t\tspec.nextSegmentId++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tspec.cycle++\n\t}\n\n\tif numPlansWithTasks <= 0 {\n\t\tt.Errorf(\"expected some plans with tasks\")\n\t}\n}\n\nfunc emit(descrip string, cycle int, step int, segments []Segment, plan *MergePlan) {\n\tif os.Getenv(\"VERBOSE\") == \"\" {\n\t\treturn\n\t}\n\n\tsuffix := \"\"\n\tif plan != nil && len(plan.Tasks) > 0 {\n\t\tsuffix = \"hasPlan\"\n\t}\n\n\tfmt.Printf(\"%s %d.%d ---------- %s\\n\", descrip, cycle, step, suffix)\n\tfmt.Printf(\"%s\\n\", ToBarChart(descrip, 100, segments, plan))\n}\n\n// -----------------------------------------------------------------------------\n// Test Vector Segment Merging\n\nfunc TestPlanMaxSegmentFileSize(t *testing.T) {\n\ttests := []struct {\n\t\tsegments []Segment\n\t\to        *MergePlanOptions\n\n\t\texpectedTasks [][]uint64\n\t}{\n\t\t{\n\t\t\t[]Segment{\n\t\t\t\t&segment{ // ineligible\n\t\t\t\t\tMyId:       1,\n\t\t\t\t\tMyFullSize: 4000,\n\t\t\t\t\tMyLiveSize: 3900,\n\n\t\t\t\t\tMyHasVector: true,\n\t\t\t\t\tMyFileSize:  3900 * 1000 * 4, // > 2MB\n\t\t\t\t},\n\t\t\t\t&segment{ // ineligible\n\t\t\t\t\tMyId:       2,\n\t\t\t\t\tMyFullSize: 6000,\n\t\t\t\t\tMyLiveSize: 5500, // > 5000\n\n\t\t\t\t\tMyHasVector: true,\n\t\t\t\t\tMyFileSize:  5500 * 1000 * 4, // > 2MB\n\t\t\t\t},\n\t\t\t\t&segment{ // eligible\n\t\t\t\t\tMyId:       3,\n\t\t\t\t\tMyFullSize: 500,\n\t\t\t\t\tMyLiveSize: 490,\n\n\t\t\t\t\tMyHasVector: true,\n\t\t\t\t\tMyFileSize:  490 * 1000 * 4,\n\t\t\t\t},\n\t\t\t\t&segment{ // eligible\n\t\t\t\t\tMyId:       4,\n\t\t\t\t\tMyFullSize: 500,\n\t\t\t\t\tMyLiveSize: 480,\n\n\t\t\t\t\tMyHasVector: true,\n\t\t\t\t\tMyFileSize:  480 * 1000 * 4,\n\t\t\t\t},\n\t\t\t\t&segment{ // eligible\n\t\t\t\t\tMyId:       5,\n\t\t\t\t\tMyFullSize: 500,\n\t\t\t\t\tMyLiveSize: 300,\n\n\t\t\t\t\tMyHasVector: true,\n\t\t\t\t\tMyFileSize:  300 * 1000 * 4,\n\t\t\t\t},\n\t\t\t\t&segment{ // eligible\n\t\t\t\t\tMyId:       6,\n\t\t\t\t\tMyFullSize: 500,\n\t\t\t\t\tMyLiveSize: 400,\n\n\t\t\t\t\tMyHasVector: true,\n\t\t\t\t\tMyFileSize:  400 * 1000 * 4,\n\t\t\t\t},\n\t\t\t},\n\t\t\t&MergePlanOptions{\n\t\t\t\tMaxSegmentSize: 5000, // number of documents\n\t\t\t\t// considering vector dimension as 1000\n\t\t\t\t// vectorBytes = 5000 * 1000 * 4 = 20MB, which is too large\n\t\t\t\t// So, let's set the fileSize limit to 4MB\n\t\t\t\tMaxSegmentFileSize:   4000000, // 4MB\n\t\t\t\tMaxSegmentsPerTier:   1,\n\t\t\t\tSegmentsPerMergeTask: 2,\n\t\t\t\tTierGrowth:           2.0,\n\t\t\t\tFloorSegmentSize:     1,\n\t\t\t},\n\t\t\t[][]uint64{\n\t\t\t\t{3, 4},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor testi, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"Test-%d\", testi), func(t *testing.T) {\n\t\t\tplans, err := Plan(test.segments, test.o)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Plan failed, err: %v\", err)\n\t\t\t}\n\n\t\t\tfor i, task := range plans.Tasks {\n\t\t\t\tvar segIDs []uint64\n\t\t\t\tfor _, seg := range task.Segments {\n\t\t\t\t\tsegIDs = append(segIDs, seg.Id())\n\t\t\t\t}\n\n\t\t\t\tif !reflect.DeepEqual(segIDs, test.expectedTasks[0]) {\n\t\t\t\t\tt.Errorf(\"expected task segments: %v, got: %v\", test.expectedTasks[i], segIDs)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestSingleTaskMergePlan(t *testing.T) {\n\to := &DefaultMergePlanOptions\n\to.FloorSegmentFileSize = 209715200\n\n\t// borrowing the spec values from MB-66112\n\t//\n\t// both segments are eligible, but the roster with a single segment is scored\n\t// higher than the roster with two segments\n\t// in this case the merge plan returns task with a single segment non-empty\n\t// segment which when introduced into the scorch system doesn't cause any change\n\t// and you'd be stuck in an infinite loop where the plan keeps generating the\n\t// same task with the same single segment, which doesn't converge the index to\n\t// a steady state\n\tspec := testCyclesSpec{\n\t\tdescrip: \"mssswdbm\",\n\t\tverbose: os.Getenv(\"VERBOSE\") == \"mssswdbm\" || os.Getenv(\"VERBOSE\") == \"y\",\n\t\to:       o,\n\t\tsegments: []Segment{\n\t\t\t&segment{\n\t\t\t\tMyId:       2,\n\t\t\t\tMyFullSize: 78059,\n\t\t\t\tMyLiveSize: 78059,\n\t\t\t\tMyFileSize: 129475914,\n\t\t\t},\n\t\t\t&segment{\n\t\t\t\tMyId:       1,\n\t\t\t\tMyFullSize: 3959,\n\t\t\t\tMyLiveSize: 3959,\n\t\t\t\tMyFileSize: 24805725,\n\t\t\t},\n\t\t},\n\t}\n\n\tplan, err := Plan(spec.segments, spec.o)\n\tif err != nil {\n\t\tt.Fatalf(\"Plan failed, err: %v\", err)\n\t}\n\n\tif len(plan.Tasks) > 0 {\n\t\tt.Fatalf(\"expected 0 tasks, got: %d\", len(plan.Tasks))\n\t}\n}\n"
  },
  {
    "path": "index/scorch/mergeplan/sort.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mergeplan\n\ntype byLiveSizeDescending []Segment\n\nfunc (a byLiveSizeDescending) Len() int { return len(a) }\n\nfunc (a byLiveSizeDescending) Swap(i, j int) { a[i], a[j] = a[j], a[i] }\n\nfunc (a byLiveSizeDescending) Less(i, j int) bool {\n\tif a[i].LiveSize() != a[j].LiveSize() {\n\t\treturn a[i].LiveSize() > a[j].LiveSize()\n\t}\n\treturn a[i].Id() < a[j].Id()\n}\n"
  },
  {
    "path": "index/scorch/optimize.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"sync/atomic\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\nvar OptimizeConjunction = true\nvar OptimizeConjunctionUnadorned = true\nvar OptimizeDisjunctionUnadorned = true\n\nfunc (s *IndexSnapshotTermFieldReader) Optimize(kind string,\n\toctx index.OptimizableContext) (index.OptimizableContext, error) {\n\tif OptimizeConjunction && kind == \"conjunction\" {\n\t\treturn s.optimizeConjunction(octx)\n\t}\n\n\tif OptimizeConjunctionUnadorned && kind == \"conjunction:unadorned\" {\n\t\treturn s.optimizeConjunctionUnadorned(octx)\n\t}\n\n\tif OptimizeDisjunctionUnadorned && kind == \"disjunction:unadorned\" {\n\t\treturn s.optimizeDisjunctionUnadorned(octx)\n\t}\n\n\treturn nil, nil\n}\n\nvar OptimizeDisjunctionUnadornedMinChildCardinality = uint64(256)\n\n// ----------------------------------------------------------------\n\nfunc (s *IndexSnapshotTermFieldReader) optimizeConjunction(\n\toctx index.OptimizableContext) (index.OptimizableContext, error) {\n\tif octx == nil {\n\t\toctx = &OptimizeTFRConjunction{snapshot: s.snapshot}\n\t}\n\n\to, ok := octx.(*OptimizeTFRConjunction)\n\tif !ok {\n\t\treturn octx, nil\n\t}\n\n\tif o.snapshot != s.snapshot {\n\t\treturn nil, fmt.Errorf(\"tried to optimize conjunction across different snapshots\")\n\t}\n\n\to.tfrs = append(o.tfrs, s)\n\n\treturn o, nil\n}\n\ntype OptimizeTFRConjunction struct {\n\tsnapshot *IndexSnapshot\n\n\ttfrs []*IndexSnapshotTermFieldReader\n}\n\nfunc (o *OptimizeTFRConjunction) Finish() (index.Optimized, error) {\n\tif len(o.tfrs) <= 1 {\n\t\treturn nil, nil\n\t}\n\n\tfor i := range o.snapshot.segment {\n\t\titr0, ok := o.tfrs[0].iterators[i].(segment.OptimizablePostingsIterator)\n\t\tif !ok || itr0.ActualBitmap() == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\titr1, ok := o.tfrs[1].iterators[i].(segment.OptimizablePostingsIterator)\n\t\tif !ok || itr1.ActualBitmap() == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tbm := roaring.And(itr0.ActualBitmap(), itr1.ActualBitmap())\n\n\t\tfor _, tfr := range o.tfrs[2:] {\n\t\t\titr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)\n\t\t\tif !ok || itr.ActualBitmap() == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tbm.And(itr.ActualBitmap())\n\t\t}\n\n\t\t// in this conjunction optimization, the postings iterators\n\t\t// will all share the same AND'ed together actual bitmap.  The\n\t\t// regular conjunction searcher machinery will still be used,\n\t\t// but the underlying bitmap will be smaller.\n\t\tfor _, tfr := range o.tfrs {\n\t\t\titr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)\n\t\t\tif ok && itr.ActualBitmap() != nil {\n\t\t\t\titr.ReplaceActual(bm)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\n// ----------------------------------------------------------------\n\n// An \"unadorned\" conjunction optimization is appropriate when\n// additional or subsidiary information like freq-norm's and\n// term-vectors are not required, and instead only the internal-id's\n// are needed.\nfunc (s *IndexSnapshotTermFieldReader) optimizeConjunctionUnadorned(\n\toctx index.OptimizableContext) (index.OptimizableContext, error) {\n\tif octx == nil {\n\t\toctx = &OptimizeTFRConjunctionUnadorned{snapshot: s.snapshot}\n\t}\n\n\to, ok := octx.(*OptimizeTFRConjunctionUnadorned)\n\tif !ok {\n\t\treturn nil, nil\n\t}\n\n\tif o.snapshot != s.snapshot {\n\t\treturn nil, fmt.Errorf(\"tried to optimize unadorned conjunction across different snapshots\")\n\t}\n\n\to.tfrs = append(o.tfrs, s)\n\n\treturn o, nil\n}\n\ntype OptimizeTFRConjunctionUnadorned struct {\n\tsnapshot *IndexSnapshot\n\n\ttfrs []*IndexSnapshotTermFieldReader\n}\n\nvar OptimizeTFRConjunctionUnadornedTerm = []byte(\"<conjunction:unadorned>\")\nvar OptimizeTFRConjunctionUnadornedField = \"*\"\n\n// Finish of an unadorned conjunction optimization will compute a\n// termFieldReader with an \"actual\" bitmap that represents the\n// constituent bitmaps AND'ed together.  This termFieldReader cannot\n// provide any freq-norm or termVector associated information.\nfunc (o *OptimizeTFRConjunctionUnadorned) Finish() (rv index.Optimized, err error) {\n\tif len(o.tfrs) <= 1 {\n\t\treturn nil, nil\n\t}\n\n\t// We use an artificial term and field because the optimized\n\t// termFieldReader can represent multiple terms and fields.\n\toTFR := o.snapshot.unadornedTermFieldReader(\n\t\tOptimizeTFRConjunctionUnadornedTerm, OptimizeTFRConjunctionUnadornedField)\n\n\tvar actualBMs []*roaring.Bitmap // Collected from regular posting lists.\n\nOUTER:\n\tfor i := range o.snapshot.segment {\n\t\tactualBMs = actualBMs[:0]\n\n\t\tvar docNum1HitLast uint64\n\t\tvar docNum1HitLastOk bool\n\n\t\tfor _, tfr := range o.tfrs {\n\t\t\tif _, ok := tfr.iterators[i].(*emptyPostingsIterator); ok {\n\t\t\t\t// An empty postings iterator means the entire AND is empty.\n\t\t\t\toTFR.iterators[i] = anEmptyPostingsIterator\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\n\t\t\titr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)\n\t\t\tif !ok {\n\t\t\t\t// We only optimize postings iterators that support this operation.\n\t\t\t\treturn nil, nil\n\t\t\t}\n\n\t\t\t// If the postings iterator is \"1-hit\" optimized, then we\n\t\t\t// can perform several optimizations up-front here.\n\t\t\tdocNum1Hit, ok := itr.DocNum1Hit()\n\t\t\tif ok {\n\t\t\t\tif docNum1HitLastOk && docNum1HitLast != docNum1Hit {\n\t\t\t\t\t// The docNum1Hit doesn't match the previous\n\t\t\t\t\t// docNum1HitLast, so the entire AND is empty.\n\t\t\t\t\toTFR.iterators[i] = anEmptyPostingsIterator\n\t\t\t\t\tcontinue OUTER\n\t\t\t\t}\n\n\t\t\t\tdocNum1HitLast = docNum1Hit\n\t\t\t\tdocNum1HitLastOk = true\n\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif itr.ActualBitmap() == nil {\n\t\t\t\t// An empty actual bitmap means the entire AND is empty.\n\t\t\t\toTFR.iterators[i] = anEmptyPostingsIterator\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\n\t\t\t// Collect the actual bitmap for more processing later.\n\t\t\tactualBMs = append(actualBMs, itr.ActualBitmap())\n\t\t}\n\n\t\tif docNum1HitLastOk {\n\t\t\t// We reach here if all the 1-hit optimized posting\n\t\t\t// iterators had the same 1-hit docNum, so we can check if\n\t\t\t// our collected actual bitmaps also have that docNum.\n\t\t\tfor _, bm := range actualBMs {\n\t\t\t\tif !bm.Contains(uint32(docNum1HitLast)) {\n\t\t\t\t\t// The docNum1Hit isn't in one of our actual\n\t\t\t\t\t// bitmaps, so the entire AND is empty.\n\t\t\t\t\toTFR.iterators[i] = anEmptyPostingsIterator\n\t\t\t\t\tcontinue OUTER\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// The actual bitmaps and docNum1Hits all contain or have\n\t\t\t// the same 1-hit docNum, so that's our AND'ed result.\n\t\t\toTFR.iterators[i] = newUnadornedPostingsIteratorFrom1Hit(docNum1HitLast)\n\n\t\t\tcontinue OUTER\n\t\t}\n\n\t\tif len(actualBMs) == 0 {\n\t\t\t// If we've collected no actual bitmaps at this point,\n\t\t\t// then the entire AND is empty.\n\t\t\toTFR.iterators[i] = anEmptyPostingsIterator\n\t\t\tcontinue OUTER\n\t\t}\n\n\t\tif len(actualBMs) == 1 {\n\t\t\t// If we've only 1 actual bitmap, then that's our result.\n\t\t\toTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(actualBMs[0])\n\n\t\t\tcontinue OUTER\n\t\t}\n\n\t\t// Else, AND together our collected bitmaps as our result.\n\t\tbm := roaring.And(actualBMs[0], actualBMs[1])\n\n\t\tfor _, actualBM := range actualBMs[2:] {\n\t\t\tbm.And(actualBM)\n\t\t}\n\n\t\toTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(bm)\n\t}\n\n\tatomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1))\n\treturn oTFR, nil\n}\n\n// ----------------------------------------------------------------\n\n// An \"unadorned\" disjunction optimization is appropriate when\n// additional or subsidiary information like freq-norm's and\n// term-vectors are not required, and instead only the internal-id's\n// are needed.\nfunc (s *IndexSnapshotTermFieldReader) optimizeDisjunctionUnadorned(\n\toctx index.OptimizableContext) (index.OptimizableContext, error) {\n\tif octx == nil {\n\t\toctx = &OptimizeTFRDisjunctionUnadorned{\n\t\t\tsnapshot: s.snapshot,\n\t\t}\n\t}\n\n\to, ok := octx.(*OptimizeTFRDisjunctionUnadorned)\n\tif !ok {\n\t\treturn nil, nil\n\t}\n\n\tif o.snapshot != s.snapshot {\n\t\treturn nil, fmt.Errorf(\"tried to optimize unadorned disjunction across different snapshots\")\n\t}\n\n\to.tfrs = append(o.tfrs, s)\n\n\treturn o, nil\n}\n\ntype OptimizeTFRDisjunctionUnadorned struct {\n\tsnapshot *IndexSnapshot\n\n\ttfrs []*IndexSnapshotTermFieldReader\n}\n\nvar OptimizeTFRDisjunctionUnadornedTerm = []byte(\"<disjunction:unadorned>\")\nvar OptimizeTFRDisjunctionUnadornedField = \"*\"\n\n// Finish of an unadorned disjunction optimization will compute a\n// termFieldReader with an \"actual\" bitmap that represents the\n// constituent bitmaps OR'ed together.  This termFieldReader cannot\n// provide any freq-norm or termVector associated information.\nfunc (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err error) {\n\tif len(o.tfrs) <= 1 {\n\t\treturn nil, nil\n\t}\n\n\tfor i := range o.snapshot.segment {\n\t\tvar cMax uint64\n\n\t\tfor _, tfr := range o.tfrs {\n\t\t\titr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)\n\t\t\tif !ok {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\n\t\t\tif itr.ActualBitmap() != nil {\n\t\t\t\tc := itr.ActualBitmap().GetCardinality()\n\t\t\t\tif cMax < c {\n\t\t\t\t\tcMax = c\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// We use an artificial term and field because the optimized\n\t// termFieldReader can represent multiple terms and fields.\n\toTFR := o.snapshot.unadornedTermFieldReader(\n\t\tOptimizeTFRDisjunctionUnadornedTerm, OptimizeTFRDisjunctionUnadornedField)\n\n\tvar docNums []uint32            // Collected docNum's from 1-hit posting lists.\n\tvar actualBMs []*roaring.Bitmap // Collected from regular posting lists.\n\n\tfor i := range o.snapshot.segment {\n\t\tdocNums = docNums[:0]\n\t\tactualBMs = actualBMs[:0]\n\n\t\tfor _, tfr := range o.tfrs {\n\t\t\titr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)\n\t\t\tif !ok {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\n\t\t\tdocNum, ok := itr.DocNum1Hit()\n\t\t\tif ok {\n\t\t\t\tdocNums = append(docNums, uint32(docNum))\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif itr.ActualBitmap() != nil {\n\t\t\t\tactualBMs = append(actualBMs, itr.ActualBitmap())\n\t\t\t}\n\t\t}\n\n\t\tvar bm *roaring.Bitmap\n\t\tif len(actualBMs) > 2 {\n\t\t\tbm = roaring.HeapOr(actualBMs...)\n\t\t} else if len(actualBMs) == 2 {\n\t\t\tbm = roaring.Or(actualBMs[0], actualBMs[1])\n\t\t} else if len(actualBMs) == 1 {\n\t\t\tbm = actualBMs[0].Clone()\n\t\t}\n\n\t\tif bm == nil {\n\t\t\tbm = roaring.New()\n\t\t}\n\n\t\tbm.AddMany(docNums)\n\n\t\toTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(bm)\n\t}\n\n\tatomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1))\n\treturn oTFR, nil\n}\n\n// ----------------------------------------------------------------\n\nfunc (i *IndexSnapshot) unadornedTermFieldReader(\n\tterm []byte, field string) *IndexSnapshotTermFieldReader {\n\t// This IndexSnapshotTermFieldReader will not be recycled, more\n\t// conversation here: https://github.com/blevesearch/bleve/pull/1438\n\treturn &IndexSnapshotTermFieldReader{\n\t\tterm:               term,\n\t\tfield:              field,\n\t\tsnapshot:           i,\n\t\titerators:          make([]segment.PostingsIterator, len(i.segment)),\n\t\tsegmentOffset:      0,\n\t\tincludeFreq:        false,\n\t\tincludeNorm:        false,\n\t\tincludeTermVectors: false,\n\t\trecycle:            false,\n\t\t// signal downstream that this is a special unadorned termFieldReader\n\t\tunadorned: true,\n\t\t// unadorned TFRs do not require bytes read tracking\n\t\tupdateBytesRead: false,\n\t}\n}\n"
  },
  {
    "path": "index/scorch/optimize_knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage scorch\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment_api \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\ntype OptimizeVR struct {\n\tctx       context.Context\n\tsnapshot  *IndexSnapshot\n\ttotalCost uint64\n\t// maps field to vector readers\n\tvrs map[string][]*IndexSnapshotVectorReader\n}\n\n// This setting _MUST_ only be changed during init and not after.\nvar BleveMaxKNNConcurrency = 10\n\nfunc (o *OptimizeVR) invokeSearcherEndCallback() {\n\tif o.ctx != nil {\n\t\tif cb := o.ctx.Value(search.SearcherEndCallbackKey); cb != nil {\n\t\t\tif cbF, ok := cb.(search.SearcherEndCallbackFn); ok {\n\t\t\t\tif o.totalCost > 0 {\n\t\t\t\t\t// notify the callback that the searcher creation etc. is finished\n\t\t\t\t\t// and report back the total cost for it to track and take actions\n\t\t\t\t\t// appropriately.\n\t\t\t\t\t_ = cbF(o.totalCost)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (o *OptimizeVR) Finish() error {\n\t// for each field, get the vector index --> invoke the zap func.\n\t// for each VR, populate postings list and iterators\n\t// by passing the obtained vector index and getting similar vectors.\n\t// defer close index - just once.\n\tvar errorsM sync.Mutex\n\tvar errors []error\n\n\tdefer o.invokeSearcherEndCallback()\n\n\twg := sync.WaitGroup{}\n\tsemaphore := make(chan struct{}, BleveMaxKNNConcurrency)\n\t// Launch goroutines to get vector index for each segment\n\tfor i, seg := range o.snapshot.segment {\n\t\tif sv, ok := seg.segment.(segment_api.VectorSegment); ok {\n\t\t\twg.Add(1)\n\t\t\tsemaphore <- struct{}{} // Acquire a semaphore slot\n\t\t\tgo func(index int, segment segment_api.VectorSegment, origSeg *SegmentSnapshot) {\n\t\t\t\tdefer func() {\n\t\t\t\t\t<-semaphore // Release the semaphore slot\n\t\t\t\t\twg.Done()\n\t\t\t\t}()\n\t\t\t\tfor field, vrs := range o.vrs {\n\t\t\t\t\t// Early exit if the field is supposed to be completely deleted or\n\t\t\t\t\t// if it's index data has been deleted\n\t\t\t\t\tif info, ok := o.snapshot.updatedFields[field]; ok && (info.Deleted || info.Index) {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\n\t\t\t\t\tvecIndex, err := segment.InterpretVectorIndex(field, origSeg.deleted)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\terrorsM.Lock()\n\t\t\t\t\t\terrors = append(errors, err)\n\t\t\t\t\t\terrorsM.Unlock()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\t\t// update the vector index size as a meta value in the segment snapshot\n\t\t\t\t\tvectorIndexSize := vecIndex.Size()\n\t\t\t\t\torigSeg.cachedMeta.updateMeta(field, vectorIndexSize)\n\t\t\t\t\tfor _, vr := range vrs {\n\t\t\t\t\t\tvar pl segment_api.VecPostingsList\n\t\t\t\t\t\tvar err error\n\n\t\t\t\t\t\t// for each VR, populate postings list and iterators\n\t\t\t\t\t\t// by passing the obtained vector index and getting similar vectors.\n\n\t\t\t\t\t\t// check if the vector reader is configured to use a pre-filter\n\t\t\t\t\t\t// to filter out ineligible documents before performing\n\t\t\t\t\t\t// kNN search.\n\t\t\t\t\t\tif vr.eligibleSelector != nil {\n\t\t\t\t\t\t\tpl, err = vecIndex.SearchWithFilter(vr.vector, vr.k,\n\t\t\t\t\t\t\t\tvr.eligibleSelector.SegmentEligibleDocuments(index), vr.searchParams)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tpl, err = vecIndex.Search(vr.vector, vr.k, vr.searchParams)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\terrorsM.Lock()\n\t\t\t\t\t\t\terrors = append(errors, err)\n\t\t\t\t\t\t\terrorsM.Unlock()\n\t\t\t\t\t\t\tgo vecIndex.Close()\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tatomic.AddUint64(&o.snapshot.parent.stats.TotKNNSearches, uint64(1))\n\n\t\t\t\t\t\t// postings and iterators are already alloc'ed when\n\t\t\t\t\t\t// IndexSnapshotVectorReader is created\n\t\t\t\t\t\tvr.postings[index] = pl\n\t\t\t\t\t\tvr.iterators[index] = pl.Iterator(vr.iterators[index])\n\t\t\t\t\t}\n\t\t\t\t\tgo vecIndex.Close()\n\t\t\t\t}\n\t\t\t}(i, sv, seg)\n\t\t}\n\t}\n\twg.Wait()\n\tclose(semaphore)\n\tif len(errors) > 0 {\n\t\treturn errors[0]\n\t}\n\treturn nil\n}\n\nfunc (s *IndexSnapshotVectorReader) VectorOptimize(ctx context.Context,\n\toctx index.VectorOptimizableContext,\n) (index.VectorOptimizableContext, error) {\n\tif s.snapshot.parent.segPlugin.Version() < VectorSearchSupportedSegmentVersion {\n\t\treturn nil, fmt.Errorf(\"vector search not supported for this index, \"+\n\t\t\t\"index's segment version %v, supported segment version for vector search %v\",\n\t\t\ts.snapshot.parent.segPlugin.Version(), VectorSearchSupportedSegmentVersion)\n\t}\n\n\tif octx == nil {\n\t\toctx = &OptimizeVR{\n\t\t\tsnapshot: s.snapshot,\n\t\t\tvrs:      make(map[string][]*IndexSnapshotVectorReader),\n\t\t}\n\t}\n\n\to, ok := octx.(*OptimizeVR)\n\tif !ok {\n\t\treturn octx, nil\n\t}\n\to.ctx = ctx\n\n\tif o.snapshot != s.snapshot {\n\t\to.invokeSearcherEndCallback()\n\t\treturn nil, fmt.Errorf(\"tried to optimize KNN across different snapshots\")\n\t}\n\n\t// for every searcher creation, consult the segment snapshot to see\n\t// what's the vector index size and since you're anyways going\n\t// to use this vector index to perform the search etc. as part of the Finish()\n\t// perform a check as to whether we allow the searcher creation (the downstream)\n\t// Finish() logic to even occur or not.\n\tvar sumVectorIndexSize uint64\n\tfor _, seg := range o.snapshot.segment {\n\t\tvecIndexSize := seg.cachedMeta.fetchMeta(s.field)\n\t\tif vecIndexSize != nil {\n\t\t\tsumVectorIndexSize += vecIndexSize.(uint64)\n\t\t}\n\t}\n\n\tif o.ctx != nil {\n\t\tif cb := o.ctx.Value(search.SearcherStartCallbackKey); cb != nil {\n\t\t\tif cbF, ok := cb.(search.SearcherStartCallbackFn); ok {\n\t\t\t\terr := cbF(sumVectorIndexSize)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// it's important to invoke the end callback at this point since\n\t\t\t\t\t// if the earlier searchers of this optimize struct were successful\n\t\t\t\t\t// the cost corresponding to it would be incremented and if the\n\t\t\t\t\t// current searcher fails the check then we end up erroring out\n\t\t\t\t\t// the overall optimized searcher creation, the cost needs to be\n\t\t\t\t\t// handled appropriately.\n\t\t\t\t\to.invokeSearcherEndCallback()\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// total cost is essentially the sum of the vector indexes' size across all the\n\t// searchers - all of them end up reading and maintaining a vector index.\n\t// misacconting this value would end up calling the \"end\" callback with a value\n\t// not equal to the value passed to \"start\" callback.\n\to.totalCost += sumVectorIndexSize\n\to.vrs[s.field] = append(o.vrs[s.field], s)\n\treturn o, nil\n}\n"
  },
  {
    "path": "index/scorch/persister.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n\tbolt \"go.etcd.io/bbolt\"\n)\n\nconst persister = \"persister\"\n\n// DefaultPersisterNapTimeMSec is kept to zero as this helps in direct\n// persistence of segments with the default safe batch option.\n// If the default safe batch option results in high number of\n// files on disk, then users may initialise this configuration parameter\n// with higher values so that the persister will nap a bit within it's\n// work loop to favour better in-memory merging of segments to result\n// in fewer segment files on disk. But that may come with an indexing\n// performance overhead.\n// Unsafe batch users are advised to override this to higher value\n// for better performance especially with high data density.\nvar DefaultPersisterNapTimeMSec int = 0 // ms\n\n// DefaultPersisterNapUnderNumFiles helps in controlling the pace of\n// persister. At times of a slow merger progress with heavy file merging\n// operations, its better to pace down the persister for letting the merger\n// to catch up within a range defined by this parameter.\n// Fewer files on disk (as per the merge plan) would result in keeping the\n// file handle usage under limit, faster disk merger and a healthier index.\n// Its been observed that such a loosely sync'ed introducer-persister-merger\n// trio results in better overall performance.\nvar DefaultPersisterNapUnderNumFiles int = 1000\n\nvar DefaultMemoryPressurePauseThreshold uint64 = math.MaxUint64\n\ntype persisterOptions struct {\n\t// PersisterNapTimeMSec controls the wait/delay injected into\n\t// persistence workloop to improve the chances for\n\t// a healthier and heavier in-memory merging\n\tPersisterNapTimeMSec int\n\n\t// PersisterNapTimeMSec > 0, and the number of files is less than\n\t// PersisterNapUnderNumFiles, then the persister will sleep\n\t// PersisterNapTimeMSec amount of time to improve the chances for\n\t// a healthier and heavier in-memory merging\n\tPersisterNapUnderNumFiles int\n\n\t// MemoryPressurePauseThreshold let persister to have a better leeway\n\t// for prudently performing the memory merge of segments on a memory\n\t// pressure situation. Here the config value is an upper threshold\n\t// for the number of paused application threads. The default value would\n\t// be a very high number to always favour the merging of memory segments.\n\tMemoryPressurePauseThreshold uint64\n\n\t// NumPersisterWorkers decides the number of parallel workers that will\n\t// perform the in-memory merge of segments followed by a flush operation.\n\tNumPersisterWorkers int\n\n\t// MaxSizeInMemoryMerge is the maximum size of data that a single persister\n\t// worker is allowed to work on\n\tMaxSizeInMemoryMergePerWorker int\n}\n\ntype notificationChan chan struct{}\n\nfunc (s *Scorch) persisterLoop() {\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\ts.fireAsyncError(NewScorchError(\n\t\t\t\tpersister,\n\t\t\t\tfmt.Sprintf(\"panic: %v, path: %s\", r, s.path),\n\t\t\t\tErrAsyncPanic,\n\t\t\t))\n\t\t}\n\n\t\ts.asyncTasks.Done()\n\t}()\n\n\tvar persistWatchers []*epochWatcher\n\tvar lastPersistedEpoch, lastMergedEpoch uint64\n\tvar ew *epochWatcher\n\n\tvar unpersistedCallbacks []index.BatchCallback\n\n\tpo, err := s.parsePersisterOptions()\n\tif err != nil {\n\t\ts.fireAsyncError(NewScorchError(\n\t\t\tpersister,\n\t\t\tfmt.Sprintf(\"persisterOptions json parsing err: %v\", err),\n\t\t\tErrOptionsParse,\n\t\t))\n\t\treturn\n\t}\n\nOUTER:\n\tfor {\n\t\tatomic.AddUint64(&s.stats.TotPersistLoopBeg, 1)\n\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\tbreak OUTER\n\t\tcase ew = <-s.persisterNotifier:\n\t\t\tpersistWatchers = append(persistWatchers, ew)\n\t\tdefault:\n\t\t}\n\t\tif ew != nil && ew.epoch > lastMergedEpoch {\n\t\t\tlastMergedEpoch = ew.epoch\n\t\t}\n\t\tlastMergedEpoch, persistWatchers = s.pausePersisterForMergerCatchUp(lastPersistedEpoch,\n\t\t\tlastMergedEpoch, persistWatchers, po)\n\n\t\tvar ourSnapshot *IndexSnapshot\n\t\tvar ourPersisted []chan error\n\t\tvar ourPersistedCallbacks []index.BatchCallback\n\n\t\t// check to see if there is a new snapshot to persist\n\t\ts.rootLock.Lock()\n\t\tif s.root != nil && s.root.epoch > lastPersistedEpoch {\n\t\t\tourSnapshot = s.root\n\t\t\tourSnapshot.AddRef()\n\t\t\tourPersisted = s.rootPersisted\n\t\t\ts.rootPersisted = nil\n\t\t\tourPersistedCallbacks = s.persistedCallbacks\n\t\t\ts.persistedCallbacks = nil\n\t\t\tatomic.StoreUint64(&s.iStats.persistSnapshotSize, uint64(ourSnapshot.Size()))\n\t\t\tatomic.StoreUint64(&s.iStats.persistEpoch, ourSnapshot.epoch)\n\t\t}\n\t\ts.rootLock.Unlock()\n\n\t\tif ourSnapshot != nil {\n\t\t\tstartTime := time.Now()\n\n\t\t\terr := s.persistSnapshot(ourSnapshot, po)\n\t\t\tfor _, ch := range ourPersisted {\n\t\t\t\tif err != nil {\n\t\t\t\t\tch <- err\n\t\t\t\t}\n\t\t\t\tclose(ch)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tatomic.StoreUint64(&s.iStats.persistEpoch, 0)\n\t\t\t\tif err == segment.ErrClosed {\n\t\t\t\t\t// index has been closed\n\t\t\t\t\t_ = ourSnapshot.DecRef()\n\t\t\t\t\tbreak OUTER\n\t\t\t\t}\n\n\t\t\t\t// save this current snapshot's persistedCallbacks, to invoke during\n\t\t\t\t// the retry attempt\n\t\t\t\tunpersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...)\n\n\t\t\t\ts.fireAsyncError(NewScorchError(\n\t\t\t\t\tpersister,\n\t\t\t\t\tfmt.Sprintf(\"got err persisting snapshot: %v\", err),\n\t\t\t\t\tErrPersist,\n\t\t\t\t))\n\t\t\t\t_ = ourSnapshot.DecRef()\n\t\t\t\tatomic.AddUint64(&s.stats.TotPersistLoopErr, 1)\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\n\t\t\tif unpersistedCallbacks != nil {\n\t\t\t\t// in the event of this being a retry attempt for persisting a snapshot\n\t\t\t\t// that had earlier failed, prepend the persistedCallbacks associated\n\t\t\t\t// with earlier segment(s) to the latest persistedCallbacks\n\t\t\t\tourPersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...)\n\t\t\t\tunpersistedCallbacks = nil\n\t\t\t}\n\n\t\t\tfor i := range ourPersistedCallbacks {\n\t\t\t\tourPersistedCallbacks[i](err)\n\t\t\t}\n\n\t\t\tatomic.StoreUint64(&s.stats.LastPersistedEpoch, ourSnapshot.epoch)\n\n\t\t\tlastPersistedEpoch = ourSnapshot.epoch\n\t\t\tfor _, ew := range persistWatchers {\n\t\t\t\tclose(ew.notifyCh)\n\t\t\t}\n\n\t\t\tpersistWatchers = nil\n\t\t\t_ = ourSnapshot.DecRef()\n\n\t\t\tchanged := false\n\t\t\ts.rootLock.RLock()\n\t\t\tif s.root != nil && s.root.epoch != lastPersistedEpoch {\n\t\t\t\tchanged = true\n\t\t\t}\n\t\t\ts.rootLock.RUnlock()\n\n\t\t\ts.fireEvent(EventKindPersisterProgress, time.Since(startTime))\n\n\t\t\tif changed {\n\t\t\t\tatomic.AddUint64(&s.stats.TotPersistLoopProgress, 1)\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\t\t}\n\n\t\t// tell the introducer we're waiting for changes\n\t\tw := &epochWatcher{\n\t\t\tepoch:    lastPersistedEpoch,\n\t\t\tnotifyCh: make(notificationChan, 1),\n\t\t}\n\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\tbreak OUTER\n\t\tcase s.introducerNotifier <- w:\n\t\t}\n\n\t\tif ok := s.fireEvent(EventKindPurgerCheck, 0); ok {\n\t\t\ts.removeOldData() // might as well cleanup while waiting\n\t\t}\n\n\t\tatomic.AddUint64(&s.stats.TotPersistLoopWait, 1)\n\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\tbreak OUTER\n\t\tcase <-w.notifyCh:\n\t\t\t// woken up, next loop should pick up work\n\t\t\tatomic.AddUint64(&s.stats.TotPersistLoopWaitNotified, 1)\n\t\tcase ew = <-s.persisterNotifier:\n\t\t\t// if the watchers are already caught up then let them wait,\n\t\t\t// else let them continue to do the catch up\n\t\t\tpersistWatchers = append(persistWatchers, ew)\n\t\t}\n\n\t\tatomic.AddUint64(&s.stats.TotPersistLoopEnd, 1)\n\t}\n}\n\nfunc notifyMergeWatchers(lastPersistedEpoch uint64,\n\tpersistWatchers []*epochWatcher,\n) []*epochWatcher {\n\tvar watchersNext []*epochWatcher\n\tfor _, w := range persistWatchers {\n\t\tif w.epoch < lastPersistedEpoch {\n\t\t\tclose(w.notifyCh)\n\t\t} else {\n\t\t\twatchersNext = append(watchersNext, w)\n\t\t}\n\t}\n\treturn watchersNext\n}\n\nfunc (s *Scorch) pausePersisterForMergerCatchUp(lastPersistedEpoch uint64,\n\tlastMergedEpoch uint64, persistWatchers []*epochWatcher,\n\tpo *persisterOptions,\n) (uint64, []*epochWatcher) {\n\t// First, let the watchers proceed if they lag behind\n\tpersistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers)\n\n\t// Check the merger lag by counting the segment files on disk,\n\tnumFilesOnDisk, _, _ := s.diskFileStats(nil)\n\n\t// On finding fewer files on disk, persister takes a short pause\n\t// for sufficient in-memory segments to pile up for the next\n\t// memory merge cum persist loop.\n\tif numFilesOnDisk < uint64(po.PersisterNapUnderNumFiles) &&\n\t\tpo.PersisterNapTimeMSec > 0 && s.NumEventsBlocking() == 0 {\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\tcase <-time.After(time.Millisecond * time.Duration(po.PersisterNapTimeMSec)):\n\t\t\tatomic.AddUint64(&s.stats.TotPersisterNapPauseCompleted, 1)\n\n\t\tcase ew := <-s.persisterNotifier:\n\t\t\t// unblock the merger in meantime\n\t\t\tpersistWatchers = append(persistWatchers, ew)\n\t\t\tlastMergedEpoch = ew.epoch\n\t\t\tpersistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers)\n\t\t\tatomic.AddUint64(&s.stats.TotPersisterMergerNapBreak, 1)\n\t\t}\n\t\treturn lastMergedEpoch, persistWatchers\n\t}\n\n\t// Finding too many files on disk could be due to two reasons.\n\t// 1. Too many older snapshots awaiting the clean up.\n\t// 2. The merger could be lagging behind on merging the disk files.\n\tif numFilesOnDisk > uint64(po.PersisterNapUnderNumFiles) {\n\t\tif ok := s.fireEvent(EventKindPurgerCheck, 0); ok {\n\t\t\ts.removeOldData()\n\t\t}\n\t\tnumFilesOnDisk, _, _ = s.diskFileStats(nil)\n\t}\n\n\t// Persister pause until the merger catches up to reduce the segment\n\t// file count under the threshold.\n\t// But if there is memory pressure, then skip this sleep maneuvers.\nOUTER:\n\tfor po.PersisterNapUnderNumFiles > 0 &&\n\t\tnumFilesOnDisk >= uint64(po.PersisterNapUnderNumFiles) &&\n\t\tlastMergedEpoch < lastPersistedEpoch {\n\t\tatomic.AddUint64(&s.stats.TotPersisterSlowMergerPause, 1)\n\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\tbreak OUTER\n\t\tcase ew := <-s.persisterNotifier:\n\t\t\tpersistWatchers = append(persistWatchers, ew)\n\t\t\tlastMergedEpoch = ew.epoch\n\t\t}\n\n\t\tatomic.AddUint64(&s.stats.TotPersisterSlowMergerResume, 1)\n\n\t\t// let the watchers proceed if they lag behind\n\t\tpersistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers)\n\n\t\tnumFilesOnDisk, _, _ = s.diskFileStats(nil)\n\t}\n\n\treturn lastMergedEpoch, persistWatchers\n}\n\nfunc (s *Scorch) parsePersisterOptions() (*persisterOptions, error) {\n\tpo := persisterOptions{\n\t\tPersisterNapTimeMSec:          DefaultPersisterNapTimeMSec,\n\t\tPersisterNapUnderNumFiles:     DefaultPersisterNapUnderNumFiles,\n\t\tMemoryPressurePauseThreshold:  DefaultMemoryPressurePauseThreshold,\n\t\tNumPersisterWorkers:           DefaultNumPersisterWorkers,\n\t\tMaxSizeInMemoryMergePerWorker: DefaultMaxSizeInMemoryMergePerWorker,\n\t}\n\tif v, ok := s.config[\"scorchPersisterOptions\"]; ok {\n\t\tb, err := util.MarshalJSON(v)\n\t\tif err != nil {\n\t\t\treturn &po, err\n\t\t}\n\n\t\terr = util.UnmarshalJSON(b, &po)\n\t\tif err != nil {\n\t\t\treturn &po, err\n\t\t}\n\t}\n\treturn &po, nil\n}\n\nfunc (s *Scorch) persistSnapshot(snapshot *IndexSnapshot,\n\tpo *persisterOptions,\n) error {\n\t// Perform in-memory segment merging only when the memory pressure is\n\t// below the configured threshold, else the persister performs the\n\t// direct persistence of segments.\n\tif s.NumEventsBlocking() < po.MemoryPressurePauseThreshold {\n\t\tpersisted, err := s.persistSnapshotMaybeMerge(snapshot, po)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif persisted {\n\t\t\treturn nil\n\t\t}\n\t}\n\n\treturn s.persistSnapshotDirect(snapshot, nil)\n}\n\n// DefaultMinSegmentsForInMemoryMerge represents the default number of\n// in-memory zap segments that persistSnapshotMaybeMerge() needs to\n// see in an IndexSnapshot before it decides to merge and persist\n// those segments\nvar DefaultMinSegmentsForInMemoryMerge = 2\n\ntype flushable struct {\n\tsegments []segment.Segment\n\tdrops    []*roaring.Bitmap\n\tsbIdxs   []int\n\ttotDocs  uint64\n}\n\n// number workers which parallelly perform an in-memory merge of the segments\n// followed by a flush operation.\nvar DefaultNumPersisterWorkers = 1\n\n// maximum size of data that a single worker is allowed to perform the in-memory\n// merge operation.\nvar DefaultMaxSizeInMemoryMergePerWorker = 0\n\nfunc legacyFlushBehaviour(maxSizeInMemoryMergePerWorker, numPersisterWorkers int) bool {\n\t// DefaultMaxSizeInMemoryMergePerWorker = 0 is a special value to preserve the legacy\n\t// one-shot in-memory merge + flush behaviour.\n\treturn maxSizeInMemoryMergePerWorker == 0 && numPersisterWorkers == 1\n}\n\n// persistSnapshotMaybeMerge examines the snapshot and might merge and\n// persist the in-memory zap segments if there are enough of them\nfunc (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot, po *persisterOptions) (\n\tbool, error) {\n\t// collect the in-memory zap segments (SegmentBase instances)\n\tvar sbs []segment.Segment\n\tvar sbsDrops []*roaring.Bitmap\n\tvar sbsIndexes []int\n\tvar oldSegIdxs []int\n\n\tflushSet := make([]*flushable, 0)\n\tvar totSize int\n\tvar numSegsToFlushOut int\n\tvar totDocs uint64\n\n\t// legacy behaviour of merge + flush of all in-memory segments in one-shot\n\tif legacyFlushBehaviour(po.MaxSizeInMemoryMergePerWorker, po.NumPersisterWorkers) {\n\t\tval := &flushable{\n\t\t\tsegments: make([]segment.Segment, 0),\n\t\t\tdrops:    make([]*roaring.Bitmap, 0),\n\t\t\tsbIdxs:   make([]int, 0),\n\t\t\ttotDocs:  totDocs,\n\t\t}\n\t\tfor i, snapshot := range snapshot.segment {\n\t\t\tif _, ok := snapshot.segment.(segment.PersistedSegment); !ok {\n\t\t\t\tval.segments = append(val.segments, snapshot.segment)\n\t\t\t\tval.drops = append(val.drops, snapshot.deleted)\n\t\t\t\tval.sbIdxs = append(val.sbIdxs, i)\n\t\t\t\toldSegIdxs = append(oldSegIdxs, i)\n\t\t\t\tval.totDocs += snapshot.segment.Count()\n\t\t\t\tnumSegsToFlushOut++\n\t\t\t}\n\t\t}\n\n\t\tflushSet = append(flushSet, val)\n\t} else {\n\t\t// constructs a flushSet where each flushable object contains a set of segments\n\t\t// to be merged and flushed out to disk.\n\t\tfor i, snapshot := range snapshot.segment {\n\t\t\tif totSize >= po.MaxSizeInMemoryMergePerWorker &&\n\t\t\t\tlen(sbs) >= DefaultMinSegmentsForInMemoryMerge {\n\t\t\t\tnumSegsToFlushOut += len(sbs)\n\t\t\t\tval := &flushable{\n\t\t\t\t\tsegments: slices.Clone(sbs),\n\t\t\t\t\tdrops:    slices.Clone(sbsDrops),\n\t\t\t\t\tsbIdxs:   slices.Clone(sbsIndexes),\n\t\t\t\t\ttotDocs:  totDocs,\n\t\t\t\t}\n\t\t\t\tflushSet = append(flushSet, val)\n\t\t\t\toldSegIdxs = append(oldSegIdxs, sbsIndexes...)\n\n\t\t\t\tsbs, sbsDrops, sbsIndexes = sbs[:0], sbsDrops[:0], sbsIndexes[:0]\n\t\t\t\ttotSize, totDocs = 0, 0\n\t\t\t}\n\n\t\t\tif len(flushSet) >= int(po.NumPersisterWorkers) {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tif _, ok := snapshot.segment.(segment.PersistedSegment); !ok {\n\t\t\t\tsbs = append(sbs, snapshot.segment)\n\t\t\t\tsbsDrops = append(sbsDrops, snapshot.deleted)\n\t\t\t\tsbsIndexes = append(sbsIndexes, i)\n\t\t\t\ttotDocs += snapshot.segment.Count()\n\t\t\t\ttotSize += snapshot.segment.Size()\n\t\t\t}\n\t\t}\n\t\t// if there were too few segments just merge them all as part of a single worker\n\t\tif len(flushSet) < po.NumPersisterWorkers {\n\t\t\tnumSegsToFlushOut += len(sbs)\n\t\t\tval := &flushable{\n\t\t\t\tsegments: slices.Clone(sbs),\n\t\t\t\tdrops:    slices.Clone(sbsDrops),\n\t\t\t\tsbIdxs:   slices.Clone(sbsIndexes),\n\t\t\t\ttotDocs:  totDocs,\n\t\t\t}\n\t\t\tflushSet = append(flushSet, val)\n\t\t\toldSegIdxs = append(oldSegIdxs, sbsIndexes...)\n\t\t}\n\t}\n\n\tif numSegsToFlushOut < DefaultMinSegmentsForInMemoryMerge {\n\t\treturn false, nil\n\t}\n\n\t// the newSnapshot at this point would contain the newly created file segments\n\t// and updated with the root.\n\tnewSnapshot, newSegmentIDs, err := s.mergeAndPersistInMemorySegments(snapshot, flushSet)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif newSnapshot == nil {\n\t\treturn false, nil\n\t}\n\n\tdefer func() {\n\t\t_ = newSnapshot.DecRef()\n\t}()\n\n\tmergedSegmentIDs := map[uint64]struct{}{}\n\tfor _, idx := range oldSegIdxs {\n\t\tmergedSegmentIDs[snapshot.segment[idx].id] = struct{}{}\n\t}\n\n\tnewMergedSegmentIDs := make(map[uint64]struct{}, len(newSegmentIDs))\n\tfor _, id := range newSegmentIDs {\n\t\tnewMergedSegmentIDs[id] = struct{}{}\n\t}\n\n\t// construct a snapshot that's logically equivalent to the input\n\t// snapshot, but with merged segments replaced by the new segment\n\tequiv := &IndexSnapshot{\n\t\tparent:   snapshot.parent,\n\t\tsegment:  make([]*SegmentSnapshot, 0, len(snapshot.segment)),\n\t\tinternal: snapshot.internal,\n\t\tepoch:    snapshot.epoch,\n\t\tcreator:  \"persistSnapshotMaybeMerge\",\n\t}\n\n\t// to track which segments haven't participated in the in-memory merge\n\t// they won't be flushed out to the disk yet, but in the next cycle will be\n\t// merged in-memory and then flushed out - this is to keep the number of\n\t// on-disk files in limit.\n\texclude := make(map[uint64]struct{})\n\n\t// copy to the equiv the segments that weren't replaced\n\tfor _, segment := range snapshot.segment {\n\t\tif _, wasMerged := mergedSegmentIDs[segment.id]; !wasMerged {\n\t\t\tequiv.segment = append(equiv.segment, segment)\n\t\t\texclude[segment.id] = struct{}{}\n\t\t}\n\t}\n\n\t// append to the equiv the newly merged segments\n\tfor _, segment := range newSnapshot.segment {\n\t\tif _, ok := newMergedSegmentIDs[segment.id]; ok {\n\t\t\tequiv.segment = append(equiv.segment, &SegmentSnapshot{\n\t\t\t\tid:      segment.id,\n\t\t\t\tsegment: segment.segment,\n\t\t\t\tdeleted: nil, // nil since merging handled deletions\n\t\t\t\tstats:   nil,\n\t\t\t})\n\t\t}\n\t}\n\n\terr = s.persistSnapshotDirect(equiv, exclude)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn true, nil\n}\n\nfunc copyToDirectory(srcPath string, d index.Directory) (int64, error) {\n\tif d == nil {\n\t\treturn 0, nil\n\t}\n\n\tdest, err := d.GetWriter(filepath.Join(\"store\", filepath.Base(srcPath)))\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"GetWriter err: %v\", err)\n\t}\n\n\tsourceFileStat, err := os.Stat(srcPath)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif !sourceFileStat.Mode().IsRegular() {\n\t\treturn 0, fmt.Errorf(\"%s is not a regular file\", srcPath)\n\t}\n\n\tsource, err := os.Open(srcPath)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tdefer source.Close()\n\tdefer dest.Close()\n\treturn io.Copy(dest, source)\n}\n\nfunc persistToDirectory(seg segment.UnpersistedSegment, d index.Directory,\n\tpath string,\n) error {\n\tif d == nil {\n\t\treturn seg.Persist(path)\n\t}\n\n\tsg, ok := seg.(io.WriterTo)\n\tif !ok {\n\t\treturn fmt.Errorf(\"no io.WriterTo segment implementation found\")\n\t}\n\n\tw, err := d.GetWriter(filepath.Join(\"store\", filepath.Base(path)))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = sg.WriteTo(w)\n\tw.Close()\n\n\treturn err\n}\n\nfunc prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string,\n\tsegPlugin SegmentPlugin, exclude map[uint64]struct{}, d index.Directory) (\n\t[]string, map[uint64]string, error) {\n\tsnapshotsBucket, err := tx.CreateBucketIfNotExists(util.BoltSnapshotsBucket)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tnewSnapshotKey := encodeUvarintAscending(nil, snapshot.epoch)\n\tsnapshotBucket, err := snapshotsBucket.CreateBucketIfNotExists(newSnapshotKey)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\t// persist meta values\n\tmetaBucket, err := snapshotBucket.CreateBucketIfNotExists(util.BoltMetaDataKey)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\terr = metaBucket.Put(util.BoltMetaDataSegmentTypeKey, []byte(segPlugin.Type()))\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tbuf := make([]byte, binary.MaxVarintLen32)\n\tbinary.BigEndian.PutUint32(buf, segPlugin.Version())\n\terr = metaBucket.Put(util.BoltMetaDataSegmentVersionKey, buf)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\t// Storing the timestamp at which the current indexSnapshot\n\t// was persisted, useful when you want to spread the\n\t// numSnapshotsToKeep reasonably better than consecutive\n\t// epochs.\n\tcurrTimeStamp := time.Now()\n\ttimeStampBinary, err := currTimeStamp.MarshalText()\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\terr = metaBucket.Put(util.BoltMetaDataTimeStamp, timeStampBinary)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\t// persist internal values\n\tinternalBucket, err := snapshotBucket.CreateBucketIfNotExists(util.BoltInternalKey)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\t// TODO optimize writing these in order?\n\tfor k, v := range snapshot.internal {\n\t\terr = internalBucket.Put([]byte(k), v)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\tif snapshot.parent != nil {\n\t\tval := make([]byte, 8)\n\t\tbytesWritten := atomic.LoadUint64(&snapshot.parent.stats.TotBytesWrittenAtIndexTime)\n\t\tbinary.LittleEndian.PutUint64(val, bytesWritten)\n\t\terr = internalBucket.Put(util.TotBytesWrittenKey, val)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\tfilenames := make([]string, 0, len(snapshot.segment))\n\tnewSegmentPaths := make(map[uint64]string, len(snapshot.segment))\n\n\t// first ensure that each segment in this snapshot has been persisted\n\tfor _, segmentSnapshot := range snapshot.segment {\n\t\tsnapshotSegmentKey := encodeUvarintAscending(nil, segmentSnapshot.id)\n\t\tsnapshotSegmentBucket, err := snapshotBucket.CreateBucketIfNotExists(snapshotSegmentKey)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tswitch seg := segmentSnapshot.segment.(type) {\n\t\tcase segment.PersistedSegment:\n\t\t\tsegPath := seg.Path()\n\t\t\t_, err = copyToDirectory(segPath, d)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, fmt.Errorf(\"segment: %s copy err: %v\", segPath, err)\n\t\t\t}\n\t\t\tfilename := filepath.Base(segPath)\n\t\t\terr = snapshotSegmentBucket.Put(util.BoltPathKey, []byte(filename))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t\tfilenames = append(filenames, filename)\n\t\tcase segment.UnpersistedSegment:\n\t\t\t// need to persist this to disk if its not part of exclude list (which\n\t\t\t// restricts which in-memory segment to be persisted to disk)\n\t\t\tif _, ok := exclude[segmentSnapshot.id]; !ok {\n\t\t\t\tfilename := zapFileName(segmentSnapshot.id)\n\t\t\t\tpath := filepath.Join(path, filename)\n\t\t\t\terr := persistToDirectory(seg, d, path)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, nil, fmt.Errorf(\"segment: %s persist err: %v\", path, err)\n\t\t\t\t}\n\t\t\t\tnewSegmentPaths[segmentSnapshot.id] = path\n\t\t\t\terr = snapshotSegmentBucket.Put(util.BoltPathKey, []byte(filename))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, nil, err\n\t\t\t\t}\n\t\t\t\tfilenames = append(filenames, filename)\n\t\t\t}\n\t\tdefault:\n\t\t\treturn nil, nil, fmt.Errorf(\"unknown segment type: %T\", seg)\n\t\t}\n\t\t// store current deleted bits\n\t\tvar roaringBuf bytes.Buffer\n\t\tif segmentSnapshot.deleted != nil {\n\t\t\t_, err = segmentSnapshot.deleted.WriteTo(&roaringBuf)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, fmt.Errorf(\"error persisting roaring bytes: %v\", err)\n\t\t\t}\n\t\t\terr = snapshotSegmentBucket.Put(util.BoltDeletedKey, roaringBuf.Bytes())\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t}\n\n\t\t// store segment stats\n\t\tif segmentSnapshot.stats != nil {\n\t\t\tb, err := json.Marshal(segmentSnapshot.stats.Fetch())\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t\terr = snapshotSegmentBucket.Put(util.BoltStatsKey, b)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t}\n\n\t\t// store updated field info\n\t\tif segmentSnapshot.updatedFields != nil {\n\t\t\tb, err := json.Marshal(segmentSnapshot.updatedFields)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t\terr = snapshotSegmentBucket.Put(util.BoltUpdatedFieldsKey, b)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn filenames, newSegmentPaths, nil\n}\n\nfunc (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot, exclude map[uint64]struct{}) (err error) {\n\t// start a write transaction\n\ttx, err := s.rootBolt.Begin(true)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// defer rollback on error\n\tdefer func() {\n\t\tif err != nil {\n\t\t\t_ = tx.Rollback()\n\t\t}\n\t}()\n\n\tfilenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin, exclude, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// we need to swap in a new root only when we've persisted 1 or\n\t// more segments -- whereby the new root would have 1-for-1\n\t// replacements of in-memory segments with file-based segments\n\t//\n\t// other cases like updates to internal values only, and/or when\n\t// there are only deletions, are already covered and persisted by\n\t// the newly populated boltdb snapshotBucket above\n\tif len(newSegmentPaths) > 0 {\n\t\t// now try to open all the new snapshots\n\t\tnewSegments := make(map[uint64]segment.Segment, len(newSegmentPaths))\n\t\tdefer func() {\n\t\t\tfor _, s := range newSegments {\n\t\t\t\tif s != nil {\n\t\t\t\t\t// cleanup segments that were opened but not\n\t\t\t\t\t// swapped into the new root\n\t\t\t\t\t_ = s.Close()\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t\tfor segmentID, path := range newSegmentPaths {\n\t\t\tnewSegments[segmentID], err = s.segPlugin.OpenUsing(path, s.segmentConfig)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"error opening new segment at %s, %v\", path, err)\n\t\t\t}\n\t\t}\n\n\t\tpersist := &persistIntroduction{\n\t\t\tpersisted: newSegments,\n\t\t\tapplied:   make(notificationChan),\n\t\t}\n\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\treturn segment.ErrClosed\n\t\tcase s.persists <- persist:\n\t\t}\n\n\t\tselect {\n\t\tcase <-s.closeCh:\n\t\t\treturn segment.ErrClosed\n\t\tcase <-persist.applied:\n\t\t}\n\t}\n\n\terr = tx.Commit()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = s.rootBolt.Sync()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// allow files to become eligible for removal after commit, such\n\t// as file segments from snapshots that came from the merger\n\ts.rootLock.Lock()\n\tfor _, filename := range filenames {\n\t\tdelete(s.ineligibleForRemoval, filename)\n\t}\n\ts.rootLock.Unlock()\n\n\treturn nil\n}\n\nfunc zapFileName(epoch uint64) string {\n\treturn fmt.Sprintf(\"%012x.zap\", epoch)\n}\n\n// bolt snapshot code\n\nfunc (s *Scorch) loadFromBolt() error {\n\terr := s.rootBolt.View(func(tx *bolt.Tx) error {\n\t\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\t\tif snapshots == nil {\n\t\t\treturn nil\n\t\t}\n\t\tfoundRoot := false\n\t\tc := snapshots.Cursor()\n\t\tfor k, _ := c.Last(); k != nil; k, _ = c.Prev() {\n\t\t\t_, snapshotEpoch, err := decodeUvarintAscending(k)\n\t\t\tif err != nil {\n\t\t\t\tlog.Printf(\"unable to parse segment epoch %x, continuing\", k)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif foundRoot {\n\t\t\t\ts.AddEligibleForRemoval(snapshotEpoch)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsnapshot := snapshots.Bucket(k)\n\t\t\tif snapshot == nil {\n\t\t\t\tlog.Printf(\"snapshot key, but bucket missing %x, continuing\", k)\n\t\t\t\ts.AddEligibleForRemoval(snapshotEpoch)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tindexSnapshot, err := s.loadSnapshot(snapshot)\n\t\t\tif err != nil {\n\t\t\t\tlog.Printf(\"unable to load snapshot, %v, continuing\", err)\n\t\t\t\ts.AddEligibleForRemoval(snapshotEpoch)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tindexSnapshot.epoch = snapshotEpoch\n\t\t\t// set the nextSegmentID\n\t\t\ts.nextSegmentID, err = s.maxSegmentIDOnDisk()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ts.nextSegmentID++\n\t\t\ts.rootLock.Lock()\n\t\t\ts.nextSnapshotEpoch = snapshotEpoch + 1\n\t\t\trootPrev := s.root\n\t\t\ts.root = indexSnapshot\n\t\t\ts.rootLock.Unlock()\n\n\t\t\tif rootPrev != nil {\n\t\t\t\t_ = rootPrev.DecRef()\n\t\t\t}\n\n\t\t\tfoundRoot = true\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpersistedSnapshots, err := s.rootBoltSnapshotMetaData()\n\tif err != nil {\n\t\treturn err\n\t}\n\ts.checkPoints = persistedSnapshots\n\treturn nil\n}\n\n// LoadSnapshot loads the segment with the specified epoch\n// NOTE: this is currently ONLY intended to be used by the command-line tool\nfunc (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) {\n\terr = s.rootBolt.View(func(tx *bolt.Tx) error {\n\t\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\t\tif snapshots == nil {\n\t\t\treturn nil\n\t\t}\n\t\tsnapshotKey := encodeUvarintAscending(nil, epoch)\n\t\tsnapshot := snapshots.Bucket(snapshotKey)\n\t\tif snapshot == nil {\n\t\t\treturn fmt.Errorf(\"snapshot with epoch: %v - doesn't exist\", epoch)\n\t\t}\n\t\trv, err = s.loadSnapshot(snapshot)\n\t\treturn err\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) {\n\trv := &IndexSnapshot{\n\t\tparent:   s,\n\t\tinternal: make(map[string][]byte),\n\t\trefs:     1,\n\t\tcreator:  \"loadSnapshot\",\n\t}\n\t// first we look for the meta-data bucket, this will tell us\n\t// which segment type/version was used for this snapshot\n\t// all operations for this scorch will use this type/version\n\tmetaBucket := snapshot.Bucket(util.BoltMetaDataKey)\n\tif metaBucket == nil {\n\t\t_ = rv.DecRef()\n\t\treturn nil, fmt.Errorf(\"meta-data bucket missing\")\n\t}\n\tsegmentType := string(metaBucket.Get(util.BoltMetaDataSegmentTypeKey))\n\tsegmentVersion := binary.BigEndian.Uint32(\n\t\tmetaBucket.Get(util.BoltMetaDataSegmentVersionKey))\n\terr := s.loadSegmentPlugin(segmentType, segmentVersion)\n\tif err != nil {\n\t\t_ = rv.DecRef()\n\t\treturn nil, fmt.Errorf(\n\t\t\t\"unable to load correct segment wrapper: %v\", err)\n\t}\n\tvar running uint64\n\tc := snapshot.Cursor()\n\tfor k, _ := c.First(); k != nil; k, _ = c.Next() {\n\t\tif k[0] == util.BoltInternalKey[0] {\n\t\t\tinternalBucket := snapshot.Bucket(k)\n\t\t\tif internalBucket == nil {\n\t\t\t\t_ = rv.DecRef()\n\t\t\t\treturn nil, fmt.Errorf(\"internal bucket missing\")\n\t\t\t}\n\t\t\terr := internalBucket.ForEach(func(key []byte, val []byte) error {\n\t\t\t\tcopiedVal := append([]byte(nil), val...)\n\t\t\t\trv.internal[string(key)] = copiedVal\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\t_ = rv.DecRef()\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t} else if k[0] != util.BoltMetaDataKey[0] {\n\t\t\tsegmentBucket := snapshot.Bucket(k)\n\t\t\tif segmentBucket == nil {\n\t\t\t\t_ = rv.DecRef()\n\t\t\t\treturn nil, fmt.Errorf(\"segment key, but bucket missing %x\", k)\n\t\t\t}\n\t\t\tsegmentSnapshot, err := s.loadSegment(segmentBucket)\n\t\t\tif err != nil {\n\t\t\t\t_ = rv.DecRef()\n\t\t\t\treturn nil, fmt.Errorf(\"failed to load segment: %v\", err)\n\t\t\t}\n\t\t\t_, segmentSnapshot.id, err = decodeUvarintAscending(k)\n\t\t\tif err != nil {\n\t\t\t\t_ = rv.DecRef()\n\t\t\t\treturn nil, fmt.Errorf(\"failed to decode segment id: %v\", err)\n\t\t\t}\n\t\t\trv.segment = append(rv.segment, segmentSnapshot)\n\t\t\trv.offsets = append(rv.offsets, running)\n\t\t\t// Merge all segment level updated field info for use during queries\n\t\t\tif segmentSnapshot.updatedFields != nil {\n\t\t\t\trv.MergeUpdateFieldsInfo(segmentSnapshot.updatedFields)\n\t\t\t}\n\t\t\trunning += segmentSnapshot.segment.Count()\n\t\t}\n\t}\n\treturn rv, nil\n}\n\nfunc (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, error) {\n\tpathBytes := segmentBucket.Get(util.BoltPathKey)\n\tif pathBytes == nil {\n\t\treturn nil, fmt.Errorf(\"segment path missing\")\n\t}\n\tsegmentPath := s.path + string(os.PathSeparator) + string(pathBytes)\n\tseg, err := s.segPlugin.OpenUsing(segmentPath, s.segmentConfig)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error opening bolt segment: %v\", err)\n\t}\n\n\trv := &SegmentSnapshot{\n\t\tsegment:    seg,\n\t\tcachedDocs: &cachedDocs{cache: nil},\n\t\tcachedMeta: &cachedMeta{meta: nil},\n\t}\n\tdeletedBytes := segmentBucket.Get(util.BoltDeletedKey)\n\tif deletedBytes != nil {\n\t\tdeletedBitmap := roaring.NewBitmap()\n\t\tr := bytes.NewReader(deletedBytes)\n\t\t_, err := deletedBitmap.ReadFrom(r)\n\t\tif err != nil {\n\t\t\t_ = seg.Close()\n\t\t\treturn nil, fmt.Errorf(\"error reading deleted bytes: %v\", err)\n\t\t}\n\t\tif !deletedBitmap.IsEmpty() {\n\t\t\trv.deleted = deletedBitmap\n\t\t}\n\t}\n\tstatBytes := segmentBucket.Get(util.BoltStatsKey)\n\tif statBytes != nil {\n\t\tvar statsMap map[string]map[string]uint64\n\n\t\terr := json.Unmarshal(statBytes, &statsMap)\n\t\tstats := &fieldStats{statMap: statsMap}\n\t\tif err != nil {\n\t\t\t_ = seg.Close()\n\t\t\treturn nil, fmt.Errorf(\"error reading stat bytes: %v\", err)\n\t\t}\n\t\trv.stats = stats\n\t}\n\tupdatedFieldBytes := segmentBucket.Get(util.BoltUpdatedFieldsKey)\n\tif updatedFieldBytes != nil {\n\t\tvar updatedFields map[string]*index.UpdateFieldInfo\n\n\t\terr := json.Unmarshal(updatedFieldBytes, &updatedFields)\n\t\tif err != nil {\n\t\t\t_ = seg.Close()\n\t\t\treturn nil, fmt.Errorf(\"error reading updated field bytes: %v\", err)\n\t\t}\n\t\trv.updatedFields = updatedFields\n\t\t// Set the value within the segment base for use during merge\n\t\trv.UpdateFieldsInfo(rv.updatedFields)\n\t}\n\n\treturn rv, nil\n}\n\nfunc (s *Scorch) removeOldData() {\n\tremoved, err := s.removeOldBoltSnapshots()\n\tif err != nil {\n\t\ts.fireAsyncError(NewScorchError(\n\t\t\tpersister,\n\t\t\tfmt.Sprintf(\"got err removing old bolt snapshots: %v\", err),\n\t\t\tErrCleanup,\n\t\t))\n\t}\n\tatomic.AddUint64(&s.stats.TotSnapshotsRemovedFromMetaStore, uint64(removed))\n\n\terr = s.removeOldZapFiles()\n\tif err != nil {\n\t\ts.fireAsyncError(NewScorchError(\n\t\t\tpersister,\n\t\t\tfmt.Sprintf(\"got err removing old zap files: %v\", err),\n\t\t\tErrCleanup,\n\t\t))\n\t}\n}\n\n// NumSnapshotsToKeep represents how many recent, old snapshots to\n// keep around per Scorch instance.  Useful for apps that require\n// rollback'ability.\nvar NumSnapshotsToKeep = 1\n\n// RollbackSamplingInterval controls how far back we are looking\n// in the history to get the rollback points.\n// For example, a value of 10 minutes ensures that the\n// protected snapshots (NumSnapshotsToKeep = 3) are:\n//\n//\tthe very latest snapshot(ie the current one),\n//\tthe snapshot that was persisted 10 minutes before the current one,\n//\tthe snapshot that was persisted 20 minutes before the current one\n//\n// By default however, the timeseries way of protecting snapshots is\n// disabled, and we protect the latest three contiguous snapshots\nvar RollbackSamplingInterval = 0 * time.Minute\n\n// Controls what portion of the earlier rollback points to retain during\n// a infrequent/sparse mutation scenario\nvar RollbackRetentionFactor = float64(0.5)\n\nfunc getTimeSeriesSnapshots(maxDataPoints int, interval time.Duration,\n\tsnapshots []*snapshotMetaData,\n) (int, map[uint64]time.Time) {\n\tif interval == 0 {\n\t\treturn len(snapshots), map[uint64]time.Time{}\n\t}\n\t// the map containing the time series snapshots, i.e the timeseries of snapshots\n\t// each of which is separated by rollbackSamplingInterval\n\trv := make(map[uint64]time.Time)\n\t// the last point in the \"time series\", i.e. the timeseries of snapshots\n\t// each of which is separated by rollbackSamplingInterval\n\tptr := len(snapshots) - 1\n\trv[snapshots[ptr].epoch] = snapshots[ptr].timeStamp\n\tnumSnapshotsProtected := 1\n\n\t// traverse the list in reverse order, older timestamps to newer ones.\n\tfor i := ptr - 1; i >= 0; i-- {\n\t\t// If we find a timeStamp which is the next datapoint in our\n\t\t// timeseries of snapshots, and newer by RollbackSamplingInterval duration\n\t\t// (comparison in terms of minutes), which is the interval of our time\n\t\t// series. In this case, add the epoch rv\n\t\tif snapshots[i].timeStamp.Sub(snapshots[ptr].timeStamp).Minutes() >\n\t\t\tinterval.Minutes() {\n\t\t\tif _, ok := rv[snapshots[i+1].epoch]; !ok {\n\t\t\t\trv[snapshots[i+1].epoch] = snapshots[i+1].timeStamp\n\t\t\t\tptr = i + 1\n\t\t\t\tnumSnapshotsProtected++\n\t\t\t}\n\t\t} else if snapshots[i].timeStamp.Sub(snapshots[ptr].timeStamp).Minutes() ==\n\t\t\tinterval.Minutes() {\n\t\t\tif _, ok := rv[snapshots[i].epoch]; !ok {\n\t\t\t\trv[snapshots[i].epoch] = snapshots[i].timeStamp\n\t\t\t\tptr = i\n\t\t\t\tnumSnapshotsProtected++\n\t\t\t}\n\t\t}\n\n\t\tif numSnapshotsProtected >= maxDataPoints {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn ptr, rv\n}\n\n// getProtectedSnapshots aims to fetch the epochs keep based on a timestamp basis.\n// It tries to get NumSnapshotsToKeep snapshots, each of which are separated\n// by a time duration of RollbackSamplingInterval.\nfunc getProtectedSnapshots(rollbackSamplingInterval time.Duration,\n\tnumSnapshotsToKeep int,\n\tpersistedSnapshots []*snapshotMetaData,\n) map[uint64]time.Time {\n\t// keep numSnapshotsToKeep - 1 worth of time series snapshots, because we always\n\t// must preserve the very latest snapshot in bolt as well to avoid accidental\n\t// deletes of bolt entries and cleanups by the purger code.\n\tlastPoint, protectedEpochs := getTimeSeriesSnapshots(numSnapshotsToKeep-1,\n\t\trollbackSamplingInterval, persistedSnapshots)\n\tif len(protectedEpochs) < numSnapshotsToKeep {\n\t\tnumSnapshotsNeeded := numSnapshotsToKeep - len(protectedEpochs)\n\t\t// we protected the contiguous snapshots from the last point in time series\n\t\tfor i := 0; i < numSnapshotsNeeded && i < lastPoint; i++ {\n\t\t\tprotectedEpochs[persistedSnapshots[i].epoch] = persistedSnapshots[i].timeStamp\n\t\t}\n\t}\n\n\treturn protectedEpochs\n}\n\nfunc newCheckPoints(snapshots map[uint64]time.Time) []*snapshotMetaData {\n\trv := make([]*snapshotMetaData, 0)\n\n\tkeys := make([]uint64, 0, len(snapshots))\n\tfor k := range snapshots {\n\t\tkeys = append(keys, k)\n\t}\n\n\tsort.SliceStable(keys, func(i, j int) bool {\n\t\treturn snapshots[keys[i]].Sub(snapshots[keys[j]]) > 0\n\t})\n\n\tfor _, key := range keys {\n\t\trv = append(rv, &snapshotMetaData{\n\t\t\tepoch:     key,\n\t\t\ttimeStamp: snapshots[key],\n\t\t})\n\t}\n\n\treturn rv\n}\n\n// Removes enough snapshots from the rootBolt so that the\n// s.eligibleForRemoval stays under the NumSnapshotsToKeep policy.\nfunc (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) {\n\tpersistedSnapshots, err := s.rootBoltSnapshotMetaData()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif len(persistedSnapshots) <= s.numSnapshotsToKeep {\n\t\t// we need to keep everything\n\t\treturn 0, nil\n\t}\n\n\tprotectedSnapshots := getProtectedSnapshots(s.rollbackSamplingInterval,\n\t\ts.numSnapshotsToKeep, persistedSnapshots)\n\n\tvar epochsToRemove []uint64\n\tvar newEligible []uint64\n\ts.rootLock.Lock()\n\tfor _, epoch := range s.eligibleForRemoval {\n\t\tif _, ok := protectedSnapshots[epoch]; ok {\n\t\t\t// protected\n\t\t\tnewEligible = append(newEligible, epoch)\n\t\t} else {\n\t\t\tepochsToRemove = append(epochsToRemove, epoch)\n\t\t}\n\t}\n\ts.eligibleForRemoval = newEligible\n\ts.rootLock.Unlock()\n\ts.checkPoints = newCheckPoints(protectedSnapshots)\n\n\tif len(epochsToRemove) == 0 {\n\t\treturn 0, nil\n\t}\n\n\ttx, err := s.rootBolt.Begin(true)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tdefer func() {\n\t\tif err == nil {\n\t\t\terr = tx.Commit()\n\t\t} else {\n\t\t\t_ = tx.Rollback()\n\t\t}\n\t\tif err == nil {\n\t\t\terr = s.rootBolt.Sync()\n\t\t}\n\t}()\n\n\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\tif snapshots == nil {\n\t\treturn 0, nil\n\t}\n\n\tfor _, epochToRemove := range epochsToRemove {\n\t\tk := encodeUvarintAscending(nil, epochToRemove)\n\t\terr = snapshots.DeleteBucket(k)\n\t\tif err == bolt.ErrBucketNotFound {\n\t\t\terr = nil\n\t\t}\n\t\tif err == nil {\n\t\t\tnumRemoved++\n\t\t}\n\t}\n\n\treturn numRemoved, err\n}\n\nfunc (s *Scorch) maxSegmentIDOnDisk() (uint64, error) {\n\tfiles, err := os.ReadDir(s.path)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tvar rv uint64\n\tfor _, f := range files {\n\t\tfname := f.Name()\n\t\tif filepath.Ext(fname) == \".zap\" {\n\t\t\tprefix := strings.TrimSuffix(fname, \".zap\")\n\t\t\tid, err2 := strconv.ParseUint(prefix, 16, 64)\n\t\t\tif err2 != nil {\n\t\t\t\treturn 0, err2\n\t\t\t}\n\t\t\tif id > rv {\n\t\t\t\trv = id\n\t\t\t}\n\t\t}\n\t}\n\treturn rv, err\n}\n\n// Removes any *.zap files which aren't listed in the rootBolt.\nfunc (s *Scorch) removeOldZapFiles() error {\n\tliveFileNames, err := s.loadZapFileNames()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfiles, err := os.ReadDir(s.path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ts.rootLock.RLock()\n\n\tfor _, f := range files {\n\t\tfname := f.Name()\n\t\tif filepath.Ext(fname) == \".zap\" {\n\t\t\tif _, exists := liveFileNames[fname]; !exists && !s.ineligibleForRemoval[fname] && (s.copyScheduled[fname] <= 0) {\n\t\t\t\terr := os.Remove(s.path + string(os.PathSeparator) + fname)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Printf(\"got err removing file: %s, err: %v\", fname, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\ts.rootLock.RUnlock()\n\n\treturn nil\n}\n\n// In sparse mutation scenario, it can so happen that all protected\n// snapshots are older than the numSnapshotsToKeep * rollbackSamplingInterval\n// duration. This results in all of them being purged from the boltDB\n// and the next iteration of the removeOldData() would end up protecting\n// latest contiguous snapshot which is a poor pattern in the rollback checkpoints.\n// Hence we try to retain at most retentionFactor portion worth of old snapshots\n// in such a scenario using the following function\nfunc getBoundaryCheckPoint(retentionFactor float64,\n\tcheckPoints []*snapshotMetaData, timeStamp time.Time,\n) time.Time {\n\tif checkPoints != nil {\n\t\tboundary := checkPoints[int(math.Floor(float64(len(checkPoints))*\n\t\t\tretentionFactor))]\n\t\tif timeStamp.Sub(boundary.timeStamp) > 0 {\n\t\t\t// return the extended boundary which will dictate the older snapshots\n\t\t\t// to be retained\n\t\t\treturn boundary.timeStamp\n\t\t}\n\t}\n\n\treturn timeStamp\n}\n\ntype snapshotMetaData struct {\n\tepoch     uint64\n\ttimeStamp time.Time\n}\n\nfunc (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) {\n\tvar rv []*snapshotMetaData\n\tcurrTime := time.Now()\n\t// including the very latest snapshot there should be n snapshots, so the\n\t// very last one would be tc - (n-1) * d\n\t// for eg for n = 3 the checkpoints preserved should be tc, tc - d, tc - 2d\n\texpirationDuration := time.Duration(s.numSnapshotsToKeep-1) * s.rollbackSamplingInterval\n\n\terr := s.rootBolt.View(func(tx *bolt.Tx) error {\n\t\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\t\tif snapshots == nil {\n\t\t\treturn nil\n\t\t}\n\t\tsc := snapshots.Cursor()\n\t\tvar found bool\n\t\t// traversal order - latest -> oldest epoch\n\t\tfor sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() {\n\t\t\t_, snapshotEpoch, err := decodeUvarintAscending(sk)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif expirationDuration == 0 {\n\t\t\t\trv = append(rv, &snapshotMetaData{\n\t\t\t\t\tepoch: snapshotEpoch,\n\t\t\t\t})\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tsnapshot := snapshots.Bucket(sk)\n\t\t\tif snapshot == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmetaBucket := snapshot.Bucket(util.BoltMetaDataKey)\n\t\t\tif metaBucket == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttimeStampBytes := metaBucket.Get(util.BoltMetaDataTimeStamp)\n\t\t\tvar timeStamp time.Time\n\t\t\terr = timeStamp.UnmarshalText(timeStampBytes)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Don't keep snapshots older than\n\t\t\t// expiration duration (numSnapshotsToKeep *\n\t\t\t// rollbackSamplingInterval, by default)\n\t\t\tif currTime.Sub(timeStamp) <= expirationDuration {\n\t\t\t\trv = append(rv, &snapshotMetaData{\n\t\t\t\t\tepoch:     snapshotEpoch,\n\t\t\t\t\ttimeStamp: timeStamp,\n\t\t\t\t})\n\t\t\t} else {\n\t\t\t\tif !found {\n\t\t\t\t\tfound = true\n\t\t\t\t\tboundary := getBoundaryCheckPoint(s.rollbackRetentionFactor,\n\t\t\t\t\t\ts.checkPoints, timeStamp)\n\t\t\t\t\texpirationDuration = currTime.Sub(boundary)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tk := encodeUvarintAscending(nil, snapshotEpoch)\n\t\t\t\terr = snapshots.DeleteBucket(k)\n\t\t\t\tif err == bolt.ErrBucketNotFound {\n\t\t\t\t\terr = nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\treturn rv, err\n}\n\nfunc (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) {\n\tvar rv []uint64\n\terr := s.rootBolt.View(func(tx *bolt.Tx) error {\n\t\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\t\tif snapshots == nil {\n\t\t\treturn nil\n\t\t}\n\t\tsc := snapshots.Cursor()\n\t\tfor sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() {\n\t\t\t_, snapshotEpoch, err := decodeUvarintAscending(sk)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trv = append(rv, snapshotEpoch)\n\t\t}\n\t\treturn nil\n\t})\n\treturn rv, err\n}\n\n// Returns the *.zap file names that are listed in the rootBolt.\nfunc (s *Scorch) loadZapFileNames() (map[string]struct{}, error) {\n\trv := map[string]struct{}{}\n\terr := s.rootBolt.View(func(tx *bolt.Tx) error {\n\t\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\t\tif snapshots == nil {\n\t\t\treturn nil\n\t\t}\n\t\tsc := snapshots.Cursor()\n\t\tfor sk, _ := sc.First(); sk != nil; sk, _ = sc.Next() {\n\t\t\tsnapshot := snapshots.Bucket(sk)\n\t\t\tif snapshot == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsegc := snapshot.Cursor()\n\t\t\tfor segk, _ := segc.First(); segk != nil; segk, _ = segc.Next() {\n\t\t\t\tif segk[0] == util.BoltInternalKey[0] {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tsegmentBucket := snapshot.Bucket(segk)\n\t\t\t\tif segmentBucket == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tpathBytes := segmentBucket.Get(util.BoltPathKey)\n\t\t\t\tif pathBytes == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tpathString := string(pathBytes)\n\t\t\t\trv[string(pathString)] = struct{}{}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\n\treturn rv, err\n}\n"
  },
  {
    "path": "index/scorch/reader_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"context\"\n\t\"encoding/binary\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestIndexReader(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexReader\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextFieldWithAnalyzer(\"name\", []uint64{}, []byte(\"test test test\"), testAnalyzer))\n\tdoc.AddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"eat more rice\"), index.IndexField|index.IncludeTermVectors, testAnalyzer))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// first look for a term that doesn't exist\n\treader, err := indexReader.TermFieldReader(context.TODO(), []byte(\"nope\"), \"name\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing term field reader: %v\", err)\n\t}\n\tcount := reader.Count()\n\tif count != 0 {\n\t\tt.Errorf(\"Expected doc count to be: %d got: %d\", 0, count)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err = indexReader.TermFieldReader(context.TODO(), []byte(\"test\"), \"name\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing term field reader: %v\", err)\n\t}\n\n\tcount = reader.Count()\n\tif count != expectedCount {\n\t\tt.Errorf(\"Expected doc count to be: %d got: %d\", expectedCount, count)\n\t}\n\n\tvar match *index.TermFieldDoc\n\tvar actualCount uint64\n\tmatch, err = reader.Next(nil)\n\tfor err == nil && match != nil {\n\t\tmatch, err = reader.Next(nil)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"unexpected error reading next\")\n\t\t}\n\t\tactualCount++\n\t}\n\tif actualCount != count {\n\t\tt.Errorf(\"count was 2, but only saw %d\", actualCount)\n\t}\n\n\tinternalIDBogus, err := indexReader.InternalID(\"a-bogus-docId\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif internalIDBogus != nil {\n\t\tt.Errorf(\"expected bogus docId to have nil InternalID\")\n\t}\n\n\tinternalID2, err := indexReader.InternalID(\"2\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texpectedMatch := &index.TermFieldDoc{\n\t\tID:   internalID2,\n\t\tFreq: 1,\n\t\tNorm: 0.5773502588272095,\n\t\tVectors: []*index.TermFieldVector{\n\t\t\t{\n\t\t\t\tField: \"desc\",\n\t\t\t\tPos:   3,\n\t\t\t\tStart: 9,\n\t\t\t\tEnd:   13,\n\t\t\t},\n\t\t},\n\t}\n\ttfr, err := indexReader.TermFieldReader(context.TODO(), []byte(\"rice\"), \"desc\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tmatch, err = tfr.Next(nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\n\tif !reflect.DeepEqual(expectedMatch, match) {\n\t\tt.Errorf(\"got %#v, expected %#v\", match, expectedMatch)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now test usage of advance\n\treader, err = indexReader.TermFieldReader(context.TODO(), []byte(\"test\"), \"name\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing term field reader: %v\", err)\n\t}\n\n\tmatch, err = reader.Advance(internalID2, nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match == nil {\n\t\tt.Fatalf(\"Expected match, got nil\")\n\t}\n\tif !match.ID.Equals(internalID2) {\n\t\tt.Errorf(\"Expected ID '2', got '%s'\", match.ID)\n\t}\n\t// have to manually construct bogus id, because it doesn't exist\n\tinternalID3 := make([]byte, 8)\n\tbinary.BigEndian.PutUint64(internalID3, 3)\n\tmatch, err = reader.Advance(index.IndexInternalID(internalID3), nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match != nil {\n\t\tt.Errorf(\"expected nil, got %v\", match)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now test creating a reader for a field that doesn't exist\n\treader, err = indexReader.TermFieldReader(context.TODO(), []byte(\"water\"), \"doesnotexist\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing term field reader: %v\", err)\n\t}\n\tcount = reader.Count()\n\tif count != 0 {\n\t\tt.Errorf(\"expected count 0 for reader of non-existent field\")\n\t}\n\tmatch, err = reader.Next(nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match != nil {\n\t\tt.Errorf(\"expected nil, got %v\", match)\n\t}\n\tmatch, err = reader.Advance(index.IndexInternalID(\"anywhere\"), nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match != nil {\n\t\tt.Errorf(\"expected nil, got %v\", match)\n\t}\n}\n\nfunc TestIndexDocIdReader(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexDocIdReader\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test test test\")))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"desc\", []uint64{}, []byte(\"eat more rice\"), index.IndexField|index.IncludeTermVectors))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\t// first get all doc ids\n\treader, err := indexReader.DocIDReaderAll()\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tid, err := reader.Next()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcount := uint64(0)\n\tfor id != nil {\n\t\tcount++\n\t\tid, err = reader.Next()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tif count != expectedCount {\n\t\tt.Errorf(\"expected %d, got %d\", expectedCount, count)\n\t}\n\n\t// try it again, but jump to the second doc this time\n\treader2, err := indexReader.DocIDReaderAll()\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader2.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\tinternalID2, err := indexReader.InternalID(\"2\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tid, err = reader2.Advance(internalID2)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(internalID2) {\n\t\tt.Errorf(\"expected to find id '2', got '%s'\", id)\n\t}\n\n\t// again 3 doesn't exist cannot use internal id for 3 as there is none\n\t// the important aspect is that this id doesn't exist, so its ok\n\tid, err = reader2.Advance(index.IndexInternalID(\"3\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif id != nil {\n\t\tt.Errorf(\"expected to find id '', got '%s'\", id)\n\t}\n}\n\nfunc TestIndexDocIdOnlyReader(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexDocIdOnlyReader\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"3\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"5\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"7\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"9\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\tonlyIds := []string{\"1\", \"5\", \"9\"}\n\treader, err := indexReader.DocIDReaderOnly(onlyIds)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tid, err := reader.Next()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcount := uint64(0)\n\tfor id != nil {\n\t\tcount++\n\t\tid, err = reader.Next()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tif count != 3 {\n\t\tt.Errorf(\"expected 3, got %d\", count)\n\t}\n\n\t// commented out because advance works with internal ids\n\t// this test presumes we see items in external doc id order\n\t// which is no longer the case, so simply converting external ids\n\t// to internal ones is not logically correct\n\t// not removing though because we need some way to test Advance()\n\n\t// // try it again, but jump\n\t// reader2, err := indexReader.DocIDReaderOnly(onlyIds)\n\t// if err != nil {\n\t// \tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t// }\n\t// defer func() {\n\t// \terr := reader2.Close()\n\t// \tif err != nil {\n\t// \t\tt.Error(err)\n\t// \t}\n\t// }()\n\t//\n\t// id, err = reader2.Advance(index.IndexInternalID(\"5\"))\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if !id.Equals(index.IndexInternalID(\"5\")) {\n\t// \tt.Errorf(\"expected to find id '5', got '%s'\", id)\n\t// }\n\t//\n\t// id, err = reader2.Advance(index.IndexInternalID(\"a\"))\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if id != nil {\n\t// \tt.Errorf(\"expected to find id '', got '%s'\", id)\n\t// }\n\n\t// some keys aren't actually there\n\tonlyIds = []string{\"0\", \"2\", \"4\", \"5\", \"6\", \"8\", \"a\"}\n\treader3, err := indexReader.DocIDReaderOnly(onlyIds)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader3.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\tid, err = reader3.Next()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcount = uint64(0)\n\tfor id != nil {\n\t\tcount++\n\t\tid, err = reader3.Next()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tif count != 1 {\n\t\tt.Errorf(\"expected 1, got %d\", count)\n\t}\n\n\t// commented out because advance works with internal ids\n\t// this test presumes we see items in external doc id order\n\t// which is no longer the case, so simply converting external ids\n\t// to internal ones is not logically correct\n\t// not removing though because we need some way to test Advance()\n\n\t// // mix advance and next\n\t// onlyIds = []string{\"0\", \"1\", \"3\", \"5\", \"6\", \"9\"}\n\t// reader4, err := indexReader.DocIDReaderOnly(onlyIds)\n\t// if err != nil {\n\t// \tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t// }\n\t// defer func() {\n\t// \terr := reader4.Close()\n\t// \tif err != nil {\n\t// \t\tt.Error(err)\n\t// \t}\n\t// }()\n\t//\n\t// // first key is \"1\"\n\t// id, err = reader4.Next()\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if !id.Equals(index.IndexInternalID(\"1\")) {\n\t// \tt.Errorf(\"expected to find id '1', got '%s'\", id)\n\t// }\n\t//\n\t// // advancing to key we dont have gives next\n\t// id, err = reader4.Advance(index.IndexInternalID(\"2\"))\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if !id.Equals(index.IndexInternalID(\"3\")) {\n\t// \tt.Errorf(\"expected to find id '3', got '%s'\", id)\n\t// }\n\t//\n\t// // next after advance works\n\t// id, err = reader4.Next()\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if !id.Equals(index.IndexInternalID(\"5\")) {\n\t// \tt.Errorf(\"expected to find id '5', got '%s'\", id)\n\t// }\n\t//\n\t// // advancing to key we do have works\n\t// id, err = reader4.Advance(index.IndexInternalID(\"9\"))\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if !id.Equals(index.IndexInternalID(\"9\")) {\n\t// \tt.Errorf(\"expected to find id '9', got '%s'\", id)\n\t// }\n\t//\n\t// // advance backwards at end\n\t// id, err = reader4.Advance(index.IndexInternalID(\"4\"))\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if !id.Equals(index.IndexInternalID(\"5\")) {\n\t// \tt.Errorf(\"expected to find id '5', got '%s'\", id)\n\t// }\n\t//\n\t// // next after advance works\n\t// id, err = reader4.Next()\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if !id.Equals(index.IndexInternalID(\"9\")) {\n\t// \tt.Errorf(\"expected to find id '9', got '%s'\", id)\n\t// }\n\t//\n\t// // advance backwards to key that exists, but not in only set\n\t// id, err = reader4.Advance(index.IndexInternalID(\"7\"))\n\t// if err != nil {\n\t// \tt.Error(err)\n\t// }\n\t// if !id.Equals(index.IndexInternalID(\"9\")) {\n\t// \tt.Errorf(\"expected to find id '9', got '%s'\", id)\n\t// }\n}\n\nfunc TestSegmentIndexAndLocalDocNumFromGlobal(t *testing.T) {\n\ttests := []struct {\n\t\toffsets      []uint64\n\t\tglobalDocNum uint64\n\t\tsegmentIndex int\n\t\tlocalDocNum  uint64\n\t}{\n\t\t// just 1 segment\n\t\t{\n\t\t\toffsets:      []uint64{0},\n\t\t\tglobalDocNum: 0,\n\t\t\tsegmentIndex: 0,\n\t\t\tlocalDocNum:  0,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0},\n\t\t\tglobalDocNum: 1,\n\t\t\tsegmentIndex: 0,\n\t\t\tlocalDocNum:  1,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0},\n\t\t\tglobalDocNum: 25,\n\t\t\tsegmentIndex: 0,\n\t\t\tlocalDocNum:  25,\n\t\t},\n\t\t// now 2 segments, 30 docs in first\n\t\t{\n\t\t\toffsets:      []uint64{0, 30},\n\t\t\tglobalDocNum: 0,\n\t\t\tsegmentIndex: 0,\n\t\t\tlocalDocNum:  0,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0, 30},\n\t\t\tglobalDocNum: 1,\n\t\t\tsegmentIndex: 0,\n\t\t\tlocalDocNum:  1,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0, 30},\n\t\t\tglobalDocNum: 25,\n\t\t\tsegmentIndex: 0,\n\t\t\tlocalDocNum:  25,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0, 30},\n\t\t\tglobalDocNum: 30,\n\t\t\tsegmentIndex: 1,\n\t\t\tlocalDocNum:  0,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0, 30},\n\t\t\tglobalDocNum: 35,\n\t\t\tsegmentIndex: 1,\n\t\t\tlocalDocNum:  5,\n\t\t},\n\t\t// lots of segments\n\t\t{\n\t\t\toffsets:      []uint64{0, 30, 40, 70, 99, 172, 800, 25000},\n\t\t\tglobalDocNum: 0,\n\t\t\tsegmentIndex: 0,\n\t\t\tlocalDocNum:  0,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0, 30, 40, 70, 99, 172, 800, 25000},\n\t\t\tglobalDocNum: 25,\n\t\t\tsegmentIndex: 0,\n\t\t\tlocalDocNum:  25,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0, 30, 40, 70, 99, 172, 800, 25000},\n\t\t\tglobalDocNum: 35,\n\t\t\tsegmentIndex: 1,\n\t\t\tlocalDocNum:  5,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0, 30, 40, 70, 99, 172, 800, 25000},\n\t\t\tglobalDocNum: 100,\n\t\t\tsegmentIndex: 4,\n\t\t\tlocalDocNum:  1,\n\t\t},\n\t\t{\n\t\t\toffsets:      []uint64{0, 30, 40, 70, 99, 172, 800, 25000},\n\t\t\tglobalDocNum: 825,\n\t\t\tsegmentIndex: 6,\n\t\t\tlocalDocNum:  25,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ti := &IndexSnapshot{\n\t\t\toffsets: test.offsets,\n\t\t\trefs:    1,\n\t\t}\n\t\tgotSegmentIndex, gotLocalDocNum := i.segmentIndexAndLocalDocNumFromGlobal(test.globalDocNum)\n\t\tif gotSegmentIndex != test.segmentIndex {\n\t\t\tt.Errorf(\"got segment index %d expected %d for offsets %v globalDocNum %d\", gotSegmentIndex, test.segmentIndex, test.offsets, test.globalDocNum)\n\t\t}\n\t\tif gotLocalDocNum != test.localDocNum {\n\t\t\tt.Errorf(\"got localDocNum %d expected %d for offsets %v globalDocNum %d\", gotLocalDocNum, test.localDocNum, test.offsets, test.globalDocNum)\n\t\t}\n\t\terr := i.DecRef()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"expected no err, got: %v\", err)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "index/scorch/regexp.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"regexp/syntax\"\n\n\t\"github.com/blevesearch/vellum/regexp\"\n)\n\nfunc parseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, err error) {\n\t// TODO: potential optimization where syntax.Regexp supports a Simplify() API?\n\n\tparsed, err := syntax.Parse(pattern, syntax.Perl)\n\tif err != nil {\n\t\treturn nil, nil, nil, err\n\t}\n\n\tre, err := regexp.NewParsedWithLimit(pattern, parsed, regexp.DefaultLimit)\n\tif err != nil {\n\t\treturn nil, nil, nil, err\n\t}\n\n\tprefix := literalPrefix(parsed)\n\tif prefix != \"\" {\n\t\tprefixBeg := []byte(prefix)\n\t\tprefixEnd := calculateExclusiveEndFromPrefix(prefixBeg)\n\t\treturn re, prefixBeg, prefixEnd, nil\n\t}\n\n\treturn re, nil, nil, nil\n}\n\n// Returns the literal prefix given the parse tree for a regexp\nfunc literalPrefix(s *syntax.Regexp) string {\n\t// traverse the left-most branch in the parse tree as long as the\n\t// node represents a concatenation\n\tfor s != nil && s.Op == syntax.OpConcat {\n\t\tif len(s.Sub) < 1 {\n\t\t\treturn \"\"\n\t\t}\n\n\t\ts = s.Sub[0]\n\t}\n\n\tif s.Op == syntax.OpLiteral && (s.Flags&syntax.FoldCase == 0) {\n\t\treturn string(s.Rune)\n\t}\n\n\treturn \"\" // no literal prefix\n}\n"
  },
  {
    "path": "index/scorch/regexp_test.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"regexp/syntax\"\n\t\"testing\"\n)\n\nfunc TestLiteralPrefix(t *testing.T) {\n\ttests := []struct {\n\t\tinput, expected string\n\t}{\n\t\t{\"\", \"\"},\n\t\t{\"hello\", \"hello\"},\n\t\t{\"hello.?\", \"hello\"},\n\t\t{\"hello$\", \"hello\"},\n\t\t{`[h][e][l][l][o].*world`, \"hello\"},\n\t\t{`[h-h][e-e][l-l][l-l][o-o].*world`, \"hello\"},\n\t\t{\".*\", \"\"},\n\t\t{\"h.*\", \"h\"},\n\t\t{\"h.?\", \"h\"},\n\t\t{\"h[a-z]\", \"h\"},\n\t\t{`h\\s`, \"h\"},\n\t\t{`(hello)world`, \"\"},\n\t\t{`日本語`, \"日本語\"},\n\t\t{`日本語\\w`, \"日本語\"},\n\t\t{`^hello`, \"\"},\n\t\t{`^`, \"\"},\n\t\t{`$`, \"\"},\n\t\t{`(?i)mArTy`, \"\"},\n\t}\n\n\tfor i, test := range tests {\n\t\ts, err := syntax.Parse(test.input, syntax.Perl)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"expected no syntax.Parse error, got: %v\", err)\n\t\t}\n\n\t\tgot := literalPrefix(s)\n\t\tif test.expected != got {\n\t\t\tt.Fatalf(\"test: %d, %+v, got: %s\", i, test, got)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "index/scorch/rollback.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tbolt \"go.etcd.io/bbolt\"\n)\n\ntype RollbackPoint struct {\n\tepoch uint64\n\tmeta  map[string][]byte\n}\n\nfunc (r *RollbackPoint) GetInternal(key []byte) []byte {\n\treturn r.meta[string(key)]\n}\n\n// RollbackPoints returns an array of rollback points available for\n// the application to rollback to, with more recent rollback points\n// (higher epochs) coming first.\nfunc RollbackPoints(path string) ([]*RollbackPoint, error) {\n\tif len(path) == 0 {\n\t\treturn nil, fmt.Errorf(\"RollbackPoints: invalid path\")\n\t}\n\n\trootBoltPath := path + string(os.PathSeparator) + \"root.bolt\"\n\trootBoltOpt := &bolt.Options{\n\t\tReadOnly: true,\n\t}\n\trootBolt, err := bolt.Open(rootBoltPath, 0600, rootBoltOpt)\n\tif err != nil || rootBolt == nil {\n\t\treturn nil, err\n\t}\n\n\t// start a read-only bolt transaction\n\ttx, err := rootBolt.Begin(false)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"RollbackPoints: failed to start\" +\n\t\t\t\" read-only transaction\")\n\t}\n\n\t// read-only bolt transactions to be rolled back\n\tdefer func() {\n\t\t_ = tx.Rollback()\n\t\t_ = rootBolt.Close()\n\t}()\n\n\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\tif snapshots == nil {\n\t\treturn nil, nil\n\t}\n\n\trollbackPoints := []*RollbackPoint{}\n\n\tc1 := snapshots.Cursor()\n\tfor k, _ := c1.Last(); k != nil; k, _ = c1.Prev() {\n\t\t_, snapshotEpoch, err := decodeUvarintAscending(k)\n\t\tif err != nil {\n\t\t\tlog.Printf(\"RollbackPoints:\"+\n\t\t\t\t\" unable to parse segment epoch %x, continuing\", k)\n\t\t\tcontinue\n\t\t}\n\n\t\tsnapshot := snapshots.Bucket(k)\n\t\tif snapshot == nil {\n\t\t\tlog.Printf(\"RollbackPoints:\"+\n\t\t\t\t\" snapshot key, but bucket missing %x, continuing\", k)\n\t\t\tcontinue\n\t\t}\n\n\t\tmeta := map[string][]byte{}\n\t\tc2 := snapshot.Cursor()\n\t\tfor j, _ := c2.First(); j != nil; j, _ = c2.Next() {\n\t\t\tif j[0] == util.BoltInternalKey[0] {\n\t\t\t\tinternalBucket := snapshot.Bucket(j)\n\t\t\t\tif internalBucket == nil {\n\t\t\t\t\terr = fmt.Errorf(\"internal bucket missing\")\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\terr = internalBucket.ForEach(func(key []byte, val []byte) error {\n\t\t\t\t\tcopiedVal := append([]byte(nil), val...)\n\t\t\t\t\tmeta[string(key)] = copiedVal\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t\tif err != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif err != nil {\n\t\t\tlog.Printf(\"RollbackPoints:\"+\n\t\t\t\t\" failed in fetching internal data: %v\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\trollbackPoints = append(rollbackPoints, &RollbackPoint{\n\t\t\tepoch: snapshotEpoch,\n\t\t\tmeta:  meta,\n\t\t})\n\t}\n\n\treturn rollbackPoints, nil\n}\n\n// Rollback atomically and durably brings the store back to the point\n// in time as represented by the RollbackPoint.\n// Rollback() should only be passed a RollbackPoint that came from the\n// same store using the RollbackPoints() API along with the index path.\nfunc Rollback(path string, to *RollbackPoint) error {\n\tif to == nil {\n\t\treturn fmt.Errorf(\"Rollback: RollbackPoint is nil\")\n\t}\n\tif len(path) == 0 {\n\t\treturn fmt.Errorf(\"Rollback: index path is empty\")\n\t}\n\n\trootBoltPath := path + string(os.PathSeparator) + \"root.bolt\"\n\trootBoltOpt := &bolt.Options{\n\t\tReadOnly: false,\n\t}\n\trootBolt, err := bolt.Open(rootBoltPath, 0600, rootBoltOpt)\n\tif err != nil || rootBolt == nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\terr1 := rootBolt.Close()\n\t\tif err1 != nil && err == nil {\n\t\t\terr = err1\n\t\t}\n\t}()\n\n\t// pick all the younger persisted epochs in bolt store\n\t// including the target one.\n\tvar found bool\n\tvar eligibleEpochs []uint64\n\terr = rootBolt.View(func(tx *bolt.Tx) error {\n\t\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\t\tif snapshots == nil {\n\t\t\treturn nil\n\t\t}\n\t\tsc := snapshots.Cursor()\n\t\tfor sk, _ := sc.Last(); sk != nil && !found; sk, _ = sc.Prev() {\n\t\t\t_, snapshotEpoch, err := decodeUvarintAscending(sk)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif snapshotEpoch == to.epoch {\n\t\t\t\tfound = true\n\t\t\t}\n\t\t\teligibleEpochs = append(eligibleEpochs, snapshotEpoch)\n\t\t}\n\t\treturn nil\n\t})\n\n\tif len(eligibleEpochs) == 0 {\n\t\treturn fmt.Errorf(\"Rollback: no persisted epochs found in bolt\")\n\t}\n\tif !found {\n\t\treturn fmt.Errorf(\"Rollback: target epoch %d not found in bolt\", to.epoch)\n\t}\n\n\t// start a write transaction\n\ttx, err := rootBolt.Begin(true)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdefer func() {\n\t\tif err == nil {\n\t\t\terr = tx.Commit()\n\t\t} else {\n\t\t\t_ = tx.Rollback()\n\t\t}\n\t\tif err == nil {\n\t\t\terr = rootBolt.Sync()\n\t\t}\n\t}()\n\n\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\tif snapshots == nil {\n\t\treturn nil\n\t}\n\tfor _, epoch := range eligibleEpochs {\n\t\tk := encodeUvarintAscending(nil, epoch)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif epoch == to.epoch {\n\t\t\t// return here as it already processed until the given epoch\n\t\t\treturn nil\n\t\t}\n\t\terr = snapshots.DeleteBucket(k)\n\t\tif err == bolt.ErrBucketNotFound {\n\t\t\terr = nil\n\t\t}\n\t}\n\n\treturn err\n}\n"
  },
  {
    "path": "index/scorch/rollback_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestIndexRollback(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexRollback\")\n\tnumSnapshotsToKeepOrig := NumSnapshotsToKeep\n\tNumSnapshotsToKeep = 1000\n\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\tNumSnapshotsToKeep = numSnapshotsToKeepOrig\n\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t_, ok := idx.(*Scorch)\n\tif !ok {\n\t\tt.Fatalf(\"Not a scorch index?\")\n\t}\n\n\tindexPath, _ := cfg[\"path\"].(string)\n\t// should have no rollback points initially\n\trollbackPoints, err := RollbackPoints(indexPath)\n\tif err == nil {\n\t\tt.Fatalf(\"expected no err, got: %v, %d\", err, len(rollbackPoints))\n\t}\n\tif len(rollbackPoints) != 0 {\n\t\tt.Fatalf(\"expected no rollbackPoints, got %d\", len(rollbackPoints))\n\t}\n\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// create a batch, insert 2 new documents\n\tbatch := index.NewBatch()\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test1\")))\n\tbatch.Update(doc)\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test2\")))\n\tbatch.Update(doc)\n\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treaderSlow, err := idx.Reader() // keep snapshot around so it's not cleaned up\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\t_ = readerSlow.Close()\n\t}()\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// fetch rollback points after first batch\n\trollbackPoints, err = RollbackPoints(indexPath)\n\tif err != nil {\n\t\tt.Fatalf(\"expected no err, got: %v, %d\", err, len(rollbackPoints))\n\t}\n\tif len(rollbackPoints) == 0 {\n\t\tt.Fatalf(\"expected some rollbackPoints, got none\")\n\t}\n\n\t// set this as a rollback point for the future\n\trollbackPoint := rollbackPoints[0]\n\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// create another batch, insert 2 new documents, and delete an existing one\n\tbatch = index.NewBatch()\n\tdoc = document.NewDocument(\"3\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test3\")))\n\tbatch.Update(doc)\n\tdoc = document.NewDocument(\"4\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test4\")))\n\tbatch.Update(doc)\n\tbatch.Delete(\"1\")\n\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\trollbackPointsB, err := RollbackPoints(indexPath)\n\tif err != nil || len(rollbackPointsB) <= len(rollbackPoints) {\n\t\tt.Fatalf(\"expected no err, got: %v, %d\", err, len(rollbackPointsB))\n\t}\n\n\tfound := false\n\tfor _, p := range rollbackPointsB {\n\t\tif rollbackPoint.epoch == p.epoch {\n\t\t\tfound = true\n\t\t}\n\t}\n\tif !found {\n\t\tt.Fatalf(\"expected rollbackPoint epoch to still be available\")\n\t}\n\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// expect docs 2, 3, 4\n\tif docCount != 3 {\n\t\tt.Fatalf(\"unexpected doc count: %v\", docCount)\n\t}\n\tret, err := reader.Document(\"1\")\n\tif err != nil || ret != nil {\n\t\tt.Fatal(ret, err)\n\t}\n\tret, err = reader.Document(\"2\")\n\tif err != nil || ret == nil {\n\t\tt.Fatal(ret, err)\n\t}\n\tret, err = reader.Document(\"3\")\n\tif err != nil || ret == nil {\n\t\tt.Fatal(ret, err)\n\t}\n\tret, err = reader.Document(\"4\")\n\tif err != nil || ret == nil {\n\t\tt.Fatal(ret, err)\n\t}\n\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// rollback to a non existing rollback point\n\terr = Rollback(indexPath, &RollbackPoint{epoch: 100})\n\tif err == nil {\n\t\tt.Fatalf(\"expected err: Rollback: target epoch 100 not found in bolt\")\n\t}\n\n\t// rollback to the selected rollback point\n\terr = Rollback(indexPath, rollbackPoint)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// expect only docs 1, 2\n\tif docCount != 2 {\n\t\tt.Fatalf(\"unexpected doc count: %v\", docCount)\n\t}\n\tret, err = reader.Document(\"1\")\n\tif err != nil || ret == nil {\n\t\tt.Fatal(ret, err)\n\t}\n\tret, err = reader.Document(\"2\")\n\tif err != nil || ret == nil {\n\t\tt.Fatal(ret, err)\n\t}\n\tret, err = reader.Document(\"3\")\n\tif err != nil || ret != nil {\n\t\tt.Fatal(ret, err)\n\t}\n\tret, err = reader.Document(\"4\")\n\tif err != nil || ret != nil {\n\t\tt.Fatal(ret, err)\n\t}\n\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestGetProtectedSnapshots(t *testing.T) {\n\torigRollbackSamplingInterval := RollbackSamplingInterval\n\tdefer func() {\n\t\tRollbackSamplingInterval = origRollbackSamplingInterval\n\t}()\n\tRollbackSamplingInterval = 10 * time.Minute\n\tcurrentTimeStamp := time.Now()\n\ttests := []struct {\n\t\ttitle              string\n\t\tmetaData           []*snapshotMetaData\n\t\tnumSnapshotsToKeep int\n\t\texpCount           int\n\t\texpEpochs          []uint64\n\t}{\n\t\t{\n\t\t\ttitle: \"epochs that have exact timestamps as per expectation for protecting\",\n\t\t\tmetaData: []*snapshotMetaData{\n\t\t\t\t{epoch: 100, timeStamp: currentTimeStamp},\n\t\t\t\t{epoch: 99, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 12))},\n\t\t\t\t{epoch: 88, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 6))},\n\t\t\t\t{epoch: 50, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval))},\n\t\t\t\t{epoch: 35, timeStamp: currentTimeStamp.Add(-(6 * RollbackSamplingInterval / 5))},\n\t\t\t\t{epoch: 10, timeStamp: currentTimeStamp.Add(-(2 * RollbackSamplingInterval))},\n\t\t\t},\n\t\t\tnumSnapshotsToKeep: 3,\n\t\t\texpCount:           3,\n\t\t\texpEpochs:          []uint64{100, 50, 10},\n\t\t},\n\t\t{\n\t\t\ttitle: \"epochs that have exact timestamps as per expectation for protecting\",\n\t\t\tmetaData: []*snapshotMetaData{\n\t\t\t\t{epoch: 100, timeStamp: currentTimeStamp},\n\t\t\t\t{epoch: 99, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 12))},\n\t\t\t\t{epoch: 88, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 6))},\n\t\t\t\t{epoch: 50, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval))},\n\t\t\t},\n\t\t\tnumSnapshotsToKeep: 2,\n\t\t\texpCount:           2,\n\t\t\texpEpochs:          []uint64{100, 50},\n\t\t},\n\t\t{\n\t\t\ttitle: \"epochs that have timestamps approximated to the expected value, \" +\n\t\t\t\t\"always retain the latest one\",\n\t\t\tmetaData: []*snapshotMetaData{\n\t\t\t\t{epoch: 100, timeStamp: currentTimeStamp},\n\t\t\t\t{epoch: 99, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 12))},\n\t\t\t\t{epoch: 88, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 6))},\n\t\t\t\t{epoch: 50, timeStamp: currentTimeStamp.Add(-(3 * RollbackSamplingInterval / 4))},\n\t\t\t\t{epoch: 35, timeStamp: currentTimeStamp.Add(-(6 * RollbackSamplingInterval / 5))},\n\t\t\t\t{epoch: 10, timeStamp: currentTimeStamp.Add(-(2 * RollbackSamplingInterval))},\n\t\t\t},\n\t\t\tnumSnapshotsToKeep: 3,\n\t\t\texpCount:           3,\n\t\t\texpEpochs:          []uint64{100, 35, 10},\n\t\t},\n\t\t{\n\t\t\ttitle: \"protecting epochs when we don't have enough snapshots with RollbackSamplingInterval\" +\n\t\t\t\t\" separated timestamps\",\n\t\t\tmetaData: []*snapshotMetaData{\n\t\t\t\t{epoch: 100, timeStamp: currentTimeStamp},\n\t\t\t\t{epoch: 99, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 12))},\n\t\t\t\t{epoch: 88, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 6))},\n\t\t\t\t{epoch: 50, timeStamp: currentTimeStamp.Add(-(3 * RollbackSamplingInterval / 4))},\n\t\t\t\t{epoch: 35, timeStamp: currentTimeStamp.Add(-(5 * RollbackSamplingInterval / 6))},\n\t\t\t\t{epoch: 10, timeStamp: currentTimeStamp.Add(-(7 * RollbackSamplingInterval / 8))},\n\t\t\t},\n\t\t\tnumSnapshotsToKeep: 4,\n\t\t\texpCount:           4,\n\t\t\texpEpochs:          []uint64{100, 99, 88, 10},\n\t\t},\n\t\t{\n\t\t\ttitle: \"epochs of which some are approximated to the expected timestamps, and\" +\n\t\t\t\t\" we don't have enough snapshots with RollbackSamplingInterval separated timestamps\",\n\t\t\tmetaData: []*snapshotMetaData{\n\t\t\t\t{epoch: 100, timeStamp: currentTimeStamp},\n\t\t\t\t{epoch: 99, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 12))},\n\t\t\t\t{epoch: 88, timeStamp: currentTimeStamp.Add(-(RollbackSamplingInterval / 6))},\n\t\t\t\t{epoch: 50, timeStamp: currentTimeStamp.Add(-(3 * RollbackSamplingInterval / 4))},\n\t\t\t\t{epoch: 35, timeStamp: currentTimeStamp.Add(-(8 * RollbackSamplingInterval / 7))},\n\t\t\t\t{epoch: 10, timeStamp: currentTimeStamp.Add(-(6 * RollbackSamplingInterval / 5))},\n\t\t\t},\n\t\t\tnumSnapshotsToKeep: 3,\n\t\t\texpCount:           3,\n\t\t\texpEpochs:          []uint64{100, 50, 10},\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tprotectedEpochs := getProtectedSnapshots(RollbackSamplingInterval,\n\t\t\ttest.numSnapshotsToKeep, test.metaData)\n\t\tif len(protectedEpochs) != test.expCount {\n\t\t\tt.Errorf(\"%d test: %s, getProtectedSnapshots expected to return %d \"+\n\t\t\t\t\"snapshots, but got: %d\", i, test.title, test.expCount, len(protectedEpochs))\n\t\t}\n\t\tfor _, e := range test.expEpochs {\n\t\t\tif _, found := protectedEpochs[e]; !found {\n\t\t\t\tt.Errorf(\"%d test: %s, %d epoch expected to be protected, \"+\n\t\t\t\t\t\"but missing from protected list: %v\", i, test.title, e, protectedEpochs)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc indexDummyData(t *testing.T, scorchi *Scorch, i int) {\n\t// create a batch, insert 2 new documents\n\tbatch := index.NewBatch()\n\tdoc := document.NewDocument(fmt.Sprintf(\"%d\", i))\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test1\")))\n\tbatch.Update(doc)\n\tdoc = document.NewDocument(fmt.Sprintf(\"%d\", i+1))\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test2\")))\n\tbatch.Update(doc)\n\n\terr := scorchi.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\ntype testFSDirector string\n\nfunc (f testFSDirector) GetWriter(filePath string) (io.WriteCloser,\n\terror) {\n\tdir, file := filepath.Split(filePath)\n\tif dir != \"\" {\n\t\terr := os.MkdirAll(filepath.Join(string(f), dir), os.ModePerm)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn os.OpenFile(filepath.Join(string(f), dir, file),\n\t\tos.O_RDWR|os.O_CREATE, 0600)\n}\n\nfunc TestLatestSnapshotProtected(t *testing.T) {\n\tcfg := CreateConfig(\"TestLatestSnapshotProtected\")\n\tnumSnapshotsToKeepOrig := NumSnapshotsToKeep\n\tNumSnapshotsToKeep = 3\n\trollbackSamplingIntervalOrig := RollbackSamplingInterval\n\tRollbackSamplingInterval = 10 * time.Second\n\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\tNumSnapshotsToKeep = numSnapshotsToKeepOrig\n\t\tRollbackSamplingInterval = rollbackSamplingIntervalOrig\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\t// disable merger and purger\n\tRegistryEventCallbacks[\"test\"] = func(e Event) bool {\n\t\tif e.Kind == EventKindPreMergeCheck || e.Kind == EventKindPurgerCheck {\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t}\n\tcfg[\"eventCallbackName\"] = \"test\"\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tscorchi, ok := idx.(*Scorch)\n\tif !ok {\n\t\tt.Fatalf(\"Not a scorch index?\")\n\t}\n\n\terr = scorchi.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// replicate the following scenario of persistence of snapshots\n\t// tc, tc - d/12, tc - d/6, tc - 3d/4, tc - 5d/6, tc - 6d/5\n\t// approximate timestamps where there's a chance that the latest snapshot\n\t// might not fit into the time-series\n\tindexDummyData(t, scorchi, 1)\n\tpersistedSnapshots, err := scorchi.rootBoltSnapshotMetaData()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(persistedSnapshots) != 1 {\n\t\tt.Fatalf(\"expected 1 persisted snapshot, got %d\", len(persistedSnapshots))\n\t}\n\ttime.Sleep(4 * RollbackSamplingInterval / 5)\n\tindexDummyData(t, scorchi, 3)\n\ttime.Sleep(9 * RollbackSamplingInterval / 20)\n\tindexDummyData(t, scorchi, 5)\n\ttime.Sleep(7 * RollbackSamplingInterval / 12)\n\tindexDummyData(t, scorchi, 7)\n\ttime.Sleep(1 * RollbackSamplingInterval / 12)\n\tindexDummyData(t, scorchi, 9)\n\n\tpersistedSnapshots, err = scorchi.rootBoltSnapshotMetaData()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tprotectedSnapshots := getProtectedSnapshots(RollbackSamplingInterval, NumSnapshotsToKeep, persistedSnapshots)\n\tif len(protectedSnapshots) != 3 {\n\t\tt.Fatalf(\"expected %d protected snapshots, got %d\", NumSnapshotsToKeep, len(protectedSnapshots))\n\t}\n\tif _, ok := protectedSnapshots[persistedSnapshots[0].epoch]; !ok {\n\t\tt.Fatalf(\"expected %d to be protected, but not found\", persistedSnapshots[0].epoch)\n\t}\n}\n\nfunc TestBackupRacingWithPurge(t *testing.T) {\n\tcfg := CreateConfig(\"TestBackupRacingWithPurge\")\n\tnumSnapshotsToKeepOrig := NumSnapshotsToKeep\n\tNumSnapshotsToKeep = 3\n\trollbackSamplingIntervalOrig := RollbackSamplingInterval\n\tRollbackSamplingInterval = 10 * time.Second\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\tNumSnapshotsToKeep = numSnapshotsToKeepOrig\n\t\tRollbackSamplingInterval = rollbackSamplingIntervalOrig\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\t// disable merger and purger\n\tRegistryEventCallbacks[\"test\"] = func(e Event) bool {\n\t\tif e.Kind == EventKindPreMergeCheck || e.Kind == EventKindPurgerCheck {\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t}\n\tcfg[\"eventCallbackName\"] = \"test\"\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer idx.Close()\n\n\tscorchi, ok := idx.(*Scorch)\n\tif !ok {\n\t\tt.Fatalf(\"Not a scorch index?\")\n\t}\n\n\terr = scorchi.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// replicate the following scenario of persistence of snapshots\n\t// tc, tc - d/12, tc - d/6, tc - 3d/4, tc - 5d/6, tc - 6d/5\n\t// approximate timestamps where there's a chance that the latest snapshot\n\t// might not fit into the time-series\n\tindexDummyData(t, scorchi, 1)\n\ttime.Sleep(4 * RollbackSamplingInterval / 5)\n\tindexDummyData(t, scorchi, 3)\n\ttime.Sleep(9 * RollbackSamplingInterval / 20)\n\tindexDummyData(t, scorchi, 5)\n\ttime.Sleep(7 * RollbackSamplingInterval / 12)\n\tindexDummyData(t, scorchi, 7)\n\ttime.Sleep(1 * RollbackSamplingInterval / 12)\n\tindexDummyData(t, scorchi, 9)\n\n\t// now if the purge code is invoked, there's a possibility of the latest snapshot\n\t// being removed from bolt and the corresponding file segment getting cleaned up.\n\tscorchi.removeOldData()\n\n\tcopyReader := scorchi.CopyReader()\n\tdefer func() { copyReader.CloseCopyReader() }()\n\n\tbackupidxConfig := CreateConfig(\"backup-directory\")\n\terr = InitTest(backupidxConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(backupidxConfig)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\t// if the latest snapshot was purged, the following will return error\n\terr = copyReader.CopyTo(testFSDirector(backupidxConfig[\"path\"].(string)))\n\tif err != nil {\n\t\tt.Fatalf(\"error copying the index: %v\", err)\n\t}\n}\n\nfunc TestSparseMutationCheckpointing(t *testing.T) {\n\tcfg := CreateConfig(\"TestSparseMutationCheckpointing\")\n\tnumSnapshotsToKeepOrig := NumSnapshotsToKeep\n\tNumSnapshotsToKeep = 3\n\trollbackSamplingIntervalOrig := RollbackSamplingInterval\n\tRollbackSamplingInterval = 2 * time.Second\n\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\tNumSnapshotsToKeep = numSnapshotsToKeepOrig\n\t\tRollbackSamplingInterval = rollbackSamplingIntervalOrig\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\t// disable merger and purger\n\tRegistryEventCallbacks[\"test\"] = func(e Event) bool {\n\t\tif e.Kind == EventKindPreMergeCheck {\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t}\n\tcfg[\"eventCallbackName\"] = \"test\"\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tscorchi, ok := idx.(*Scorch)\n\tif !ok {\n\t\tt.Fatalf(\"Not a scorch index?\")\n\t}\n\n\terr = scorchi.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// create 4 snapshots every 2 seconds\n\tindexDummyData(t, scorchi, 1)\n\ttime.Sleep(RollbackSamplingInterval)\n\tindexDummyData(t, scorchi, 3)\n\ttime.Sleep(RollbackSamplingInterval)\n\tindexDummyData(t, scorchi, 5)\n\ttime.Sleep(RollbackSamplingInterval)\n\tindexDummyData(t, scorchi, 7)\n\n\t// now the another snapshot is persisted outside of the window of checkpointing\n\t// and we should be able to retain some older checkpoints as well along with\n\t// the latest one\n\ttime.Sleep(time.Duration(NumSnapshotsToKeep) * RollbackSamplingInterval)\n\tindexDummyData(t, scorchi, 9)\n\n\tpersistedSnapshots, err := scorchi.rootBoltSnapshotMetaData()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// should have more than 1 snapshots\n\tprotectedSnapshots := getProtectedSnapshots(RollbackSamplingInterval, NumSnapshotsToKeep, persistedSnapshots)\n\tif len(protectedSnapshots) <= 1 {\n\t\tt.Fatalf(\"expected %d protected snapshots, got %d\", NumSnapshotsToKeep, len(protectedSnapshots))\n\t}\n}\n"
  },
  {
    "path": "index/scorch/scorch.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n\tbolt \"go.etcd.io/bbolt\"\n)\n\nconst Name = \"scorch\"\n\nconst Version uint8 = 2\n\nvar ErrClosed = fmt.Errorf(\"scorch closed\")\n\ntype Scorch struct {\n\tnextSegmentID uint64\n\tstats         Stats\n\tiStats        internalStats\n\n\treadOnly      bool\n\tversion       uint8\n\tconfig        map[string]interface{}\n\tsegmentConfig map[string]interface{}\n\tanalysisQueue *index.AnalysisQueue\n\tpath          string\n\n\tunsafeBatch bool\n\n\trootLock sync.RWMutex\n\n\troot                 *IndexSnapshot // holds 1 ref-count on the root\n\trootPersisted        []chan error   // closed when root is persisted\n\tpersistedCallbacks   []index.BatchCallback\n\tnextSnapshotEpoch    uint64\n\teligibleForRemoval   []uint64        // Index snapshot epochs that are safe to GC.\n\tineligibleForRemoval map[string]bool // Filenames that should not be GC'ed yet.\n\n\t// keeps track of segments scheduled for online copy/backup operation. Each segment's filename maps to\n\t// the count of copy schedules. Segments with non-zero counts are protected from removal by the cleanup\n\t// operation. Counts decrement upon successful copy, allowing removal of segments with zero or absent counts.\n\t// must be accessed within the rootLock as it is accessed by the asynchronous cleanup routine.\n\tcopyScheduled map[string]int\n\n\tnumSnapshotsToKeep       int\n\trollbackRetentionFactor  float64\n\tcheckPoints              []*snapshotMetaData\n\trollbackSamplingInterval time.Duration\n\tcloseCh                  chan struct{}\n\tintroductions            chan *segmentIntroduction\n\tpersists                 chan *persistIntroduction\n\tmerges                   chan *segmentMerge\n\tintroducerNotifier       chan *epochWatcher\n\tpersisterNotifier        chan *epochWatcher\n\trootBolt                 *bolt.DB\n\tasyncTasks               sync.WaitGroup\n\n\tonEvent      func(event Event) bool\n\tonAsyncError func(err error, path string)\n\n\tforceMergeRequestCh chan *mergerCtrl\n\n\tsegPlugin SegmentPlugin\n\n\tspatialPlugin index.SpatialAnalyzerPlugin\n}\n\ntype ScorchErrorType string\n\nfunc (t ScorchErrorType) Error() string {\n\treturn string(t)\n}\n\n// ErrType values for ScorchError\nconst (\n\tErrAsyncPanic   = ScorchErrorType(\"async panic error\")\n\tErrPersist      = ScorchErrorType(\"persist error\")\n\tErrCleanup      = ScorchErrorType(\"cleanup error\")\n\tErrOptionsParse = ScorchErrorType(\"options parse error\")\n)\n\n// ScorchError is passed to onAsyncError when errors are\n// fired from scorch background processes\ntype ScorchError struct {\n\tSource  string\n\tErrMsg  string\n\tErrType ScorchErrorType\n}\n\nfunc (e *ScorchError) Error() string {\n\treturn fmt.Sprintf(\"source: %s, %v: %s\", e.Source, e.ErrType, e.ErrMsg)\n}\n\n// Lets the onAsyncError function verify what type of\n// error is fired using errors.Is(...). This lets the function\n// handle errors differently.\nfunc (e *ScorchError) Unwrap() error {\n\treturn e.ErrType\n}\n\nfunc NewScorchError(source, errMsg string, errType ScorchErrorType) error {\n\treturn &ScorchError{\n\t\tSource:  source,\n\t\tErrMsg:  errMsg,\n\t\tErrType: errType,\n\t}\n}\n\ntype internalStats struct {\n\tpersistEpoch          uint64\n\tpersistSnapshotSize   uint64\n\tmergeEpoch            uint64\n\tmergeSnapshotSize     uint64\n\tnewSegBufBytesAdded   uint64\n\tnewSegBufBytesRemoved uint64\n\tanalysisBytesAdded    uint64\n\tanalysisBytesRemoved  uint64\n}\n\nfunc NewScorch(storeName string,\n\tconfig map[string]interface{},\n\tanalysisQueue *index.AnalysisQueue,\n) (index.Index, error) {\n\trv := &Scorch{\n\t\tversion:              Version,\n\t\tconfig:               config,\n\t\tanalysisQueue:        analysisQueue,\n\t\tnextSnapshotEpoch:    1,\n\t\tcloseCh:              make(chan struct{}),\n\t\tineligibleForRemoval: map[string]bool{},\n\t\tforceMergeRequestCh:  make(chan *mergerCtrl, 1),\n\t\tsegPlugin:            defaultSegmentPlugin,\n\t\tcopyScheduled:        map[string]int{},\n\t\tsegmentConfig:        make(map[string]interface{}),\n\t}\n\n\tforcedSegmentType, forcedSegmentVersion, err := configForceSegmentTypeVersion(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif forcedSegmentType != \"\" && forcedSegmentVersion != 0 {\n\t\terr := rv.loadSegmentPlugin(forcedSegmentType,\n\t\t\tuint32(forcedSegmentVersion))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\ttyp, ok := config[\"spatialPlugin\"].(string)\n\tif ok {\n\t\tif err := rv.loadSpatialAnalyzerPlugin(typ); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\trv.root = &IndexSnapshot{parent: rv, refs: 1, creator: \"NewScorch\"}\n\tro, ok := config[\"read_only\"].(bool)\n\tif ok {\n\t\trv.readOnly = ro\n\t}\n\tub, ok := config[\"unsafe_batch\"].(bool)\n\tif ok {\n\t\trv.unsafeBatch = ub\n\t}\n\tecbName, ok := config[\"eventCallbackName\"].(string)\n\tif ok {\n\t\trv.onEvent = RegistryEventCallbacks[ecbName]\n\t}\n\taecbName, ok := config[\"asyncErrorCallbackName\"].(string)\n\tif ok {\n\t\trv.onAsyncError = RegistryAsyncErrorCallbacks[aecbName]\n\t}\n\t// validate any custom persistor options to\n\t// prevent an async error in the persistor routine\n\t_, err = rv.parsePersisterOptions()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// validate any custom merge planner options to\n\t// prevent an async error in the merger routine\n\t_, err = rv.parseMergePlannerOptions()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rv, nil\n}\n\n// configForceSegmentTypeVersion checks if the caller has requested a\n// specific segment type/version\nfunc configForceSegmentTypeVersion(config map[string]interface{}) (string, uint32, error) {\n\tforcedSegmentVersion, err := parseToInteger(config[\"forceSegmentVersion\"])\n\tif err != nil {\n\t\treturn \"\", 0, nil\n\t}\n\n\tforcedSegmentType, ok := config[\"forceSegmentType\"].(string)\n\tif !ok {\n\t\treturn \"\", 0, fmt.Errorf(\n\t\t\t\"forceSegmentVersion set to %d, must also specify forceSegmentType\", forcedSegmentVersion)\n\t}\n\n\treturn forcedSegmentType, uint32(forcedSegmentVersion), nil\n}\n\nfunc (s *Scorch) NumEventsBlocking() uint64 {\n\teventsCompleted := atomic.LoadUint64(&s.stats.TotEventTriggerCompleted)\n\teventsStarted := atomic.LoadUint64(&s.stats.TotEventTriggerStarted)\n\treturn eventsStarted - eventsCompleted\n}\n\nfunc (s *Scorch) fireEvent(kind EventKind, dur time.Duration) bool {\n\tres := true\n\tif s.onEvent != nil {\n\t\tatomic.AddUint64(&s.stats.TotEventTriggerStarted, 1)\n\t\tres = s.onEvent(Event{Kind: kind, Scorch: s, Duration: dur})\n\t\tatomic.AddUint64(&s.stats.TotEventTriggerCompleted, 1)\n\t}\n\treturn res\n}\n\nfunc (s *Scorch) fireAsyncError(err error) {\n\tif s.onAsyncError != nil {\n\t\ts.onAsyncError(err, s.path)\n\t}\n\tatomic.AddUint64(&s.stats.TotOnErrors, 1)\n}\n\nfunc (s *Scorch) Open() error {\n\tif s.rootBolt == nil {\n\t\terr := s.openBolt()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\ts.asyncTasks.Add(1)\n\tgo s.introducerLoop()\n\n\tif !s.readOnly && s.path != \"\" {\n\t\ts.asyncTasks.Add(1)\n\t\tgo s.persisterLoop()\n\t\ts.asyncTasks.Add(1)\n\t\tgo s.mergerLoop()\n\t}\n\n\treturn nil\n}\n\nfunc (s *Scorch) openBolt() error {\n\tvar ok bool\n\ts.path, ok = s.config[\"path\"].(string)\n\tif !ok {\n\t\treturn fmt.Errorf(\"must specify path\")\n\t}\n\tif s.path == \"\" {\n\t\ts.unsafeBatch = true\n\t}\n\n\trootBoltOpt := *bolt.DefaultOptions\n\tif s.readOnly {\n\t\trootBoltOpt.ReadOnly = true\n\t\trootBoltOpt.OpenFile = func(path string, flag int, mode os.FileMode) (*os.File, error) {\n\t\t\t// Bolt appends an O_CREATE flag regardless.\n\t\t\t// See - https://github.com/etcd-io/bbolt/blob/v1.3.5/db.go#L210\n\t\t\t// Use os.O_RDONLY only if path exists (#1623)\n\t\t\tif _, err := os.Stat(path); os.IsNotExist(err) {\n\t\t\t\treturn os.OpenFile(path, flag, mode)\n\t\t\t}\n\t\t\treturn os.OpenFile(path, os.O_RDONLY, mode)\n\t\t}\n\t} else {\n\t\tif s.path != \"\" {\n\t\t\terr := os.MkdirAll(s.path, 0o700)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\tif boltTimeoutStr, ok := s.config[\"bolt_timeout\"].(string); ok {\n\t\tvar err error\n\t\tboltTimeout, err := time.ParseDuration(boltTimeoutStr)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"invalid duration specified for bolt_timeout: %v\", err)\n\t\t}\n\t\trootBoltOpt.Timeout = boltTimeout\n\t}\n\n\trootBoltPath := s.path + string(os.PathSeparator) + \"root.bolt\"\n\tvar err error\n\tif s.path != \"\" {\n\t\ts.rootBolt, err = bolt.Open(rootBoltPath, 0o600, &rootBoltOpt)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// now see if there is any existing state to load\n\t\terr = s.loadFromBolt()\n\t\tif err != nil {\n\t\t\t_ = s.Close()\n\t\t\treturn err\n\t\t}\n\t}\n\n\tatomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, uint64(len(s.root.segment)))\n\n\ts.introductions = make(chan *segmentIntroduction)\n\ts.persists = make(chan *persistIntroduction)\n\ts.merges = make(chan *segmentMerge)\n\ts.introducerNotifier = make(chan *epochWatcher, 1)\n\ts.persisterNotifier = make(chan *epochWatcher, 1)\n\ts.closeCh = make(chan struct{})\n\ts.forceMergeRequestCh = make(chan *mergerCtrl, 1)\n\n\tif !s.readOnly && s.path != \"\" {\n\t\terr := s.removeOldZapFiles() // Before persister or merger create any new files.\n\t\tif err != nil {\n\t\t\t_ = s.Close()\n\t\t\treturn err\n\t\t}\n\t}\n\n\ts.numSnapshotsToKeep = NumSnapshotsToKeep\n\tif v, ok := s.config[\"numSnapshotsToKeep\"]; ok {\n\t\tvar t int\n\t\tif t, err = parseToInteger(v); err != nil {\n\t\t\treturn fmt.Errorf(\"numSnapshotsToKeep parse err: %v\", err)\n\t\t}\n\t\tif t > 0 {\n\t\t\ts.numSnapshotsToKeep = t\n\t\t}\n\t}\n\n\ts.rollbackSamplingInterval = RollbackSamplingInterval\n\tif v, ok := s.config[\"rollbackSamplingInterval\"]; ok {\n\t\tvar t time.Duration\n\t\tif t, err = parseToTimeDuration(v); err != nil {\n\t\t\treturn fmt.Errorf(\"rollbackSamplingInterval parse err: %v\", err)\n\t\t}\n\t\ts.rollbackSamplingInterval = t\n\t}\n\n\ts.rollbackRetentionFactor = RollbackRetentionFactor\n\tif v, ok := s.config[\"rollbackRetentionFactor\"]; ok {\n\t\tvar r float64\n\t\tif r, ok = v.(float64); ok {\n\t\t\treturn fmt.Errorf(\"rollbackRetentionFactor parse err: %v\", err)\n\t\t}\n\t\ts.rollbackRetentionFactor = r\n\t}\n\n\ttyp, ok := s.config[\"spatialPlugin\"].(string)\n\tif ok {\n\t\tif err := s.loadSpatialAnalyzerPlugin(typ); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (s *Scorch) Close() (err error) {\n\tstartTime := time.Now()\n\tdefer func() {\n\t\ts.fireEvent(EventKindClose, time.Since(startTime))\n\t}()\n\n\ts.fireEvent(EventKindCloseStart, 0)\n\n\t// signal to async tasks we want to close\n\tclose(s.closeCh)\n\t// wait for them to close\n\ts.asyncTasks.Wait()\n\t// now close the root bolt\n\tif s.rootBolt != nil {\n\t\terr = s.rootBolt.Close()\n\t\ts.rootLock.Lock()\n\t\tif s.root != nil {\n\t\t\terr2 := s.root.DecRef()\n\t\t\tif err == nil {\n\t\t\t\terr = err2\n\t\t\t}\n\t\t}\n\t\ts.root = nil\n\t\ts.rootBolt = nil\n\t\ts.rootLock.Unlock()\n\t}\n\n\treturn\n}\n\nfunc (s *Scorch) Update(doc index.Document) error {\n\tb := index.NewBatch()\n\tb.Update(doc)\n\treturn s.Batch(b)\n}\n\nfunc (s *Scorch) Delete(id string) error {\n\tb := index.NewBatch()\n\tb.Delete(id)\n\treturn s.Batch(b)\n}\n\n// Batch applices a batch of changes to the index atomically\nfunc (s *Scorch) Batch(batch *index.Batch) (err error) {\n\tstart := time.Now()\n\n\t// notify handlers that we're about to index a batch of data\n\ts.fireEvent(EventKindBatchIntroductionStart, 0)\n\tdefer func() {\n\t\ts.fireEvent(EventKindBatchIntroduction, time.Since(start))\n\t}()\n\n\tresultChan := make(chan index.Document, len(batch.IndexOps))\n\n\tvar numUpdates uint64\n\tvar numDeletes uint64\n\tvar numPlainTextBytes uint64\n\tvar ids []string\n\tfor docID, doc := range batch.IndexOps {\n\t\tif doc != nil {\n\t\t\t// insert _id field\n\t\t\tdoc.AddIDField()\n\t\t\tnumUpdates++\n\t\t\tnumPlainTextBytes += doc.NumPlainTextBytes()\n\t\t} else {\n\t\t\tnumDeletes++\n\t\t}\n\t\tids = append(ids, docID)\n\t}\n\n\t// FIXME could sort ids list concurrent with analysis?\n\n\tif numUpdates > 0 {\n\t\tgo func() {\n\t\t\tfor k := range batch.IndexOps {\n\t\t\t\tdoc := batch.IndexOps[k]\n\t\t\t\tif doc != nil {\n\t\t\t\t\t// put the work on the queue\n\t\t\t\t\ts.analysisQueue.Queue(func() {\n\t\t\t\t\t\tanalyze(doc, s.setSpatialAnalyzerPlugin)\n\t\t\t\t\t\tresultChan <- doc\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\t// wait for analysis result\n\tanalysisResults := make([]index.Document, int(numUpdates))\n\tvar itemsDeQueued uint64\n\tvar totalAnalysisSize int\n\tfor itemsDeQueued < numUpdates {\n\t\tresult := <-resultChan\n\t\tresultSize := result.Size()\n\t\t// check if the document is searchable by the index\n\t\tif result.Indexed() {\n\t\t\tatomic.AddUint64(&s.stats.TotMutationsFiltered, 1)\n\t\t}\n\t\tatomic.AddUint64(&s.iStats.analysisBytesAdded, uint64(resultSize))\n\t\ttotalAnalysisSize += resultSize\n\t\tanalysisResults[itemsDeQueued] = result\n\t\titemsDeQueued++\n\t}\n\tclose(resultChan)\n\tdefer atomic.AddUint64(&s.iStats.analysisBytesRemoved, uint64(totalAnalysisSize))\n\n\tatomic.AddUint64(&s.stats.TotAnalysisTime, uint64(time.Since(start)))\n\n\tindexStart := time.Now()\n\n\tvar newSegment segment.Segment\n\tvar bufBytes uint64\n\tstats := newFieldStats()\n\n\tif len(analysisResults) > 0 {\n\t\tnewSegment, bufBytes, err = s.segPlugin.NewUsing(analysisResults, s.segmentConfig)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif segB, ok := newSegment.(segment.DiskStatsReporter); ok {\n\t\t\tatomic.AddUint64(&s.stats.TotBytesWrittenAtIndexTime,\n\t\t\t\tsegB.BytesWritten())\n\t\t}\n\t\tatomic.AddUint64(&s.iStats.newSegBufBytesAdded, bufBytes)\n\t\tif fsr, ok := newSegment.(segment.FieldStatsReporter); ok {\n\t\t\tfsr.UpdateFieldStats(stats)\n\t\t}\n\t} else {\n\t\tatomic.AddUint64(&s.stats.TotBatchesEmpty, 1)\n\t}\n\n\terr = s.prepareSegment(newSegment, ids, batch.InternalOps, batch.PersistedCallback(), stats)\n\tif err != nil {\n\t\tif newSegment != nil {\n\t\t\t_ = newSegment.Close()\n\t\t}\n\t\tatomic.AddUint64(&s.stats.TotOnErrors, 1)\n\t} else {\n\t\tatomic.AddUint64(&s.stats.TotUpdates, numUpdates)\n\t\tatomic.AddUint64(&s.stats.TotDeletes, numDeletes)\n\t\tatomic.AddUint64(&s.stats.TotBatches, 1)\n\t\tatomic.AddUint64(&s.stats.TotIndexedPlainTextBytes, numPlainTextBytes)\n\t}\n\n\tatomic.AddUint64(&s.iStats.newSegBufBytesRemoved, bufBytes)\n\tatomic.AddUint64(&s.stats.TotIndexTime, uint64(time.Since(indexStart)))\n\n\treturn err\n}\n\nfunc (s *Scorch) prepareSegment(newSegment segment.Segment, ids []string,\n\tinternalOps map[string][]byte, persistedCallback index.BatchCallback, stats *fieldStats,\n) error {\n\t// new introduction\n\tintroduction := &segmentIntroduction{\n\t\tid:                atomic.AddUint64(&s.nextSegmentID, 1),\n\t\tdata:              newSegment,\n\t\tids:               ids,\n\t\tinternal:          internalOps,\n\t\tstats:             stats,\n\t\tapplied:           make(chan error),\n\t\tpersistedCallback: persistedCallback,\n\t}\n\n\tif !s.unsafeBatch {\n\t\tintroduction.persisted = make(chan error, 1)\n\t}\n\n\t// optimistically prepare obsoletes outside of rootLock\n\ts.rootLock.RLock()\n\troot := s.root\n\troot.AddRef()\n\ts.rootLock.RUnlock()\n\n\tdefer func() { _ = root.DecRef() }()\n\n\tintroduction.obsoletes = make(map[uint64]*roaring.Bitmap, len(root.segment))\n\n\tfor _, seg := range root.segment {\n\t\tdelta, err := seg.segment.DocNumbers(ids)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tintroduction.obsoletes[seg.id] = delta\n\t}\n\n\tintroStartTime := time.Now()\n\n\ts.introductions <- introduction\n\n\t// block until this segment is applied\n\terr := <-introduction.applied\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif introduction.persisted != nil {\n\t\terr = <-introduction.persisted\n\t}\n\n\tintroTime := uint64(time.Since(introStartTime))\n\tatomic.AddUint64(&s.stats.TotBatchIntroTime, introTime)\n\tif atomic.LoadUint64(&s.stats.MaxBatchIntroTime) < introTime {\n\t\tatomic.StoreUint64(&s.stats.MaxBatchIntroTime, introTime)\n\t}\n\n\treturn err\n}\n\nfunc (s *Scorch) SetInternal(key, val []byte) error {\n\tb := index.NewBatch()\n\tb.SetInternal(key, val)\n\treturn s.Batch(b)\n}\n\nfunc (s *Scorch) DeleteInternal(key []byte) error {\n\tb := index.NewBatch()\n\tb.DeleteInternal(key)\n\treturn s.Batch(b)\n}\n\n// Reader returns a low-level accessor on the index data. Close it to\n// release associated resources.\nfunc (s *Scorch) Reader() (index.IndexReader, error) {\n\treturn s.currentSnapshot(), nil\n}\n\nfunc (s *Scorch) currentSnapshot() *IndexSnapshot {\n\ts.rootLock.RLock()\n\trv := s.root\n\tif rv != nil {\n\t\trv.AddRef()\n\t}\n\ts.rootLock.RUnlock()\n\treturn rv\n}\n\nfunc (s *Scorch) Stats() json.Marshaler {\n\treturn &s.stats\n}\n\nfunc (s *Scorch) BytesReadQueryTime() uint64 {\n\treturn s.stats.TotBytesReadAtQueryTime\n}\n\nfunc (s *Scorch) diskFileStats(rootSegmentPaths map[string]struct{}) (uint64,\n\tuint64, uint64,\n) {\n\tvar numFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot uint64\n\tif s.path != \"\" {\n\t\tfiles, err := os.ReadDir(s.path)\n\t\tif err == nil {\n\t\t\tfor _, f := range files {\n\t\t\t\tif !f.IsDir() {\n\t\t\t\t\tif finfo, err := f.Info(); err == nil {\n\t\t\t\t\t\tnumBytesUsedDisk += uint64(finfo.Size())\n\t\t\t\t\t\tnumFilesOnDisk++\n\t\t\t\t\t\tif rootSegmentPaths != nil {\n\t\t\t\t\t\t\tfname := s.path + string(os.PathSeparator) + finfo.Name()\n\t\t\t\t\t\t\tif _, fileAtRoot := rootSegmentPaths[fname]; fileAtRoot {\n\t\t\t\t\t\t\t\tnumBytesOnDiskByRoot += uint64(finfo.Size())\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// if no root files path given, then consider all disk files.\n\tif rootSegmentPaths == nil {\n\t\treturn numFilesOnDisk, numBytesUsedDisk, numBytesUsedDisk\n\t}\n\n\treturn numFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot\n}\n\nfunc (s *Scorch) StatsMap() map[string]interface{} {\n\tm := s.stats.ToMap()\n\n\tindexSnapshot := s.currentSnapshot()\n\tif indexSnapshot == nil {\n\t\treturn nil\n\t}\n\n\tdefer func() {\n\t\t_ = indexSnapshot.Close()\n\t}()\n\n\trootSegPaths := indexSnapshot.diskSegmentsPaths()\n\n\ts.rootLock.RLock()\n\tm[\"CurFilesIneligibleForRemoval\"] = uint64(len(s.ineligibleForRemoval))\n\ts.rootLock.RUnlock()\n\n\tnumFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot := s.diskFileStats(rootSegPaths)\n\n\tm[\"CurOnDiskBytes\"] = numBytesUsedDisk\n\tm[\"CurOnDiskFiles\"] = numFilesOnDisk\n\n\t// TODO: consider one day removing these backwards compatible\n\t// names for apps using the old names\n\tm[\"updates\"] = m[\"TotUpdates\"]\n\tm[\"deletes\"] = m[\"TotDeletes\"]\n\tm[\"batches\"] = m[\"TotBatches\"]\n\tm[\"errors\"] = m[\"TotOnErrors\"]\n\tm[\"analysis_time\"] = m[\"TotAnalysisTime\"]\n\tm[\"index_time\"] = m[\"TotIndexTime\"]\n\tm[\"term_searchers_started\"] = m[\"TotTermSearchersStarted\"]\n\tm[\"term_searchers_finished\"] = m[\"TotTermSearchersFinished\"]\n\tm[\"knn_searches\"] = m[\"TotKNNSearches\"]\n\tm[\"synonym_searches\"] = m[\"TotSynonymSearches\"]\n\tm[\"total_mutations_filtered\"] = m[\"TotMutationsFiltered\"]\n\n\tm[\"num_bytes_read_at_query_time\"] = m[\"TotBytesReadAtQueryTime\"]\n\tm[\"num_plain_text_bytes_indexed\"] = m[\"TotIndexedPlainTextBytes\"]\n\tm[\"num_bytes_written_at_index_time\"] = m[\"TotBytesWrittenAtIndexTime\"]\n\tm[\"num_items_introduced\"] = m[\"TotIntroducedItems\"]\n\tm[\"num_items_persisted\"] = m[\"TotPersistedItems\"]\n\tm[\"num_recs_to_persist\"] = m[\"TotItemsToPersist\"]\n\t// total disk bytes found in index directory inclusive of older snapshots\n\tm[\"num_bytes_used_disk\"] = numBytesUsedDisk\n\t// total disk bytes by the latest root index, exclusive of older snapshots\n\tm[\"num_bytes_used_disk_by_root\"] = numBytesOnDiskByRoot\n\t// num_bytes_used_disk_by_root_reclaimable is an approximation about the\n\t// reclaimable disk space in an index. (eg: from a full compaction)\n\tm[\"num_bytes_used_disk_by_root_reclaimable\"] = uint64(float64(numBytesOnDiskByRoot) *\n\t\tindexSnapshot.reClaimableDocsRatio())\n\tm[\"num_files_on_disk\"] = numFilesOnDisk\n\tm[\"num_root_memorysegments\"] = m[\"TotMemorySegmentsAtRoot\"]\n\tm[\"num_root_filesegments\"] = m[\"TotFileSegmentsAtRoot\"]\n\tm[\"num_persister_nap_pause_completed\"] = m[\"TotPersisterNapPauseCompleted\"]\n\tm[\"num_persister_nap_merger_break\"] = m[\"TotPersisterMergerNapBreak\"]\n\tm[\"total_compaction_written_bytes\"] = m[\"TotFileMergeWrittenBytes\"]\n\n\t// the bool stat `index_bgthreads_active` indicates whether the background routines\n\t// (which are responsible for the index to attain a steady state) are still\n\t// doing some work.\n\tif rootEpoch, ok := m[\"CurRootEpoch\"].(uint64); ok {\n\t\tif lastMergedEpoch, ok := m[\"LastMergedEpoch\"].(uint64); ok {\n\t\t\tif lastPersistedEpoch, ok := m[\"LastPersistedEpoch\"].(uint64); ok {\n\t\t\t\tm[\"index_bgthreads_active\"] = !(lastMergedEpoch == rootEpoch && lastPersistedEpoch == rootEpoch)\n\t\t\t}\n\t\t}\n\t}\n\n\t// calculate the aggregate of all the segment's field stats\n\taggFieldStats := newFieldStats()\n\tfor _, segmentSnapshot := range indexSnapshot.Segments() {\n\t\tif segmentSnapshot.stats != nil {\n\t\t\taggFieldStats.Aggregate(segmentSnapshot.stats)\n\t\t}\n\t}\n\n\taggFieldStatsMap := aggFieldStats.Fetch()\n\tfor statName, stats := range aggFieldStatsMap {\n\t\tfor fieldName, val := range stats {\n\t\t\tm[\"field:\"+fieldName+\":\"+statName] = val\n\t\t}\n\t}\n\treturn m\n}\n\nfunc (s *Scorch) Analyze(d index.Document) {\n\tanalyze(d, s.setSpatialAnalyzerPlugin)\n}\n\ntype customAnalyzerPluginInitFunc func(field index.Field)\n\nfunc (s *Scorch) setSpatialAnalyzerPlugin(f index.Field) {\n\tif s.segPlugin != nil {\n\t\t// check whether the current field is a custom tokenizable\n\t\t// spatial field then set the spatial analyser plugin for\n\t\t// overriding the tokenisation during the analysis stage.\n\t\tif sf, ok := f.(index.TokenizableSpatialField); ok {\n\t\t\tsf.SetSpatialAnalyzerPlugin(s.spatialPlugin)\n\t\t}\n\t}\n}\n\nfunc analyze(d index.Document, fn customAnalyzerPluginInitFunc) {\n\td.VisitFields(func(field index.Field) {\n\t\tif field.Options().IsIndexed() {\n\t\t\tif fn != nil {\n\t\t\t\tfn(field)\n\t\t\t}\n\n\t\t\tfield.Analyze()\n\n\t\t\tif d.HasComposite() && field.Name() != \"_id\" {\n\t\t\t\t// see if any of the composite fields need this\n\t\t\t\td.VisitComposite(func(cf index.CompositeField) {\n\t\t\t\t\tcf.Compose(field.Name(), field.AnalyzedLength(), field.AnalyzedTokenFrequencies())\n\t\t\t\t})\n\t\t\t\t// Since the encoded geoShape is only necessary within the doc values\n\t\t\t\t// of the geoShapeField, it has been removed from the field's term dictionary.\n\t\t\t\t// However, '_all' field uses its term dictionary as its docValues, so it\n\t\t\t\t// becomes necessary to add the geoShape into the '_all' field's term dictionary\n\t\t\t\tif f, ok := field.(index.GeoShapeField); ok {\n\t\t\t\t\td.VisitComposite(func(cf index.CompositeField) {\n\t\t\t\t\t\tgeoshape := f.EncodedShape()\n\t\t\t\t\t\tcf.Compose(field.Name(), 1, index.TokenFrequencies{\n\t\t\t\t\t\t\tstring(geoshape): &index.TokenFreq{\n\t\t\t\t\t\t\t\tTerm: geoshape,\n\t\t\t\t\t\t\t\tLocations: []*index.TokenLocation{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tStart:    0,\n\t\t\t\t\t\t\t\t\t\tEnd:      len(geoshape),\n\t\t\t\t\t\t\t\t\t\tPosition: 1,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t})\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t})\n\tif nd, ok := d.(index.NestedDocument); ok {\n\t\tnd.VisitNestedDocuments(func(doc index.Document) {\n\t\t\tdoc.AddIDField()\n\t\t\tanalyze(doc, fn)\n\t\t})\n\t}\n}\n\nfunc (s *Scorch) AddEligibleForRemoval(epoch uint64) {\n\ts.rootLock.Lock()\n\tif s.root == nil || s.root.epoch != epoch {\n\t\ts.eligibleForRemoval = append(s.eligibleForRemoval, epoch)\n\t}\n\ts.rootLock.Unlock()\n}\n\nfunc (s *Scorch) MemoryUsed() (memUsed uint64) {\n\tindexSnapshot := s.currentSnapshot()\n\tif indexSnapshot == nil {\n\t\treturn\n\t}\n\n\tdefer func() {\n\t\t_ = indexSnapshot.Close()\n\t}()\n\n\t// Account for current root snapshot overhead\n\tmemUsed += uint64(indexSnapshot.Size())\n\n\t// Account for snapshot that the persister may be working on\n\tpersistEpoch := atomic.LoadUint64(&s.iStats.persistEpoch)\n\tpersistSnapshotSize := atomic.LoadUint64(&s.iStats.persistSnapshotSize)\n\tif persistEpoch != 0 && indexSnapshot.epoch > persistEpoch {\n\t\t// the snapshot that the persister is working on isn't the same as\n\t\t// the current snapshot\n\t\tmemUsed += persistSnapshotSize\n\t}\n\n\t// Account for snapshot that the merger may be working on\n\tmergeEpoch := atomic.LoadUint64(&s.iStats.mergeEpoch)\n\tmergeSnapshotSize := atomic.LoadUint64(&s.iStats.mergeSnapshotSize)\n\tif mergeEpoch != 0 && indexSnapshot.epoch > mergeEpoch {\n\t\t// the snapshot that the merger is working on isn't the same as\n\t\t// the current snapshot\n\t\tmemUsed += mergeSnapshotSize\n\t}\n\n\tmemUsed += (atomic.LoadUint64(&s.iStats.newSegBufBytesAdded) -\n\t\tatomic.LoadUint64(&s.iStats.newSegBufBytesRemoved))\n\n\tmemUsed += (atomic.LoadUint64(&s.iStats.analysisBytesAdded) -\n\t\tatomic.LoadUint64(&s.iStats.analysisBytesRemoved))\n\n\treturn memUsed\n}\n\nfunc (s *Scorch) markIneligibleForRemoval(filename string) {\n\ts.rootLock.Lock()\n\ts.ineligibleForRemoval[filename] = true\n\ts.rootLock.Unlock()\n}\n\nfunc (s *Scorch) unmarkIneligibleForRemoval(filename string) {\n\ts.rootLock.Lock()\n\tdelete(s.ineligibleForRemoval, filename)\n\ts.rootLock.Unlock()\n}\n\nfunc init() {\n\terr := registry.RegisterIndexType(Name, NewScorch)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc parseToTimeDuration(i interface{}) (time.Duration, error) {\n\tswitch v := i.(type) {\n\tcase string:\n\t\treturn time.ParseDuration(v)\n\n\tdefault:\n\t\treturn 0, fmt.Errorf(\"expects a duration string\")\n\t}\n}\n\nfunc parseToInteger(i interface{}) (int, error) {\n\tswitch v := i.(type) {\n\tcase float64:\n\t\treturn int(v), nil\n\tcase int:\n\t\treturn v, nil\n\n\tdefault:\n\t\treturn 0, fmt.Errorf(\"expects int or float64 value\")\n\t}\n}\n\n// Holds Zap's field level stats at a segment level\ntype fieldStats struct {\n\t// StatName -> FieldName -> value\n\tstatMap map[string]map[string]uint64\n}\n\n// Add the data into the map after checking if the statname is valid\nfunc (fs *fieldStats) Store(statName, fieldName string, value uint64) {\n\tif _, exists := fs.statMap[statName]; !exists {\n\t\tfs.statMap[statName] = make(map[string]uint64)\n\t}\n\tfs.statMap[statName][fieldName] = value\n}\n\n// Combine the given stats map with the existing map\nfunc (fs *fieldStats) Aggregate(stats segment.FieldStats) {\n\tstatMap := stats.Fetch()\n\tif statMap == nil {\n\t\treturn\n\t}\n\tfor statName, statMap := range statMap {\n\t\tif _, exists := fs.statMap[statName]; !exists {\n\t\t\tfs.statMap[statName] = make(map[string]uint64)\n\t\t}\n\t\tfor fieldName, val := range statMap {\n\t\t\tif _, exists := fs.statMap[statName][fieldName]; !exists {\n\t\t\t\tfs.statMap[statName][fieldName] = 0\n\t\t\t}\n\t\t\tfs.statMap[statName][fieldName] += val\n\t\t}\n\t}\n}\n\n// Returns the stats map\nfunc (fs *fieldStats) Fetch() map[string]map[string]uint64 {\n\tif fs == nil {\n\t\treturn nil\n\t}\n\n\treturn fs.statMap\n}\n\n// Initializes an empty stats map\nfunc newFieldStats() *fieldStats {\n\trv := &fieldStats{\n\t\tstatMap: map[string]map[string]uint64{},\n\t}\n\treturn rv\n}\n\n// CopyReader returns a low-level accessor for index data, ensuring persisted segments\n// remain on disk for backup, preventing race conditions with the persister/merger cleanup.\n// Close the reader after backup to allow segment removal by the persister/merger.\nfunc (s *Scorch) CopyReader() index.CopyReader {\n\ts.rootLock.Lock()\n\trv := s.root\n\tif rv != nil {\n\t\trv.AddRef()\n\t\tvar fileName string\n\t\t// schedule a backup for all the segments from the root. Note that the\n\t\t// both the unpersisted and persisted segments are scheduled for backup.\n\t\t// because during the backup, the unpersisted segments may get persisted and\n\t\t// hence we need to protect both the unpersisted and persisted segments from removal\n\t\t// by the cleanup routine during the online backup\n\t\tfor _, seg := range rv.segment {\n\t\t\tif perSeg, ok := seg.segment.(segment.PersistedSegment); ok {\n\t\t\t\t// segment is persisted\n\t\t\t\tfileName = filepath.Base(perSeg.Path())\n\t\t\t} else {\n\t\t\t\t// segment is not persisted\n\t\t\t\t// the name of the segment file that is generated if the\n\t\t\t\t// the segment is persisted in the future.\n\t\t\t\tfileName = zapFileName(seg.id)\n\t\t\t}\n\t\t\trv.parent.copyScheduled[fileName]++\n\t\t}\n\t}\n\ts.rootLock.Unlock()\n\treturn rv\n}\n\n// external API to fire a scorch event (EventKindIndexStart) externally from bleve\nfunc (s *Scorch) FireIndexEvent() {\n\ts.fireEvent(EventKindIndexStart, 0)\n}\n\n// Updates bolt db with the given field info. Existing field info already in bolt\n// will be merged before persisting. The index mapping is also overwritted both\n// in bolt as well as the index snapshot\nfunc (s *Scorch) UpdateFields(fieldInfo map[string]*index.UpdateFieldInfo, mappingBytes []byte) error {\n\terr := s.updateBolt(fieldInfo, mappingBytes)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Pass the update field info to all snapshots and segment bases\n\ts.root.UpdateFieldsInfo(fieldInfo)\n\treturn nil\n}\n\nfunc (s *Scorch) OpenMeta() error {\n\tif s.rootBolt == nil {\n\t\terr := s.openBolt()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Merge and update deleted field info and rewrite index mapping\nfunc (s *Scorch) updateBolt(fieldInfo map[string]*index.UpdateFieldInfo, mappingBytes []byte) error {\n\treturn s.rootBolt.Update(func(tx *bolt.Tx) error {\n\t\tsnapshots := tx.Bucket(util.BoltSnapshotsBucket)\n\t\tif snapshots == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\tc := snapshots.Cursor()\n\t\tfor k, _ := c.Last(); k != nil; k, _ = c.Prev() {\n\t\t\t_, _, err := decodeUvarintAscending(k)\n\t\t\tif err != nil {\n\t\t\t\tfmt.Printf(\"unable to parse segment epoch %x, continuing\", k)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsnapshot := snapshots.Bucket(k)\n\t\t\tcc := snapshot.Cursor()\n\t\t\tfor kk, _ := cc.First(); kk != nil; kk, _ = cc.Next() {\n\t\t\t\tif kk[0] == util.BoltInternalKey[0] {\n\t\t\t\t\tinternalBucket := snapshot.Bucket(kk)\n\t\t\t\t\tif internalBucket == nil {\n\t\t\t\t\t\treturn fmt.Errorf(\"segment key, but bucket missing %x\", kk)\n\t\t\t\t\t}\n\t\t\t\t\terr = internalBucket.Put(util.MappingInternalKey, mappingBytes)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t} else if kk[0] != util.BoltMetaDataKey[0] {\n\t\t\t\t\tsegmentBucket := snapshot.Bucket(kk)\n\t\t\t\t\tif segmentBucket == nil {\n\t\t\t\t\t\treturn fmt.Errorf(\"segment key, but bucket missing %x\", kk)\n\t\t\t\t\t}\n\t\t\t\t\tvar updatedFields map[string]*index.UpdateFieldInfo\n\t\t\t\t\tupdatedFieldBytes := segmentBucket.Get(util.BoltUpdatedFieldsKey)\n\t\t\t\t\tif updatedFieldBytes != nil {\n\t\t\t\t\t\terr := json.Unmarshal(updatedFieldBytes, &updatedFields)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn fmt.Errorf(\"error reading updated field bytes: %v\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tfor field, info := range fieldInfo {\n\t\t\t\t\t\t\tif val, ok := updatedFields[field]; ok {\n\t\t\t\t\t\t\t\tupdatedFields[field] = &index.UpdateFieldInfo{\n\t\t\t\t\t\t\t\t\tDeleted:   info.Deleted || val.Deleted,\n\t\t\t\t\t\t\t\t\tStore:     info.Store || val.Store,\n\t\t\t\t\t\t\t\t\tDocValues: info.DocValues || val.DocValues,\n\t\t\t\t\t\t\t\t\tIndex:     info.Index || val.Index,\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tupdatedFields[field] = info\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tupdatedFields = fieldInfo\n\t\t\t\t\t}\n\t\t\t\t\tb, err := json.Marshal(updatedFields)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\terr = segmentBucket.Put(util.BoltUpdatedFieldsKey, b)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n"
  },
  {
    "path": "index/scorch/scorch_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\tregexpTokenizer \"github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch/mergeplan\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc init() {\n\t// override for tests\n\tDefaultPersisterNapTimeMSec = 1\n}\n\nfunc InitTest(cfg map[string]interface{}) error {\n\treturn os.RemoveAll(cfg[\"path\"].(string))\n}\n\nfunc DestroyTest(cfg map[string]interface{}) error {\n\treturn os.RemoveAll(cfg[\"path\"].(string))\n}\n\nfunc CreateConfig(name string) map[string]interface{} {\n\t// TODO: Use t.Name() when Go 1.7 support terminates.\n\trv := make(map[string]interface{})\n\trv[\"path\"] = os.TempDir() + \"/bleve-scorch-test-\" + name\n\treturn rv\n}\n\nvar testAnalyzer = &analysis.DefaultAnalyzer{\n\tTokenizer: regexpTokenizer.NewRegexpTokenizer(regexp.MustCompile(`\\w+`)),\n}\n\nfunc TestIndexOpenReopen(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexOpenReopen\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// insert a doc\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now close it\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err = NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\n\t// check the doc count again after reopening it\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now close it\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexOpenReopenWithInsert(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexOpenReopen\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// insert a doc\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now close it\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// try to open the index and insert data\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\n\t// insert a doc\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test2\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\t// check the doc count again after reopening it\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now close it\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsert(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexInsert\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsertThenDelete(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexInsertThenDelete\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc2 := document.NewDocument(\"2\")\n\tdoc2.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc2)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\tiid, err := reader.InternalID(\"1\")\n\tif err != nil || iid == nil {\n\t\tt.Errorf(\"unexpected on doc id 1\")\n\t}\n\tiid, err = reader.InternalID(\"2\")\n\tif err != nil || iid == nil {\n\t\tt.Errorf(\"unexpected on doc id 2\")\n\t}\n\tiid, err = reader.InternalID(\"3\")\n\tif err != nil || iid != nil {\n\t\tt.Errorf(\"unexpected on doc id 3\")\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Delete(\"1\")\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\texpectedCount--\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\tstoredDoc, err := reader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif storedDoc != nil {\n\t\tt.Errorf(\"expected nil for deleted stored doc #1, got %v\", storedDoc)\n\t}\n\tstoredDoc, err = reader.Document(\"2\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif storedDoc == nil {\n\t\tt.Errorf(\"expected stored doc for #2, got nil\")\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now close it\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err = NewScorch(Name, cfg, analysisQueue) // reopen\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error reopening index: %v\", err)\n\t}\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\tstoredDoc, err = reader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif storedDoc != nil {\n\t\tt.Errorf(\"expected nil for deleted stored doc #1, got %v\", storedDoc)\n\t}\n\tstoredDoc, err = reader.Document(\"2\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif storedDoc == nil {\n\t\tt.Errorf(\"expected stored doc for #2, got nil\")\n\t}\n\tiid, err = reader.InternalID(\"1\")\n\tif err != nil || iid != nil {\n\t\tt.Errorf(\"unexpected on doc id 1\")\n\t}\n\tiid, err = reader.InternalID(\"2\")\n\tif err != nil || iid == nil {\n\t\tt.Errorf(\"unexpected on doc id 2, should exist\")\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Delete(\"2\")\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\texpectedCount--\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\tstoredDoc, err = reader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif storedDoc != nil {\n\t\tt.Errorf(\"expected nil for deleted stored doc #1, got %v\", storedDoc)\n\t}\n\tstoredDoc, err = reader.Document(\"2\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif storedDoc != nil {\n\t\tt.Errorf(\"expected nil for deleted stored doc #2, got nil\")\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsertThenUpdate(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexInsertThenUpdate\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar expectedCount uint64\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\t// this update should overwrite one term, and introduce one new one\n\tdoc = document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithAnalyzer(\"name\", []uint64{}, []byte(\"test fail\"), testAnalyzer))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\n\t// now do another update that should remove one of the terms\n\tdoc = document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"fail\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsertMultiple(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexInsertMultiple\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"3\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsertWithStore(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexInsertWithStore\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tstoredDocInt, err := indexReader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstoredDoc := storedDocInt.(*document.Document)\n\n\tif len(storedDoc.Fields) != 1 {\n\t\tt.Errorf(\"expected 1 stored field, got %d\", len(storedDoc.Fields))\n\t}\n\tfor _, field := range storedDoc.Fields {\n\t\tif field.Name() == \"name\" {\n\t\t\ttextField, ok := field.(*document.TextField)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"expected text field\")\n\t\t\t}\n\t\t\tif string(textField.Value()) != \"test\" {\n\t\t\t\tt.Errorf(\"expected field content 'test', got '%s'\", string(textField.Value()))\n\t\t\t}\n\t\t} else if field.Name() == \"_id\" {\n\t\t\tt.Errorf(\"not expecting _id field\")\n\t\t}\n\t}\n}\n\nfunc TestIndexInternalCRUD(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexInternalCRUD\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(indexReader.(*IndexSnapshot).segment) != 0 {\n\t\tt.Errorf(\"expected 0 segments\")\n\t}\n\n\t// get something that doesn't exist yet\n\tval, err := indexReader.GetInternal([]byte(\"key\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got %s\", val)\n\t}\n\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// set\n\terr = idx.SetInternal([]byte(\"key\"), []byte(\"abc\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tindexReader2, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(indexReader2.(*IndexSnapshot).segment) != 0 {\n\t\tt.Errorf(\"expected 0 segments\")\n\t}\n\n\t// get\n\tval, err = indexReader2.GetInternal([]byte(\"key\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif string(val) != \"abc\" {\n\t\tt.Errorf(\"expected %s, got '%s'\", \"abc\", val)\n\t}\n\n\terr = indexReader2.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// delete\n\terr = idx.DeleteInternal([]byte(\"key\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tindexReader3, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(indexReader3.(*IndexSnapshot).segment) != 0 {\n\t\tt.Errorf(\"expected 0 segments\")\n\t}\n\n\t// get again\n\tval, err = indexReader3.GetInternal([]byte(\"key\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got %s\", val)\n\t}\n\n\terr = indexReader3.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexBatch(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexBatch\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\n\t// first create 2 docs the old fashioned way\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test2\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\t// now create a batch which does 3 things\n\t// insert new doc\n\t// update existing doc\n\t// delete existing doc\n\t// net document count change 0\n\n\tbatch := index.NewBatch()\n\tdoc = document.NewDocument(\"3\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test3\")))\n\tbatch.Update(doc)\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test2updated\")))\n\tbatch.Update(doc)\n\tbatch.Delete(\"1\")\n\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tnumSegments := len(indexReader.(*IndexSnapshot).segment)\n\tif numSegments <= 0 {\n\t\tt.Errorf(\"expected some segments, got: %d\", numSegments)\n\t}\n\n\tdocCount, err := indexReader.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\n\tdocIDReader, err := indexReader.DocIDReaderAll()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tvar docIds []index.IndexInternalID\n\tdocID, err := docIDReader.Next()\n\tfor docID != nil && err == nil {\n\t\tdocIds = append(docIds, docID)\n\t\tdocID, err = docIDReader.Next()\n\t}\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texternalDocIds := map[string]struct{}{}\n\t// convert back to external doc ids\n\tfor _, id := range docIds {\n\t\texternalID, err := indexReader.ExternalID(id)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\texternalDocIds[externalID] = struct{}{}\n\t}\n\texpectedDocIds := map[string]struct{}{\n\t\t\"2\": {},\n\t\t\"3\": {},\n\t}\n\tif !reflect.DeepEqual(externalDocIds, expectedDocIds) {\n\t\tt.Errorf(\"expected ids: %v, got ids: %v\", expectedDocIds, externalDocIds)\n\t}\n}\n\nfunc TestIndexBatchWithCallbacks(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexBatchWithCallbacks\")\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\tcerr := idx.Close()\n\t\tif cerr != nil {\n\t\t\tt.Fatal(cerr)\n\t\t}\n\t}()\n\n\t// Check that callback function works\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\tbatch := index.NewBatch()\n\tdoc := document.NewDocument(\"3\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test3\")))\n\tbatch.Update(doc)\n\tbatch.SetPersistedCallback(func(e error) {\n\t\twg.Done()\n\t})\n\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\twg.Wait()\n\t// test has no assertion but will timeout if callback doesn't fire\n}\n\nfunc TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexInsertUpdateDeleteWithMultipleTypesStored\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewNumericFieldWithIndexingOptions(\"age\", []uint64{}, 35.99, index.IndexField|index.StoreField))\n\tdf, err := document.NewDateTimeFieldWithIndexingOptions(\"unixEpoch\", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdoc.AddField(df)\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstoredDocInt, err := indexReader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstoredDoc := storedDocInt.(*document.Document)\n\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(storedDoc.Fields) != 3 {\n\t\tt.Errorf(\"expected 3 stored field, got %d\", len(storedDoc.Fields))\n\t}\n\tfor _, field := range storedDoc.Fields {\n\t\tif field.Name() == \"name\" {\n\t\t\ttextField, ok := field.(*document.TextField)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"expected text field\")\n\t\t\t}\n\t\t\tif string(textField.Value()) != \"test\" {\n\t\t\t\tt.Errorf(\"expected field content 'test', got '%s'\", string(textField.Value()))\n\t\t\t}\n\t\t} else if field.Name() == \"age\" {\n\t\t\tnumField, ok := field.(*document.NumericField)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"expected numeric field\")\n\t\t\t}\n\t\t\tnumFieldNumer, err := numField.Number()\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t} else {\n\t\t\t\tif numFieldNumer != 35.99 {\n\t\t\t\t\tt.Errorf(\"expected numeric value 35.99, got %f\", numFieldNumer)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if field.Name() == \"unixEpoch\" {\n\t\t\tdateField, ok := field.(*document.DateTimeField)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"expected date field\")\n\t\t\t}\n\t\t\tdateFieldDate, _, err := dateField.DateTime()\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t} else {\n\t\t\t\tif dateFieldDate != time.Unix(0, 0).UTC() {\n\t\t\t\t\tt.Errorf(\"expected date value unix epoch, got %v\", dateFieldDate)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if field.Name() == \"_id\" {\n\t\t\tt.Errorf(\"not expecting _id field\")\n\t\t}\n\t}\n\n\t// now update the document, but omit one of the fields\n\tdoc = document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"testup\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewNumericFieldWithIndexingOptions(\"age\", []uint64{}, 36.99, index.IndexField|index.StoreField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader2, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\t// expected doc count shouldn't have changed\n\tdocCount, err = indexReader2.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\n\t// should only get 2 fields back now though\n\tstoredDocInt, err = indexReader2.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstoredDoc = storedDocInt.(*document.Document)\n\n\terr = indexReader2.Close()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(storedDoc.Fields) != 2 {\n\t\tt.Errorf(\"expected 2 stored field, got %d\", len(storedDoc.Fields))\n\t}\n\n\tfor _, field := range storedDoc.Fields {\n\t\tif field.Name() == \"name\" {\n\t\t\ttextField, ok := field.(*document.TextField)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"expected text field\")\n\t\t\t}\n\t\t\tif string(textField.Value()) != \"testup\" {\n\t\t\t\tt.Errorf(\"expected field content 'testup', got '%s'\", string(textField.Value()))\n\t\t\t}\n\t\t} else if field.Name() == \"age\" {\n\t\t\tnumField, ok := field.(*document.NumericField)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"expected numeric field\")\n\t\t\t}\n\t\t\tnumFieldNumer, err := numField.Number()\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t} else {\n\t\t\t\tif numFieldNumer != 36.99 {\n\t\t\t\t\tt.Errorf(\"expected numeric value 36.99, got %f\", numFieldNumer)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if field.Name() == \"_id\" {\n\t\t\tt.Errorf(\"not expecting _id field\")\n\t\t}\n\t}\n\n\t// now delete the document\n\terr = idx.Delete(\"1\")\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\texpectedCount--\n\n\t// expected doc count shouldn't have changed\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsertFields(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexInsertFields\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewNumericFieldWithIndexingOptions(\"age\", []uint64{}, 35.99, index.IndexField|index.StoreField))\n\tdateField, err := document.NewDateTimeFieldWithIndexingOptions(\"unixEpoch\", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdoc.AddField(dateField)\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfields, err := indexReader.Fields()\n\tif err != nil {\n\t\tt.Error(err)\n\t} else {\n\t\tfieldsMap := map[string]struct{}{}\n\t\tfor _, field := range fields {\n\t\t\tfieldsMap[field] = struct{}{}\n\t\t}\n\t\texpectedFieldsMap := map[string]struct{}{\n\t\t\t\"_id\":       {},\n\t\t\t\"name\":      {},\n\t\t\t\"age\":       {},\n\t\t\t\"unixEpoch\": {},\n\t\t}\n\t\tif !reflect.DeepEqual(fieldsMap, expectedFieldsMap) {\n\t\t\tt.Errorf(\"expected fields: %v, got %v\", expectedFieldsMap, fieldsMap)\n\t\t}\n\t}\n}\n\nfunc TestIndexUpdateComposites(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexUpdateComposites\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewCompositeFieldWithIndexingOptions(\"_all\", true, nil, nil, index.IndexField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\t// now lets update it\n\tdoc = document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"testupdated\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"misterupdated\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewCompositeFieldWithIndexingOptions(\"_all\", true, nil, nil, index.IndexField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// make sure new values are in index\n\tstoredDocInt, err := indexReader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstoredDoc := storedDocInt.(*document.Document)\n\tif len(storedDoc.Fields) != 2 {\n\t\tt.Errorf(\"expected 2 stored field, got %d\", len(storedDoc.Fields))\n\t}\n\tfor _, field := range storedDoc.Fields {\n\t\tif field.Name() == \"name\" {\n\t\t\ttextField, ok := field.(*document.TextField)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"expected text field\")\n\t\t\t}\n\t\t\tif string(textField.Value()) != \"testupdated\" {\n\t\t\t\tt.Errorf(\"expected field content 'test', got '%s'\", string(textField.Value()))\n\t\t\t}\n\t\t} else if field.Name() == \"_id\" {\n\t\t\tt.Errorf(\"not expecting _id field\")\n\t\t}\n\t}\n}\n\nfunc TestIndexTermReaderCompositeFields(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexTermReaderCompositeFields\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc.AddField(document.NewCompositeFieldWithIndexingOptions(\"_all\", true, nil, nil, index.IndexField|index.IncludeTermVectors))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermFieldReader, err := indexReader.TermFieldReader(context.TODO(), []byte(\"mister\"), \"_all\", true, true, true)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\ttfd, err := termFieldReader.Next(nil)\n\tfor tfd != nil && err == nil {\n\t\texternalID, err := indexReader.ExternalID(tfd.ID)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif externalID != \"1\" {\n\t\t\tt.Errorf(\"expected to find document id 1\")\n\t\t}\n\n\t\ttfd, err = termFieldReader.Next(nil)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n}\n\nfunc TestIndexDocValueReader(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexDocumentVisitFieldTerms\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tactualFieldTerms := make(fieldTerms)\n\n\tinternalID, err := indexReader.InternalID(\"1\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdvr, err := indexReader.DocValueReader([]string{\"name\", \"title\"})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\terr = dvr.VisitDocValues(internalID, func(field string, term []byte) {\n\t\tactualFieldTerms[field] = append(actualFieldTerms[field], string(term))\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texpectedFieldTerms := fieldTerms{\n\t\t\"name\":  []string{\"test\"},\n\t\t\"title\": []string{\"mister\"},\n\t}\n\tif !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) {\n\t\tt.Errorf(\"expected field terms: %#v, got: %#v\", expectedFieldTerms, actualFieldTerms)\n\t}\n}\n\nfunc TestDocValueReaderConcurrent(t *testing.T) {\n\tcfg := CreateConfig(\"TestFieldTermsConcurrent\")\n\n\t// setting path to empty string disables persistence/merging\n\t// which ensures we have in-memory segments\n\t// which is important for this test, to trigger the right code\n\t// path, where fields exist, but have NOT been uninverted by\n\t// the Segment impl (in memory segments are still SegmentBase)\n\tcfg[\"path\"] = \"\"\n\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tmp := mapping.NewIndexMapping()\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\tcerr := idx.Close()\n\t\tif cerr != nil {\n\t\t\tt.Fatal(cerr)\n\t\t}\n\t}()\n\n\t// create a single bath (leading to 1 in-memory segment)\n\t// have one field named \"name\" and 100 others named f0-f99\n\tbatch := index.NewBatch()\n\tfor i := 0; i < 1000; i++ {\n\t\tdata := map[string]string{\n\t\t\t\"name\": fmt.Sprintf(\"doc-%d\", i),\n\t\t}\n\t\tfor j := 0; j < 100; j++ {\n\t\t\tdata[fmt.Sprintf(\"f%d\", j)] = fmt.Sprintf(\"v%d\", i)\n\t\t}\n\t\tdoc := document.NewDocument(fmt.Sprintf(\"%d\", i))\n\t\terr = mp.MapDocument(doc, data)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"error mapping doc: %v\", err)\n\t\t}\n\t\tbatch.Update(doc)\n\t}\n\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now have 10 goroutines try to visit field values for doc 1\n\t// in a random field\n\tvar wg sync.WaitGroup\n\tfor j := 0; j < 10; j++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tr, err := idx.Reader()\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"error getting reader: %v\", err)\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdocNumber, err := r.InternalID(\"1\")\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"error getting internal ID: %v\", err)\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdvr, err := r.DocValueReader([]string{fmt.Sprintf(\"f%d\", rand.Intn(100))})\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"error getting doc value reader: %v\", err)\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\t}\n\t\t\terr = dvr.VisitDocValues(docNumber, func(field string, term []byte) {})\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"error visiting doc values: %v\", err)\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\t}\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\twg.Wait()\n}\n\nfunc TestConcurrentUpdate(t *testing.T) {\n\tcfg := CreateConfig(\"TestConcurrentUpdate\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// do some concurrent updates\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(1)\n\t\tgo func(i int) {\n\t\t\tdoc := document.NewDocument(\"1\")\n\t\t\tdoc.AddField(document.NewTextFieldWithIndexingOptions(strconv.Itoa(i), []uint64{}, []byte(strconv.Itoa(i)), index.StoreField))\n\t\t\terr := idx.Update(doc)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Error updating index: %v\", err)\n\t\t\t}\n\t\t\twg.Done()\n\t\t}(i)\n\t}\n\twg.Wait()\n\n\t// now load the name field and see what we get\n\tr, err := idx.Reader()\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := r.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocInt, err := r.Document(\"1\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tdoc := docInt.(*document.Document)\n\n\tif len(doc.Fields) > 2 {\n\t\tt.Errorf(\"expected no more than 2 fields, found %d\", len(doc.Fields))\n\t}\n}\n\nfunc TestLargeField(t *testing.T) {\n\tcfg := CreateConfig(\"TestLargeField\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar largeFieldValue []byte\n\tfor len(largeFieldValue) < 4096 {\n\t\tlargeFieldValue = append(largeFieldValue, bleveWikiArticle1K...)\n\t}\n\n\td := document.NewDocument(\"large\")\n\tf := document.NewTextFieldWithIndexingOptions(\"desc\", nil, largeFieldValue, index.IndexField|index.StoreField)\n\td.AddField(f)\n\n\terr = idx.Update(d)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nvar bleveWikiArticle1K = []byte(`Boiling liquid expanding vapor explosion\nFrom Wikipedia, the free encyclopedia\nSee also: Boiler explosion and Steam explosion\n\nFlames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire.\n\nThis article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013)\nA boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1]\nContents  [hide]\n1 Mechanism\n1.1 Water example\n1.2 BLEVEs without chemical reactions\n2 Fires\n3 Incidents\n4 Safety measures\n5 See also\n6 References\n7 External links\nMechanism[edit]\n\nThis section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013)\nThere are three characteristics of liquids which are relevant to the discussion of a BLEVE:`)\n\nfunc TestIndexDocValueReaderWithMultipleDocs(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexDocumentVisitFieldTermsWithMultipleDocs\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tactualFieldTerms := make(fieldTerms)\n\tdocNumber, err := indexReader.InternalID(\"1\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdvr, err := indexReader.DocValueReader([]string{\"name\", \"title\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = dvr.VisitDocValues(docNumber, func(field string, term []byte) {\n\t\tactualFieldTerms[field] = append(actualFieldTerms[field], string(term))\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texpectedFieldTerms := fieldTerms{\n\t\t\"name\":  []string{\"test\"},\n\t\t\"title\": []string{\"mister\"},\n\t}\n\tif !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) {\n\t\tt.Errorf(\"expected field terms: %#v, got: %#v\", expectedFieldTerms, actualFieldTerms)\n\t}\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc2 := document.NewDocument(\"2\")\n\tdoc2.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test2\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc2.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister2\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\terr = idx.Update(doc2)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\tindexReader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tactualFieldTerms = make(fieldTerms)\n\tdocNumber, err = indexReader.InternalID(\"2\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdvr, err = indexReader.DocValueReader([]string{\"name\", \"title\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = dvr.VisitDocValues(docNumber, func(field string, term []byte) {\n\t\tactualFieldTerms[field] = append(actualFieldTerms[field], string(term))\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texpectedFieldTerms = fieldTerms{\n\t\t\"name\":  []string{\"test2\"},\n\t\t\"title\": []string{\"mister2\"},\n\t}\n\tif !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) {\n\t\tt.Errorf(\"expected field terms: %#v, got: %#v\", expectedFieldTerms, actualFieldTerms)\n\t}\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc3 := document.NewDocument(\"3\")\n\tdoc3.AddField(document.NewTextFieldWithIndexingOptions(\"name3\", []uint64{}, []byte(\"test3\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc3.AddField(document.NewTextFieldWithIndexingOptions(\"title3\", []uint64{}, []byte(\"mister3\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\terr = idx.Update(doc3)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\tindexReader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tactualFieldTerms = make(fieldTerms)\n\tdocNumber, err = indexReader.InternalID(\"3\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdvr, err = indexReader.DocValueReader([]string{\"name3\", \"title3\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = dvr.VisitDocValues(docNumber, func(field string, term []byte) {\n\t\tactualFieldTerms[field] = append(actualFieldTerms[field], string(term))\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texpectedFieldTerms = fieldTerms{\n\t\t\"name3\":  []string{\"test3\"},\n\t\t\"title3\": []string{\"mister3\"},\n\t}\n\tif !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) {\n\t\tt.Errorf(\"expected field terms: %#v, got: %#v\", expectedFieldTerms, actualFieldTerms)\n\t}\n\n\tactualFieldTerms = make(fieldTerms)\n\tdocNumber, err = indexReader.InternalID(\"1\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdvr, err = indexReader.DocValueReader([]string{\"name\", \"title\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = dvr.VisitDocValues(docNumber, func(field string, term []byte) {\n\t\tactualFieldTerms[field] = append(actualFieldTerms[field], string(term))\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texpectedFieldTerms = fieldTerms{\n\t\t\"name\":  []string{\"test\"},\n\t\t\"title\": []string{\"mister\"},\n\t}\n\tif !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) {\n\t\tt.Errorf(\"expected field terms: %#v, got: %#v\", expectedFieldTerms, actualFieldTerms)\n\t}\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexDocValueReaderWithMultipleFieldOptions(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexDocumentVisitFieldTermsWithMultipleFieldOptions\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// mix of field options, this exercises the run time/ on the fly un inverting of\n\t// doc values for custom options enabled field like designation, dept.\n\toptions := index.IndexField | index.StoreField | index.IncludeTermVectors\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))    // default doc value persisted\n\tdoc.AddField(document.NewTextField(\"title\", []uint64{}, []byte(\"mister\"))) // default doc value persisted\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"designation\", []uint64{}, []byte(\"engineer\"), options))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"dept\", []uint64{}, []byte(\"bleve\"), options))\n\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tactualFieldTerms := make(fieldTerms)\n\tdocNumber, err := indexReader.InternalID(\"1\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdvr, err := indexReader.DocValueReader([]string{\"name\", \"designation\", \"dept\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = dvr.VisitDocValues(docNumber, func(field string, term []byte) {\n\t\tactualFieldTerms[field] = append(actualFieldTerms[field], string(term))\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texpectedFieldTerms := fieldTerms{\n\t\t\"name\":        []string{\"test\"},\n\t\t\"designation\": []string{\"engineer\"},\n\t\t\"dept\":        []string{\"bleve\"},\n\t}\n\tif !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) {\n\t\tt.Errorf(\"expected field terms: %#v, got: %#v\", expectedFieldTerms, actualFieldTerms)\n\t}\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestAllFieldWithDifferentTermVectorsEnabled(t *testing.T) {\n\t// Based on https://github.com/blevesearch/bleve/issues/895 from xeizmendi\n\tcfg := CreateConfig(\"TestAllFieldWithDifferentTermVectorsEnabled\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\ttestConfig := cfg\n\tmp := mapping.NewIndexMapping()\n\n\tkeywordMapping := mapping.NewTextFieldMapping()\n\tkeywordMapping.Analyzer = keyword.Name\n\tkeywordMapping.IncludeTermVectors = false\n\tkeywordMapping.IncludeInAll = true\n\n\ttextMapping := mapping.NewTextFieldMapping()\n\ttextMapping.Analyzer = standard.Name\n\ttextMapping.IncludeTermVectors = true\n\ttextMapping.IncludeInAll = true\n\n\tdocMapping := mapping.NewDocumentStaticMapping()\n\tdocMapping.AddFieldMappingsAt(\"keyword\", keywordMapping)\n\tdocMapping.AddFieldMappingsAt(\"text\", textMapping)\n\n\tmp.DefaultMapping = docMapping\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(\"storeName\", testConfig, analysisQueue)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdata := map[string]string{\n\t\t\"keyword\": \"something\",\n\t\t\"text\":    \"A sentence that includes something within.\",\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\terr = mp.MapDocument(doc, data)\n\tif err != nil {\n\t\tt.Errorf(\"error mapping doc: %v\", err)\n\t}\n\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n}\n\nfunc TestForceVersion(t *testing.T) {\n\tcfg := map[string]interface{}{}\n\tcfg[\"forceSegmentType\"] = \"zap\"\n\tcfg[\"forceSegmentVersion\"] = 11\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatalf(\"error opening a supported version: %v\", err)\n\t}\n\ts := idx.(*Scorch)\n\tif s.segPlugin.Version() != 11 {\n\t\tt.Fatalf(\"wrong segment wrapper version loaded, expected %d got %d\", 11, s.segPlugin.Version())\n\t}\n\tcfg[\"forceSegmentVersion\"] = 12\n\tidx, err = NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatalf(\"error opening a supported version: %v\", err)\n\t}\n\ts = idx.(*Scorch)\n\tif s.segPlugin.Version() != 12 {\n\t\tt.Fatalf(\"wrong segment wrapper version loaded, expected %d got %d\", 12, s.segPlugin.Version())\n\t}\n\tcfg[\"forceSegmentVersion\"] = 10\n\t_, err = NewScorch(Name, cfg, analysisQueue)\n\tif err == nil {\n\t\tt.Fatalf(\"expected an error opening an unsupported version, got nil\")\n\t}\n}\n\nfunc TestIndexForceMerge(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexForceMerge\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\ttmp := struct {\n\t\tMaxSegmentsPerTier   int   `json:\"maxSegmentsPerTier\"`\n\t\tSegmentsPerMergeTask int   `json:\"segmentsPerMergeTask\"`\n\t\tFloorSegmentSize     int64 `json:\"floorSegmentSize\"`\n\t}{\n\t\tint(1),\n\t\tint(1),\n\t\tint64(2),\n\t}\n\tcfg[\"scorchMergePlanOptions\"] = &tmp\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\n\tvar expectedCount uint64\n\tbatch := index.NewBatch()\n\tfor i := 0; i < 10; i++ {\n\t\tdoc := document.NewDocument(fmt.Sprintf(\"doc1-%d\", i))\n\t\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(fmt.Sprintf(\"text1-%d\", i))))\n\t\tbatch.Update(doc)\n\t\tdoc = document.NewDocument(fmt.Sprintf(\"doc2-%d\", i))\n\t\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(fmt.Sprintf(\"text2-%d\", i))))\n\t\tbatch.Update(doc)\n\t\terr = idx.Batch(batch)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tbatch.Reset()\n\t\texpectedCount += 2\n\t}\n\n\t// verify doc count\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocCount, err := indexReader.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar si *Scorch\n\tvar ok bool\n\tif si, ok = idx.(*Scorch); !ok {\n\t\tt.Errorf(\"expects a scorch index\")\n\t}\n\n\tnfs := atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot)\n\tif nfs != 10 {\n\t\tt.Errorf(\"expected 10 root file segments, got: %d\", nfs)\n\t}\n\n\tctx := context.Background()\n\tfor atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) != 1 {\n\t\terr := si.ForceMerge(ctx, &mergeplan.MergePlanOptions{\n\t\t\tMaxSegmentsPerTier:   1,\n\t\t\tMaxSegmentSize:       10000,\n\t\t\tSegmentsPerMergeTask: 10,\n\t\t\tFloorSegmentSize:     10000,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Errorf(\"ForceMerge failed, err: %v\", err)\n\t\t}\n\t}\n\n\t// verify the final root segment count\n\tif atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) != 1 {\n\t\tt.Errorf(\"expected a single root file segments, got: %d\",\n\t\t\tatomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot))\n\t}\n\n\t// verify with an invalid merge plan\n\terr = si.ForceMerge(ctx, &mergeplan.MergePlanOptions{\n\t\tMaxSegmentsPerTier:   1,\n\t\tMaxSegmentSize:       1 << 33,\n\t\tSegmentsPerMergeTask: 10,\n\t\tFloorSegmentSize:     10000,\n\t})\n\tif err != mergeplan.ErrMaxSegmentSizeTooLarge {\n\t\tt.Errorf(\"ForceMerge expected to fail with ErrMaxSegmentSizeTooLarge\")\n\t}\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestCancelIndexForceMerge(t *testing.T) {\n\tcfg := CreateConfig(\"TestCancelIndexForceMerge\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\ttmp := struct {\n\t\tMaxSegmentsPerTier   int   `json:\"maxSegmentsPerTier\"`\n\t\tSegmentsPerMergeTask int   `json:\"segmentsPerMergeTask\"`\n\t\tFloorSegmentSize     int64 `json:\"floorSegmentSize\"`\n\t}{\n\t\tint(1),\n\t\tint(1),\n\t\tint64(2),\n\t}\n\tcfg[\"scorchMergePlanOptions\"] = &tmp\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\n\tvar expectedCount uint64\n\tbatch := index.NewBatch()\n\tfor i := 0; i < 20; i++ {\n\t\tdoc := document.NewDocument(fmt.Sprintf(\"doc1-%d\", i))\n\t\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(fmt.Sprintf(\"text1-%d\", i))))\n\t\tbatch.Update(doc)\n\t\tdoc = document.NewDocument(fmt.Sprintf(\"doc2-%d\", i))\n\t\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(fmt.Sprintf(\"text2-%d\", i))))\n\t\tbatch.Update(doc)\n\t\terr = idx.Batch(batch)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tbatch.Reset()\n\t\texpectedCount += 2\n\t}\n\n\t// verify doc count\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdocCount, err := indexReader.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar si *Scorch\n\tvar ok bool\n\tif si, ok = idx.(*Scorch); !ok {\n\t\tt.Fatal(\"expects a scorch index\")\n\t}\n\n\t// no merge operations are expected as per the original merge policy.\n\tnfsr := atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot)\n\tif nfsr != 20 {\n\t\tt.Errorf(\"expected 20 root file segments, got: %d\", nfsr)\n\t}\n\n\tctx := context.Background()\n\tctx, cancel := context.WithCancel(ctx)\n\n\t// cancel the force merge operation once the root has some new merge\n\t// introductions. ie if the root has lesser file segments than earlier.\n\tgo func() {\n\t\tfor {\n\t\t\tnval := atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot)\n\t\t\tif nval < nfsr {\n\t\t\t\tcancel()\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttime.Sleep(time.Millisecond * 5)\n\t\t}\n\t}()\n\n\terr = si.ForceMerge(ctx, &mergeplan.MergePlanOptions{\n\t\tMaxSegmentsPerTier:   1,\n\t\tMaxSegmentSize:       10000,\n\t\tSegmentsPerMergeTask: 5,\n\t\tFloorSegmentSize:     10000,\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"ForceMerge failed, err: %v\", err)\n\t}\n\n\t// verify the final root file segment count or forceMerge completion\n\tif atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) == 1 {\n\t\tt.Errorf(\"expected many files at root, but got: %d segments\",\n\t\t\tatomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot))\n\t}\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexSeekBackwardsStats(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexOpenReopen\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// insert a doc\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"cat\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Fatalf(\"error updating index: %v\", err)\n\t}\n\n\t// insert another doc\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"cat\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Fatalf(\"error updating index: %v\", err)\n\t}\n\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatalf(\"error getting index reader: %v\", err)\n\t}\n\tdefer reader.Close()\n\n\ttfr, err := reader.TermFieldReader(context.TODO(), []byte(\"cat\"), \"name\", false, false, false)\n\tif err != nil {\n\t\tt.Fatalf(\"error getting term field readyer for name/cat: %v\", err)\n\t}\n\n\ttfdFirst, err := tfr.Next(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"error getting first tfd: %v\", err)\n\t}\n\n\t_, err = tfr.Next(nil)\n\tif err != nil {\n\t\tt.Fatalf(\"error getting second tfd: %v\", err)\n\t}\n\n\t// seek backwards to the first\n\t_, err = tfr.Advance(tfdFirst.ID, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"error adancing backwards: %v\", err)\n\t}\n\n\terr = tfr.Close()\n\tif err != nil {\n\t\tt.Fatalf(\"error closing term field reader: %v\", err)\n\t}\n\n\tif idx.(*Scorch).stats.TotTermSearchersStarted != idx.(*Scorch).stats.TotTermSearchersFinished {\n\t\tt.Errorf(\"expected term searchers started %d to equal term searchers finished %d\",\n\t\t\tidx.(*Scorch).stats.TotTermSearchersStarted,\n\t\t\tidx.(*Scorch).stats.TotTermSearchersFinished)\n\t}\n}\n\n// fieldTerms contains the terms used by a document, keyed by field\ntype fieldTerms map[string][]string\n\n// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields\nfunc (f fieldTerms) FieldsNotYetCached(fields []string) []string {\n\trv := make([]string, 0, len(fields))\n\tfor _, field := range fields {\n\t\tif _, ok := f[field]; !ok {\n\t\t\trv = append(rv, field)\n\t\t}\n\t}\n\treturn rv\n}\n\n// Merge will combine two fieldTerms\n// it assumes that the terms lists are complete (thus do not need to be merged)\n// field terms from the other list always replace the ones in the receiver\nfunc (f fieldTerms) Merge(other fieldTerms) {\n\tfor field, terms := range other {\n\t\tf[field] = terms\n\t}\n}\n\nfunc TestOpenBoltTimeout(t *testing.T) {\n\tcfg := CreateConfig(\"TestIndexOpenReopen\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewScorch(\"storeName\", cfg, analysisQueue)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\n\t// new config\n\tcfg2 := CreateConfig(\"TestIndexOpenReopen\")\n\t// copy path from original config\n\tcfg2[\"path\"] = cfg[\"path\"]\n\t// set timeout in this cfg\n\tcfg2[\"bolt_timeout\"] = \"100ms\"\n\n\tidx2, err := NewScorch(\"storeName\", cfg2, analysisQueue)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\terr = idx2.Open()\n\tif err == nil {\n\t\tt.Error(\"expected timeout error opening index again\")\n\t}\n}\n\nfunc TestReadOnlyIndex(t *testing.T) {\n\t// https://github.com/blevesearch/bleve/issues/1623\n\tcfg := CreateConfig(\"TestReadOnlyIndex\")\n\terr := InitTest(cfg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := DestroyTest(cfg)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\twriteIdx, err := NewScorch(Name, cfg, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = writeIdx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\twriteIdxClosed := false\n\tdefer func() {\n\t\tif !writeIdxClosed {\n\t\t\terr := writeIdx.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\t}()\n\n\t// Add a single document to the index.\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = writeIdx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\twriteIdx.Close()\n\twriteIdxClosed = true\n\n\t// After the index is written, change permissions on every file\n\t// in the index to read-only.\n\tvar permissionsFunc func(folder string)\n\tpermissionsFunc = func(folder string) {\n\t\tentries, _ := os.ReadDir(folder)\n\t\tfor _, entry := range entries {\n\t\t\tfullName := filepath.Join(folder, entry.Name())\n\t\t\tif entry.IsDir() {\n\t\t\t\tpermissionsFunc(fullName)\n\t\t\t} else {\n\t\t\t\tif err := os.Chmod(fullName, 0o555); err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tpermissionsFunc(cfg[\"path\"].(string))\n\n\t// Now reopen the index in read-only mode and attempt to read from it.\n\tcfg[\"read_only\"] = true\n\treadIdx, err := NewScorch(Name, cfg, analysisQueue)\n\tdefer func() {\n\t\terr := readIdx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = readIdx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\treader, err := readIdx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docCount != 1 {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", 1, docCount)\n\t}\n}\n\nfunc BenchmarkAggregateFieldStats(b *testing.B) {\n\tfieldStatsArray := make([]*fieldStats, 1000)\n\n\tfor i := range fieldStatsArray {\n\t\tfieldStatsArray[i] = newFieldStats()\n\n\t\tfieldStatsArray[i].Store(\"num_vectors\", \"vector\", uint64(rand.Intn(1000)))\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\taggFieldStats := newFieldStats()\n\n\t\tfor _, fs := range fieldStatsArray {\n\t\t\taggFieldStats.Aggregate(fs)\n\t\t}\n\t}\n}\n\nfunc TestPersistorMergerOptions(t *testing.T) {\n\ttype test struct {\n\t\tconfig    string\n\t\texpectErr bool\n\t}\n\ttests := []test{\n\t\t{\n\t\t\t// valid config and no error expected\n\t\t\tconfig: `{\n\t\t\t\t\"scorchPersisterOptions\": {\n\t\t\t\t\t\"persisterNapTimeMSec\": 1110,\n\t\t\t\t\t\"memoryPressurePauseThreshold\" : 333\n\t\t\t\t}\n\t\t\t}`,\n\t\t\texpectErr: false,\n\t\t},\n\t\t{\n\t\t\t// valid json with invalid config values\n\t\t\t// and error expected\n\t\t\tconfig: `{\n\t\t\t\t\"scorchPersisterOptions\": {\n\t\t\t\t\t\"persisterNapTimeMSec\": \"1110\",\n\t\t\t\t\t\"memoryPressurePauseThreshold\" : [333]\n\t\t\t\t}\n\t\t\t}`,\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\t// valid json with invalid config values\n\t\t\t// and error expected\n\t\t\tconfig: `{\n\t\t\t\t\"scorchPersisterOptions\": {\n\t\t\t\t\t\"persisterNapTimeMSec\": 1110.2,\n\t\t\t\t\t\"memoryPressurePauseThreshold\" : 333\n\t\t\t\t}\n\t\t\t}`,\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\t// invalid setting for scorchMergePlanOptions\n\t\t\tconfig: `{\n\t\t\t\t\"scorchPersisterOptions\": {\n\t\t\t\t\t\"persisterNapTimeMSec\": 1110,\n\t\t\t\t\t\"memoryPressurePauseThreshold\" : 333\n\t\t\t\t},\n\t\t\t\t\"scorchMergePlanOptions\": [{\n\t\t\t\t\t\"maxSegmentSize\": 10000,\n\t\t\t\t\t\"maxSegmentsPerTier\": 10,\n\t\t\t\t\t\"segmentsPerMergeTask\": 10\n\t\t\t\t}]\n\t\t\t}`,\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\t// valid setting\n\t\t\tconfig: `{\n\t\t\t\t\"scorchPersisterOptions\": {\n\t\t\t\t\t\"persisterNapTimeMSec\": 1110,\n\t\t\t\t\t\"memoryPressurePauseThreshold\" : 333\n\t\t\t\t},\n\t\t\t\t\"scorchMergePlanOptions\": {\n\t\t\t\t\t\"maxSegmentSize\": 10000,\n\t\t\t\t\t\"maxSegmentsPerTier\": 10,\n\t\t\t\t\t\"segmentsPerMergeTask\": 10\n\t\t\t\t}\n\t\t\t}`,\n\t\t\texpectErr: false,\n\t\t},\n\t\t{\n\t\t\tconfig: `{\n\t\t\t\t\"scorchPersisterOptions\": {\n\t\t\t\t\t\"persisterNapTimeMSec\": 1110,\n\t\t\t\t\t\"memoryPressurePauseThreshold\" : 333\n\t\t\t\t},\n\t\t\t\t\"scorchMergePlanOptions\": {\n\t\t\t\t\t\"maxSegmentSize\": 5.6,\n\t\t\t\t\t\"maxSegmentsPerTier\": 10,\n\t\t\t\t\t\"segmentsPerMergeTask\": 10\n\t\t\t\t}\n\t\t\t}`,\n\t\t\texpectErr: true,\n\t\t},\n\t}\n\tfor i, test := range tests {\n\t\tcfg := map[string]interface{}{}\n\t\terr := json.Unmarshal([]byte(test.config), &cfg)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"test %d: error unmarshalling config: %v\", i, err)\n\t\t}\n\t\tanalysisQueue := index.NewAnalysisQueue(1)\n\t\t_, err = NewScorch(Name, cfg, analysisQueue)\n\t\tif test.expectErr {\n\t\t\tif err == nil {\n\t\t\t\tt.Errorf(\"test %d: expected error, got nil\", i)\n\t\t\t}\n\t\t} else {\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"test %d: unexpected error: %v\", i, err)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "index/scorch/segment_plugin.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n\n\tzapv11 \"github.com/blevesearch/zapx/v11\"\n\tzapv12 \"github.com/blevesearch/zapx/v12\"\n\tzapv13 \"github.com/blevesearch/zapx/v13\"\n\tzapv14 \"github.com/blevesearch/zapx/v14\"\n\tzapv15 \"github.com/blevesearch/zapx/v15\"\n\tzapv16 \"github.com/blevesearch/zapx/v16\"\n\tzapv17 \"github.com/blevesearch/zapx/v17\"\n)\n\n// SegmentPlugin represents the essential functions required by a package to plug in\n// it's segment implementation\ntype SegmentPlugin interface {\n\n\t// Type is the name for this segment plugin\n\tType() string\n\n\t// Version is a numeric value identifying a specific version of this type.\n\t// When incompatible changes are made to a particular type of plugin, the\n\t// version must be incremented.\n\tVersion() uint32\n\n\t// New takes a set of Documents and turns them into a new Segment\n\tNew(results []index.Document) (segment.Segment, uint64, error)\n\n\tNewUsing(results []index.Document, config map[string]interface{}) (segment.Segment, uint64, error)\n\n\t// Open attempts to open the file at the specified path and\n\t// return the corresponding Segment\n\tOpen(path string) (segment.Segment, error)\n\n\tOpenUsing(path string, config map[string]interface{}) (segment.Segment, error)\n\n\t// Merge takes a set of Segments, and creates a new segment on disk at\n\t// the specified path.\n\t// Drops is a set of bitmaps (one for each segment) indicating which\n\t// documents can be dropped from the segments during the merge.\n\t// If the closeCh channel is closed, Merge will cease doing work at\n\t// the next opportunity, and return an error (closed).\n\t// StatsReporter can optionally be provided, in which case progress\n\t// made during the merge is reported while operation continues.\n\t// Returns:\n\t// A slice of new document numbers (one for each input segment),\n\t// this allows the caller to know a particular document's new\n\t// document number in the newly merged segment.\n\t// The number of bytes written to the new segment file.\n\t// An error, if any occurred.\n\tMerge(segments []segment.Segment, drops []*roaring.Bitmap, path string,\n\t\tcloseCh chan struct{}, s segment.StatsReporter) (\n\t\t[][]uint64, uint64, error)\n\n\tMergeUsing(segments []segment.Segment, drops []*roaring.Bitmap, path string,\n\t\tcloseCh chan struct{}, s segment.StatsReporter, config map[string]interface{}) (\n\t\t[][]uint64, uint64, error)\n}\n\nvar supportedSegmentPlugins map[string]map[uint32]SegmentPlugin\nvar defaultSegmentPlugin SegmentPlugin\n\nfunc init() {\n\tResetSegmentPlugins()\n\tRegisterSegmentPlugin(&zapv17.ZapPlugin{}, true)\n\tRegisterSegmentPlugin(&zapv16.ZapPlugin{}, false)\n\tRegisterSegmentPlugin(&zapv15.ZapPlugin{}, false)\n\tRegisterSegmentPlugin(&zapv14.ZapPlugin{}, false)\n\tRegisterSegmentPlugin(&zapv13.ZapPlugin{}, false)\n\tRegisterSegmentPlugin(&zapv12.ZapPlugin{}, false)\n\tRegisterSegmentPlugin(&zapv11.ZapPlugin{}, false)\n}\n\nfunc ResetSegmentPlugins() {\n\tsupportedSegmentPlugins = map[string]map[uint32]SegmentPlugin{}\n}\n\nfunc RegisterSegmentPlugin(plugin SegmentPlugin, makeDefault bool) {\n\tif _, ok := supportedSegmentPlugins[plugin.Type()]; !ok {\n\t\tsupportedSegmentPlugins[plugin.Type()] = map[uint32]SegmentPlugin{}\n\t}\n\tsupportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin\n\tif makeDefault {\n\t\tdefaultSegmentPlugin = plugin\n\t}\n}\n\nfunc SupportedSegmentTypes() (rv []string) {\n\tfor k := range supportedSegmentPlugins {\n\t\trv = append(rv, k)\n\t}\n\treturn\n}\n\nfunc SupportedSegmentTypeVersions(typ string) (rv []uint32) {\n\tfor k := range supportedSegmentPlugins[typ] {\n\t\trv = append(rv, k)\n\t}\n\treturn rv\n}\n\nfunc chooseSegmentPlugin(forcedSegmentType string,\n\tforcedSegmentVersion uint32) (SegmentPlugin, error) {\n\tif versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok {\n\t\tif segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok {\n\t\t\treturn segPlugin, nil\n\t\t}\n\t\treturn nil, fmt.Errorf(\n\t\t\t\"unsupported version %d for segment type: %s, supported: %v\",\n\t\t\tforcedSegmentVersion, forcedSegmentType,\n\t\t\tSupportedSegmentTypeVersions(forcedSegmentType))\n\t}\n\treturn nil, fmt.Errorf(\"unsupported segment type: %s, supported: %v\",\n\t\tforcedSegmentType, SupportedSegmentTypes())\n}\n\nfunc (s *Scorch) loadSegmentPlugin(forcedSegmentType string,\n\tforcedSegmentVersion uint32) error {\n\tsegPlugin, err := chooseSegmentPlugin(forcedSegmentType,\n\t\tforcedSegmentVersion)\n\tif err != nil {\n\t\treturn err\n\t}\n\ts.segPlugin = segPlugin\n\treturn nil\n}\n\nfunc (s *Scorch) loadSpatialAnalyzerPlugin(typ string) error {\n\ts.spatialPlugin = geo.GetSpatialAnalyzerPlugin(typ)\n\tif s.spatialPlugin == nil {\n\t\treturn fmt.Errorf(\"unsupported spatial plugin type: %s\", typ)\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "index/scorch/snapshot_index.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"container/heap\"\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"sort\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n\t\"github.com/blevesearch/vellum\"\n\tlev \"github.com/blevesearch/vellum/levenshtein\"\n\tbolt \"go.etcd.io/bbolt\"\n)\n\n// re usable, threadsafe levenshtein builders\nvar lb1, lb2 *lev.LevenshteinAutomatonBuilder\n\ntype asynchSegmentResult struct {\n\tdict    segment.TermDictionary\n\tdictItr segment.DictionaryIterator\n\n\tindex int\n\tdocs  *roaring.Bitmap\n\n\tthesItr segment.ThesaurusIterator\n\n\terr error\n}\n\nvar reflectStaticSizeIndexSnapshot int\n\nfunc init() {\n\tvar is interface{} = IndexSnapshot{}\n\treflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size())\n\tvar err error\n\tlb1, err = lev.NewLevenshteinAutomatonBuilder(1, true)\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"levenshtein automaton ed1 builder err: %v\", err))\n\t}\n\tlb2, err = lev.NewLevenshteinAutomatonBuilder(2, true)\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"levenshtein automaton ed2 builder err: %v\", err))\n\t}\n}\n\ntype IndexSnapshot struct {\n\tparent   *Scorch\n\tsegment  []*SegmentSnapshot\n\toffsets  []uint64\n\tinternal map[string][]byte\n\tepoch    uint64\n\tsize     uint64\n\tcreator  string\n\n\tm    sync.Mutex // Protects the fields that follow.\n\trefs int64\n\n\tm2        sync.Mutex                                 // Protects the fields that follow.\n\tfieldTFRs map[string][]*IndexSnapshotTermFieldReader // keyed by field, recycled TFR's\n\n\tm3               sync.RWMutex // bm25 metrics specific - not to interfere with TFR creation\n\tfieldCardinality map[string]int\n\n\t// Stores information about zapx fields that have been\n\t// fully deleted (indicated by UpdateFieldInfo.Deleted) or\n\t// partially deleted index, store or docvalues (indicated by\n\t// UpdateFieldInfo.Index or .Store or .DocValues).\n\t// Used to short circuit queries trying to read stale data\n\tupdatedFields map[string]*index.UpdateFieldInfo\n}\n\nfunc (i *IndexSnapshot) Segments() []*SegmentSnapshot {\n\treturn i.segment\n}\n\nfunc (i *IndexSnapshot) Internal() map[string][]byte {\n\treturn i.internal\n}\n\nfunc (i *IndexSnapshot) AddRef() {\n\ti.m.Lock()\n\ti.refs++\n\ti.m.Unlock()\n}\n\nfunc (i *IndexSnapshot) DecRef() (err error) {\n\ti.m.Lock()\n\ti.refs--\n\tif i.refs == 0 {\n\t\tfor _, s := range i.segment {\n\t\t\tif s != nil {\n\t\t\t\terr2 := s.segment.DecRef()\n\t\t\t\tif err == nil {\n\t\t\t\t\terr = err2\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif i.parent != nil {\n\t\t\tgo i.parent.AddEligibleForRemoval(i.epoch)\n\t\t}\n\t}\n\ti.m.Unlock()\n\treturn err\n}\n\nfunc (i *IndexSnapshot) Close() error {\n\treturn i.DecRef()\n}\n\nfunc (i *IndexSnapshot) Size() int {\n\treturn int(i.size)\n}\n\nfunc (i *IndexSnapshot) updateSize() {\n\ti.size += uint64(reflectStaticSizeIndexSnapshot)\n\tfor _, s := range i.segment {\n\t\ti.size += uint64(s.Size())\n\t}\n}\n\nfunc (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,\n\tmakeItr func(i segment.TermDictionary) segment.DictionaryIterator,\n\trandomLookup bool,\n) (*IndexSnapshotFieldDict, error) {\n\tresults := make(chan *asynchSegmentResult, len(is.segment))\n\tvar totalBytesRead uint64\n\tvar fieldCardinality int64\n\tfor _, s := range is.segment {\n\t\tgo func(s *SegmentSnapshot) {\n\t\t\tdict, err := s.segment.Dictionary(field)\n\t\t\tif err != nil {\n\t\t\t\tresults <- &asynchSegmentResult{err: err}\n\t\t\t} else {\n\t\t\t\tif dictStats, ok := dict.(segment.DiskStatsReporter); ok {\n\t\t\t\t\tatomic.AddUint64(&totalBytesRead, dictStats.BytesRead())\n\t\t\t\t}\n\t\t\t\tatomic.AddInt64(&fieldCardinality, int64(dict.Cardinality()))\n\t\t\t\tif randomLookup {\n\t\t\t\t\tresults <- &asynchSegmentResult{dict: dict}\n\t\t\t\t} else {\n\t\t\t\t\tresults <- &asynchSegmentResult{dictItr: makeItr(dict)}\n\t\t\t\t}\n\t\t\t}\n\t\t}(s)\n\t}\n\n\tvar err error\n\trv := &IndexSnapshotFieldDict{\n\t\tsnapshot: is,\n\t\tcursors:  make([]*segmentDictCursor, 0, len(is.segment)),\n\t}\n\n\tfor count := 0; count < len(is.segment); count++ {\n\t\tasr := <-results\n\t\tif asr.err != nil && err == nil {\n\t\t\terr = asr.err\n\t\t} else {\n\t\t\tif !randomLookup {\n\t\t\t\tnext, err2 := asr.dictItr.Next()\n\t\t\t\tif err2 != nil && err == nil {\n\t\t\t\t\terr = err2\n\t\t\t\t}\n\t\t\t\tif next != nil {\n\t\t\t\t\trv.cursors = append(rv.cursors, &segmentDictCursor{\n\t\t\t\t\t\titr:  asr.dictItr,\n\t\t\t\t\t\tcurr: *next,\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\trv.cursors = append(rv.cursors, &segmentDictCursor{\n\t\t\t\t\tdict: asr.dict,\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n\trv.cardinality = int(fieldCardinality)\n\trv.bytesRead = totalBytesRead\n\t// after ensuring we've read all items on channel\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif !randomLookup {\n\t\t// prepare heap\n\t\theap.Init(rv)\n\t}\n\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) FieldCardinality(field string) (rv int, err error) {\n\tis.m3.RLock()\n\trv, ok := is.fieldCardinality[field]\n\tis.m3.RUnlock()\n\tif ok {\n\t\treturn rv, nil\n\t}\n\n\tis.m3.Lock()\n\tdefer is.m3.Unlock()\n\tif is.fieldCardinality == nil {\n\t\tis.fieldCardinality = make(map[string]int)\n\t}\n\t// check again to avoid redundant fieldDict creation\n\tif rv, ok := is.fieldCardinality[field]; ok {\n\t\treturn rv, nil\n\t}\n\n\tfd, err := is.FieldDict(field)\n\tif err != nil {\n\t\treturn rv, err\n\t}\n\trv = fd.Cardinality()\n\tis.fieldCardinality[field] = rv\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) FieldDict(field string) (index.FieldDict, error) {\n\treturn is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {\n\t\treturn is.AutomatonIterator(nil, nil, nil)\n\t}, false)\n}\n\n// calculateExclusiveEndFromInclusiveEnd produces the next key\n// when sorting using memcmp style comparisons, suitable to\n// use as the end key in a traditional (inclusive, exclusive]\n// start/end range\nfunc calculateExclusiveEndFromInclusiveEnd(inclusiveEnd []byte) []byte {\n\trv := inclusiveEnd\n\tif len(inclusiveEnd) > 0 {\n\t\trv = make([]byte, len(inclusiveEnd))\n\t\tcopy(rv, inclusiveEnd)\n\t\tif rv[len(rv)-1] < 0xff {\n\t\t\t// last byte can be incremented by one\n\t\t\trv[len(rv)-1]++\n\t\t} else {\n\t\t\t// last byte is already 0xff, so append 0\n\t\t\t// next key is simply one byte longer\n\t\t\trv = append(rv, 0x0)\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (is *IndexSnapshot) FieldDictRange(field string, startTerm []byte,\n\tendTerm []byte,\n) (index.FieldDict, error) {\n\treturn is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {\n\t\tendTermExclusive := calculateExclusiveEndFromInclusiveEnd(endTerm)\n\t\treturn is.AutomatonIterator(nil, startTerm, endTermExclusive)\n\t}, false)\n}\n\n// calculateExclusiveEndFromPrefix produces the first key that\n// does not have the same prefix as the input bytes, suitable\n// to use as the end key in a traditional (inclusive, exclusive]\n// start/end range\nfunc calculateExclusiveEndFromPrefix(in []byte) []byte {\n\tif len(in) == 0 {\n\t\treturn nil\n\t}\n\trv := make([]byte, len(in))\n\tcopy(rv, in)\n\tfor i := len(rv) - 1; i >= 0; i-- {\n\t\trv[i]++\n\t\tif rv[i] != 0 {\n\t\t\treturn rv // didn't overflow, so stop\n\t\t}\n\t}\n\t// all bytes were 0xff, so return nil\n\t// as there is no end key for this prefix\n\treturn nil\n}\n\nfunc (is *IndexSnapshot) FieldDictPrefix(field string,\n\ttermPrefix []byte,\n) (index.FieldDict, error) {\n\ttermPrefixEnd := calculateExclusiveEndFromPrefix(termPrefix)\n\treturn is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {\n\t\treturn is.AutomatonIterator(nil, termPrefix, termPrefixEnd)\n\t}, false)\n}\n\nfunc (is *IndexSnapshot) FieldDictRegexp(field string,\n\ttermRegex string,\n) (index.FieldDict, error) {\n\tfd, _, err := is.FieldDictRegexpAutomaton(field, termRegex)\n\treturn fd, err\n}\n\nfunc (is *IndexSnapshot) FieldDictRegexpAutomaton(field string,\n\ttermRegex string,\n) (index.FieldDict, index.RegexAutomaton, error) {\n\treturn is.fieldDictRegexp(field, termRegex)\n}\n\nfunc (is *IndexSnapshot) fieldDictRegexp(field string,\n\ttermRegex string,\n) (index.FieldDict, index.RegexAutomaton, error) {\n\t// TODO: potential optimization where the literal prefix represents the,\n\t//       entire regexp, allowing us to use PrefixIterator(prefixTerm)?\n\n\ta, prefixBeg, prefixEnd, err := parseRegexp(termRegex)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tfd, err := is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {\n\t\treturn is.AutomatonIterator(a, prefixBeg, prefixEnd)\n\t}, false)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\treturn fd, a, nil\n}\n\nfunc (is *IndexSnapshot) getLevAutomaton(term string,\n\tfuzziness uint8,\n) (vellum.Automaton, error) {\n\tswitch fuzziness {\n\tcase 1:\n\t\treturn lb1.BuildDfa(term, fuzziness)\n\tcase 2:\n\t\treturn lb2.BuildDfa(term, fuzziness)\n\t}\n\treturn nil, fmt.Errorf(\"fuzziness exceeds the max limit\")\n}\n\nfunc (is *IndexSnapshot) FieldDictFuzzy(field string,\n\tterm string, fuzziness int, prefix string,\n) (index.FieldDict, error) {\n\tfd, _, err := is.FieldDictFuzzyAutomaton(field, term, fuzziness, prefix)\n\treturn fd, err\n}\n\nfunc (is *IndexSnapshot) FieldDictFuzzyAutomaton(field string,\n\tterm string, fuzziness int, prefix string,\n) (index.FieldDict, index.FuzzyAutomaton, error) {\n\treturn is.fieldDictFuzzy(field, term, fuzziness, prefix)\n}\n\nfunc (is *IndexSnapshot) fieldDictFuzzy(field string,\n\tterm string, fuzziness int, prefix string,\n) (index.FieldDict, index.FuzzyAutomaton, error) {\n\ta, err := is.getLevAutomaton(term, uint8(fuzziness))\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tvar fa index.FuzzyAutomaton\n\tif vfa, ok := a.(vellum.FuzzyAutomaton); ok {\n\t\tfa = vfa\n\t}\n\tvar prefixBeg, prefixEnd []byte\n\tif prefix != \"\" {\n\t\tprefixBeg = []byte(prefix)\n\t\tprefixEnd = calculateExclusiveEndFromPrefix(prefixBeg)\n\t}\n\tfd, err := is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {\n\t\treturn is.AutomatonIterator(a, prefixBeg, prefixEnd)\n\t}, false)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\treturn fd, fa, nil\n}\n\nfunc (is *IndexSnapshot) FieldDictContains(field string) (index.FieldDictContains, error) {\n\treturn is.newIndexSnapshotFieldDict(field, nil, true)\n}\n\nfunc (is *IndexSnapshot) DocIDReaderAll() (index.DocIDReader, error) {\n\tresults := make(chan *asynchSegmentResult, len(is.segment))\n\tfor index, segment := range is.segment {\n\t\tgo func(index int, segment *SegmentSnapshot) {\n\t\t\tresults <- &asynchSegmentResult{\n\t\t\t\tindex: index,\n\t\t\t\tdocs:  segment.DocNumbersLive(),\n\t\t\t}\n\t\t}(index, segment)\n\t}\n\n\treturn is.newDocIDReader(results)\n}\n\nfunc (is *IndexSnapshot) DocIDReaderOnly(ids []string) (index.DocIDReader, error) {\n\tresults := make(chan *asynchSegmentResult, len(is.segment))\n\tfor index, segment := range is.segment {\n\t\tgo func(index int, segment *SegmentSnapshot) {\n\t\t\tdocs, err := segment.DocNumbers(ids)\n\t\t\tif err != nil {\n\t\t\t\tresults <- &asynchSegmentResult{err: err}\n\t\t\t} else {\n\t\t\t\tresults <- &asynchSegmentResult{\n\t\t\t\t\tindex: index,\n\t\t\t\t\tdocs:  docs,\n\t\t\t\t}\n\t\t\t}\n\t\t}(index, segment)\n\t}\n\n\treturn is.newDocIDReader(results)\n}\n\nfunc (is *IndexSnapshot) newDocIDReader(results chan *asynchSegmentResult) (index.DocIDReader, error) {\n\trv := &IndexSnapshotDocIDReader{\n\t\tsnapshot:  is,\n\t\titerators: make([]roaring.IntIterable, len(is.segment)),\n\t}\n\tvar err error\n\tfor count := 0; count < len(is.segment); count++ {\n\t\tasr := <-results\n\t\tif asr.err != nil {\n\t\t\tif err == nil {\n\t\t\t\t// returns the first error encountered\n\t\t\t\terr = asr.err\n\t\t\t}\n\t\t} else if err == nil {\n\t\t\trv.iterators[asr.index] = asr.docs.Iterator()\n\t\t}\n\t}\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) Fields() ([]string, error) {\n\t// FIXME not making this concurrent for now as it's not used in hot path\n\t// of any searches at the moment (just a debug aid)\n\tfieldsMap := make(map[string]struct{})\n\tfor _, segment := range is.segment {\n\t\tfields := segment.Fields()\n\t\tfor _, field := range fields {\n\t\t\tfieldsMap[field] = struct{}{}\n\t\t}\n\t}\n\trv := make([]string, 0, len(fieldsMap))\n\tfor k := range fieldsMap {\n\t\trv = append(rv, k)\n\t}\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) GetInternal(key []byte) ([]byte, error) {\n\treturn is.internal[string(key)], nil\n}\n\nfunc (is *IndexSnapshot) DocCount() (uint64, error) {\n\tvar rv uint64\n\tfor _, segment := range is.segment {\n\t\trv += segment.CountRoot()\n\t}\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) Document(id string) (rv index.Document, err error) {\n\t// FIXME could be done more efficiently directly, but reusing for simplicity\n\ttfr, err := is.TermFieldReader(context.TODO(), []byte(id), \"_id\", false, false, false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := tfr.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tnext, err := tfr.Next(nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif next == nil {\n\t\t// no such doc exists\n\t\treturn nil, nil\n\t}\n\n\tdocNum, err := next.ID.Value()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsegmentIndex, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum)\n\n\trvd := document.NewDocument(id)\n\n\terr = is.segment[segmentIndex].VisitDocument(localDocNum, func(name string, typ byte, val []byte, pos []uint64) bool {\n\t\tif name == \"_id\" {\n\t\t\treturn true\n\t\t}\n\n\t\t// track uncompressed stored fields bytes as part of IO stats.\n\t\t// However, ideally we'd need to track the compressed on-disk value\n\t\t// Keeping that TODO for now until we have a cleaner way.\n\t\trvd.StoredFieldsSize += uint64(len(val))\n\n\t\t// Skip fields that have been completely deleted or had their\n\t\t// store data deleted\n\t\tif info, ok := is.updatedFields[name]; ok &&\n\t\t\t(info.Deleted || info.Store) {\n\t\t\treturn true\n\t\t}\n\n\t\t// copy value, array positions to preserve them beyond the scope of this callback\n\t\tvalue := append([]byte(nil), val...)\n\t\tarrayPos := append([]uint64(nil), pos...)\n\n\t\tswitch typ {\n\t\tcase 't':\n\t\t\trvd.AddField(document.NewTextField(name, arrayPos, value))\n\t\tcase 'n':\n\t\t\trvd.AddField(document.NewNumericFieldFromBytes(name, arrayPos, value))\n\t\tcase 'i':\n\t\t\trvd.AddField(document.NewIPFieldFromBytes(name, arrayPos, value))\n\t\tcase 'd':\n\t\t\trvd.AddField(document.NewDateTimeFieldFromBytes(name, arrayPos, value))\n\t\tcase 'b':\n\t\t\trvd.AddField(document.NewBooleanFieldFromBytes(name, arrayPos, value))\n\t\tcase 'g':\n\t\t\trvd.AddField(document.NewGeoPointFieldFromBytes(name, arrayPos, value))\n\t\tcase 's':\n\t\t\trvd.AddField(document.NewGeoShapeFieldFromBytes(name, arrayPos, value))\n\t\t}\n\n\t\treturn true\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rvd, nil\n}\n\n// In a multi-segment index, each document has:\n// 1. a local docnum - local to the segment\n// 2. a global docnum - unique identifier across the index\n// This function returns the segment index(the segment in which the docnum is present)\n// and local docnum of a document.\nfunc (is *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (int, uint64) {\n\tsegmentIndex := sort.Search(len(is.offsets),\n\t\tfunc(x int) bool {\n\t\t\treturn is.offsets[x] > docNum\n\t\t}) - 1\n\n\treturn int(segmentIndex), docNum - is.offsets[segmentIndex]\n}\n\nfunc (is *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) {\n\tdocNum, err := id.Value()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tsegmentIndex, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum)\n\n\tv, err := is.segment[segmentIndex].DocID(localDocNum)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif v == nil {\n\t\treturn \"\", fmt.Errorf(\"document number %d not found\", docNum)\n\t}\n\n\treturn string(v), nil\n}\n\nfunc (is *IndexSnapshot) segmentIndexAndLocalDocNum(id index.IndexInternalID) (int, uint64, error) {\n\tdocNum, err := id.Value()\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\tsegIdx, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum)\n\treturn segIdx, localDocNum, nil\n}\n\nfunc (is *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err error) {\n\t// FIXME could be done more efficiently directly, but reusing for simplicity\n\ttfr, err := is.TermFieldReader(context.TODO(), []byte(id), \"_id\", false, false, false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := tfr.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tnext, err := tfr.Next(nil)\n\tif err != nil || next == nil {\n\t\treturn nil, err\n\t}\n\n\treturn next.ID, nil\n}\n\nfunc (is *IndexSnapshot) TermFieldReader(ctx context.Context, term []byte, field string, includeFreq,\n\tincludeNorm, includeTermVectors bool,\n) (index.TermFieldReader, error) {\n\trv := is.allocTermFieldReaderDicts(field)\n\n\trv.ctx = ctx\n\trv.term = term\n\trv.field = field\n\trv.snapshot = is\n\tif rv.postings == nil {\n\t\trv.postings = make([]segment.PostingsList, len(is.segment))\n\t}\n\tif rv.iterators == nil {\n\t\trv.iterators = make([]segment.PostingsIterator, len(is.segment))\n\t}\n\trv.segmentOffset = 0\n\trv.includeFreq = includeFreq\n\trv.includeNorm = includeNorm\n\trv.includeTermVectors = includeTermVectors\n\trv.currPosting = nil\n\trv.currID = rv.currID[:0]\n\n\tif rv.dicts == nil {\n\t\trv.dicts = make([]segment.TermDictionary, len(is.segment))\n\t\tfor i, s := range is.segment {\n\t\t\t// the intention behind this compare and swap operation is\n\t\t\t// to make sure that the accounting of the metadata is happening\n\t\t\t// only once(which corresponds to this persisted segment's most\n\t\t\t// recent segPlugin.Open() call), and any subsequent queries won't\n\t\t\t// incur this cost which would essentially be a double counting.\n\t\t\tif atomic.CompareAndSwapUint32(&s.mmaped, 1, 0) {\n\t\t\t\tsegBytesRead := s.segment.BytesRead()\n\t\t\t\trv.incrementBytesRead(segBytesRead)\n\t\t\t}\n\n\t\t\tvar dict segment.TermDictionary\n\t\t\tvar err error\n\n\t\t\t// Skip fields that have been completely deleted or had their\n\t\t\t// index data deleted\n\t\t\tif info, ok := is.updatedFields[field]; ok &&\n\t\t\t\t(info.Index || info.Deleted) {\n\t\t\t\tdict, err = s.segment.Dictionary(\"\")\n\t\t\t} else {\n\t\t\t\tdict, err = s.segment.Dictionary(field)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tif dictStats, ok := dict.(segment.DiskStatsReporter); ok {\n\t\t\t\tbytesRead := dictStats.BytesRead()\n\t\t\t\trv.incrementBytesRead(bytesRead)\n\t\t\t}\n\t\t\trv.dicts[i] = dict\n\t\t}\n\t}\n\n\tfor i, s := range is.segment {\n\t\tvar prevBytesReadPL uint64\n\t\tif rv.postings[i] != nil {\n\t\t\tprevBytesReadPL = rv.postings[i].BytesRead()\n\t\t}\n\t\tpl, err := rv.dicts[i].PostingsList(term, s.deleted, rv.postings[i])\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trv.postings[i] = pl\n\n\t\tvar prevBytesReadItr uint64\n\t\tif rv.iterators[i] != nil {\n\t\t\tprevBytesReadItr = rv.iterators[i].BytesRead()\n\t\t}\n\t\trv.iterators[i] = pl.Iterator(includeFreq, includeNorm, includeTermVectors, rv.iterators[i])\n\n\t\tif bytesRead := rv.postings[i].BytesRead(); prevBytesReadPL < bytesRead {\n\t\t\trv.incrementBytesRead(bytesRead - prevBytesReadPL)\n\t\t}\n\n\t\tif bytesRead := rv.iterators[i].BytesRead(); prevBytesReadItr < bytesRead {\n\t\t\trv.incrementBytesRead(bytesRead - prevBytesReadItr)\n\t\t}\n\t}\n\t// ONLY update the bytes read value beyond this point for this TFR if scoring is enabled\n\trv.updateBytesRead = rv.includeFreq || rv.includeNorm || rv.includeTermVectors\n\tatomic.AddUint64(&is.parent.stats.TotTermSearchersStarted, uint64(1))\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnapshotTermFieldReader) {\n\tis.m2.Lock()\n\tif is.fieldTFRs != nil {\n\t\ttfrs := is.fieldTFRs[field]\n\t\tlast := len(tfrs) - 1\n\t\tif last >= 0 {\n\t\t\ttfr = tfrs[last]\n\t\t\ttfrs[last] = nil\n\t\t\tis.fieldTFRs[field] = tfrs[:last]\n\t\t\tis.m2.Unlock()\n\t\t\treturn\n\t\t}\n\t}\n\tis.m2.Unlock()\n\treturn &IndexSnapshotTermFieldReader{\n\t\trecycle: true,\n\t}\n}\n\n// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for\n// a field in an index snapshot. Without this limit, when recycling TFRs, it is\n// possible that a very large number of TFRs may be added to the recycle\n// cache, which could eventually lead to significant memory consumption.\n// This threshold can be overwritten by users at the library level by changing the\n// exported variable, or at the index level by setting the \"fieldTFRCacheThreshold\"\n// in the kvConfig.\nvar DefaultFieldTFRCacheThreshold int = 0 // disabled because it causes MB-64604\n\nfunc (is *IndexSnapshot) getFieldTFRCacheThreshold() int {\n\tif is.parent.config != nil {\n\t\tif val, exists := is.parent.config[\"fieldTFRCacheThreshold\"]; exists {\n\t\t\tif x, ok := val.(float64); ok {\n\t\t\t\t// JSON unmarshal-ed into a map[string]interface{} will default\n\t\t\t\t// to float64 for numbers, so we need to check for float64 first.\n\t\t\t\treturn int(x)\n\t\t\t} else if x, ok := val.(int); ok {\n\t\t\t\t// If library users provided an int in the config, we'll honor it.\n\t\t\t\treturn x\n\t\t\t}\n\t\t}\n\t}\n\treturn DefaultFieldTFRCacheThreshold\n}\n\nfunc (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) {\n\tif !tfr.recycle {\n\t\t// Do not recycle an optimized unadorned term field reader (used for\n\t\t// ConjunctionUnadorned or DisjunctionUnadorned), during when a fresh\n\t\t// roaring.Bitmap is built by AND-ing or OR-ing individual bitmaps,\n\t\t// and we'll need to release them for GC. (See MB-40916)\n\t\treturn\n\t}\n\n\tis.parent.rootLock.RLock()\n\tobsolete := is.parent.root != is\n\tis.parent.rootLock.RUnlock()\n\tif obsolete {\n\t\t// if we're not the current root (mutations happened), don't bother recycling\n\t\treturn\n\t}\n\n\tis.m2.Lock()\n\tif is.fieldTFRs == nil {\n\t\tis.fieldTFRs = make(map[string][]*IndexSnapshotTermFieldReader)\n\t}\n\tif len(is.fieldTFRs[tfr.field]) < is.getFieldTFRCacheThreshold() {\n\t\ttfr.bytesRead = 0\n\t\tis.fieldTFRs[tfr.field] = append(is.fieldTFRs[tfr.field], tfr)\n\t}\n\tis.m2.Unlock()\n}\n\nfunc (is *IndexSnapshot) documentVisitFieldTermsOnSegment(\n\tsegmentIndex int, localDocNum uint64, fields []string, cFields []string,\n\tvisitor index.DocValueVisitor, dvs segment.DocVisitState) (\n\tcFieldsOut []string, dvsOut segment.DocVisitState, err error,\n) {\n\tss := is.segment[segmentIndex]\n\n\tvar vFields []string // fields that are visitable via the segment\n\n\tssv, ssvOk := ss.segment.(segment.DocValueVisitable)\n\tif ssvOk && ssv != nil {\n\t\tvFields, err = ssv.VisitableDocValueFields()\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\t// Filter out fields that have been completely deleted or had their\n\t// docvalues data deleted from both visitable fields and required fields\n\tfilterUpdatedFields := func(fields []string) []string {\n\t\tfilteredFields := make([]string, 0, len(fields))\n\t\tfor _, field := range fields {\n\t\t\tif info, ok := is.updatedFields[field]; ok &&\n\t\t\t\t(info.DocValues || info.Deleted) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfilteredFields = append(filteredFields, field)\n\t\t}\n\t\treturn filteredFields\n\t}\n\n\tif len(is.updatedFields) > 0 {\n\t\tfields = filterUpdatedFields(fields)\n\t\tvFields = filterUpdatedFields(vFields)\n\t}\n\n\tvar errCh chan error\n\n\t// cFields represents the fields that we'll need from the\n\t// cachedDocs, and might be optionally be provided by the caller,\n\t// if the caller happens to know we're on the same segmentIndex\n\t// from a previous invocation\n\tif cFields == nil {\n\t\tcFields = subtractStrings(fields, vFields)\n\n\t\tif len(cFields) > 0 && !ss.cachedDocs.hasFields(cFields) {\n\t\t\terrCh = make(chan error, 1)\n\n\t\t\tgo func() {\n\t\t\t\terr := ss.cachedDocs.prepareFields(cFields, ss)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrCh <- err\n\t\t\t\t}\n\t\t\t\tclose(errCh)\n\t\t\t}()\n\t\t}\n\t}\n\n\tif ssvOk && ssv != nil && len(vFields) > 0 {\n\t\tdvs, err = ssv.VisitDocValues(localDocNum, fields, visitor, dvs)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\tif errCh != nil {\n\t\terr = <-errCh\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\tif len(cFields) > 0 {\n\t\tss.cachedDocs.visitDoc(localDocNum, cFields, visitor)\n\t}\n\n\treturn cFields, dvs, nil\n}\n\nfunc (is *IndexSnapshot) DocValueReader(fields []string) (\n\tindex.DocValueReader, error,\n) {\n\treturn &DocValueReader{i: is, fields: fields, currSegmentIndex: -1}, nil\n}\n\ntype DocValueReader struct {\n\ti      *IndexSnapshot\n\tfields []string\n\tdvs    segment.DocVisitState\n\n\tcurrSegmentIndex int\n\tcurrCachedFields []string\n\n\ttotalBytesRead uint64\n\tbytesRead      uint64\n}\n\nfunc (dvr *DocValueReader) BytesRead() uint64 {\n\treturn dvr.totalBytesRead + dvr.bytesRead\n}\n\nfunc (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID,\n\tvisitor index.DocValueVisitor,\n) (err error) {\n\tdocNum, err := id.Value()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tsegmentIndex, localDocNum := dvr.i.segmentIndexAndLocalDocNumFromGlobal(docNum)\n\tif segmentIndex >= len(dvr.i.segment) {\n\t\treturn nil\n\t}\n\n\tif dvr.currSegmentIndex != segmentIndex {\n\t\tdvr.currSegmentIndex = segmentIndex\n\t\tdvr.currCachedFields = nil\n\t\tdvr.totalBytesRead += dvr.bytesRead\n\t\tdvr.bytesRead = 0\n\t}\n\n\tdvr.currCachedFields, dvr.dvs, err = dvr.i.documentVisitFieldTermsOnSegment(\n\t\tdvr.currSegmentIndex, localDocNum, dvr.fields, dvr.currCachedFields, visitor, dvr.dvs)\n\n\tif dvr.dvs != nil {\n\t\tdvr.bytesRead = dvr.dvs.BytesRead()\n\t}\n\treturn err\n}\n\nfunc (is *IndexSnapshot) DumpAll() chan interface{} {\n\trv := make(chan interface{})\n\tgo func() {\n\t\tclose(rv)\n\t}()\n\treturn rv\n}\n\nfunc (is *IndexSnapshot) DumpDoc(id string) chan interface{} {\n\trv := make(chan interface{})\n\tgo func() {\n\t\tclose(rv)\n\t}()\n\treturn rv\n}\n\nfunc (is *IndexSnapshot) DumpFields() chan interface{} {\n\trv := make(chan interface{})\n\tgo func() {\n\t\tclose(rv)\n\t}()\n\treturn rv\n}\n\nfunc (is *IndexSnapshot) diskSegmentsPaths() map[string]struct{} {\n\trv := make(map[string]struct{}, len(is.segment))\n\tfor _, s := range is.segment {\n\t\tif seg, ok := s.segment.(segment.PersistedSegment); ok {\n\t\t\trv[seg.Path()] = struct{}{}\n\t\t}\n\t}\n\treturn rv\n}\n\n// reClaimableDocsRatio gives a ratio about the obsoleted or\n// reclaimable documents present in a given index snapshot.\nfunc (is *IndexSnapshot) reClaimableDocsRatio() float64 {\n\tvar totalCount, liveCount uint64\n\tfor _, s := range is.segment {\n\t\tif _, ok := s.segment.(segment.PersistedSegment); ok {\n\t\t\ttotalCount += uint64(s.FullSize())\n\t\t\tliveCount += uint64(s.Count())\n\t\t}\n\t}\n\n\tif totalCount > 0 {\n\t\treturn float64(totalCount-liveCount) / float64(totalCount)\n\t}\n\treturn 0\n}\n\n// subtractStrings returns set a minus elements of set b.\nfunc subtractStrings(a, b []string) []string {\n\tif len(b) == 0 {\n\t\treturn a\n\t}\n\n\trv := make([]string, 0, len(a))\nOUTER:\n\tfor _, as := range a {\n\t\tfor _, bs := range b {\n\t\t\tif as == bs {\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\t\t}\n\t\trv = append(rv, as)\n\t}\n\treturn rv\n}\n\nfunc (is *IndexSnapshot) CopyTo(d index.Directory) error {\n\t// get the root bolt file.\n\tw, err := d.GetWriter(filepath.Join(\"store\", \"root.bolt\"))\n\tif err != nil || w == nil {\n\t\treturn fmt.Errorf(\"failed to create the root.bolt file, err: %v\", err)\n\t}\n\trootFile, ok := w.(*os.File)\n\tif !ok {\n\t\treturn fmt.Errorf(\"invalid root.bolt file found\")\n\t}\n\n\tcopyBolt, err := bolt.Open(rootFile.Name(), 0o600, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tw.Close()\n\t\tif cerr := copyBolt.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\t// start a write transaction\n\ttx, err := copyBolt.Begin(true)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, _, err = prepareBoltSnapshot(is, tx, \"\", is.parent.segPlugin, nil, d)\n\tif err != nil {\n\t\t_ = tx.Rollback()\n\t\treturn fmt.Errorf(\"error backing up index snapshot: %v\", err)\n\t}\n\n\t// commit bolt data\n\terr = tx.Commit()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error commit tx to backup root bolt: %v\", err)\n\t}\n\n\treturn copyBolt.Sync()\n}\n\nfunc (is *IndexSnapshot) UpdateIOStats(val uint64) {\n\tatomic.AddUint64(&is.parent.stats.TotBytesReadAtQueryTime, val)\n}\n\nfunc (is *IndexSnapshot) GetSpatialAnalyzerPlugin(typ string) (\n\tindex.SpatialAnalyzerPlugin, error,\n) {\n\tvar rv index.SpatialAnalyzerPlugin\n\tis.m.Lock()\n\trv = is.parent.spatialPlugin\n\tis.m.Unlock()\n\n\tif rv == nil {\n\t\treturn nil, fmt.Errorf(\"no spatial plugin type: %s found\", typ)\n\t}\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) CloseCopyReader() error {\n\t// first unmark the segments that were marked for backup by this index snapshot\n\tis.parent.rootLock.Lock()\n\tfor _, seg := range is.segment {\n\t\tvar fileName string\n\t\tif perSeg, ok := seg.segment.(segment.PersistedSegment); ok {\n\t\t\t// segment is persisted\n\t\t\tfileName = filepath.Base(perSeg.Path())\n\t\t} else {\n\t\t\t// segment is not persisted\n\t\t\t// the name of the segment file that is generated if the\n\t\t\t// the segment is persisted in the future.\n\t\t\tfileName = zapFileName(seg.id)\n\t\t}\n\t\tif is.parent.copyScheduled[fileName]--; is.parent.copyScheduled[fileName] <= 0 {\n\t\t\tdelete(is.parent.copyScheduled, fileName)\n\t\t}\n\t}\n\tis.parent.rootLock.Unlock()\n\t// close the index snapshot normally\n\treturn is.Close()\n}\n\nfunc (is *IndexSnapshot) ThesaurusTermReader(ctx context.Context, thesaurusName string, term []byte) (index.ThesaurusTermReader, error) {\n\trv := &IndexSnapshotThesaurusTermReader{\n\t\tname:          thesaurusName,\n\t\tsnapshot:      is,\n\t\tpostings:      make([]segment.SynonymsList, len(is.segment)),\n\t\titerators:     make([]segment.SynonymsIterator, len(is.segment)),\n\t\tthesauri:      make([]segment.Thesaurus, len(is.segment)),\n\t\tsegmentOffset: 0,\n\t}\n\n\tfor i, s := range is.segment {\n\t\tif synSeg, ok := s.segment.(segment.ThesaurusSegment); ok {\n\t\t\tthes, err := synSeg.Thesaurus(thesaurusName)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\trv.thesauri[i] = thes\n\t\t\tpl, err := rv.thesauri[i].SynonymsList(term, s.deleted, rv.postings[i])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\trv.postings[i] = pl\n\n\t\t\trv.iterators[i] = pl.Iterator(rv.iterators[i])\n\t\t}\n\t}\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) newIndexSnapshotThesaurusKeys(name string,\n\tmakeItr func(i segment.Thesaurus) segment.ThesaurusIterator,\n) (*IndexSnapshotThesaurusKeys, error) {\n\tresults := make(chan *asynchSegmentResult, len(is.segment))\n\tvar wg sync.WaitGroup\n\twg.Add(len(is.segment))\n\tfor _, s := range is.segment {\n\t\tgo func(s *SegmentSnapshot) {\n\t\t\tdefer wg.Done()\n\t\t\tif synSeg, ok := s.segment.(segment.ThesaurusSegment); ok {\n\t\t\t\tthes, err := synSeg.Thesaurus(name)\n\t\t\t\tif err != nil {\n\t\t\t\t\tresults <- &asynchSegmentResult{err: err}\n\t\t\t\t} else {\n\t\t\t\t\tresults <- &asynchSegmentResult{thesItr: makeItr(thes)}\n\t\t\t\t}\n\t\t\t}\n\t\t}(s)\n\t}\n\t// Close the channel after all goroutines complete\n\tgo func() {\n\t\twg.Wait()\n\t\tclose(results)\n\t}()\n\n\tvar err error\n\trv := &IndexSnapshotThesaurusKeys{\n\t\tsnapshot: is,\n\t\tcursors:  make([]*segmentThesCursor, 0, len(is.segment)),\n\t}\n\tfor asr := range results {\n\t\tif asr.err != nil && err == nil {\n\t\t\terr = asr.err\n\t\t} else {\n\t\t\tnext, err2 := asr.thesItr.Next()\n\t\t\tif err2 != nil && err == nil {\n\t\t\t\terr = err2\n\t\t\t}\n\t\t\tif next != nil {\n\t\t\t\trv.cursors = append(rv.cursors, &segmentThesCursor{\n\t\t\t\t\titr:  asr.thesItr,\n\t\t\t\t\tcurr: *next,\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n\t// after ensuring we've read all items on channel\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rv, nil\n}\n\nfunc (is *IndexSnapshot) ThesaurusKeys(name string) (index.ThesaurusKeys, error) {\n\treturn is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator {\n\t\treturn is.AutomatonIterator(nil, nil, nil)\n\t})\n}\n\nfunc (is *IndexSnapshot) ThesaurusKeysFuzzy(name string,\n\tterm string, fuzziness int, prefix string,\n) (index.ThesaurusKeys, error) {\n\ta, err := is.getLevAutomaton(term, uint8(fuzziness))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar prefixBeg, prefixEnd []byte\n\tif prefix != \"\" {\n\t\tprefixBeg = []byte(prefix)\n\t\tprefixEnd = calculateExclusiveEndFromPrefix(prefixBeg)\n\t}\n\treturn is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator {\n\t\treturn is.AutomatonIterator(a, prefixBeg, prefixEnd)\n\t})\n}\n\nfunc (is *IndexSnapshot) ThesaurusKeysPrefix(name string,\n\ttermPrefix []byte,\n) (index.ThesaurusKeys, error) {\n\ttermPrefixEnd := calculateExclusiveEndFromPrefix(termPrefix)\n\treturn is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator {\n\t\treturn is.AutomatonIterator(nil, termPrefix, termPrefixEnd)\n\t})\n}\n\nfunc (is *IndexSnapshot) ThesaurusKeysRegexp(name string,\n\ttermRegex string,\n) (index.ThesaurusKeys, error) {\n\ta, prefixBeg, prefixEnd, err := parseRegexp(termRegex)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator {\n\t\treturn is.AutomatonIterator(a, prefixBeg, prefixEnd)\n\t})\n}\n\nfunc (is *IndexSnapshot) UpdateSynonymSearchCount(delta uint64) {\n\tatomic.AddUint64(&is.parent.stats.TotSynonymSearches, delta)\n}\n\n// Update current snapshot updated field data as well as pass it on to all segments and segment bases\nfunc (is *IndexSnapshot) UpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) {\n\tis.m.Lock()\n\tdefer is.m.Unlock()\n\n\tis.MergeUpdateFieldsInfo(updatedFields)\n\n\tfor _, segmentSnapshot := range is.segment {\n\t\tsegmentSnapshot.UpdateFieldsInfo(is.updatedFields)\n\t}\n}\n\n// Merge given updated field information with existing updated field information\nfunc (is *IndexSnapshot) MergeUpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) {\n\tif is.updatedFields == nil {\n\t\tis.updatedFields = updatedFields\n\t} else {\n\t\tfor fieldName, info := range updatedFields {\n\t\t\tif val, ok := is.updatedFields[fieldName]; ok {\n\t\t\t\tval.Deleted = val.Deleted || info.Deleted\n\t\t\t\tval.Index = val.Index || info.Index\n\t\t\t\tval.DocValues = val.DocValues || info.DocValues\n\t\t\t\tval.Store = val.Store || info.Store\n\t\t\t} else {\n\t\t\t\tis.updatedFields[fieldName] = info\n\t\t\t}\n\t\t}\n\t}\n}\n\n// TermFrequencies returns the top N terms ordered by the frequencies\n// for a given field across all segments in the index snapshot.\nfunc (is *IndexSnapshot) TermFrequencies(field string, limit int, descending bool) (\n\ttermFreqs []index.TermFreq, err error) {\n\tif len(is.segment) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tif limit <= 0 {\n\t\treturn nil, fmt.Errorf(\"limit must be positive\")\n\t}\n\n\t// Use FieldDict which aggregates term frequencies across all segments\n\tfieldDict, err := is.FieldDict(field)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to get field dictionary for field %s: %v\", field, err)\n\t}\n\tdefer fieldDict.Close()\n\n\t// Preallocate slice with capacity equal to the number of unique terms\n\t// in the field dictionary\n\ttermFreqs = make([]index.TermFreq, 0, fieldDict.Cardinality())\n\n\t// Iterate through all terms using FieldDict\n\tfor {\n\t\tdictEntry, err := fieldDict.Next()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error iterating field dictionary: %v\", err)\n\t\t}\n\t\tif dictEntry == nil {\n\t\t\tbreak // End of terms\n\t\t}\n\n\t\ttermFreqs = append(termFreqs, index.TermFreq{\n\t\t\tTerm:      dictEntry.Term,\n\t\t\tFrequency: dictEntry.Count,\n\t\t})\n\t}\n\n\t// Sort by frequency (descending or ascending)\n\tsort.Slice(termFreqs, func(i, j int) bool {\n\t\tif termFreqs[i].Frequency == termFreqs[j].Frequency {\n\t\t\t// If frequencies are equal, sort by term lexicographically\n\t\t\treturn termFreqs[i].Term < termFreqs[j].Term\n\t\t}\n\t\tif descending {\n\t\t\treturn termFreqs[i].Frequency > termFreqs[j].Frequency\n\t\t}\n\t\treturn termFreqs[i].Frequency < termFreqs[j].Frequency\n\t})\n\n\tif limit >= len(termFreqs) {\n\t\treturn termFreqs, nil\n\t}\n\n\treturn termFreqs[:limit], nil\n}\n\n// Ancestors returns the ancestor IDs for the given document ID. The prealloc\n// slice can be provided to avoid allocations downstream, and MUST be empty.\nfunc (i *IndexSnapshot) Ancestors(ID index.IndexInternalID, prealloc []index.AncestorID) ([]index.AncestorID, error) {\n\t// get segment and local doc num for the ID\n\tseg, ldoc, err := i.segmentIndexAndLocalDocNum(ID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// get ancestors from the segment\n\tprealloc = i.segment[seg].Ancestors(ldoc, prealloc)\n\t// get global offset for the segment (correcting factor for multi-segment indexes)\n\tglobalOffset := i.offsets[seg]\n\t// adjust ancestors to global doc numbers, not local to segment\n\tfor idx := range prealloc {\n\t\tprealloc[idx] = prealloc[idx].Add(globalOffset)\n\t}\n\t// return adjusted ancestors\n\treturn prealloc, nil\n}\n"
  },
  {
    "path": "index/scorch/snapshot_index_dict.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"container/heap\"\n\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\ntype segmentDictCursor struct {\n\tdict segment.TermDictionary\n\titr  segment.DictionaryIterator\n\tcurr index.DictEntry\n}\n\ntype IndexSnapshotFieldDict struct {\n\tcardinality int\n\tbytesRead   uint64\n\n\tsnapshot *IndexSnapshot\n\tcursors  []*segmentDictCursor\n\tentry    index.DictEntry\n}\n\nfunc (i *IndexSnapshotFieldDict) BytesRead() uint64 {\n\treturn i.bytesRead\n}\n\nfunc (i *IndexSnapshotFieldDict) Len() int { return len(i.cursors) }\nfunc (i *IndexSnapshotFieldDict) Less(a, b int) bool {\n\treturn i.cursors[a].curr.Term < i.cursors[b].curr.Term\n}\nfunc (i *IndexSnapshotFieldDict) Swap(a, b int) {\n\ti.cursors[a], i.cursors[b] = i.cursors[b], i.cursors[a]\n}\n\nfunc (i *IndexSnapshotFieldDict) Push(x interface{}) {\n\ti.cursors = append(i.cursors, x.(*segmentDictCursor))\n}\n\nfunc (i *IndexSnapshotFieldDict) Pop() interface{} {\n\tn := len(i.cursors)\n\tx := i.cursors[n-1]\n\ti.cursors = i.cursors[0 : n-1]\n\treturn x\n}\n\nfunc (i *IndexSnapshotFieldDict) Next() (*index.DictEntry, error) {\n\tif len(i.cursors) == 0 {\n\t\treturn nil, nil\n\t}\n\ti.entry = i.cursors[0].curr\n\tnext, err := i.cursors[0].itr.Next()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif next == nil {\n\t\t// at end of this cursor, remove it\n\t\theap.Pop(i)\n\t} else {\n\t\t// modified heap, fix it\n\t\ti.cursors[0].curr = *next\n\t\theap.Fix(i, 0)\n\t}\n\t// look for any other entries with the exact same term\n\tfor len(i.cursors) > 0 && i.cursors[0].curr.Term == i.entry.Term {\n\t\ti.entry.Count += i.cursors[0].curr.Count\n\t\tnext, err := i.cursors[0].itr.Next()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif next == nil {\n\t\t\t// at end of this cursor, remove it\n\t\t\theap.Pop(i)\n\t\t} else {\n\t\t\t// modified heap, fix it\n\t\t\ti.cursors[0].curr = *next\n\t\t\theap.Fix(i, 0)\n\t\t}\n\t}\n\n\treturn &i.entry, nil\n}\n\nfunc (i *IndexSnapshotFieldDict) Cardinality() int {\n\treturn i.cardinality\n}\n\nfunc (i *IndexSnapshotFieldDict) Close() error {\n\treturn nil\n}\n\nfunc (i *IndexSnapshotFieldDict) Contains(key []byte) (bool, error) {\n\tif len(i.cursors) == 0 {\n\t\treturn false, nil\n\t}\n\n\tfor _, cursor := range i.cursors {\n\t\tif found, _ := cursor.dict.Contains(key); found {\n\t\t\treturn true, nil\n\t\t}\n\t}\n\n\treturn false, nil\n}\n"
  },
  {
    "path": "index/scorch/snapshot_index_doc.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"reflect\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeIndexSnapshotDocIDReader int\n\nfunc init() {\n\tvar isdr IndexSnapshotDocIDReader\n\treflectStaticSizeIndexSnapshotDocIDReader = int(reflect.TypeOf(isdr).Size())\n}\n\ntype IndexSnapshotDocIDReader struct {\n\tsnapshot      *IndexSnapshot\n\titerators     []roaring.IntIterable\n\tsegmentOffset int\n}\n\nfunc (i *IndexSnapshotDocIDReader) Size() int {\n\treturn reflectStaticSizeIndexSnapshotDocIDReader + size.SizeOfPtr\n}\n\nfunc (i *IndexSnapshotDocIDReader) Next() (index.IndexInternalID, error) {\n\tfor i.segmentOffset < len(i.iterators) {\n\t\tif !i.iterators[i.segmentOffset].HasNext() {\n\t\t\ti.segmentOffset++\n\t\t\tcontinue\n\t\t}\n\t\tnext := i.iterators[i.segmentOffset].Next()\n\t\t// make segment number into global number by adding offset\n\t\tglobalOffset := i.snapshot.offsets[i.segmentOffset]\n\t\treturn index.NewIndexInternalID(nil, uint64(next)+globalOffset), nil\n\t}\n\treturn nil, nil\n}\n\nfunc (i *IndexSnapshotDocIDReader) Advance(ID index.IndexInternalID) (index.IndexInternalID, error) {\n\t// FIXME do something better\n\tnext, err := i.Next()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif next == nil {\n\t\treturn nil, nil\n\t}\n\tfor next.Compare(ID) < 0 {\n\t\tnext, err = i.Next()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif next == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn next, nil\n}\n\nfunc (i *IndexSnapshotDocIDReader) Close() error {\n\treturn nil\n}\n"
  },
  {
    "path": "index/scorch/snapshot_index_str.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\nvar reflectStaticSizeIndexSnapshotThesaurusTermReader int\n\nfunc init() {\n\tvar istr IndexSnapshotThesaurusTermReader\n\treflectStaticSizeIndexSnapshotThesaurusTermReader = int(reflect.TypeOf(istr).Size())\n}\n\ntype IndexSnapshotThesaurusTermReader struct {\n\tname          string\n\tsnapshot      *IndexSnapshot\n\tthesauri      []segment.Thesaurus\n\tpostings      []segment.SynonymsList\n\titerators     []segment.SynonymsIterator\n\tsegmentOffset int\n}\n\nfunc (i *IndexSnapshotThesaurusTermReader) Size() int {\n\tsizeInBytes := reflectStaticSizeIndexSnapshotThesaurusTermReader + size.SizeOfPtr +\n\t\tlen(i.name) + size.SizeOfString\n\n\tfor _, postings := range i.postings {\n\t\tif postings != nil {\n\t\t\tsizeInBytes += postings.Size()\n\t\t}\n\t}\n\n\tfor _, iterator := range i.iterators {\n\t\tif iterator != nil {\n\t\t\tsizeInBytes += iterator.Size()\n\t\t}\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (i *IndexSnapshotThesaurusTermReader) Next() (string, error) {\n\t// find the next hit\n\tfor i.segmentOffset < len(i.iterators) {\n\t\tif i.iterators[i.segmentOffset] != nil {\n\t\t\tnext, err := i.iterators[i.segmentOffset].Next()\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", err\n\t\t\t}\n\t\t\tif next != nil {\n\t\t\t\tsynTerm := next.Term()\n\t\t\t\treturn synTerm, nil\n\t\t\t}\n\t\t}\n\t\ti.segmentOffset++\n\t}\n\treturn \"\", nil\n}\n\nfunc (i *IndexSnapshotThesaurusTermReader) Close() error {\n\treturn nil\n}\n"
  },
  {
    "path": "index/scorch/snapshot_index_test.go",
    "content": "package scorch\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/vellum\"\n)\n\nfunc TestIndexSnapshot_getLevAutomaton(t *testing.T) {\n\t// Create a dummy IndexSnapshot (parent doesn't matter for this method)\n\tis := &IndexSnapshot{}\n\n\ttests := []struct {\n\t\tname        string\n\t\tterm        string\n\t\tfuzziness   uint8\n\t\texpectError bool\n\t\terrorMsg    string // Optional: check specific error message\n\t}{\n\t\t{\n\t\t\tname:        \"fuzziness 1\",\n\t\t\tterm:        \"test\",\n\t\t\tfuzziness:   1,\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname:        \"fuzziness 2\",\n\t\t\tterm:        \"another\",\n\t\t\tfuzziness:   2,\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname:        \"fuzziness 0\",\n\t\t\tterm:        \"zero\",\n\t\t\tfuzziness:   0,\n\t\t\texpectError: true,\n\t\t\terrorMsg:    \"fuzziness exceeds the max limit\",\n\t\t},\n\t\t{\n\t\t\tname:        \"fuzziness 3\",\n\t\t\tterm:        \"three\",\n\t\t\tfuzziness:   3,\n\t\t\texpectError: true,\n\t\t\terrorMsg:    \"fuzziness exceeds the max limit\",\n\t\t},\n\t\t{\n\t\t\tname:        \"empty term fuzziness 1\",\n\t\t\tterm:        \"\",\n\t\t\tfuzziness:   1,\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname:        \"empty term fuzziness 2\",\n\t\t\tterm:        \"\",\n\t\t\tfuzziness:   2,\n\t\t\texpectError: false,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgotAutomaton, err := is.getLevAutomaton(tt.term, tt.fuzziness)\n\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"getLevAutomaton() expected an error but got nil\")\n\t\t\t\t} else if tt.errorMsg != \"\" && err.Error() != tt.errorMsg {\n\t\t\t\t\tt.Errorf(\"getLevAutomaton() expected error msg %q but got %q\", tt.errorMsg, err.Error())\n\t\t\t\t}\n\t\t\t\tif gotAutomaton != nil {\n\t\t\t\t\tt.Errorf(\"getLevAutomaton() expected nil automaton on error but got %v\", gotAutomaton)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"getLevAutomaton() got unexpected error: %v\", err)\n\t\t\t\t}\n\t\t\t\tif gotAutomaton == nil {\n\t\t\t\t\tt.Errorf(\"getLevAutomaton() expected a valid automaton but got nil\")\n\t\t\t\t}\n\t\t\t\t// Optional: Check type if needed, though non-nil is usually sufficient\n\t\t\t\t_, ok := gotAutomaton.(vellum.Automaton)\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Errorf(\"getLevAutomaton() returned type is not vellum.Automaton\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Add other tests for snapshot_index.go below if needed...\n"
  },
  {
    "path": "index/scorch/snapshot_index_tfr.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"sync/atomic\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\nvar reflectStaticSizeIndexSnapshotTermFieldReader int\n\nfunc init() {\n\tvar istfr IndexSnapshotTermFieldReader\n\treflectStaticSizeIndexSnapshotTermFieldReader = int(reflect.TypeOf(istfr).Size())\n}\n\ntype IndexSnapshotTermFieldReader struct {\n\tterm               []byte\n\tfield              string\n\tsnapshot           *IndexSnapshot\n\tdicts              []segment.TermDictionary\n\tpostings           []segment.PostingsList\n\titerators          []segment.PostingsIterator\n\tsegmentOffset      int\n\tincludeFreq        bool\n\tincludeNorm        bool\n\tincludeTermVectors bool\n\tcurrPosting        segment.Posting\n\tcurrID             index.IndexInternalID\n\trecycle            bool\n\tbytesRead          uint64\n\tctx                context.Context\n\tunadorned          bool\n\t// flag to indicate whether to increment our bytesRead\n\t// value after creation of the TFR while iterating our postings\n\t// lists\n\tupdateBytesRead bool\n}\n\nfunc (i *IndexSnapshotTermFieldReader) incrementBytesRead(val uint64) {\n\ti.bytesRead += val\n}\n\nfunc (i *IndexSnapshotTermFieldReader) Size() int {\n\tsizeInBytes := reflectStaticSizeIndexSnapshotTermFieldReader + size.SizeOfPtr +\n\t\tlen(i.term) +\n\t\tlen(i.field) +\n\t\tlen(i.currID)\n\n\tfor _, entry := range i.postings {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tfor _, entry := range i.iterators {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tif i.currPosting != nil {\n\t\tsizeInBytes += i.currPosting.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (i *IndexSnapshotTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) {\n\trv := preAlloced\n\tif rv == nil {\n\t\trv = &index.TermFieldDoc{}\n\t}\n\tvar prevBytesRead uint64\n\t// find the next hit\n\tfor i.segmentOffset < len(i.iterators) {\n\t\t// get our current postings iterator\n\t\tcurItr := i.iterators[i.segmentOffset]\n\t\tif i.updateBytesRead {\n\t\t\tprevBytesRead = curItr.BytesRead()\n\t\t}\n\t\tnext, err := curItr.Next()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif next != nil {\n\t\t\t// make segment number into global number by adding offset\n\t\t\tglobalOffset := i.snapshot.offsets[i.segmentOffset]\n\t\t\tnnum := next.Number()\n\t\t\trv.ID = index.NewIndexInternalID(rv.ID, nnum+globalOffset)\n\t\t\ti.postingToTermFieldDoc(next, rv)\n\n\t\t\ti.currID = rv.ID\n\t\t\ti.currPosting = next\n\t\t\tif i.updateBytesRead {\n\t\t\t\t// postingsIterators maintains the bytesRead stat in a cumulative fashion.\n\t\t\t\t// this is because there are chances of having a series of loadChunk calls,\n\t\t\t\t// and they have to be added together before sending the bytesRead at this point\n\t\t\t\t// upstream.\n\t\t\t\tbytesRead := curItr.BytesRead()\n\t\t\t\tif bytesRead > prevBytesRead {\n\t\t\t\t\ti.incrementBytesRead(bytesRead - prevBytesRead)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn rv, nil\n\t\t}\n\t\ti.segmentOffset++\n\t}\n\treturn nil, nil\n}\n\nfunc (i *IndexSnapshotTermFieldReader) postingToTermFieldDoc(next segment.Posting, rv *index.TermFieldDoc) {\n\tif i.includeFreq {\n\t\trv.Freq = next.Frequency()\n\t}\n\tif i.includeNorm {\n\t\trv.Norm = next.Norm()\n\t}\n\tif i.includeTermVectors {\n\t\tlocs := next.Locations()\n\t\tif cap(rv.Vectors) < len(locs) {\n\t\t\trv.Vectors = make([]*index.TermFieldVector, len(locs))\n\t\t\tbacking := make([]index.TermFieldVector, len(locs))\n\t\t\tfor i := range backing {\n\t\t\t\trv.Vectors[i] = &backing[i]\n\t\t\t}\n\t\t}\n\t\trv.Vectors = rv.Vectors[:len(locs)]\n\t\tfor i, loc := range locs {\n\t\t\t*rv.Vectors[i] = index.TermFieldVector{\n\t\t\t\tStart:          loc.Start(),\n\t\t\t\tEnd:            loc.End(),\n\t\t\t\tPos:            loc.Pos(),\n\t\t\t\tArrayPositions: loc.ArrayPositions(),\n\t\t\t\tField:          loc.Field(),\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (i *IndexSnapshotTermFieldReader) Advance(ID index.IndexInternalID, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) {\n\t// FIXME do something better\n\t// for now, if we need to seek backwards, then restart from the beginning\n\tif i.currPosting != nil && i.currID.Compare(ID) >= 0 {\n\t\t// Check if the TFR is a special unadorned composite optimization.\n\t\t// Such a TFR will NOT have a valid `term` or `field` set, making it\n\t\t// impossible for the TFR to replace itself with a new one.\n\t\tif !i.unadorned {\n\t\t\ti2, err := i.snapshot.TermFieldReader(context.TODO(), i.term, i.field,\n\t\t\t\ti.includeFreq, i.includeNorm, i.includeTermVectors)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\t// close the current term field reader before replacing it with a new one\n\t\t\t_ = i.Close()\n\t\t\t*i = *(i2.(*IndexSnapshotTermFieldReader))\n\t\t} else {\n\t\t\t// unadorned composite optimization\n\t\t\t// we need to reset all the iterators\n\t\t\t// back to the beginning, which effectively\n\t\t\t// achieves the same thing as the above\n\t\t\tfor _, iter := range i.iterators {\n\t\t\t\tif optimizedIterator, ok := iter.(ResetablePostingsIterator); ok {\n\t\t\t\t\toptimizedIterator.ResetIterator()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tnum, err := ID.Value()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error converting to doc number % x - %v\", ID, err)\n\t}\n\tsegIndex, ldocNum := i.snapshot.segmentIndexAndLocalDocNumFromGlobal(num)\n\tif segIndex >= len(i.snapshot.segment) {\n\t\treturn nil, fmt.Errorf(\"computed segment index %d out of bounds %d\",\n\t\t\tsegIndex, len(i.snapshot.segment))\n\t}\n\t// skip directly to the target segment\n\ti.segmentOffset = segIndex\n\tnext, err := i.iterators[i.segmentOffset].Advance(ldocNum)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif next == nil {\n\t\t// we jumped directly to the segment that should have contained it\n\t\t// but it wasn't there, so reuse Next() which should correctly\n\t\t// get the next hit after it (we moved i.segmentOffset)\n\t\treturn i.Next(preAlloced)\n\t}\n\n\tif preAlloced == nil {\n\t\tpreAlloced = &index.TermFieldDoc{}\n\t}\n\tpreAlloced.ID = index.NewIndexInternalID(preAlloced.ID, next.Number()+\n\t\ti.snapshot.offsets[segIndex])\n\ti.postingToTermFieldDoc(next, preAlloced)\n\ti.currID = preAlloced.ID\n\ti.currPosting = next\n\treturn preAlloced, nil\n}\n\nfunc (i *IndexSnapshotTermFieldReader) Count() uint64 {\n\tvar rv uint64\n\tfor _, posting := range i.postings {\n\t\trv += posting.Count()\n\t}\n\treturn rv\n}\n\nfunc (i *IndexSnapshotTermFieldReader) Close() error {\n\tif i.ctx != nil {\n\t\tstatsCallbackFn := i.ctx.Value(search.SearchIOStatsCallbackKey)\n\t\tif statsCallbackFn != nil {\n\t\t\t// essentially before you close the TFR, you must report this\n\t\t\t// reader's bytesRead value\n\t\t\tstatsCallbackFn.(search.SearchIOStatsCallbackFunc)(i.bytesRead)\n\t\t}\n\n\t\tsearch.RecordSearchCost(i.ctx, search.AddM, i.bytesRead)\n\t}\n\n\tif i.snapshot != nil {\n\t\tatomic.AddUint64(&i.snapshot.parent.stats.TotTermSearchersFinished, uint64(1))\n\t\ti.snapshot.recycleTermFieldReader(i)\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "index/scorch/snapshot_index_thes.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"container/heap\"\n\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\ntype segmentThesCursor struct {\n\tthes segment.Thesaurus\n\titr  segment.ThesaurusIterator\n\tcurr index.ThesaurusEntry\n}\n\ntype IndexSnapshotThesaurusKeys struct {\n\tsnapshot *IndexSnapshot\n\tcursors  []*segmentThesCursor\n\tentry    index.ThesaurusEntry\n}\n\nfunc (i *IndexSnapshotThesaurusKeys) Len() int { return len(i.cursors) }\nfunc (i *IndexSnapshotThesaurusKeys) Less(a, b int) bool {\n\treturn i.cursors[a].curr.Term < i.cursors[b].curr.Term\n}\nfunc (i *IndexSnapshotThesaurusKeys) Swap(a, b int) {\n\ti.cursors[a], i.cursors[b] = i.cursors[b], i.cursors[a]\n}\n\nfunc (i *IndexSnapshotThesaurusKeys) Push(x interface{}) {\n\ti.cursors = append(i.cursors, x.(*segmentThesCursor))\n}\n\nfunc (i *IndexSnapshotThesaurusKeys) Pop() interface{} {\n\tn := len(i.cursors)\n\tx := i.cursors[n-1]\n\ti.cursors = i.cursors[0 : n-1]\n\treturn x\n}\n\nfunc (i *IndexSnapshotThesaurusKeys) Next() (*index.ThesaurusEntry, error) {\n\tif len(i.cursors) == 0 {\n\t\treturn nil, nil\n\t}\n\ti.entry = i.cursors[0].curr\n\tnext, err := i.cursors[0].itr.Next()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif next == nil {\n\t\t// at end of this cursor, remove it\n\t\theap.Pop(i)\n\t} else {\n\t\t// modified heap, fix it\n\t\ti.cursors[0].curr = *next\n\t\theap.Fix(i, 0)\n\t}\n\t// look for any other entries with the exact same term\n\tfor len(i.cursors) > 0 && i.cursors[0].curr.Term == i.entry.Term {\n\t\tnext, err := i.cursors[0].itr.Next()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif next == nil {\n\t\t\t// at end of this cursor, remove it\n\t\t\theap.Pop(i)\n\t\t} else {\n\t\t\t// modified heap, fix it\n\t\t\ti.cursors[0].curr = *next\n\t\t\theap.Fix(i, 0)\n\t\t}\n\t}\n\n\treturn &i.entry, nil\n}\n\nfunc (i *IndexSnapshotThesaurusKeys) Close() error {\n\treturn nil\n}\n\nfunc (i *IndexSnapshotThesaurusKeys) Contains(key []byte) (bool, error) {\n\tif len(i.cursors) == 0 {\n\t\treturn false, nil\n\t}\n\n\tfor _, cursor := range i.cursors {\n\t\tif found, _ := cursor.thes.Contains(key); found {\n\t\t\treturn true, nil\n\t\t}\n\t}\n\n\treturn false, nil\n}\n"
  },
  {
    "path": "index/scorch/snapshot_index_vr.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage scorch\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment_api \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\nconst VectorSearchSupportedSegmentVersion = 16\n\nvar reflectStaticSizeIndexSnapshotVectorReader int\n\nfunc init() {\n\tvar istfr IndexSnapshotVectorReader\n\treflectStaticSizeIndexSnapshotVectorReader = int(reflect.TypeOf(istfr).Size())\n}\n\ntype IndexSnapshotVectorReader struct {\n\tvector        []float32\n\tfield         string\n\tk             int64\n\tsnapshot      *IndexSnapshot\n\tpostings      []segment_api.VecPostingsList\n\titerators     []segment_api.VecPostingsIterator\n\tsegmentOffset int\n\tcurrPosting   segment_api.VecPosting\n\tcurrID        index.IndexInternalID\n\tctx           context.Context\n\n\tsearchParams     json.RawMessage\n\teligibleSelector index.EligibleDocumentSelector\n}\n\nfunc (i *IndexSnapshotVectorReader) Size() int {\n\tsizeInBytes := reflectStaticSizeIndexSnapshotVectorReader + size.SizeOfPtr +\n\t\tlen(i.vector)*size.SizeOfFloat32 +\n\t\tlen(i.field) +\n\t\tlen(i.currID)\n\n\tfor _, entry := range i.postings {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tfor _, entry := range i.iterators {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tif i.currPosting != nil {\n\t\tsizeInBytes += i.currPosting.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (i *IndexSnapshotVectorReader) Next(preAlloced *index.VectorDoc) (\n\t*index.VectorDoc, error) {\n\trv := preAlloced\n\tif rv == nil {\n\t\trv = &index.VectorDoc{}\n\t}\n\n\tfor i.segmentOffset < len(i.iterators) {\n\t\tif i.iterators[i.segmentOffset] == nil {\n\t\t\ti.segmentOffset++\n\t\t\tcontinue\n\t\t}\n\t\tnext, err := i.iterators[i.segmentOffset].Next()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif next != nil {\n\t\t\t// make segment number into global number by adding offset\n\t\t\tglobalOffset := i.snapshot.offsets[i.segmentOffset]\n\t\t\tnnum := next.Number()\n\t\t\trv.ID = index.NewIndexInternalID(rv.ID, nnum+globalOffset)\n\t\t\trv.Score = float64(next.Score())\n\n\t\t\ti.currID = rv.ID\n\t\t\ti.currPosting = next\n\n\t\t\treturn rv, nil\n\t\t}\n\t\ti.segmentOffset++\n\t}\n\n\treturn nil, nil\n}\n\nfunc (i *IndexSnapshotVectorReader) Advance(ID index.IndexInternalID,\n\tpreAlloced *index.VectorDoc) (*index.VectorDoc, error) {\n\n\tif i.currPosting != nil && i.currID.Compare(ID) >= 0 {\n\t\ti2, err := i.snapshot.VectorReader(i.ctx, i.vector, i.field, i.k,\n\t\t\ti.searchParams, i.eligibleSelector)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// close the current term field reader before replacing it with a new one\n\t\t_ = i.Close()\n\t\t*i = *(i2.(*IndexSnapshotVectorReader))\n\t}\n\n\tnum, err := ID.Value()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error converting to doc number % x - %v\", ID, err)\n\t}\n\tsegIndex, ldocNum := i.snapshot.segmentIndexAndLocalDocNumFromGlobal(num)\n\tif segIndex >= len(i.snapshot.segment) {\n\t\treturn nil, fmt.Errorf(\"computed segment index %d out of bounds %d\",\n\t\t\tsegIndex, len(i.snapshot.segment))\n\t}\n\t// skip directly to the target segment\n\ti.segmentOffset = segIndex\n\tnext, err := i.iterators[i.segmentOffset].Advance(ldocNum)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif next == nil {\n\t\t// we jumped directly to the segment that should have contained it\n\t\t// but it wasn't there, so reuse Next() which should correctly\n\t\t// get the next hit after it (we moved i.segmentOffset)\n\t\treturn i.Next(preAlloced)\n\t}\n\n\tif preAlloced == nil {\n\t\tpreAlloced = &index.VectorDoc{}\n\t}\n\tpreAlloced.ID = index.NewIndexInternalID(preAlloced.ID, next.Number()+\n\t\ti.snapshot.offsets[segIndex])\n\ti.currID = preAlloced.ID\n\ti.currPosting = next\n\treturn preAlloced, nil\n}\n\nfunc (i *IndexSnapshotVectorReader) Count() uint64 {\n\tvar rv uint64\n\tfor _, posting := range i.postings {\n\t\trv += posting.Count()\n\t}\n\treturn rv\n}\n\nfunc (i *IndexSnapshotVectorReader) Close() error {\n\t// TODO Consider if any scope of recycling here.\n\treturn nil\n}\n\nfunc (i *IndexSnapshot) CentroidCardinalities(field string, limit int, descending bool) (\n\t[]index.CentroidCardinality, error) {\n\tif len(i.segment) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tif limit <= 0 {\n\t\treturn nil, fmt.Errorf(\"limit must be positive\")\n\t}\n\n\tcentroids := make([]index.CentroidCardinality, 0, limit*len(i.segment))\n\n\tfor _, segment := range i.segment {\n\t\tif sv, ok := segment.segment.(segment_api.VectorSegment); ok {\n\t\t\tvecIndex, err := sv.InterpretVectorIndex(field, segment.deleted)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to interpret vector index for field %s in segment: %v\", field, err)\n\t\t\t}\n\n\t\t\tcentroidCardinalities, err := vecIndex.ObtainKCentroidCardinalitiesFromIVFIndex(limit, descending)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to obtain top k centroid cardinalities for field %s in segment: %v\", field, err)\n\t\t\t}\n\n\t\t\tif len(centroidCardinalities) > 0 {\n\t\t\t\tcentroids = append(centroids, centroidCardinalities...)\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(centroids) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tsort.Slice(centroids, func(i, j int) bool {\n\t\tif descending {\n\t\t\treturn centroids[i].Cardinality > centroids[j].Cardinality\n\t\t}\n\t\treturn centroids[i].Cardinality < centroids[j].Cardinality\n\t})\n\n\tif limit >= len(centroids) {\n\t\treturn centroids, nil\n\t}\n\n\treturn centroids[:limit], nil\n}\n"
  },
  {
    "path": "index/scorch/snapshot_segment.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\ntype SegmentSnapshot struct {\n\t// this flag is needed to identify whether this\n\t// segment was mmaped recently, in which case\n\t// we consider the loading cost of the metadata\n\t// as part of IO stats.\n\tmmaped        uint32\n\tid            uint64\n\tsegment       segment.Segment\n\tdeleted       *roaring.Bitmap\n\tcreator       string\n\tstats         *fieldStats\n\tupdatedFields map[string]*index.UpdateFieldInfo\n\n\tcachedMeta *cachedMeta\n\n\tcachedDocs *cachedDocs\n}\n\nfunc (s *SegmentSnapshot) Segment() segment.Segment {\n\treturn s.segment\n}\n\nfunc (s *SegmentSnapshot) Deleted() *roaring.Bitmap {\n\treturn s.deleted\n}\n\nfunc (s *SegmentSnapshot) Id() uint64 {\n\treturn s.id\n}\n\nfunc (s *SegmentSnapshot) FullSize() int64 {\n\treturn int64(s.segment.Count())\n}\n\nfunc (s *SegmentSnapshot) LiveSize() int64 {\n\treturn int64(s.Count())\n}\n\nfunc (s *SegmentSnapshot) HasVector() bool {\n\t// number of vectors, for each vector field in the segment\n\tnumVecs := s.stats.Fetch()[\"num_vectors\"]\n\treturn len(numVecs) > 0\n}\n\nfunc (s *SegmentSnapshot) FileSize() int64 {\n\tps, ok := s.segment.(segment.PersistedSegment)\n\tif !ok {\n\t\treturn 0\n\t}\n\n\tpath := ps.Path()\n\tif path == \"\" {\n\t\treturn 0\n\t}\n\n\tfi, err := os.Stat(path)\n\tif err != nil {\n\t\treturn 0\n\t}\n\n\treturn fi.Size()\n}\n\nfunc (s *SegmentSnapshot) Close() error {\n\treturn s.segment.Close()\n}\n\nfunc (s *SegmentSnapshot) VisitDocument(num uint64, visitor segment.StoredFieldValueVisitor) error {\n\treturn s.segment.VisitStoredFields(num, visitor)\n}\n\nfunc (s *SegmentSnapshot) DocID(num uint64) ([]byte, error) {\n\treturn s.segment.DocID(num)\n}\n\nfunc (s *SegmentSnapshot) Count() uint64 {\n\trv := s.segment.Count()\n\tif s.deleted != nil {\n\t\trv -= s.deleted.GetCardinality()\n\t}\n\treturn rv\n}\n\n// this counts the root documents in the segment this differs from Count() in that\n// Count() counts all live documents including nested children, whereas this method\n// counts only root live documents\nfunc (s *SegmentSnapshot) CountRoot() uint64 {\n\tvar rv uint64\n\tif nsb, ok := s.segment.(segment.NestedSegment); ok {\n\t\trv = nsb.CountRoot(s.deleted)\n\t} else {\n\t\trv = s.Count()\n\t}\n\treturn rv\n}\n\nfunc (s *SegmentSnapshot) DocNumbers(docIDs []string) (*roaring.Bitmap, error) {\n\trv, err := s.segment.DocNumbers(docIDs)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif s.deleted != nil {\n\t\trv.AndNot(s.deleted)\n\t}\n\treturn rv, nil\n}\n\n// DocNumbersLive returns a bitmap containing doc numbers for all live docs\nfunc (s *SegmentSnapshot) DocNumbersLive() *roaring.Bitmap {\n\trv := roaring.NewBitmap()\n\trv.AddRange(0, s.segment.Count())\n\tif s.deleted != nil {\n\t\trv.AndNot(s.deleted)\n\t}\n\treturn rv\n}\n\nfunc (s *SegmentSnapshot) Fields() []string {\n\treturn s.segment.Fields()\n}\n\nfunc (s *SegmentSnapshot) Size() (rv int) {\n\trv = s.segment.Size()\n\tif s.deleted != nil {\n\t\trv += int(s.deleted.GetSizeInBytes())\n\t}\n\trv += s.cachedDocs.Size()\n\treturn\n}\n\n// Merge given updated field information with existing and pass it on to the segment base\nfunc (s *SegmentSnapshot) UpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) {\n\tif s.updatedFields == nil {\n\t\ts.updatedFields = updatedFields\n\t} else {\n\t\tfor fieldName, info := range updatedFields {\n\t\t\tif val, ok := s.updatedFields[fieldName]; ok {\n\t\t\t\tval.Deleted = val.Deleted || info.Deleted\n\t\t\t\tval.Index = val.Index || info.Index\n\t\t\t\tval.DocValues = val.DocValues || info.DocValues\n\t\t\t\tval.Store = val.Store || info.Store\n\t\t\t} else {\n\t\t\t\ts.updatedFields[fieldName] = info\n\t\t\t}\n\t\t}\n\t}\n\n\tif segment, ok := s.segment.(segment.UpdatableSegment); ok {\n\t\tsegment.SetUpdatedFields(s.updatedFields)\n\t}\n}\n\ntype cachedFieldDocs struct {\n\tm       sync.Mutex\n\treadyCh chan struct{}     // closed when the cachedFieldDocs.docs is ready to be used.\n\terr     error             // Non-nil if there was an error when preparing this cachedFieldDocs.\n\tdocs    map[uint64][]byte // Keyed by localDocNum, value is a list of terms delimited by 0xFF.\n\tsize    uint64\n}\n\nfunc (cfd *cachedFieldDocs) Size() int {\n\tvar rv int\n\tcfd.m.Lock()\n\tfor _, entry := range cfd.docs {\n\t\trv += 8 /* size of uint64 */ + len(entry)\n\t}\n\tcfd.m.Unlock()\n\treturn rv\n}\n\nfunc (cfd *cachedFieldDocs) prepareField(field string, ss *SegmentSnapshot) {\n\tcfd.m.Lock()\n\tdefer func() {\n\t\tclose(cfd.readyCh)\n\t\tcfd.m.Unlock()\n\t}()\n\n\tcfd.size += uint64(size.SizeOfUint64) /* size field */\n\tdict, err := ss.segment.Dictionary(field)\n\tif err != nil {\n\t\tcfd.err = err\n\t\treturn\n\t}\n\n\tvar postings segment.PostingsList\n\tvar postingsItr segment.PostingsIterator\n\n\tdictItr := dict.AutomatonIterator(nil, nil, nil)\n\tnext, err := dictItr.Next()\n\tfor err == nil && next != nil {\n\t\tvar err1 error\n\t\tpostings, err1 = dict.PostingsList([]byte(next.Term), nil, postings)\n\t\tif err1 != nil {\n\t\t\tcfd.err = err1\n\t\t\treturn\n\t\t}\n\n\t\tcfd.size += uint64(size.SizeOfUint64) /* map key */\n\t\tpostingsItr = postings.Iterator(false, false, false, postingsItr)\n\t\tnextPosting, err2 := postingsItr.Next()\n\t\tfor err2 == nil && nextPosting != nil {\n\t\t\tdocNum := nextPosting.Number()\n\t\t\tcfd.docs[docNum] = append(cfd.docs[docNum], []byte(next.Term)...)\n\t\t\tcfd.docs[docNum] = append(cfd.docs[docNum], index.DocValueTermSeparator)\n\t\t\tcfd.size += uint64(len(next.Term) + 1) // map value\n\t\t\tnextPosting, err2 = postingsItr.Next()\n\t\t}\n\n\t\tif err2 != nil {\n\t\t\tcfd.err = err2\n\t\t\treturn\n\t\t}\n\n\t\tnext, err = dictItr.Next()\n\t}\n\n\tif err != nil {\n\t\tcfd.err = err\n\t\treturn\n\t}\n}\n\ntype cachedDocs struct {\n\tsize  uint64\n\tm     sync.RWMutex                // As the cache is asynchronously prepared, need a lock\n\tcache map[string]*cachedFieldDocs // Keyed by field\n}\n\nfunc (c *cachedDocs) prepareFields(wantedFields []string, ss *SegmentSnapshot) error {\n\tc.m.Lock()\n\n\tif c.cache == nil {\n\t\tc.cache = make(map[string]*cachedFieldDocs, len(ss.Fields()))\n\t}\n\n\tfor _, field := range wantedFields {\n\t\t_, exists := c.cache[field]\n\t\tif !exists {\n\t\t\tc.cache[field] = &cachedFieldDocs{\n\t\t\t\treadyCh: make(chan struct{}),\n\t\t\t\tdocs:    make(map[uint64][]byte),\n\t\t\t}\n\n\t\t\tgo c.cache[field].prepareField(field, ss)\n\t\t}\n\t}\n\n\tfor _, field := range wantedFields {\n\t\tcachedFieldDocs := c.cache[field]\n\t\tc.m.Unlock()\n\t\t<-cachedFieldDocs.readyCh\n\n\t\tif cachedFieldDocs.err != nil {\n\t\t\treturn cachedFieldDocs.err\n\t\t}\n\t\tc.m.Lock()\n\t}\n\n\tc.updateSizeLOCKED()\n\n\tc.m.Unlock()\n\treturn nil\n}\n\n// hasFields returns true if the cache has all the given fields\nfunc (c *cachedDocs) hasFields(fields []string) bool {\n\tc.m.RLock()\n\tfor _, field := range fields {\n\t\tif _, exists := c.cache[field]; !exists {\n\t\t\tc.m.RUnlock()\n\t\t\treturn false // found a field not in cache\n\t\t}\n\t}\n\tc.m.RUnlock()\n\treturn true\n}\n\nfunc (c *cachedDocs) Size() int {\n\treturn int(atomic.LoadUint64(&c.size))\n}\n\nfunc (c *cachedDocs) updateSizeLOCKED() {\n\tsizeInBytes := 0\n\tfor k, v := range c.cache { // cachedFieldDocs\n\t\tsizeInBytes += len(k)\n\t\tif v != nil {\n\t\t\tsizeInBytes += v.Size()\n\t\t}\n\t}\n\tatomic.StoreUint64(&c.size, uint64(sizeInBytes))\n}\n\nfunc (c *cachedDocs) visitDoc(localDocNum uint64,\n\tfields []string, visitor index.DocValueVisitor) {\n\tc.m.RLock()\n\n\tfor _, field := range fields {\n\t\tif cachedFieldDocs, exists := c.cache[field]; exists {\n\t\t\tc.m.RUnlock()\n\t\t\t<-cachedFieldDocs.readyCh\n\t\t\tc.m.RLock()\n\n\t\t\tif tlist, exists := cachedFieldDocs.docs[localDocNum]; exists {\n\t\t\t\tfor {\n\t\t\t\t\ti := bytes.IndexByte(tlist, index.DocValueTermSeparator)\n\t\t\t\t\tif i < 0 {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tvisitor(field, tlist[0:i])\n\t\t\t\t\ttlist = tlist[i+1:]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tc.m.RUnlock()\n}\n\n// the purpose of the cachedMeta is to simply allow the user of this type to record\n// and cache certain meta data information (specific to the segment) that can be\n// used across calls to save compute on the same.\n// for example searcher creations on the same index snapshot can use this struct\n// to help and fetch the backing index size information which can be used in\n// memory usage calculation thereby deciding whether to allow a query or not.\ntype cachedMeta struct {\n\tm    sync.RWMutex\n\tmeta map[string]interface{}\n}\n\nfunc (c *cachedMeta) updateMeta(field string, val interface{}) {\n\tc.m.Lock()\n\tif c.meta == nil {\n\t\tc.meta = make(map[string]interface{})\n\t}\n\tc.meta[field] = val\n\tc.m.Unlock()\n}\n\nfunc (c *cachedMeta) fetchMeta(field string) (rv interface{}) {\n\tc.m.RLock()\n\trv = c.meta[field]\n\tc.m.RUnlock()\n\treturn rv\n}\n\nfunc (s *SegmentSnapshot) Ancestors(docNum uint64, prealloc []index.AncestorID) []index.AncestorID {\n\tnsb, ok := s.segment.(segment.NestedSegment)\n\tif !ok {\n\t\treturn append(prealloc, index.NewAncestorID(docNum))\n\t}\n\treturn nsb.Ancestors(docNum, prealloc)\n}\n"
  },
  {
    "path": "index/scorch/snapshot_vector_index.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage scorch\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/bits-and-blooms/bitset\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tsegment_api \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\nfunc (is *IndexSnapshot) VectorReader(ctx context.Context, vector []float32,\n\tfield string, k int64, searchParams json.RawMessage,\n\teligibleSelector index.EligibleDocumentSelector) (\n\tindex.VectorReader, error) {\n\trv := &IndexSnapshotVectorReader{\n\t\tvector:           vector,\n\t\tfield:            field,\n\t\tk:                k,\n\t\tsnapshot:         is,\n\t\tsearchParams:     searchParams,\n\t\teligibleSelector: eligibleSelector,\n\t\tpostings:         make([]segment_api.VecPostingsList, len(is.segment)),\n\t\titerators:        make([]segment_api.VecPostingsIterator, len(is.segment)),\n\t}\n\n\t// initialize postings and iterators within the OptimizeVR's Finish()\n\treturn rv, nil\n}\n\n// eligibleDocumentList represents the list of eligible documents within a segment.\ntype eligibleDocumentList struct {\n\tbs *bitset.BitSet\n}\n\n// Iterator returns an iterator for the eligible document IDs.\nfunc (edl *eligibleDocumentList) Iterator() index.EligibleDocumentIterator {\n\tif edl.bs == nil {\n\t\t// no eligible documents\n\t\treturn emptyEligibleIterator\n\t}\n\t// return the iterator\n\treturn &eligibleDocumentIterator{\n\t\tbs: edl.bs,\n\t}\n}\n\n// Count returns the number of eligible document IDs.\nfunc (edl *eligibleDocumentList) Count() uint64 {\n\tif edl.bs == nil {\n\t\treturn 0\n\t}\n\treturn uint64(edl.bs.Count())\n}\n\n// emptyEligibleDocumentList is a reusable empty eligible document list.\nvar emptyEligibleDocumentList = &eligibleDocumentList{}\n\n// eligibleDocumentIterator iterates over eligible document IDs within a segment.\ntype eligibleDocumentIterator struct {\n\tbs      *bitset.BitSet\n\tcurrent uint\n}\n\n// Next returns the next eligible document ID and whether it exists.\nfunc (it *eligibleDocumentIterator) Next() (id uint64, ok bool) {\n\tnext, found := it.bs.NextSet(it.current)\n\tif !found {\n\t\treturn 0, false\n\t}\n\tit.current = next + 1\n\treturn uint64(next), true\n}\n\n// emptyEligibleIterator is a reusable empty eligible document iterator.\nvar emptyEligibleIterator = &emptyEligibleDocumentIterator{}\n\n// emptyEligibleDocumentIterator is an iterator that always returns no documents.\ntype emptyEligibleDocumentIterator struct{}\n\n// Next always returns false for empty iterator.\nfunc (it *emptyEligibleDocumentIterator) Next() (id uint64, ok bool) {\n\treturn 0, false\n}\n\n// eligibleDocumentSelector is used to filter out documents that are eligible for\n// the KNN search from a pre-filter query.\ntype eligibleDocumentSelector struct {\n\t// segment ID -> segment local doc nums in a bitset\n\teligibleDocNums []*bitset.BitSet\n\tis              *IndexSnapshot\n}\n\n// SegmentEligibleDocuments returns an EligibleDocumentList for the specified segment ID.\nfunc (eds *eligibleDocumentSelector) SegmentEligibleDocuments(segmentID int) index.EligibleDocumentList {\n\tif eds.eligibleDocNums == nil || segmentID < 0 || segmentID >= len(eds.eligibleDocNums) {\n\t\treturn emptyEligibleDocumentList\n\t}\n\tbs := eds.eligibleDocNums[segmentID]\n\tif bs == nil {\n\t\t// no eligible documents for this segment\n\t\treturn emptyEligibleDocumentList\n\t}\n\treturn &eligibleDocumentList{\n\t\tbs: bs,\n\t}\n}\n\n// AddEligibleDocumentMatch adds a document match to the list of eligible documents.\nfunc (eds *eligibleDocumentSelector) AddEligibleDocumentMatch(id index.IndexInternalID) error {\n\tif eds.is == nil {\n\t\treturn fmt.Errorf(\"eligibleDocumentSelector is not initialized with IndexSnapshot\")\n\t}\n\t// Get the segment number and the local doc number for this document.\n\tsegIdx, docNum, err := eds.is.segmentIndexAndLocalDocNum(id)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// allocate a bitset for this segment if needed\n\tif eds.eligibleDocNums[segIdx] == nil {\n\t\t// the size of the bitset is the full size of the segment (which is the max local doc num + 1)\n\t\teds.eligibleDocNums[segIdx] = bitset.New(uint(eds.is.segment[segIdx].FullSize()))\n\t}\n\t// Add the local doc number to the list of eligible doc numbers for this segment.\n\teds.eligibleDocNums[segIdx].Set(uint(docNum))\n\treturn nil\n}\n\nfunc (is *IndexSnapshot) NewEligibleDocumentSelector() index.EligibleDocumentSelector {\n\treturn &eligibleDocumentSelector{\n\t\teligibleDocNums: make([]*bitset.BitSet, len(is.segment)),\n\t\tis:              is,\n\t}\n}\n"
  },
  {
    "path": "index/scorch/stats.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"reflect\"\n\t\"sync/atomic\"\n\n\t\"github.com/blevesearch/bleve/v2/util\"\n)\n\n// Stats tracks statistics about the index, fields that are\n// prefixed like CurXxxx are gauges (can go up and down),\n// and fields that are prefixed like TotXxxx are monotonically\n// increasing counters.\ntype Stats struct {\n\tTotUpdates uint64\n\tTotDeletes uint64\n\n\tTotBatches        uint64\n\tTotBatchesEmpty   uint64\n\tTotBatchIntroTime uint64\n\tMaxBatchIntroTime uint64\n\n\tCurRootEpoch       uint64\n\tLastPersistedEpoch uint64\n\tLastMergedEpoch    uint64\n\n\tTotOnErrors uint64\n\n\tTotAnalysisTime uint64\n\tTotIndexTime    uint64\n\n\tTotIndexedPlainTextBytes uint64\n\n\tTotBytesReadAtQueryTime    uint64\n\tTotBytesWrittenAtIndexTime uint64\n\n\tTotTermSearchersStarted  uint64\n\tTotTermSearchersFinished uint64\n\n\tTotKNNSearches     uint64\n\tTotSynonymSearches uint64\n\n\tTotEventTriggerStarted   uint64\n\tTotEventTriggerCompleted uint64\n\n\tTotIntroduceLoop       uint64\n\tTotIntroduceSegmentBeg uint64\n\tTotIntroduceSegmentEnd uint64\n\tTotIntroducePersistBeg uint64\n\tTotIntroducePersistEnd uint64\n\tTotIntroduceMergeBeg   uint64\n\tTotIntroduceMergeEnd   uint64\n\tTotIntroduceRevertBeg  uint64\n\tTotIntroduceRevertEnd  uint64\n\n\tTotIntroducedItems         uint64\n\tTotIntroducedSegmentsBatch uint64\n\tTotIntroducedSegmentsMerge uint64\n\n\tTotPersistLoopBeg          uint64\n\tTotPersistLoopErr          uint64\n\tTotPersistLoopProgress     uint64\n\tTotPersistLoopWait         uint64\n\tTotPersistLoopWaitNotified uint64\n\tTotPersistLoopEnd          uint64\n\n\tTotPersistedItems    uint64\n\tTotItemsToPersist    uint64\n\tTotPersistedSegments uint64\n\tTotMutationsFiltered uint64\n\n\tTotPersisterSlowMergerPause  uint64\n\tTotPersisterSlowMergerResume uint64\n\n\tTotPersisterNapPauseCompleted uint64\n\tTotPersisterMergerNapBreak    uint64\n\n\tTotFileMergeLoopBeg uint64\n\tTotFileMergeLoopErr uint64\n\tTotFileMergeLoopEnd uint64\n\n\tTotFileMergeForceOpsStarted   uint64\n\tTotFileMergeForceOpsCompleted uint64\n\n\tTotFileMergePlan     uint64\n\tTotFileMergePlanErr  uint64\n\tTotFileMergePlanNone uint64\n\tTotFileMergePlanOk   uint64\n\n\tTotFileMergePlanTasks              uint64\n\tTotFileMergePlanTasksDone          uint64\n\tTotFileMergePlanTasksErr           uint64\n\tTotFileMergePlanTasksSegments      uint64\n\tTotFileMergePlanTasksSegmentsEmpty uint64\n\n\tTotFileMergeSegmentsEmpty uint64\n\tTotFileMergeSegments      uint64\n\tTotFileSegmentsAtRoot     uint64\n\tTotFileMergeWrittenBytes  uint64\n\n\tTotFileMergeZapBeg              uint64\n\tTotFileMergeZapEnd              uint64\n\tTotFileMergeZapTime             uint64\n\tMaxFileMergeZapTime             uint64\n\tTotFileMergeZapIntroductionTime uint64\n\tMaxFileMergeZapIntroductionTime uint64\n\n\tTotFileMergeIntroductions          uint64\n\tTotFileMergeIntroductionsDone      uint64\n\tTotFileMergeIntroductionsSkipped   uint64\n\tTotFileMergeIntroductionsObsoleted uint64\n\n\tCurFilesIneligibleForRemoval     uint64\n\tTotSnapshotsRemovedFromMetaStore uint64\n\n\tTotMemMergeBeg          uint64\n\tTotMemMergeErr          uint64\n\tTotMemMergeDone         uint64\n\tTotMemMergeZapBeg       uint64\n\tTotMemMergeZapEnd       uint64\n\tTotMemMergeZapTime      uint64\n\tMaxMemMergeZapTime      uint64\n\tTotMemMergeSegments     uint64\n\tTotMemorySegmentsAtRoot uint64\n}\n\n// atomically populates the returned map\nfunc (s *Stats) ToMap() map[string]interface{} {\n\tm := map[string]interface{}{}\n\tsve := reflect.ValueOf(s).Elem()\n\tsvet := sve.Type()\n\tfor i := 0; i < svet.NumField(); i++ {\n\t\tsvef := sve.Field(i)\n\t\tif svef.CanAddr() {\n\t\t\tsvefp := svef.Addr().Interface()\n\t\t\tm[svet.Field(i).Name] = atomic.LoadUint64(svefp.(*uint64))\n\t\t}\n\t}\n\treturn m\n}\n\n// MarshalJSON implements json.Marshaler, and in contrast to standard\n// json marshaling provides atomic safety\nfunc (s *Stats) MarshalJSON() ([]byte, error) {\n\treturn util.MarshalJSON(s.ToMap())\n}\n"
  },
  {
    "path": "index/scorch/unadorned.go",
    "content": "//  Copyright (c) 2020 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorch\n\nimport (\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/RoaringBitmap/roaring/v2\"\n\tsegment \"github.com/blevesearch/scorch_segment_api/v2\"\n)\n\nvar reflectStaticSizeUnadornedPostingsIteratorBitmap int\nvar reflectStaticSizeUnadornedPostingsIterator1Hit int\nvar reflectStaticSizeUnadornedPosting int\n\nfunc init() {\n\tvar pib unadornedPostingsIteratorBitmap\n\treflectStaticSizeUnadornedPostingsIteratorBitmap = int(reflect.TypeOf(pib).Size())\n\tvar pi1h unadornedPostingsIterator1Hit\n\treflectStaticSizeUnadornedPostingsIterator1Hit = int(reflect.TypeOf(pi1h).Size())\n\tvar up UnadornedPosting\n\treflectStaticSizeUnadornedPosting = int(reflect.TypeOf(up).Size())\n}\n\ntype unadornedPostingsIteratorBitmap struct {\n\tactual   roaring.IntPeekable\n\tactualBM *roaring.Bitmap\n\tnext     UnadornedPosting // reused across Next() calls\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) Next() (segment.Posting, error) {\n\treturn i.nextAtOrAfter(0)\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) Advance(docNum uint64) (segment.Posting, error) {\n\treturn i.nextAtOrAfter(docNum)\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) {\n\tdocNum, exists := i.nextDocNumAtOrAfter(atOrAfter)\n\tif !exists {\n\t\treturn nil, nil\n\t}\n\ti.next = UnadornedPosting{} // clear the struct\n\trv := &i.next\n\trv.docNum = docNum\n\treturn rv, nil\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) {\n\tif i.actual == nil || !i.actual.HasNext() {\n\t\treturn 0, false\n\t}\n\ti.actual.AdvanceIfNeeded(uint32(atOrAfter))\n\n\tif !i.actual.HasNext() {\n\t\treturn 0, false // couldn't find anything\n\t}\n\n\treturn uint64(i.actual.Next()), true\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) Size() int {\n\treturn reflectStaticSizeUnadornedPostingsIteratorBitmap\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) BytesRead() uint64 {\n\treturn 0\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) BytesWritten() uint64 {\n\treturn 0\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) ResetBytesRead(uint64) {}\n\nfunc (i *unadornedPostingsIteratorBitmap) ActualBitmap() *roaring.Bitmap {\n\treturn i.actualBM\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) DocNum1Hit() (uint64, bool) {\n\treturn 0, false\n}\n\nfunc (i *unadornedPostingsIteratorBitmap) ReplaceActual(actual *roaring.Bitmap) {\n\ti.actualBM = actual\n\ti.actual = actual.Iterator()\n}\n\n// Resets the iterator to the beginning of the postings list.\n// by resetting the actual iterator.\nfunc (i *unadornedPostingsIteratorBitmap) ResetIterator() {\n\ti.actual = i.actualBM.Iterator()\n}\n\nfunc newUnadornedPostingsIteratorFromBitmap(bm *roaring.Bitmap) segment.PostingsIterator {\n\treturn &unadornedPostingsIteratorBitmap{\n\t\tactualBM: bm,\n\t\tactual:   bm.Iterator(),\n\t}\n}\n\nconst docNum1HitFinished = math.MaxUint64\n\ntype unadornedPostingsIterator1Hit struct {\n\tdocNumOrig uint64           // original 1-hit docNum used to create this iterator\n\tdocNum     uint64           // current docNum\n\tnext       UnadornedPosting // reused across Next() calls\n}\n\nfunc (i *unadornedPostingsIterator1Hit) Next() (segment.Posting, error) {\n\treturn i.nextAtOrAfter(0)\n}\n\nfunc (i *unadornedPostingsIterator1Hit) Advance(docNum uint64) (segment.Posting, error) {\n\treturn i.nextAtOrAfter(docNum)\n}\n\nfunc (i *unadornedPostingsIterator1Hit) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) {\n\tdocNum, exists := i.nextDocNumAtOrAfter(atOrAfter)\n\tif !exists {\n\t\treturn nil, nil\n\t}\n\ti.next = UnadornedPosting{} // clear the struct\n\trv := &i.next\n\trv.docNum = docNum\n\treturn rv, nil\n}\n\nfunc (i *unadornedPostingsIterator1Hit) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) {\n\tif i.docNum == docNum1HitFinished {\n\t\treturn 0, false\n\t}\n\tif i.docNum < atOrAfter {\n\t\t// advanced past our 1-hit\n\t\ti.docNum = docNum1HitFinished // consume our 1-hit docNum\n\t\treturn 0, false\n\t}\n\tdocNum := i.docNum\n\ti.docNum = docNum1HitFinished // consume our 1-hit docNum\n\treturn docNum, true\n}\n\nfunc (i *unadornedPostingsIterator1Hit) Size() int {\n\treturn reflectStaticSizeUnadornedPostingsIterator1Hit\n}\n\nfunc (i *unadornedPostingsIterator1Hit) BytesRead() uint64 {\n\treturn 0\n}\n\nfunc (i *unadornedPostingsIterator1Hit) BytesWritten() uint64 {\n\treturn 0\n}\n\nfunc (i *unadornedPostingsIterator1Hit) ResetBytesRead(uint64) {}\n\n// ResetIterator resets the iterator to the original state.\nfunc (i *unadornedPostingsIterator1Hit) ResetIterator() {\n\ti.docNum = i.docNumOrig\n}\n\nfunc newUnadornedPostingsIteratorFrom1Hit(docNum1Hit uint64) segment.PostingsIterator {\n\treturn &unadornedPostingsIterator1Hit{\n\t\tdocNumOrig: docNum1Hit,\n\t\tdocNum:     docNum1Hit,\n\t}\n}\n\ntype ResetablePostingsIterator interface {\n\tResetIterator()\n}\n\ntype UnadornedPosting struct {\n\tdocNum uint64\n}\n\nfunc (p *UnadornedPosting) Number() uint64 {\n\treturn p.docNum\n}\n\nfunc (p *UnadornedPosting) Frequency() uint64 {\n\treturn 0\n}\n\nfunc (p *UnadornedPosting) Norm() float64 {\n\treturn 0\n}\n\nfunc (p *UnadornedPosting) Locations() []segment.Location {\n\treturn nil\n}\n\nfunc (p *UnadornedPosting) Size() int {\n\treturn reflectStaticSizeUnadornedPosting\n}\n"
  },
  {
    "path": "index/upsidedown/analysis.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype IndexRow interface {\n\tKeySize() int\n\tKeyTo([]byte) (int, error)\n\tKey() []byte\n\n\tValueSize() int\n\tValueTo([]byte) (int, error)\n\tValue() []byte\n}\n\ntype AnalysisResult struct {\n\tDocID string\n\tRows  []IndexRow\n}\n\nfunc (udc *UpsideDownCouch) Analyze(d index.Document) *AnalysisResult {\n\treturn udc.analyze(d)\n}\n\nfunc (udc *UpsideDownCouch) analyze(d index.Document) *AnalysisResult {\n\trv := &AnalysisResult{\n\t\tDocID: d.ID(),\n\t\tRows:  make([]IndexRow, 0, 100),\n\t}\n\n\tdocIDBytes := []byte(d.ID())\n\n\t// track our back index entries\n\tbackIndexStoredEntries := make([]*BackIndexStoreEntry, 0)\n\n\t// information we collate as we merge fields with same name\n\tfieldTermFreqs := make(map[uint16]index.TokenFrequencies)\n\tfieldLengths := make(map[uint16]int)\n\tfieldIncludeTermVectors := make(map[uint16]bool)\n\tfieldNames := make(map[uint16]string)\n\n\tanalyzeField := func(field index.Field, storable bool) {\n\t\tfieldIndex, newFieldRow := udc.fieldIndexOrNewRow(field.Name())\n\t\tif newFieldRow != nil {\n\t\t\trv.Rows = append(rv.Rows, newFieldRow)\n\t\t}\n\t\tfieldNames[fieldIndex] = field.Name()\n\n\t\tif field.Options().IsIndexed() {\n\t\t\tfield.Analyze()\n\t\t\tfieldLength := field.AnalyzedLength()\n\t\t\ttokenFreqs := field.AnalyzedTokenFrequencies()\n\t\t\texistingFreqs := fieldTermFreqs[fieldIndex]\n\t\t\tif existingFreqs == nil {\n\t\t\t\tfieldTermFreqs[fieldIndex] = tokenFreqs\n\t\t\t} else {\n\t\t\t\texistingFreqs.MergeAll(field.Name(), tokenFreqs)\n\t\t\t\tfieldTermFreqs[fieldIndex] = existingFreqs\n\t\t\t}\n\t\t\tfieldLengths[fieldIndex] += fieldLength\n\t\t\tfieldIncludeTermVectors[fieldIndex] = field.Options().IncludeTermVectors()\n\t\t}\n\n\t\tif storable && field.Options().IsStored() {\n\t\t\trv.Rows, backIndexStoredEntries = udc.storeField(docIDBytes, field, fieldIndex, rv.Rows, backIndexStoredEntries)\n\t\t}\n\t}\n\n\t// walk all the fields, record stored fields now\n\t// place information about indexed fields into map\n\t// this collates information across fields with\n\t// same names (arrays)\n\td.VisitFields(func(field index.Field) {\n\t\tanalyzeField(field, true)\n\t})\n\n\tif d.HasComposite() {\n\t\tfor fieldIndex, tokenFreqs := range fieldTermFreqs {\n\t\t\t// see if any of the composite fields need this\n\t\t\td.VisitComposite(func(field index.CompositeField) {\n\t\t\t\tfield.Compose(fieldNames[fieldIndex], fieldLengths[fieldIndex], tokenFreqs)\n\t\t\t})\n\t\t}\n\n\t\td.VisitComposite(func(field index.CompositeField) {\n\t\t\tanalyzeField(field, false)\n\t\t})\n\t}\n\n\trowsCapNeeded := len(rv.Rows) + 1\n\tfor _, tokenFreqs := range fieldTermFreqs {\n\t\trowsCapNeeded += len(tokenFreqs)\n\t}\n\n\trv.Rows = append(make([]IndexRow, 0, rowsCapNeeded), rv.Rows...)\n\n\tbackIndexTermsEntries := make([]*BackIndexTermsEntry, 0, len(fieldTermFreqs))\n\n\t// walk through the collated information and process\n\t// once for each indexed field (unique name)\n\tfor fieldIndex, tokenFreqs := range fieldTermFreqs {\n\t\tfieldLength := fieldLengths[fieldIndex]\n\t\tincludeTermVectors := fieldIncludeTermVectors[fieldIndex]\n\n\t\t// encode this field\n\t\trv.Rows, backIndexTermsEntries = udc.indexField(docIDBytes, includeTermVectors, fieldIndex, fieldLength, tokenFreqs, rv.Rows, backIndexTermsEntries)\n\t}\n\n\t// build the back index row\n\tbackIndexRow := NewBackIndexRow(docIDBytes, backIndexTermsEntries, backIndexStoredEntries)\n\trv.Rows = append(rv.Rows, backIndexRow)\n\n\treturn rv\n}\n"
  },
  {
    "path": "index/upsidedown/analysis_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/null\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestAnalysisBug328(t *testing.T) {\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(standard.Name)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\td := document.NewDocument(\"1\")\n\tf := document.NewTextFieldCustom(\"title\", nil, []byte(\"bleve\"), index.IndexField|index.IncludeTermVectors, analyzer)\n\td.AddField(f)\n\tf = document.NewTextFieldCustom(\"body\", nil, []byte(\"bleve\"), index.IndexField|index.IncludeTermVectors, analyzer)\n\td.AddField(f)\n\tcf := document.NewCompositeFieldWithIndexingOptions(\"_all\", true, []string{}, []string{}, index.IndexField|index.IncludeTermVectors)\n\td.AddField(cf)\n\n\trv := idx.(*UpsideDownCouch).analyze(d)\n\tfieldIndexes := make(map[uint16]string)\n\tfor _, row := range rv.Rows {\n\t\tif row, ok := row.(*FieldRow); ok {\n\t\t\tfieldIndexes[row.index] = row.name\n\t\t}\n\t\tif row, ok := row.(*TermFrequencyRow); ok && string(row.term) == \"bleve\" {\n\t\t\tfor _, vec := range row.vectors {\n\t\t\t\tif vec.field != row.field {\n\t\t\t\t\tif fieldIndexes[row.field] != \"_all\" {\n\t\t\t\t\t\tt.Errorf(\"row named %s field %d - vector field %d\", fieldIndexes[row.field], row.field, vec.field)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc BenchmarkAnalyze(b *testing.B) {\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(standard.Name)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\td := document.NewDocument(\"1\")\n\tf := document.NewTextFieldWithAnalyzer(\"desc\", nil, bleveWikiArticle1K, analyzer)\n\td.AddField(f)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trv := idx.(*UpsideDownCouch).analyze(d)\n\t\tif len(rv.Rows) < 92 || len(rv.Rows) > 93 {\n\t\t\tb.Fatalf(\"expected 512-13 rows, got %d\", len(rv.Rows))\n\t\t}\n\t}\n}\n\nvar bleveWikiArticle1K = []byte(`Boiling liquid expanding vapor explosion\nFrom Wikipedia, the free encyclopedia\nSee also: Boiler explosion and Steam explosion\n\nFlames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire.\n\nThis article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013)\nA boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1]\nContents  [hide]\n1 Mechanism\n1.1 Water example\n1.2 BLEVEs without chemical reactions\n2 Fires\n3 Incidents\n4 Safety measures\n5 See also\n6 References\n7 External links\nMechanism[edit]\n\nThis section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013)\nThere are three characteristics of liquids which are relevant to the discussion of a BLEVE:`)\n"
  },
  {
    "path": "index/upsidedown/benchmark_all.sh",
    "content": "#!/bin/sh\n\nBENCHMARKS=`grep \"func Benchmark\" *_test.go  | sed 's/.*func //' | sed s/\\(.*{//`\n\nfor BENCHMARK in $BENCHMARKS\ndo\n\tgo test -v -run=xxx -bench=^$BENCHMARK$ -benchtime=10s -tags 'forestdb leveldb'  | grep -v ok | grep -v PASS\ndone\n"
  },
  {
    "path": "index/upsidedown/benchmark_boltdb_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n)\n\nvar boltTestConfig = map[string]interface{}{\n\t\"path\": \"test\",\n}\n\nfunc BenchmarkBoltDBIndexing1Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 1)\n}\n\nfunc BenchmarkBoltDBIndexing2Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 2)\n}\n\nfunc BenchmarkBoltDBIndexing4Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 4)\n}\n\n// batches\n\nfunc BenchmarkBoltDBIndexing1Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 10)\n}\n\nfunc BenchmarkBoltDBIndexing2Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 10)\n}\n\nfunc BenchmarkBoltDBIndexing4Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 10)\n}\n\nfunc BenchmarkBoltDBIndexing1Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 100)\n}\n\nfunc BenchmarkBoltDBIndexing2Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 100)\n}\n\nfunc BenchmarkBoltDBIndexing4Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 100)\n}\n\nfunc BenchmarkBoltBIndexing1Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 1000)\n}\n\nfunc BenchmarkBoltBIndexing2Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 1000)\n}\n\nfunc BenchmarkBoltBIndexing4Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 1000)\n}\n"
  },
  {
    "path": "index/upsidedown/benchmark_common_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\n\t_ \"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar benchmarkDocBodies = []string{\n\t\"A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.\",\n\t\"A boiler explosion is a catastrophic failure of a boiler. As seen today, boiler explosions are of two kinds. One kind is a failure of the pressure parts of the steam and water sides. There can be many different causes, such as failure of the safety valve, corrosion of critical parts of the boiler, or low water level. Corrosion along the edges of lap joints was a common cause of early boiler explosions.\",\n\t\"A boiler is a closed vessel in which water or other fluid is heated. The fluid does not necessarily boil. (In North America the term \\\"furnace\\\" is normally used if the purpose is not actually to boil the fluid.) The heated or vaporized fluid exits the boiler for use in various processes or heating applications,[1][2] including central heating, boiler-based power generation, cooking, and sanitation.\",\n\t\"A pressure vessel is a closed container designed to hold gases or liquids at a pressure substantially different from the ambient pressure.\",\n\t\"Pressure (symbol: p or P) is the ratio of force to the area over which that force is distributed.\",\n\t\"Liquid is one of the four fundamental states of matter (the others being solid, gas, and plasma), and is the only state with a definite volume but no fixed shape.\",\n\t\"The boiling point of a substance is the temperature at which the vapor pressure of the liquid equals the pressure surrounding the liquid[1][2] and the liquid changes into a vapor.\",\n\t\"Vapor pressure or equilibrium vapor pressure is defined as the pressure exerted by a vapor in thermodynamic equilibrium with its condensed phases (solid or liquid) at a given temperature in a closed system.\",\n\t\"Industrial gases are a group of gases that are specifically manufactured for use in a wide range of industries, which include oil and gas, petrochemicals, chemicals, power, mining, steelmaking, metals, environmental protection, medicine, pharmaceuticals, biotechnology, food, water, fertilizers, nuclear power, electronics and aerospace.\",\n\t\"The expansion ratio of a liquefied and cryogenic substance is the volume of a given amount of that substance in liquid form compared to the volume of the same amount of substance in gaseous form, at room temperature and normal atmospheric pressure.\",\n}\n\ntype KVStoreDestroy func() error\n\nfunc DestroyTest() error {\n\treturn os.RemoveAll(\"test\")\n}\n\nfunc CommonBenchmarkIndex(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers int) {\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(\"standard\")\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tindexDocument := document.NewDocument(\"\").\n\t\tAddField(document.NewTextFieldWithAnalyzer(\"body\", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer))\n\n\tb.ResetTimer()\n\tb.StopTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tanalysisQueue := index.NewAnalysisQueue(analysisWorkers)\n\t\tidx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\n\t\terr = idx.Open()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tindexDocument.SetID(strconv.Itoa(i))\n\t\t// just time the indexing portion\n\t\tb.StartTimer()\n\t\terr = idx.Update(indexDocument)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tb.StopTimer()\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\terr = destroy()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tanalysisQueue.Close()\n\t}\n}\n\nfunc CommonBenchmarkIndexBatch(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers, batchSize int) {\n\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(\"standard\")\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tb.ResetTimer()\n\tb.StopTimer()\n\tfor i := 0; i < b.N; i++ {\n\n\t\tanalysisQueue := index.NewAnalysisQueue(analysisWorkers)\n\t\tidx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\n\t\terr = idx.Open()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\n\t\tb.StartTimer()\n\t\tbatch := index.NewBatch()\n\t\tfor j := 0; j < 1000; j++ {\n\t\t\tif j%batchSize == 0 {\n\t\t\t\tif len(batch.IndexOps) > 0 {\n\t\t\t\t\terr := idx.Batch(batch)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tb.Fatal(err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tbatch = index.NewBatch()\n\t\t\t}\n\t\t\tindexDocument := document.NewDocument(\"\").\n\t\t\t\tAddField(document.NewTextFieldWithAnalyzer(\"body\", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer))\n\t\t\tindexDocument.SetID(strconv.Itoa(i) + \"-\" + strconv.Itoa(j))\n\t\t\tbatch.Update(indexDocument)\n\t\t}\n\t\t// close last batch\n\t\tif len(batch.IndexOps) > 0 {\n\t\t\terr := idx.Batch(batch)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t}\n\t\tb.StopTimer()\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\terr = destroy()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tanalysisQueue.Close()\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/benchmark_gtreap_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n)\n\nfunc BenchmarkGTreapIndexing1Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 1)\n}\n\nfunc BenchmarkGTreapIndexing2Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 2)\n}\n\nfunc BenchmarkGTreapIndexing4Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 4)\n}\n\n// batches\n\nfunc BenchmarkGTreapIndexing1Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 10)\n}\n\nfunc BenchmarkGTreapIndexing2Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 10)\n}\n\nfunc BenchmarkGTreapIndexing4Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 10)\n}\n\nfunc BenchmarkGTreapIndexing1Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 100)\n}\n\nfunc BenchmarkGTreapIndexing2Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 100)\n}\n\nfunc BenchmarkGTreapIndexing4Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 100)\n}\n\nfunc BenchmarkGTreapIndexing1Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 1000)\n}\n\nfunc BenchmarkGTreapIndexing2Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 1000)\n}\n\nfunc BenchmarkGTreapIndexing4Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 1000)\n}\n"
  },
  {
    "path": "index/upsidedown/benchmark_null_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/null\"\n)\n\nfunc BenchmarkNullIndexing1Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 1)\n}\n\nfunc BenchmarkNullIndexing2Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 2)\n}\n\nfunc BenchmarkNullIndexing4Workers(b *testing.B) {\n\tCommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 4)\n}\n\n// batches\n\nfunc BenchmarkNullIndexing1Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 10)\n}\n\nfunc BenchmarkNullIndexing2Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 10)\n}\n\nfunc BenchmarkNullIndexing4Workers10Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 10)\n}\n\nfunc BenchmarkNullIndexing1Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 100)\n}\n\nfunc BenchmarkNullIndexing2Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 100)\n}\n\nfunc BenchmarkNullIndexing4Workers100Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 100)\n}\n\nfunc BenchmarkNullIndexing1Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 1000)\n}\n\nfunc BenchmarkNullIndexing2Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 1000)\n}\n\nfunc BenchmarkNullIndexing4Workers1000Batch(b *testing.B) {\n\tCommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 1000)\n}\n"
  },
  {
    "path": "index/upsidedown/dump.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"bytes\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/upsidedown_store_api\"\n)\n\n// the functions in this file are only intended to be used by\n// the bleve_dump utility and the debug http handlers\n// if your application relies on them, you're doing something wrong\n// they may change or be removed at any time\n\nfunc dumpPrefix(kvreader store.KVReader, rv chan interface{}, prefix []byte) {\n\tstart := prefix\n\tif start == nil {\n\t\tstart = []byte{0}\n\t}\n\tit := kvreader.PrefixIterator(start)\n\tdefer func() {\n\t\tcerr := it.Close()\n\t\tif cerr != nil {\n\t\t\trv <- cerr\n\t\t}\n\t}()\n\tkey, val, valid := it.Current()\n\tfor valid {\n\t\tck := make([]byte, len(key))\n\t\tcopy(ck, key)\n\t\tcv := make([]byte, len(val))\n\t\tcopy(cv, val)\n\t\trow, err := ParseFromKeyValue(ck, cv)\n\t\tif err != nil {\n\t\t\trv <- err\n\t\t\treturn\n\t\t}\n\t\trv <- row\n\n\t\tit.Next()\n\t\tkey, val, valid = it.Current()\n\t}\n}\n\nfunc dumpRange(kvreader store.KVReader, rv chan interface{}, start, end []byte) {\n\tit := kvreader.RangeIterator(start, end)\n\tdefer func() {\n\t\tcerr := it.Close()\n\t\tif cerr != nil {\n\t\t\trv <- cerr\n\t\t}\n\t}()\n\tkey, val, valid := it.Current()\n\tfor valid {\n\t\tck := make([]byte, len(key))\n\t\tcopy(ck, key)\n\t\tcv := make([]byte, len(val))\n\t\tcopy(cv, val)\n\t\trow, err := ParseFromKeyValue(ck, cv)\n\t\tif err != nil {\n\t\t\trv <- err\n\t\t\treturn\n\t\t}\n\t\trv <- row\n\n\t\tit.Next()\n\t\tkey, val, valid = it.Current()\n\t}\n}\n\nfunc (i *IndexReader) DumpAll() chan interface{} {\n\trv := make(chan interface{})\n\tgo func() {\n\t\tdefer close(rv)\n\t\tdumpRange(i.kvreader, rv, nil, nil)\n\t}()\n\treturn rv\n}\n\nfunc (i *IndexReader) DumpFields() chan interface{} {\n\trv := make(chan interface{})\n\tgo func() {\n\t\tdefer close(rv)\n\t\tdumpPrefix(i.kvreader, rv, []byte{'f'})\n\t}()\n\treturn rv\n}\n\ntype keyset [][]byte\n\nfunc (k keyset) Len() int           { return len(k) }\nfunc (k keyset) Swap(i, j int)      { k[i], k[j] = k[j], k[i] }\nfunc (k keyset) Less(i, j int) bool { return bytes.Compare(k[i], k[j]) < 0 }\n\n// DumpDoc returns all rows in the index related to this doc id\nfunc (i *IndexReader) DumpDoc(id string) chan interface{} {\n\tidBytes := []byte(id)\n\n\trv := make(chan interface{})\n\n\tgo func() {\n\t\tdefer close(rv)\n\n\t\tback, err := backIndexRowForDoc(i.kvreader, []byte(id))\n\t\tif err != nil {\n\t\t\trv <- err\n\t\t\treturn\n\t\t}\n\n\t\t// no such doc\n\t\tif back == nil {\n\t\t\treturn\n\t\t}\n\t\t// build sorted list of term keys\n\t\tkeys := make(keyset, 0)\n\t\tfor _, entry := range back.termsEntries {\n\t\t\tfor i := range entry.Terms {\n\t\t\t\ttfr := NewTermFrequencyRow([]byte(entry.Terms[i]), uint16(*entry.Field), idBytes, 0, 0)\n\t\t\t\tkey := tfr.Key()\n\t\t\t\tkeys = append(keys, key)\n\t\t\t}\n\t\t}\n\t\tsort.Sort(keys)\n\n\t\t// first add all the stored rows\n\t\tstoredRowPrefix := NewStoredRow(idBytes, 0, []uint64{}, 'x', []byte{}).ScanPrefixForDoc()\n\t\tdumpPrefix(i.kvreader, rv, storedRowPrefix)\n\n\t\t// now walk term keys in order and add them as well\n\t\tif len(keys) > 0 {\n\t\t\tit := i.kvreader.RangeIterator(keys[0], nil)\n\t\t\tdefer func() {\n\t\t\t\tcerr := it.Close()\n\t\t\t\tif cerr != nil {\n\t\t\t\t\trv <- cerr\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\tfor _, key := range keys {\n\t\t\t\tit.Seek(key)\n\t\t\t\trkey, rval, valid := it.Current()\n\t\t\t\tif !valid {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\trck := make([]byte, len(rkey))\n\t\t\t\tcopy(rck, key)\n\t\t\t\trcv := make([]byte, len(rval))\n\t\t\t\tcopy(rcv, rval)\n\t\t\t\trow, err := ParseFromKeyValue(rck, rcv)\n\t\t\t\tif err != nil {\n\t\t\t\t\trv <- err\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\trv <- row\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn rv\n}\n"
  },
  {
    "path": "index/upsidedown/dump_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n)\n\nfunc TestDump(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewNumericFieldWithIndexingOptions(\"age\", []uint64{}, 35.99, index.IndexField|index.StoreField))\n\tdateField, err := document.NewDateTimeFieldWithIndexingOptions(\"unixEpoch\", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdoc.AddField(dateField)\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test2\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewNumericFieldWithIndexingOptions(\"age\", []uint64{}, 35.99, index.IndexField|index.StoreField))\n\tdateField, err = document.NewDateTimeFieldWithIndexingOptions(\"unixEpoch\", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdoc.AddField(dateField)\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tfieldsCount := 0\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tupsideDownReader, ok := reader.(*IndexReader)\n\tif !ok {\n\t\tt.Fatal(\"dump is only supported by index type upsidedown\")\n\t}\n\tfieldsRows := upsideDownReader.DumpFields()\n\tfor range fieldsRows {\n\t\tfieldsCount++\n\t}\n\tif fieldsCount != 3 {\n\t\tt.Errorf(\"expected 3 fields, got %d\", fieldsCount)\n\t}\n\n\t// 1 text term\n\t// 16 numeric terms\n\t// 16 date terms\n\t// 3 stored fields\n\texpectedDocRowCount := int(1 + (2 * (64 / document.DefaultPrecisionStep)) + 3)\n\tdocRowCount := 0\n\tdocRows := upsideDownReader.DumpDoc(\"1\")\n\tfor range docRows {\n\t\tdocRowCount++\n\t}\n\tif docRowCount != expectedDocRowCount {\n\t\tt.Errorf(\"expected %d rows for document, got %d\", expectedDocRowCount, docRowCount)\n\t}\n\n\tdocRowCount = 0\n\tdocRows = upsideDownReader.DumpDoc(\"2\")\n\tfor range docRows {\n\t\tdocRowCount++\n\t}\n\tif docRowCount != expectedDocRowCount {\n\t\tt.Errorf(\"expected %d rows for document, got %d\", expectedDocRowCount, docRowCount)\n\t}\n\n\t// 1 version\n\t// fieldsCount field rows\n\t// 2 docs * expectedDocRowCount\n\t// 2 back index rows\n\t// 2 text term row count (2 different text terms)\n\t// 16 numeric term row counts (shared for both docs, same numeric value)\n\t// 16 date term row counts (shared for both docs, same date value)\n\texpectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DefaultPrecisionStep))))\n\tallRowCount := 0\n\tallRows := upsideDownReader.DumpAll()\n\tfor range allRows {\n\t\tallRowCount++\n\t}\n\tif allRowCount != expectedAllRowCount {\n\t\tt.Errorf(\"expected %d rows for all, got %d\", expectedAllRowCount, allRowCount)\n\t}\n\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/field_cache.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"sync\"\n)\n\ntype FieldCache struct {\n\tfieldIndexes   map[string]uint16\n\tindexFields    []string\n\tlastFieldIndex int\n\tmutex          sync.RWMutex\n}\n\nfunc NewFieldCache() *FieldCache {\n\treturn &FieldCache{\n\t\tfieldIndexes:   make(map[string]uint16),\n\t\tlastFieldIndex: -1,\n\t}\n}\n\nfunc (f *FieldCache) AddExisting(field string, index uint16) {\n\tf.mutex.Lock()\n\tf.addLOCKED(field, index)\n\tf.mutex.Unlock()\n}\n\nfunc (f *FieldCache) addLOCKED(field string, index uint16) uint16 {\n\tf.fieldIndexes[field] = index\n\tif len(f.indexFields) < int(index)+1 {\n\t\tprevIndexFields := f.indexFields\n\t\tf.indexFields = make([]string, int(index)+16)\n\t\tcopy(f.indexFields, prevIndexFields)\n\t}\n\tf.indexFields[int(index)] = field\n\tif int(index) > f.lastFieldIndex {\n\t\tf.lastFieldIndex = int(index)\n\t}\n\treturn index\n}\n\n// FieldNamed returns the index of the field, and whether or not it existed\n// before this call.  if createIfMissing is true, and new field index is assigned\n// but the second return value will still be false\nfunc (f *FieldCache) FieldNamed(field string, createIfMissing bool) (uint16, bool) {\n\tf.mutex.RLock()\n\tif index, ok := f.fieldIndexes[field]; ok {\n\t\tf.mutex.RUnlock()\n\t\treturn index, true\n\t} else if !createIfMissing {\n\t\tf.mutex.RUnlock()\n\t\treturn 0, false\n\t}\n\t// trade read lock for write lock\n\tf.mutex.RUnlock()\n\tf.mutex.Lock()\n\t// need to check again with write lock\n\tif index, ok := f.fieldIndexes[field]; ok {\n\t\tf.mutex.Unlock()\n\t\treturn index, true\n\t}\n\t// assign next field id\n\tindex := f.addLOCKED(field, uint16(f.lastFieldIndex+1))\n\tf.mutex.Unlock()\n\treturn index, false\n}\n\nfunc (f *FieldCache) FieldIndexed(index uint16) (field string) {\n\tf.mutex.RLock()\n\tif int(index) < len(f.indexFields) {\n\t\tfield = f.indexFields[int(index)]\n\t}\n\tf.mutex.RUnlock()\n\treturn field\n}\n"
  },
  {
    "path": "index/upsidedown/field_dict.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"fmt\"\n\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype UpsideDownCouchFieldDict struct {\n\tindexReader *IndexReader\n\titerator    store.KVIterator\n\tdictRow     *DictionaryRow\n\tdictEntry   *index.DictEntry\n\tfield       uint16\n}\n\nfunc newUpsideDownCouchFieldDict(indexReader *IndexReader, field uint16, startTerm, endTerm []byte) (*UpsideDownCouchFieldDict, error) {\n\n\tstartKey := NewDictionaryRow(startTerm, field, 0).Key()\n\tif endTerm == nil {\n\t\tendTerm = []byte{ByteSeparator}\n\t} else {\n\t\tendTerm = incrementBytes(endTerm)\n\t}\n\tendKey := NewDictionaryRow(endTerm, field, 0).Key()\n\n\tit := indexReader.kvreader.RangeIterator(startKey, endKey)\n\n\treturn &UpsideDownCouchFieldDict{\n\t\tindexReader: indexReader,\n\t\titerator:    it,\n\t\tdictRow:     &DictionaryRow{},   // Pre-alloced, reused row.\n\t\tdictEntry:   &index.DictEntry{}, // Pre-alloced, reused entry.\n\t\tfield:       field,\n\t}, nil\n\n}\n\nfunc (r *UpsideDownCouchFieldDict) BytesRead() uint64 {\n\treturn 0\n}\n\nfunc (r *UpsideDownCouchFieldDict) Next() (*index.DictEntry, error) {\n\tkey, val, valid := r.iterator.Current()\n\tif !valid {\n\t\treturn nil, nil\n\t}\n\n\terr := r.dictRow.parseDictionaryK(key)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unexpected error parsing dictionary row key: %v\", err)\n\t}\n\terr = r.dictRow.parseDictionaryV(val)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unexpected error parsing dictionary row val: %v\", err)\n\t}\n\tr.dictEntry.Term = string(r.dictRow.term)\n\tr.dictEntry.Count = r.dictRow.count\n\t// advance the iterator to the next term\n\tr.iterator.Next()\n\treturn r.dictEntry, nil\n\n}\n\nfunc (r *UpsideDownCouchFieldDict) Cardinality() int {\n\treturn 0\n}\n\nfunc (r *UpsideDownCouchFieldDict) Close() error {\n\treturn r.iterator.Close()\n}\n"
  },
  {
    "path": "index/upsidedown/field_dict_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestIndexFieldDict(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextFieldWithAnalyzer(\"name\", []uint64{}, []byte(\"test test test\"), testAnalyzer))\n\tdoc.AddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"eat more rice\"), index.IndexField|index.IncludeTermVectors, testAnalyzer))\n\tdoc.AddField(document.NewTextFieldCustom(\"prefix\", []uint64{}, []byte(\"bob cat cats catting dog doggy zoo\"), index.IndexField|index.IncludeTermVectors, testAnalyzer))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdict, err := indexReader.FieldDict(\"name\")\n\tif err != nil {\n\t\tt.Errorf(\"error creating reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := dict.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermCount := 0\n\tcurr, err := dict.Next()\n\tfor err == nil && curr != nil {\n\t\ttermCount++\n\t\tif curr.Term != \"test\" {\n\t\t\tt.Errorf(\"expected term to be 'test', got '%s'\", curr.Term)\n\t\t}\n\t\tcurr, err = dict.Next()\n\t}\n\tif termCount != 1 {\n\t\tt.Errorf(\"expected 1 term for this field, got %d\", termCount)\n\t}\n\n\tdict2, err := indexReader.FieldDict(\"desc\")\n\tif err != nil {\n\t\tt.Errorf(\"error creating reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := dict2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermCount = 0\n\tterms := make([]string, 0)\n\tcurr, err = dict2.Next()\n\tfor err == nil && curr != nil {\n\t\ttermCount++\n\t\tterms = append(terms, curr.Term)\n\t\tcurr, err = dict2.Next()\n\t}\n\tif termCount != 3 {\n\t\tt.Errorf(\"expected 3 term for this field, got %d\", termCount)\n\t}\n\texpectedTerms := []string{\"eat\", \"more\", \"rice\"}\n\tif !reflect.DeepEqual(expectedTerms, terms) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedTerms, terms)\n\t}\n\n\t// test start and end range\n\tdict3, err := indexReader.FieldDictRange(\"desc\", []byte(\"fun\"), []byte(\"nice\"))\n\tif err != nil {\n\t\tt.Errorf(\"error creating reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := dict3.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermCount = 0\n\tterms = make([]string, 0)\n\tcurr, err = dict3.Next()\n\tfor err == nil && curr != nil {\n\t\ttermCount++\n\t\tterms = append(terms, curr.Term)\n\t\tcurr, err = dict3.Next()\n\t}\n\tif termCount != 1 {\n\t\tt.Errorf(\"expected 1 term for this field, got %d\", termCount)\n\t}\n\texpectedTerms = []string{\"more\"}\n\tif !reflect.DeepEqual(expectedTerms, terms) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedTerms, terms)\n\t}\n\n\t// test use case for prefix\n\tdict4, err := indexReader.FieldDictPrefix(\"prefix\", []byte(\"cat\"))\n\tif err != nil {\n\t\tt.Errorf(\"error creating reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := dict4.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermCount = 0\n\tterms = make([]string, 0)\n\tcurr, err = dict4.Next()\n\tfor err == nil && curr != nil {\n\t\ttermCount++\n\t\tterms = append(terms, curr.Term)\n\t\tcurr, err = dict4.Next()\n\t}\n\tif termCount != 3 {\n\t\tt.Errorf(\"expected 3 term for this field, got %d\", termCount)\n\t}\n\texpectedTerms = []string{\"cat\", \"cats\", \"catting\"}\n\tif !reflect.DeepEqual(expectedTerms, terms) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedTerms, terms)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/index_reader.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\nvar reflectStaticSizeIndexReader int\n\nfunc init() {\n\tvar ir IndexReader\n\treflectStaticSizeIndexReader = int(reflect.TypeOf(ir).Size())\n}\n\ntype IndexReader struct {\n\tindex    *UpsideDownCouch\n\tkvreader store.KVReader\n\tdocCount uint64\n}\n\nfunc (i *IndexReader) TermFieldReader(ctx context.Context, term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {\n\tfieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false)\n\tif fieldExists {\n\t\treturn newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex), includeFreq, includeNorm, includeTermVectors)\n\t}\n\treturn newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0), includeFreq, includeNorm, includeTermVectors)\n}\n\nfunc (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) {\n\treturn i.FieldDictRange(fieldName, nil, nil)\n}\n\nfunc (i *IndexReader) FieldDictRange(fieldName string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {\n\tfieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false)\n\tif fieldExists {\n\t\treturn newUpsideDownCouchFieldDict(i, uint16(fieldIndex), startTerm, endTerm)\n\t}\n\treturn newUpsideDownCouchFieldDict(i, ^uint16(0), []byte{ByteSeparator}, []byte{})\n}\n\nfunc (i *IndexReader) FieldDictPrefix(fieldName string, termPrefix []byte) (index.FieldDict, error) {\n\treturn i.FieldDictRange(fieldName, termPrefix, termPrefix)\n}\n\nfunc (i *IndexReader) DocIDReaderAll() (index.DocIDReader, error) {\n\treturn newUpsideDownCouchDocIDReader(i)\n}\n\nfunc (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) {\n\treturn newUpsideDownCouchDocIDReaderOnly(i, ids)\n}\n\nfunc (i *IndexReader) Document(id string) (doc index.Document, err error) {\n\t// first hit the back index to confirm doc exists\n\tvar backIndexRow *BackIndexRow\n\tbackIndexRow, err = backIndexRowForDoc(i.kvreader, []byte(id))\n\tif err != nil {\n\t\treturn\n\t}\n\tif backIndexRow == nil {\n\t\treturn\n\t}\n\trvd := document.NewDocument(id)\n\tstoredRow := NewStoredRow([]byte(id), 0, []uint64{}, 'x', nil)\n\tstoredRowScanPrefix := storedRow.ScanPrefixForDoc()\n\tit := i.kvreader.PrefixIterator(storedRowScanPrefix)\n\tdefer func() {\n\t\tif cerr := it.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\tkey, val, valid := it.Current()\n\tfor valid {\n\t\tsafeVal := make([]byte, len(val))\n\t\tcopy(safeVal, val)\n\t\tvar row *StoredRow\n\t\trow, err = NewStoredRowKV(key, safeVal)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif row != nil {\n\t\t\tfieldName := i.index.fieldCache.FieldIndexed(row.field)\n\t\t\tfield := decodeFieldType(row.typ, fieldName, row.arrayPositions, row.value)\n\t\t\tif field != nil {\n\t\t\t\trvd.AddField(field)\n\t\t\t}\n\t\t}\n\n\t\tit.Next()\n\t\tkey, val, valid = it.Current()\n\t}\n\treturn rvd, nil\n}\n\nfunc (i *IndexReader) documentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocValueVisitor) error {\n\tfieldsMap := make(map[uint16]string, len(fields))\n\tfor _, f := range fields {\n\t\tid, ok := i.index.fieldCache.FieldNamed(f, false)\n\t\tif ok {\n\t\t\tfieldsMap[id] = f\n\t\t}\n\t}\n\n\ttempRow := BackIndexRow{\n\t\tdoc: id,\n\t}\n\n\tkeyBuf := GetRowBuffer()\n\tif tempRow.KeySize() > len(keyBuf.buf) {\n\t\tkeyBuf.buf = make([]byte, 2*tempRow.KeySize())\n\t}\n\tdefer PutRowBuffer(keyBuf)\n\tkeySize, err := tempRow.KeyTo(keyBuf.buf)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvalue, err := i.kvreader.Get(keyBuf.buf[:keySize])\n\tif err != nil {\n\t\treturn err\n\t}\n\tif value == nil {\n\t\treturn nil\n\t}\n\n\treturn visitBackIndexRow(value, func(field uint32, term []byte) {\n\t\tif field, ok := fieldsMap[uint16(field)]; ok {\n\t\t\tvisitor(field, term)\n\t\t}\n\t})\n}\n\nfunc (i *IndexReader) Fields() (fields []string, err error) {\n\tfields = make([]string, 0)\n\tit := i.kvreader.PrefixIterator([]byte{'f'})\n\tdefer func() {\n\t\tif cerr := it.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\tkey, val, valid := it.Current()\n\tfor valid {\n\t\tvar row UpsideDownCouchRow\n\t\trow, err = ParseFromKeyValue(key, val)\n\t\tif err != nil {\n\t\t\tfields = nil\n\t\t\treturn\n\t\t}\n\t\tif row != nil {\n\t\t\tfieldRow, ok := row.(*FieldRow)\n\t\t\tif ok {\n\t\t\t\tfields = append(fields, fieldRow.name)\n\t\t\t}\n\t\t}\n\n\t\tit.Next()\n\t\tkey, val, valid = it.Current()\n\t}\n\treturn\n}\n\nfunc (i *IndexReader) GetInternal(key []byte) ([]byte, error) {\n\tinternalRow := NewInternalRow(key, nil)\n\treturn i.kvreader.Get(internalRow.Key())\n}\n\nfunc (i *IndexReader) DocCount() (uint64, error) {\n\treturn i.docCount, nil\n}\n\nfunc (i *IndexReader) Close() error {\n\treturn i.kvreader.Close()\n}\n\nfunc (i *IndexReader) ExternalID(id index.IndexInternalID) (string, error) {\n\treturn string(id), nil\n}\n\nfunc (i *IndexReader) InternalID(id string) (index.IndexInternalID, error) {\n\treturn index.IndexInternalID(id), nil\n}\n\nfunc incrementBytes(in []byte) []byte {\n\trv := make([]byte, len(in))\n\tcopy(rv, in)\n\tfor i := len(rv) - 1; i >= 0; i-- {\n\t\trv[i] = rv[i] + 1\n\t\tif rv[i] != 0 {\n\t\t\t// didn't overflow, so stop\n\t\t\tbreak\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (i *IndexReader) DocValueReader(fields []string) (index.DocValueReader, error) {\n\treturn &DocValueReader{i: i, fields: fields}, nil\n}\n\ntype DocValueReader struct {\n\ti      *IndexReader\n\tfields []string\n}\n\nfunc (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID,\n\tvisitor index.DocValueVisitor) error {\n\treturn dvr.i.documentVisitFieldTerms(id, dvr.fields, visitor)\n}\n\nfunc (dvr *DocValueReader) BytesRead() uint64 { return 0 }\n"
  },
  {
    "path": "index/upsidedown/protoc-README.md",
    "content": "## Instructions for generating new go stubs using upsidedown.proto\n\n1. Download latest of protoc-gen-go\n```\ngo install google.golang.org/protobuf/cmd/protoc-gen-go@latest\n```\n\n2. To generate `upsidedown.pb.go` using upsdidedown.proto:\n```\nprotoc --go_out=. --go_opt=Mindex/upsidedown/upsidedown.proto=index/upsidedown/ index/upsidedown/upsidedown.proto\n```\n\n3. Manually add back Size and MarshalTo methods for BackIndexRowValue, BackIndexTermsEntry, BackIndexStoreEntry to support upside_down.\n"
  },
  {
    "path": "index/upsidedown/reader.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"bytes\"\n\t\"reflect\"\n\t\"sort\"\n\t\"sync/atomic\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"github.com/blevesearch/upsidedown_store_api\"\n)\n\nvar reflectStaticSizeUpsideDownCouchTermFieldReader int\nvar reflectStaticSizeUpsideDownCouchDocIDReader int\n\nfunc init() {\n\tvar tfr UpsideDownCouchTermFieldReader\n\treflectStaticSizeUpsideDownCouchTermFieldReader =\n\t\tint(reflect.TypeOf(tfr).Size())\n\tvar cdr UpsideDownCouchDocIDReader\n\treflectStaticSizeUpsideDownCouchDocIDReader =\n\t\tint(reflect.TypeOf(cdr).Size())\n}\n\ntype UpsideDownCouchTermFieldReader struct {\n\tcount              uint64\n\tindexReader        *IndexReader\n\titerator           store.KVIterator\n\tterm               []byte\n\ttfrNext            *TermFrequencyRow\n\ttfrPrealloc        TermFrequencyRow\n\tkeyBuf             []byte\n\tfield              uint16\n\tincludeTermVectors bool\n}\n\nfunc (r *UpsideDownCouchTermFieldReader) Size() int {\n\tsizeInBytes := reflectStaticSizeUpsideDownCouchTermFieldReader + size.SizeOfPtr +\n\t\tlen(r.term) +\n\t\tr.tfrPrealloc.Size() +\n\t\tlen(r.keyBuf)\n\n\tif r.tfrNext != nil {\n\t\tsizeInBytes += r.tfrNext.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) {\n\tbufNeeded := termFrequencyRowKeySize(term, nil)\n\tif bufNeeded < dictionaryRowKeySize(term) {\n\t\tbufNeeded = dictionaryRowKeySize(term)\n\t}\n\tbuf := make([]byte, bufNeeded)\n\n\tbufUsed := dictionaryRowKeyTo(buf, field, term)\n\tval, err := indexReader.kvreader.Get(buf[:bufUsed])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif val == nil {\n\t\tatomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1))\n\t\trv := &UpsideDownCouchTermFieldReader{\n\t\t\tcount:              0,\n\t\t\tterm:               term,\n\t\t\tfield:              field,\n\t\t\tincludeTermVectors: includeTermVectors,\n\t\t}\n\t\trv.tfrNext = &rv.tfrPrealloc\n\t\treturn rv, nil\n\t}\n\n\tcount, err := dictionaryRowParseV(val)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tbufUsed = termFrequencyRowKeyTo(buf, field, term, nil)\n\tit := indexReader.kvreader.PrefixIterator(buf[:bufUsed])\n\n\tatomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1))\n\treturn &UpsideDownCouchTermFieldReader{\n\t\tindexReader:        indexReader,\n\t\titerator:           it,\n\t\tcount:              count,\n\t\tterm:               term,\n\t\tfield:              field,\n\t\tincludeTermVectors: includeTermVectors,\n\t}, nil\n}\n\nfunc (r *UpsideDownCouchTermFieldReader) Count() uint64 {\n\treturn r.count\n}\n\nfunc (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) {\n\tif r.iterator != nil {\n\t\t// We treat tfrNext also like an initialization flag, which\n\t\t// tells us whether we need to invoke the underlying\n\t\t// iterator.Next().  The first time, don't call iterator.Next().\n\t\tif r.tfrNext != nil {\n\t\t\tr.iterator.Next()\n\t\t} else {\n\t\t\tr.tfrNext = &r.tfrPrealloc\n\t\t}\n\t\tkey, val, valid := r.iterator.Current()\n\t\tif valid {\n\t\t\ttfr := r.tfrNext\n\t\t\terr := tfr.parseKDoc(key, r.term)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\terr = tfr.parseV(val, r.includeTermVectors)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\trv := preAlloced\n\t\t\tif rv == nil {\n\t\t\t\trv = &index.TermFieldDoc{}\n\t\t\t}\n\t\t\trv.ID = append(rv.ID, tfr.doc...)\n\t\t\trv.Freq = tfr.freq\n\t\t\trv.Norm = float64(tfr.norm)\n\t\t\tif tfr.vectors != nil {\n\t\t\t\trv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors)\n\t\t\t}\n\t\t\treturn rv, nil\n\t\t}\n\t}\n\treturn nil, nil\n}\n\nfunc (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (rv *index.TermFieldDoc, err error) {\n\tif r.iterator != nil {\n\t\tif r.tfrNext == nil {\n\t\t\tr.tfrNext = &TermFrequencyRow{}\n\t\t}\n\t\ttfr := InitTermFrequencyRow(r.tfrNext, r.term, r.field, docID, 0, 0)\n\t\tr.keyBuf, err = tfr.KeyAppendTo(r.keyBuf[:0])\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tr.iterator.Seek(r.keyBuf)\n\t\tkey, val, valid := r.iterator.Current()\n\t\tif valid {\n\t\t\terr := tfr.parseKDoc(key, r.term)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\terr = tfr.parseV(val, r.includeTermVectors)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\trv = preAlloced\n\t\t\tif rv == nil {\n\t\t\t\trv = &index.TermFieldDoc{}\n\t\t\t}\n\t\t\trv.ID = append(rv.ID, tfr.doc...)\n\t\t\trv.Freq = tfr.freq\n\t\t\trv.Norm = float64(tfr.norm)\n\t\t\tif tfr.vectors != nil {\n\t\t\t\trv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors)\n\t\t\t}\n\t\t\treturn rv, nil\n\t\t}\n\t}\n\treturn nil, nil\n}\n\nfunc (r *UpsideDownCouchTermFieldReader) Close() error {\n\tif r.indexReader != nil {\n\t\tatomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1))\n\t}\n\tif r.iterator != nil {\n\t\treturn r.iterator.Close()\n\t}\n\treturn nil\n}\n\ntype UpsideDownCouchDocIDReader struct {\n\tindexReader *IndexReader\n\titerator    store.KVIterator\n\tonly        []string\n\tonlyPos     int\n\tonlyMode    bool\n}\n\nfunc (r *UpsideDownCouchDocIDReader) Size() int {\n\tsizeInBytes := reflectStaticSizeUpsideDownCouchDocIDReader +\n\t\treflectStaticSizeIndexReader + size.SizeOfPtr\n\n\tfor _, entry := range r.only {\n\t\tsizeInBytes += size.SizeOfString + len(entry)\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDocIDReader, error) {\n\tstartBytes := []byte{0x0}\n\tendBytes := []byte{0xff}\n\n\tbisr := NewBackIndexRow(startBytes, nil, nil)\n\tbier := NewBackIndexRow(endBytes, nil, nil)\n\tit := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key())\n\n\treturn &UpsideDownCouchDocIDReader{\n\t\tindexReader: indexReader,\n\t\titerator:    it,\n\t}, nil\n}\n\nfunc newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) {\n\t// we don't actually own the list of ids, so if before we sort we must copy\n\tidsCopy := make([]string, len(ids))\n\tcopy(idsCopy, ids)\n\t// ensure ids are sorted\n\tsort.Strings(idsCopy)\n\tstartBytes := []byte{0x0}\n\tif len(idsCopy) > 0 {\n\t\tstartBytes = []byte(idsCopy[0])\n\t}\n\tendBytes := []byte{0xff}\n\tif len(idsCopy) > 0 {\n\t\tendBytes = incrementBytes([]byte(idsCopy[len(idsCopy)-1]))\n\t}\n\tbisr := NewBackIndexRow(startBytes, nil, nil)\n\tbier := NewBackIndexRow(endBytes, nil, nil)\n\tit := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key())\n\n\treturn &UpsideDownCouchDocIDReader{\n\t\tindexReader: indexReader,\n\t\titerator:    it,\n\t\tonly:        idsCopy,\n\t\tonlyMode:    true,\n\t}, nil\n}\n\nfunc (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) {\n\tkey, val, valid := r.iterator.Current()\n\n\tif r.onlyMode {\n\t\tvar rv index.IndexInternalID\n\t\tfor valid && r.onlyPos < len(r.only) {\n\t\t\tbr, err := NewBackIndexRowKV(key, val)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) {\n\t\t\t\tok := r.nextOnly()\n\t\t\t\tif !ok {\n\t\t\t\t\treturn nil, nil\n\t\t\t\t}\n\t\t\t\tr.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())\n\t\t\t\tkey, val, valid = r.iterator.Current()\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\trv = append([]byte(nil), br.doc...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif valid && r.onlyPos < len(r.only) {\n\t\t\tok := r.nextOnly()\n\t\t\tif ok {\n\t\t\t\tr.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())\n\t\t\t}\n\t\t\treturn rv, nil\n\t\t}\n\n\t} else {\n\t\tif valid {\n\t\t\tbr, err := NewBackIndexRowKV(key, val)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\trv := append([]byte(nil), br.doc...)\n\t\t\tr.iterator.Next()\n\t\t\treturn rv, nil\n\t\t}\n\t}\n\treturn nil, nil\n}\n\nfunc (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) {\n\n\tif r.onlyMode {\n\t\tr.onlyPos = sort.SearchStrings(r.only, string(docID))\n\t\tif r.onlyPos >= len(r.only) {\n\t\t\t// advanced to key after our last only key\n\t\t\treturn nil, nil\n\t\t}\n\t\tr.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())\n\t\tkey, val, valid := r.iterator.Current()\n\n\t\tvar rv index.IndexInternalID\n\t\tfor valid && r.onlyPos < len(r.only) {\n\t\t\tbr, err := NewBackIndexRowKV(key, val)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) {\n\t\t\t\t// the only key we seek'd to didn't exist\n\t\t\t\t// now look for the closest key that did exist in only\n\t\t\t\tr.onlyPos = sort.SearchStrings(r.only, string(br.doc))\n\t\t\t\tif r.onlyPos >= len(r.only) {\n\t\t\t\t\t// advanced to key after our last only key\n\t\t\t\t\treturn nil, nil\n\t\t\t\t}\n\t\t\t\t// now seek to this new only key\n\t\t\t\tr.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())\n\t\t\t\tkey, val, valid = r.iterator.Current()\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\trv = append([]byte(nil), br.doc...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif valid && r.onlyPos < len(r.only) {\n\t\t\tok := r.nextOnly()\n\t\t\tif ok {\n\t\t\t\tr.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())\n\t\t\t}\n\t\t\treturn rv, nil\n\t\t}\n\t} else {\n\t\tbir := NewBackIndexRow(docID, nil, nil)\n\t\tr.iterator.Seek(bir.Key())\n\t\tkey, val, valid := r.iterator.Current()\n\t\tif valid {\n\t\t\tbr, err := NewBackIndexRowKV(key, val)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\trv := append([]byte(nil), br.doc...)\n\t\t\tr.iterator.Next()\n\t\t\treturn rv, nil\n\t\t}\n\t}\n\treturn nil, nil\n}\n\nfunc (r *UpsideDownCouchDocIDReader) Close() error {\n\treturn r.iterator.Close()\n}\n\n// move the r.only pos forward one, skipping duplicates\n// return true if there is more data, or false if we got to the end of the list\nfunc (r *UpsideDownCouchDocIDReader) nextOnly() bool {\n\n\t// advance 1 position, until we see a different key\n\t//   it's already sorted, so this skips duplicates\n\tstart := r.onlyPos\n\tr.onlyPos++\n\tfor r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] {\n\t\tstart = r.onlyPos\n\t\tr.onlyPos++\n\t}\n\t// indicate if we got to the end of the list\n\treturn r.onlyPos < len(r.only)\n}\n"
  },
  {
    "path": "index/upsidedown/reader_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestIndexReader(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextFieldWithAnalyzer(\"name\", []uint64{}, []byte(\"test test test\"), testAnalyzer))\n\tdoc.AddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"eat more rice\"), index.IndexField|index.IncludeTermVectors, testAnalyzer))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// first look for a term that doesn't exist\n\treader, err := indexReader.TermFieldReader(context.TODO(), []byte(\"nope\"), \"name\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing term field reader: %v\", err)\n\t}\n\tcount := reader.Count()\n\tif count != 0 {\n\t\tt.Errorf(\"Expected doc count to be: %d got: %d\", 0, count)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err = indexReader.TermFieldReader(context.TODO(), []byte(\"test\"), \"name\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing term field reader: %v\", err)\n\t}\n\n\tcount = reader.Count()\n\tif count != expectedCount {\n\t\tt.Errorf(\"Expected doc count to be: %d got: %d\", expectedCount, count)\n\t}\n\n\tvar match *index.TermFieldDoc\n\tvar actualCount uint64\n\tmatch, err = reader.Next(nil)\n\tfor err == nil && match != nil {\n\t\tmatch, err = reader.Next(nil)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"unexpected error reading next\")\n\t\t}\n\t\tactualCount++\n\t}\n\tif actualCount != count {\n\t\tt.Errorf(\"count was 2, but only saw %d\", actualCount)\n\t}\n\n\texpectedMatch := &index.TermFieldDoc{\n\t\tID:   index.IndexInternalID(\"2\"),\n\t\tFreq: 1,\n\t\tNorm: 0.5773502588272095,\n\t\tVectors: []*index.TermFieldVector{\n\t\t\t{\n\t\t\t\tField: \"desc\",\n\t\t\t\tPos:   3,\n\t\t\t\tStart: 9,\n\t\t\t\tEnd:   13,\n\t\t\t},\n\t\t},\n\t}\n\ttfr, err := indexReader.TermFieldReader(context.TODO(), []byte(\"rice\"), \"desc\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tmatch, err = tfr.Next(nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif !reflect.DeepEqual(expectedMatch, match) {\n\t\tt.Errorf(\"got %#v, expected %#v\", match, expectedMatch)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now test usage of advance\n\treader, err = indexReader.TermFieldReader(context.TODO(), []byte(\"test\"), \"name\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing term field reader: %v\", err)\n\t}\n\n\tmatch, err = reader.Advance(index.IndexInternalID(\"2\"), nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match == nil {\n\t\tt.Fatalf(\"Expected match, got nil\")\n\t}\n\tif !match.ID.Equals(index.IndexInternalID(\"2\")) {\n\t\tt.Errorf(\"Expected ID '2', got '%s'\", match.ID)\n\t}\n\tmatch, err = reader.Advance(index.IndexInternalID(\"3\"), nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match != nil {\n\t\tt.Errorf(\"expected nil, got %v\", match)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now test creating a reader for a field that doesn't exist\n\treader, err = indexReader.TermFieldReader(context.TODO(), []byte(\"water\"), \"doesnotexist\", true, true, true)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing term field reader: %v\", err)\n\t}\n\tcount = reader.Count()\n\tif count != 0 {\n\t\tt.Errorf(\"expected count 0 for reader of non-existent field\")\n\t}\n\tmatch, err = reader.Next(nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match != nil {\n\t\tt.Errorf(\"expected nil, got %v\", match)\n\t}\n\tmatch, err = reader.Advance(index.IndexInternalID(\"anywhere\"), nil)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match != nil {\n\t\tt.Errorf(\"expected nil, got %v\", match)\n\t}\n}\n\nfunc TestIndexDocIdReader(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test test test\")))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"desc\", []uint64{}, []byte(\"eat more rice\"), index.IndexField|index.IncludeTermVectors))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\t// first get all doc ids\n\treader, err := indexReader.DocIDReaderAll()\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tid, err := reader.Next()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcount := uint64(0)\n\tfor id != nil {\n\t\tcount++\n\t\tid, err = reader.Next()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tif count != expectedCount {\n\t\tt.Errorf(\"expected %d, got %d\", expectedCount, count)\n\t}\n\n\t// try it again, but jump to the second doc this time\n\treader2, err := indexReader.DocIDReaderAll()\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader2.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\tid, err = reader2.Advance(index.IndexInternalID(\"2\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"2\")) {\n\t\tt.Errorf(\"expected to find id '2', got '%s'\", id)\n\t}\n\n\tid, err = reader2.Advance(index.IndexInternalID(\"3\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif id != nil {\n\t\tt.Errorf(\"expected to find id '', got '%s'\", id)\n\t}\n}\n\nfunc TestCrashBadBackIndexRow(t *testing.T) {\n\tbr, err := NewBackIndexRowKV([]byte{byte('b'), byte('a'), ByteSeparator}, []byte{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(br.doc) != \"a\" {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexDocIdOnlyReader(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"3\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"5\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"7\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tdoc = document.NewDocument(\"9\")\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\tonlyIds := []string{\"1\", \"5\", \"9\"}\n\treader, err := indexReader.DocIDReaderOnly(onlyIds)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tid, err := reader.Next()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcount := uint64(0)\n\tfor id != nil {\n\t\tcount++\n\t\tid, err = reader.Next()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tif count != 3 {\n\t\tt.Errorf(\"expected 3, got %d\", count)\n\t}\n\n\t// try it again, but jump\n\treader2, err := indexReader.DocIDReaderOnly(onlyIds)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader2.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\tid, err = reader2.Advance(index.IndexInternalID(\"5\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"5\")) {\n\t\tt.Errorf(\"expected to find id '5', got '%s'\", id)\n\t}\n\n\tid, err = reader2.Advance(index.IndexInternalID(\"a\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif id != nil {\n\t\tt.Errorf(\"expected to find id '', got '%s'\", id)\n\t}\n\n\t// some keys aren't actually there\n\tonlyIds = []string{\"0\", \"2\", \"4\", \"5\", \"6\", \"8\", \"a\"}\n\treader3, err := indexReader.DocIDReaderOnly(onlyIds)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader3.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\tid, err = reader3.Next()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcount = uint64(0)\n\tfor id != nil {\n\t\tcount++\n\t\tid, err = reader3.Next()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tif count != 1 {\n\t\tt.Errorf(\"expected 1, got %d\", count)\n\t}\n\n\t// mix advance and next\n\tonlyIds = []string{\"0\", \"1\", \"3\", \"5\", \"6\", \"9\"}\n\treader4, err := indexReader.DocIDReaderOnly(onlyIds)\n\tif err != nil {\n\t\tt.Errorf(\"Error accessing doc id reader: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := reader4.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\t// first key is \"1\"\n\tid, err = reader4.Next()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"1\")) {\n\t\tt.Errorf(\"expected to find id '1', got '%s'\", id)\n\t}\n\n\t// advancing to key we dont have gives next\n\tid, err = reader4.Advance(index.IndexInternalID(\"2\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"3\")) {\n\t\tt.Errorf(\"expected to find id '3', got '%s'\", id)\n\t}\n\n\t// next after advance works\n\tid, err = reader4.Next()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"5\")) {\n\t\tt.Errorf(\"expected to find id '5', got '%s'\", id)\n\t}\n\n\t// advancing to key we do have works\n\tid, err = reader4.Advance(index.IndexInternalID(\"9\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"9\")) {\n\t\tt.Errorf(\"expected to find id '9', got '%s'\", id)\n\t}\n\n\t// advance backwards at end\n\tid, err = reader4.Advance(index.IndexInternalID(\"4\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"5\")) {\n\t\tt.Errorf(\"expected to find id '5', got '%s'\", id)\n\t}\n\n\t// next after advance works\n\tid, err = reader4.Next()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"9\")) {\n\t\tt.Errorf(\"expected to find id '9', got '%s'\", id)\n\t}\n\n\t// advance backwards to key that exists, but not in only set\n\tid, err = reader4.Advance(index.IndexInternalID(\"7\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif !id.Equals(index.IndexInternalID(\"9\")) {\n\t\tt.Errorf(\"expected to find id '9', got '%s'\", id)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/row.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\t\"google.golang.org/protobuf/proto\"\n)\n\nvar (\n\treflectStaticSizeTermFrequencyRow int\n\treflectStaticSizeTermVector       int\n)\n\nfunc init() {\n\tvar tfr TermFrequencyRow\n\treflectStaticSizeTermFrequencyRow = int(reflect.TypeOf(tfr).Size())\n\tvar tv TermVector\n\treflectStaticSizeTermVector = int(reflect.TypeOf(tv).Size())\n}\n\nconst ByteSeparator byte = 0xff\n\ntype UpsideDownCouchRowStream chan UpsideDownCouchRow\n\ntype UpsideDownCouchRow interface {\n\tKeySize() int\n\tKeyTo([]byte) (int, error)\n\tKey() []byte\n\tValue() []byte\n\tValueSize() int\n\tValueTo([]byte) (int, error)\n}\n\nfunc ParseFromKeyValue(key, value []byte) (UpsideDownCouchRow, error) {\n\tif len(key) > 0 {\n\t\tswitch key[0] {\n\t\tcase 'v':\n\t\t\treturn NewVersionRowKV(key, value)\n\t\tcase 'f':\n\t\t\treturn NewFieldRowKV(key, value)\n\t\tcase 'd':\n\t\t\treturn NewDictionaryRowKV(key, value)\n\t\tcase 't':\n\t\t\treturn NewTermFrequencyRowKV(key, value)\n\t\tcase 'b':\n\t\t\treturn NewBackIndexRowKV(key, value)\n\t\tcase 's':\n\t\t\treturn NewStoredRowKV(key, value)\n\t\tcase 'i':\n\t\t\treturn NewInternalRowKV(key, value)\n\t\t}\n\t\treturn nil, fmt.Errorf(\"Unknown field type '%s'\", string(key[0]))\n\t}\n\treturn nil, fmt.Errorf(\"Invalid empty key\")\n}\n\n// VERSION\n\ntype VersionRow struct {\n\tversion uint8\n}\n\nfunc (v *VersionRow) Key() []byte {\n\treturn []byte{'v'}\n}\n\nfunc (v *VersionRow) KeySize() int {\n\treturn 1\n}\n\nfunc (v *VersionRow) KeyTo(buf []byte) (int, error) {\n\tbuf[0] = 'v'\n\treturn 1, nil\n}\n\nfunc (v *VersionRow) Value() []byte {\n\treturn []byte{byte(v.version)}\n}\n\nfunc (v *VersionRow) ValueSize() int {\n\treturn 1\n}\n\nfunc (v *VersionRow) ValueTo(buf []byte) (int, error) {\n\tbuf[0] = v.version\n\treturn 1, nil\n}\n\nfunc (v *VersionRow) String() string {\n\treturn fmt.Sprintf(\"Version: %d\", v.version)\n}\n\nfunc NewVersionRow(version uint8) *VersionRow {\n\treturn &VersionRow{\n\t\tversion: version,\n\t}\n}\n\nfunc NewVersionRowKV(key, value []byte) (*VersionRow, error) {\n\trv := VersionRow{}\n\tbuf := bytes.NewBuffer(value)\n\terr := binary.Read(buf, binary.LittleEndian, &rv.version)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &rv, nil\n}\n\n// INTERNAL STORAGE\n\ntype InternalRow struct {\n\tkey []byte\n\tval []byte\n}\n\nfunc (i *InternalRow) Key() []byte {\n\tbuf := make([]byte, i.KeySize())\n\tsize, _ := i.KeyTo(buf)\n\treturn buf[:size]\n}\n\nfunc (i *InternalRow) KeySize() int {\n\treturn len(i.key) + 1\n}\n\nfunc (i *InternalRow) KeyTo(buf []byte) (int, error) {\n\tbuf[0] = 'i'\n\tactual := copy(buf[1:], i.key)\n\treturn 1 + actual, nil\n}\n\nfunc (i *InternalRow) Value() []byte {\n\treturn i.val\n}\n\nfunc (i *InternalRow) ValueSize() int {\n\treturn len(i.val)\n}\n\nfunc (i *InternalRow) ValueTo(buf []byte) (int, error) {\n\tactual := copy(buf, i.val)\n\treturn actual, nil\n}\n\nfunc (i *InternalRow) String() string {\n\treturn fmt.Sprintf(\"InternalStore - Key: %s (% x) Val: %s (% x)\", i.key, i.key, i.val, i.val)\n}\n\nfunc NewInternalRow(key, val []byte) *InternalRow {\n\treturn &InternalRow{\n\t\tkey: key,\n\t\tval: val,\n\t}\n}\n\nfunc NewInternalRowKV(key, value []byte) (*InternalRow, error) {\n\trv := InternalRow{}\n\trv.key = key[1:]\n\trv.val = value\n\treturn &rv, nil\n}\n\n// FIELD definition\n\ntype FieldRow struct {\n\tindex uint16\n\tname  string\n}\n\nfunc (f *FieldRow) Key() []byte {\n\tbuf := make([]byte, f.KeySize())\n\tsize, _ := f.KeyTo(buf)\n\treturn buf[:size]\n}\n\nfunc (f *FieldRow) KeySize() int {\n\treturn 3\n}\n\nfunc (f *FieldRow) KeyTo(buf []byte) (int, error) {\n\tbuf[0] = 'f'\n\tbinary.LittleEndian.PutUint16(buf[1:3], f.index)\n\treturn 3, nil\n}\n\nfunc (f *FieldRow) Value() []byte {\n\treturn append([]byte(f.name), ByteSeparator)\n}\n\nfunc (f *FieldRow) ValueSize() int {\n\treturn len(f.name) + 1\n}\n\nfunc (f *FieldRow) ValueTo(buf []byte) (int, error) {\n\tsize := copy(buf, f.name)\n\tbuf[size] = ByteSeparator\n\treturn size + 1, nil\n}\n\nfunc (f *FieldRow) String() string {\n\treturn fmt.Sprintf(\"Field: %d Name: %s\", f.index, f.name)\n}\n\nfunc NewFieldRow(index uint16, name string) *FieldRow {\n\treturn &FieldRow{\n\t\tindex: index,\n\t\tname:  name,\n\t}\n}\n\nfunc NewFieldRowKV(key, value []byte) (*FieldRow, error) {\n\trv := FieldRow{}\n\n\tbuf := bytes.NewBuffer(key)\n\t_, err := buf.ReadByte() // type\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\terr = binary.Read(buf, binary.LittleEndian, &rv.index)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tbuf = bytes.NewBuffer(value)\n\trv.name, err = buf.ReadString(ByteSeparator)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv.name = rv.name[:len(rv.name)-1] // trim off separator byte\n\n\treturn &rv, nil\n}\n\n// DICTIONARY\n\nconst DictionaryRowMaxValueSize = binary.MaxVarintLen64\n\ntype DictionaryRow struct {\n\tterm  []byte\n\tcount uint64\n\tfield uint16\n}\n\nfunc (dr *DictionaryRow) Key() []byte {\n\tbuf := make([]byte, dr.KeySize())\n\tsize, _ := dr.KeyTo(buf)\n\treturn buf[:size]\n}\n\nfunc (dr *DictionaryRow) KeySize() int {\n\treturn dictionaryRowKeySize(dr.term)\n}\n\nfunc dictionaryRowKeySize(term []byte) int {\n\treturn len(term) + 3\n}\n\nfunc (dr *DictionaryRow) KeyTo(buf []byte) (int, error) {\n\treturn dictionaryRowKeyTo(buf, dr.field, dr.term), nil\n}\n\nfunc dictionaryRowKeyTo(buf []byte, field uint16, term []byte) int {\n\tbuf[0] = 'd'\n\tbinary.LittleEndian.PutUint16(buf[1:3], field)\n\tsize := copy(buf[3:], term)\n\treturn size + 3\n}\n\nfunc (dr *DictionaryRow) Value() []byte {\n\tbuf := make([]byte, dr.ValueSize())\n\tsize, _ := dr.ValueTo(buf)\n\treturn buf[:size]\n}\n\nfunc (dr *DictionaryRow) ValueSize() int {\n\treturn DictionaryRowMaxValueSize\n}\n\nfunc (dr *DictionaryRow) ValueTo(buf []byte) (int, error) {\n\tused := binary.PutUvarint(buf, dr.count)\n\treturn used, nil\n}\n\nfunc (dr *DictionaryRow) String() string {\n\treturn fmt.Sprintf(\"Dictionary Term: `%s` Field: %d Count: %d \", string(dr.term), dr.field, dr.count)\n}\n\nfunc NewDictionaryRow(term []byte, field uint16, count uint64) *DictionaryRow {\n\treturn &DictionaryRow{\n\t\tterm:  term,\n\t\tfield: field,\n\t\tcount: count,\n\t}\n}\n\nfunc NewDictionaryRowKV(key, value []byte) (*DictionaryRow, error) {\n\trv, err := NewDictionaryRowK(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\terr = rv.parseDictionaryV(value)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc NewDictionaryRowK(key []byte) (*DictionaryRow, error) {\n\trv := &DictionaryRow{}\n\terr := rv.parseDictionaryK(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc (dr *DictionaryRow) parseDictionaryK(key []byte) error {\n\tdr.field = binary.LittleEndian.Uint16(key[1:3])\n\tif dr.term != nil {\n\t\tdr.term = dr.term[:0]\n\t}\n\tdr.term = append(dr.term, key[3:]...)\n\treturn nil\n}\n\nfunc (dr *DictionaryRow) parseDictionaryV(value []byte) error {\n\tcount, err := dictionaryRowParseV(value)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdr.count = count\n\treturn nil\n}\n\nfunc dictionaryRowParseV(value []byte) (uint64, error) {\n\tcount, nread := binary.Uvarint(value)\n\tif nread <= 0 {\n\t\treturn 0, fmt.Errorf(\"DictionaryRow parse Uvarint error, nread: %d\", nread)\n\t}\n\treturn count, nil\n}\n\n// TERM FIELD FREQUENCY\n\ntype TermVector struct {\n\tfield          uint16\n\tarrayPositions []uint64\n\tpos            uint64\n\tstart          uint64\n\tend            uint64\n}\n\nfunc (tv *TermVector) Size() int {\n\treturn reflectStaticSizeTermVector + size.SizeOfPtr +\n\t\tlen(tv.arrayPositions)*size.SizeOfUint64\n}\n\nfunc (tv *TermVector) String() string {\n\treturn fmt.Sprintf(\"Field: %d Pos: %d Start: %d End %d ArrayPositions: %#v\", tv.field, tv.pos, tv.start, tv.end, tv.arrayPositions)\n}\n\ntype TermFrequencyRow struct {\n\tterm    []byte\n\tdoc     []byte\n\tfreq    uint64\n\tvectors []*TermVector\n\tnorm    float32\n\tfield   uint16\n}\n\nfunc (tfr *TermFrequencyRow) Size() int {\n\tsizeInBytes := reflectStaticSizeTermFrequencyRow +\n\t\tlen(tfr.term) +\n\t\tlen(tfr.doc)\n\n\tfor _, entry := range tfr.vectors {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (tfr *TermFrequencyRow) Term() []byte {\n\treturn tfr.term\n}\n\nfunc (tfr *TermFrequencyRow) Freq() uint64 {\n\treturn tfr.freq\n}\n\nfunc (tfr *TermFrequencyRow) ScanPrefixForField() []byte {\n\tbuf := make([]byte, 3)\n\tbuf[0] = 't'\n\tbinary.LittleEndian.PutUint16(buf[1:3], tfr.field)\n\treturn buf\n}\n\nfunc (tfr *TermFrequencyRow) ScanPrefixForFieldTermPrefix() []byte {\n\tbuf := make([]byte, 3+len(tfr.term))\n\tbuf[0] = 't'\n\tbinary.LittleEndian.PutUint16(buf[1:3], tfr.field)\n\tcopy(buf[3:], tfr.term)\n\treturn buf\n}\n\nfunc (tfr *TermFrequencyRow) ScanPrefixForFieldTerm() []byte {\n\tbuf := make([]byte, 3+len(tfr.term)+1)\n\tbuf[0] = 't'\n\tbinary.LittleEndian.PutUint16(buf[1:3], tfr.field)\n\ttermLen := copy(buf[3:], tfr.term)\n\tbuf[3+termLen] = ByteSeparator\n\treturn buf\n}\n\nfunc (tfr *TermFrequencyRow) Key() []byte {\n\tbuf := make([]byte, tfr.KeySize())\n\tsize, _ := tfr.KeyTo(buf)\n\treturn buf[:size]\n}\n\nfunc (tfr *TermFrequencyRow) KeySize() int {\n\treturn termFrequencyRowKeySize(tfr.term, tfr.doc)\n}\n\nfunc termFrequencyRowKeySize(term, doc []byte) int {\n\treturn 3 + len(term) + 1 + len(doc)\n}\n\nfunc (tfr *TermFrequencyRow) KeyTo(buf []byte) (int, error) {\n\treturn termFrequencyRowKeyTo(buf, tfr.field, tfr.term, tfr.doc), nil\n}\n\nfunc termFrequencyRowKeyTo(buf []byte, field uint16, term, doc []byte) int {\n\tbuf[0] = 't'\n\tbinary.LittleEndian.PutUint16(buf[1:3], field)\n\ttermLen := copy(buf[3:], term)\n\tbuf[3+termLen] = ByteSeparator\n\tdocLen := copy(buf[3+termLen+1:], doc)\n\treturn 3 + termLen + 1 + docLen\n}\n\nfunc (tfr *TermFrequencyRow) KeyAppendTo(buf []byte) ([]byte, error) {\n\tkeySize := tfr.KeySize()\n\tif cap(buf) < keySize {\n\t\tbuf = make([]byte, keySize)\n\t}\n\tactualSize, err := tfr.KeyTo(buf[0:keySize])\n\treturn buf[0:actualSize], err\n}\n\nfunc (tfr *TermFrequencyRow) DictionaryRowKey() []byte {\n\tdr := NewDictionaryRow(tfr.term, tfr.field, 0)\n\treturn dr.Key()\n}\n\nfunc (tfr *TermFrequencyRow) DictionaryRowKeySize() int {\n\tdr := NewDictionaryRow(tfr.term, tfr.field, 0)\n\treturn dr.KeySize()\n}\n\nfunc (tfr *TermFrequencyRow) DictionaryRowKeyTo(buf []byte) (int, error) {\n\tdr := NewDictionaryRow(tfr.term, tfr.field, 0)\n\treturn dr.KeyTo(buf)\n}\n\nfunc (tfr *TermFrequencyRow) Value() []byte {\n\tbuf := make([]byte, tfr.ValueSize())\n\tsize, _ := tfr.ValueTo(buf)\n\treturn buf[:size]\n}\n\nfunc (tfr *TermFrequencyRow) ValueSize() int {\n\tbufLen := binary.MaxVarintLen64 + binary.MaxVarintLen64\n\tfor _, vector := range tfr.vectors {\n\t\tbufLen += (binary.MaxVarintLen64 * 4) + (1+len(vector.arrayPositions))*binary.MaxVarintLen64\n\t}\n\treturn bufLen\n}\n\nfunc (tfr *TermFrequencyRow) ValueTo(buf []byte) (int, error) {\n\tused := binary.PutUvarint(buf[:binary.MaxVarintLen64], tfr.freq)\n\n\tnormuint32 := math.Float32bits(tfr.norm)\n\tnewbuf := buf[used : used+binary.MaxVarintLen64]\n\tused += binary.PutUvarint(newbuf, uint64(normuint32))\n\n\tfor _, vector := range tfr.vectors {\n\t\tused += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], uint64(vector.field))\n\t\tused += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.pos)\n\t\tused += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.start)\n\t\tused += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.end)\n\t\tused += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], uint64(len(vector.arrayPositions)))\n\t\tfor _, arrayPosition := range vector.arrayPositions {\n\t\t\tused += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], arrayPosition)\n\t\t}\n\t}\n\treturn used, nil\n}\n\nfunc (tfr *TermFrequencyRow) String() string {\n\treturn fmt.Sprintf(\"Term: `%s` Field: %d DocId: `%s` Frequency: %d Norm: %f Vectors: %v\", string(tfr.term), tfr.field, string(tfr.doc), tfr.freq, tfr.norm, tfr.vectors)\n}\n\nfunc InitTermFrequencyRow(tfr *TermFrequencyRow, term []byte, field uint16, docID []byte, freq uint64, norm float32) *TermFrequencyRow {\n\ttfr.term = term\n\ttfr.field = field\n\ttfr.doc = docID\n\ttfr.freq = freq\n\ttfr.norm = norm\n\treturn tfr\n}\n\nfunc NewTermFrequencyRow(term []byte, field uint16, docID []byte, freq uint64, norm float32) *TermFrequencyRow {\n\treturn &TermFrequencyRow{\n\t\tterm:  term,\n\t\tfield: field,\n\t\tdoc:   docID,\n\t\tfreq:  freq,\n\t\tnorm:  norm,\n\t}\n}\n\nfunc NewTermFrequencyRowWithTermVectors(term []byte, field uint16, docID []byte, freq uint64, norm float32, vectors []*TermVector) *TermFrequencyRow {\n\treturn &TermFrequencyRow{\n\t\tterm:    term,\n\t\tfield:   field,\n\t\tdoc:     docID,\n\t\tfreq:    freq,\n\t\tnorm:    norm,\n\t\tvectors: vectors,\n\t}\n}\n\nfunc NewTermFrequencyRowK(key []byte) (*TermFrequencyRow, error) {\n\trv := &TermFrequencyRow{}\n\terr := rv.parseK(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc (tfr *TermFrequencyRow) parseK(key []byte) error {\n\tkeyLen := len(key)\n\tif keyLen < 3 {\n\t\treturn fmt.Errorf(\"invalid term frequency key, no valid field\")\n\t}\n\ttfr.field = binary.LittleEndian.Uint16(key[1:3])\n\n\ttermEndPos := bytes.IndexByte(key[3:], ByteSeparator)\n\tif termEndPos < 0 {\n\t\treturn fmt.Errorf(\"invalid term frequency key, no byte separator terminating term\")\n\t}\n\ttfr.term = key[3 : 3+termEndPos]\n\n\tdocLen := keyLen - (3 + termEndPos + 1)\n\tif docLen < 1 {\n\t\treturn fmt.Errorf(\"invalid term frequency key, empty docid\")\n\t}\n\ttfr.doc = key[3+termEndPos+1:]\n\n\treturn nil\n}\n\nfunc (tfr *TermFrequencyRow) parseKDoc(key []byte, term []byte) error {\n\ttfr.doc = key[3+len(term)+1:]\n\tif len(tfr.doc) == 0 {\n\t\treturn fmt.Errorf(\"invalid term frequency key, empty docid\")\n\t}\n\n\treturn nil\n}\n\nfunc (tfr *TermFrequencyRow) parseV(value []byte, includeTermVectors bool) error {\n\tvar bytesRead int\n\ttfr.freq, bytesRead = binary.Uvarint(value)\n\tif bytesRead <= 0 {\n\t\treturn fmt.Errorf(\"invalid term frequency value, invalid frequency\")\n\t}\n\tcurrOffset := bytesRead\n\n\tvar norm uint64\n\tnorm, bytesRead = binary.Uvarint(value[currOffset:])\n\tif bytesRead <= 0 {\n\t\treturn fmt.Errorf(\"invalid term frequency value, no norm\")\n\t}\n\tcurrOffset += bytesRead\n\n\ttfr.norm = math.Float32frombits(uint32(norm))\n\n\ttfr.vectors = nil\n\tif !includeTermVectors {\n\t\treturn nil\n\t}\n\n\tvar field uint64\n\tfield, bytesRead = binary.Uvarint(value[currOffset:])\n\tfor bytesRead > 0 {\n\t\tcurrOffset += bytesRead\n\t\ttv := TermVector{}\n\t\ttv.field = uint16(field)\n\t\t// at this point we expect at least one term vector\n\t\tif tfr.vectors == nil {\n\t\t\ttfr.vectors = make([]*TermVector, 0)\n\t\t}\n\n\t\ttv.pos, bytesRead = binary.Uvarint(value[currOffset:])\n\t\tif bytesRead <= 0 {\n\t\t\treturn fmt.Errorf(\"invalid term frequency value, vector contains no position\")\n\t\t}\n\t\tcurrOffset += bytesRead\n\n\t\ttv.start, bytesRead = binary.Uvarint(value[currOffset:])\n\t\tif bytesRead <= 0 {\n\t\t\treturn fmt.Errorf(\"invalid term frequency value, vector contains no start\")\n\t\t}\n\t\tcurrOffset += bytesRead\n\n\t\ttv.end, bytesRead = binary.Uvarint(value[currOffset:])\n\t\tif bytesRead <= 0 {\n\t\t\treturn fmt.Errorf(\"invalid term frequency value, vector contains no end\")\n\t\t}\n\t\tcurrOffset += bytesRead\n\n\t\tvar arrayPositionsLen uint64\n\t\tarrayPositionsLen, bytesRead = binary.Uvarint(value[currOffset:])\n\t\tif bytesRead <= 0 {\n\t\t\treturn fmt.Errorf(\"invalid term frequency value, vector contains no arrayPositionLen\")\n\t\t}\n\t\tcurrOffset += bytesRead\n\n\t\tif arrayPositionsLen > 0 {\n\t\t\ttv.arrayPositions = make([]uint64, arrayPositionsLen)\n\t\t\tfor i := 0; uint64(i) < arrayPositionsLen; i++ {\n\t\t\t\ttv.arrayPositions[i], bytesRead = binary.Uvarint(value[currOffset:])\n\t\t\t\tif bytesRead <= 0 {\n\t\t\t\t\treturn fmt.Errorf(\"invalid term frequency value, vector contains no arrayPosition of index %d\", i)\n\t\t\t\t}\n\t\t\t\tcurrOffset += bytesRead\n\t\t\t}\n\t\t}\n\n\t\ttfr.vectors = append(tfr.vectors, &tv)\n\t\t// try to read next record (may not exist)\n\t\tfield, bytesRead = binary.Uvarint(value[currOffset:])\n\t}\n\tif len(value[currOffset:]) > 0 && bytesRead <= 0 {\n\t\treturn fmt.Errorf(\"invalid term frequency value, vector field invalid\")\n\t}\n\n\treturn nil\n}\n\nfunc NewTermFrequencyRowKV(key, value []byte) (*TermFrequencyRow, error) {\n\trv, err := NewTermFrequencyRowK(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\terr = rv.parseV(value, true)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\ntype BackIndexRow struct {\n\tdoc           []byte\n\ttermsEntries  []*BackIndexTermsEntry\n\tstoredEntries []*BackIndexStoreEntry\n}\n\nfunc (br *BackIndexRow) AllTermKeys() [][]byte {\n\tif br == nil {\n\t\treturn nil\n\t}\n\trv := make([][]byte, 0, len(br.termsEntries)) // FIXME this underestimates severely\n\tfor _, termsEntry := range br.termsEntries {\n\t\tfor i := range termsEntry.Terms {\n\t\t\ttermRow := NewTermFrequencyRow([]byte(termsEntry.Terms[i]), uint16(termsEntry.GetField()), br.doc, 0, 0)\n\t\t\trv = append(rv, termRow.Key())\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (br *BackIndexRow) AllStoredKeys() [][]byte {\n\tif br == nil {\n\t\treturn nil\n\t}\n\trv := make([][]byte, len(br.storedEntries))\n\tfor i, storedEntry := range br.storedEntries {\n\t\tstoredRow := NewStoredRow(br.doc, uint16(storedEntry.GetField()), storedEntry.GetArrayPositions(), 'x', []byte{})\n\t\trv[i] = storedRow.Key()\n\t}\n\treturn rv\n}\n\nfunc (br *BackIndexRow) Key() []byte {\n\tbuf := make([]byte, br.KeySize())\n\tsize, _ := br.KeyTo(buf)\n\treturn buf[:size]\n}\n\nfunc (br *BackIndexRow) KeySize() int {\n\treturn len(br.doc) + 1\n}\n\nfunc (br *BackIndexRow) KeyTo(buf []byte) (int, error) {\n\tbuf[0] = 'b'\n\tused := copy(buf[1:], br.doc)\n\treturn used + 1, nil\n}\n\nfunc (br *BackIndexRow) Value() []byte {\n\tbuf := make([]byte, br.ValueSize())\n\tsize, _ := br.ValueTo(buf)\n\treturn buf[:size]\n}\n\nfunc (br *BackIndexRow) ValueSize() int {\n\tbirv := &BackIndexRowValue{\n\t\tTermsEntries:  br.termsEntries,\n\t\tStoredEntries: br.storedEntries,\n\t}\n\treturn birv.Size()\n}\n\nfunc (br *BackIndexRow) ValueTo(buf []byte) (int, error) {\n\tbirv := &BackIndexRowValue{\n\t\tTermsEntries:  br.termsEntries,\n\t\tStoredEntries: br.storedEntries,\n\t}\n\treturn birv.MarshalTo(buf)\n}\n\nfunc (br *BackIndexRow) String() string {\n\treturn fmt.Sprintf(\"Backindex DocId: `%s` Terms Entries: %v, Stored Entries: %v\", string(br.doc), br.termsEntries, br.storedEntries)\n}\n\nfunc NewBackIndexRow(docID []byte, entries []*BackIndexTermsEntry, storedFields []*BackIndexStoreEntry) *BackIndexRow {\n\treturn &BackIndexRow{\n\t\tdoc:           docID,\n\t\ttermsEntries:  entries,\n\t\tstoredEntries: storedFields,\n\t}\n}\n\nfunc NewBackIndexRowKV(key, value []byte) (*BackIndexRow, error) {\n\trv := BackIndexRow{}\n\n\tbuf := bytes.NewBuffer(key)\n\t_, err := buf.ReadByte() // type\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trv.doc, err = buf.ReadBytes(ByteSeparator)\n\tif err == io.EOF && len(rv.doc) < 1 {\n\t\terr = fmt.Errorf(\"invalid doc length 0 - % x\", key)\n\t}\n\tif err != nil && err != io.EOF {\n\t\treturn nil, err\n\t} else if err == nil {\n\t\trv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte\n\t}\n\n\tvar birv BackIndexRowValue\n\terr = proto.Unmarshal(value, &birv)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv.termsEntries = birv.TermsEntries\n\trv.storedEntries = birv.StoredEntries\n\n\treturn &rv, nil\n}\n\n// STORED\n\ntype StoredRow struct {\n\tdoc            []byte\n\tfield          uint16\n\tarrayPositions []uint64\n\ttyp            byte\n\tvalue          []byte\n}\n\nfunc (s *StoredRow) Key() []byte {\n\tbuf := make([]byte, s.KeySize())\n\tsize, _ := s.KeyTo(buf)\n\treturn buf[0:size]\n}\n\nfunc (s *StoredRow) KeySize() int {\n\treturn 1 + len(s.doc) + 1 + 2 + (binary.MaxVarintLen64 * len(s.arrayPositions))\n}\n\nfunc (s *StoredRow) KeyTo(buf []byte) (int, error) {\n\tdocLen := len(s.doc)\n\tbuf[0] = 's'\n\tcopy(buf[1:], s.doc)\n\tbuf[1+docLen] = ByteSeparator\n\tbinary.LittleEndian.PutUint16(buf[1+docLen+1:], s.field)\n\tbytesUsed := 1 + docLen + 1 + 2\n\tfor _, arrayPosition := range s.arrayPositions {\n\t\tvarbytes := binary.PutUvarint(buf[bytesUsed:], arrayPosition)\n\t\tbytesUsed += varbytes\n\t}\n\treturn bytesUsed, nil\n}\n\nfunc (s *StoredRow) Value() []byte {\n\tbuf := make([]byte, s.ValueSize())\n\tsize, _ := s.ValueTo(buf)\n\treturn buf[:size]\n}\n\nfunc (s *StoredRow) ValueSize() int {\n\treturn len(s.value) + 1\n}\n\nfunc (s *StoredRow) ValueTo(buf []byte) (int, error) {\n\tbuf[0] = s.typ\n\tused := copy(buf[1:], s.value)\n\treturn used + 1, nil\n}\n\nfunc (s *StoredRow) String() string {\n\treturn fmt.Sprintf(\"Document: %s Field %d, Array Positions: %v, Type: %s Value: %s\", s.doc, s.field, s.arrayPositions, string(s.typ), s.value)\n}\n\nfunc (s *StoredRow) ScanPrefixForDoc() []byte {\n\tdocLen := len(s.doc)\n\tbuf := make([]byte, 1+docLen+1)\n\tbuf[0] = 's'\n\tcopy(buf[1:], s.doc)\n\tbuf[1+docLen] = ByteSeparator\n\treturn buf\n}\n\nfunc NewStoredRow(docID []byte, field uint16, arrayPositions []uint64, typ byte, value []byte) *StoredRow {\n\treturn &StoredRow{\n\t\tdoc:            docID,\n\t\tfield:          field,\n\t\tarrayPositions: arrayPositions,\n\t\ttyp:            typ,\n\t\tvalue:          value,\n\t}\n}\n\nfunc NewStoredRowK(key []byte) (*StoredRow, error) {\n\trv := StoredRow{}\n\n\tbuf := bytes.NewBuffer(key)\n\t_, err := buf.ReadByte() // type\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trv.doc, err = buf.ReadBytes(ByteSeparator)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator\n\t\terr = fmt.Errorf(\"invalid doc length 0\")\n\t\treturn nil, err\n\t}\n\n\trv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte\n\n\terr = binary.Read(buf, binary.LittleEndian, &rv.field)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trv.arrayPositions = make([]uint64, 0)\n\tnextArrayPos, err := binary.ReadUvarint(buf)\n\tfor err == nil {\n\t\trv.arrayPositions = append(rv.arrayPositions, nextArrayPos)\n\t\tnextArrayPos, err = binary.ReadUvarint(buf)\n\t}\n\treturn &rv, nil\n}\n\nfunc NewStoredRowKV(key, value []byte) (*StoredRow, error) {\n\trv, err := NewStoredRowK(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv.typ = value[0]\n\trv.value = value[1:]\n\treturn rv, nil\n}\n\ntype backIndexFieldTermVisitor func(field uint32, term []byte)\n\n// visitBackIndexRow is designed to process a protobuf encoded\n// value, without creating unnecessary garbage.  Instead values are passed\n// to a callback, inspected first, and only copied if necessary.\n// Due to the fact that this borrows from generated code, it must be marnually\n// updated if the protobuf definition changes.\n//\n// This code originates from:\n// func (m *BackIndexRowValue) Unmarshal(data []byte) error\n// the sections which create garbage or parse uninteresting sections\n// have been commented out.  This was done by design to allow for easier\n// merging in the future if that original function is regenerated\nfunc visitBackIndexRow(data []byte, callback backIndexFieldTermVisitor) error {\n\tl := len(data)\n\tiNdEx := 0\n\tfor iNdEx < l {\n\t\tvar wire uint64\n\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\tif iNdEx >= l {\n\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t}\n\t\t\tb := data[iNdEx]\n\t\t\tiNdEx++\n\t\t\twire |= (uint64(b) & 0x7F) << shift\n\t\t\tif b < 0x80 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tfieldNum := int32(wire >> 3)\n\t\twireType := int(wire & 0x7)\n\t\tswitch fieldNum {\n\t\tcase 1:\n\t\t\tif wireType != 2 {\n\t\t\t\treturn fmt.Errorf(\"proto: wrong wireType = %d for field TermsEntries\", wireType)\n\t\t\t}\n\t\t\tvar msglen int\n\t\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\t\tif iNdEx >= l {\n\t\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t\t}\n\t\t\t\tb := data[iNdEx]\n\t\t\t\tiNdEx++\n\t\t\t\tmsglen |= (int(b) & 0x7F) << shift\n\t\t\t\tif b < 0x80 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tpostIndex := iNdEx + msglen\n\t\t\tif msglen < 0 {\n\t\t\t\treturn ErrInvalidLengthUpsidedown\n\t\t\t}\n\t\t\tif postIndex > l {\n\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t}\n\t\t\t// dont parse term entries\n\t\t\t// m.TermsEntries = append(m.TermsEntries, &BackIndexTermsEntry{})\n\t\t\t// if err := m.TermsEntries[len(m.TermsEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {\n\t\t\t// \treturn err\n\t\t\t// }\n\t\t\t// instead, inspect them\n\t\t\tif err := visitBackIndexRowFieldTerms(data[iNdEx:postIndex], callback); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tiNdEx = postIndex\n\t\tcase 2:\n\t\t\tif wireType != 2 {\n\t\t\t\treturn fmt.Errorf(\"proto: wrong wireType = %d for field StoredEntries\", wireType)\n\t\t\t}\n\t\t\tvar msglen int\n\t\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\t\tif iNdEx >= l {\n\t\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t\t}\n\t\t\t\tb := data[iNdEx]\n\t\t\t\tiNdEx++\n\t\t\t\tmsglen |= (int(b) & 0x7F) << shift\n\t\t\t\tif b < 0x80 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tpostIndex := iNdEx + msglen\n\t\t\tif msglen < 0 {\n\t\t\t\treturn ErrInvalidLengthUpsidedown\n\t\t\t}\n\t\t\tif postIndex > l {\n\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t}\n\t\t\t// don't parse stored entries\n\t\t\t// m.StoredEntries = append(m.StoredEntries, &BackIndexStoreEntry{})\n\t\t\t// if err := m.StoredEntries[len(m.StoredEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {\n\t\t\t// \treturn err\n\t\t\t// }\n\t\t\tiNdEx = postIndex\n\t\tdefault:\n\t\t\tvar sizeOfWire int\n\t\t\tfor {\n\t\t\t\tsizeOfWire++\n\t\t\t\twire >>= 7\n\t\t\t\tif wire == 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tiNdEx -= sizeOfWire\n\t\t\tskippy, err := skipUpsidedown(data[iNdEx:])\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif skippy < 0 {\n\t\t\t\treturn ErrInvalidLengthUpsidedown\n\t\t\t}\n\t\t\tif (iNdEx + skippy) > l {\n\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t}\n\t\t\t// don't track unrecognized data\n\t\t\t// m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)\n\t\t\tiNdEx += skippy\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// visitBackIndexRowFieldTerms is designed to process a protobuf encoded\n// sub-value within the BackIndexRowValue, without creating unnecessary garbage.\n// Instead values are passed to a callback, inspected first, and only copied if\n// necessary.  Due to the fact that this borrows from generated code, it must\n// be marnually updated if the protobuf definition changes.\n//\n// This code originates from:\n// func (m *BackIndexTermsEntry) Unmarshal(data []byte) error {\n// the sections which create garbage or parse uninteresting sections\n// have been commented out.  This was done by design to allow for easier\n// merging in the future if that original function is regenerated\nfunc visitBackIndexRowFieldTerms(data []byte, callback backIndexFieldTermVisitor) error {\n\tvar theField uint32\n\n\tvar hasFields [1]uint64\n\tl := len(data)\n\tiNdEx := 0\n\tfor iNdEx < l {\n\t\tvar wire uint64\n\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\tif iNdEx >= l {\n\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t}\n\t\t\tb := data[iNdEx]\n\t\t\tiNdEx++\n\t\t\twire |= (uint64(b) & 0x7F) << shift\n\t\t\tif b < 0x80 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tfieldNum := int32(wire >> 3)\n\t\twireType := int(wire & 0x7)\n\t\tswitch fieldNum {\n\t\tcase 1:\n\t\t\tif wireType != 0 {\n\t\t\t\treturn fmt.Errorf(\"proto: wrong wireType = %d for field Field\", wireType)\n\t\t\t}\n\t\t\tvar v uint32\n\t\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\t\tif iNdEx >= l {\n\t\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t\t}\n\t\t\t\tb := data[iNdEx]\n\t\t\t\tiNdEx++\n\t\t\t\tv |= (uint32(b) & 0x7F) << shift\n\t\t\t\tif b < 0x80 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\t// m.Field = &v\n\t\t\ttheField = v\n\t\t\thasFields[0] |= uint64(0x00000001)\n\t\tcase 2:\n\t\t\tif wireType != 2 {\n\t\t\t\treturn fmt.Errorf(\"proto: wrong wireType = %d for field Terms\", wireType)\n\t\t\t}\n\t\t\tvar stringLen uint64\n\t\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\t\tif iNdEx >= l {\n\t\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t\t}\n\t\t\t\tb := data[iNdEx]\n\t\t\t\tiNdEx++\n\t\t\t\tstringLen |= (uint64(b) & 0x7F) << shift\n\t\t\t\tif b < 0x80 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tpostIndex := iNdEx + int(stringLen)\n\t\t\tif postIndex > l {\n\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t}\n\t\t\t// m.Terms = append(m.Terms, string(data[iNdEx:postIndex]))\n\t\t\tcallback(theField, data[iNdEx:postIndex])\n\t\t\tiNdEx = postIndex\n\t\tdefault:\n\t\t\tvar sizeOfWire int\n\t\t\tfor {\n\t\t\t\tsizeOfWire++\n\t\t\t\twire >>= 7\n\t\t\t\tif wire == 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tiNdEx -= sizeOfWire\n\t\t\tskippy, err := skipUpsidedown(data[iNdEx:])\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif skippy < 0 {\n\t\t\t\treturn ErrInvalidLengthUpsidedown\n\t\t\t}\n\t\t\tif (iNdEx + skippy) > l {\n\t\t\t\treturn io.ErrUnexpectedEOF\n\t\t\t}\n\t\t\t// m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)\n\t\t\tiNdEx += skippy\n\t\t}\n\t}\n\t// if hasFields[0]&uint64(0x00000001) == 0 {\n\t// \treturn new(github_com_golang_protobuf_proto.RequiredNotSetError)\n\t// }\n\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/row_merge.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"encoding/binary\"\n)\n\nvar mergeOperator upsideDownMerge\n\nvar dictionaryTermIncr []byte\nvar dictionaryTermDecr []byte\n\nfunc init() {\n\tdictionaryTermIncr = make([]byte, 8)\n\tbinary.LittleEndian.PutUint64(dictionaryTermIncr, uint64(1))\n\tdictionaryTermDecr = make([]byte, 8)\n\tvar negOne = int64(-1)\n\tbinary.LittleEndian.PutUint64(dictionaryTermDecr, uint64(negOne))\n}\n\ntype upsideDownMerge struct{}\n\nfunc (m *upsideDownMerge) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) {\n\t// set up record based on key\n\tdr, err := NewDictionaryRowK(key)\n\tif err != nil {\n\t\treturn nil, false\n\t}\n\tif len(existingValue) > 0 {\n\t\t// if existing value, parse it\n\t\terr = dr.parseDictionaryV(existingValue)\n\t\tif err != nil {\n\t\t\treturn nil, false\n\t\t}\n\t}\n\n\t// now process operands\n\tfor _, operand := range operands {\n\t\tnext := int64(binary.LittleEndian.Uint64(operand))\n\t\tif next < 0 && uint64(-next) > dr.count {\n\t\t\t// subtracting next from existing would overflow\n\t\t\tdr.count = 0\n\t\t} else if next < 0 {\n\t\t\tdr.count -= uint64(-next)\n\t\t} else {\n\t\t\tdr.count += uint64(next)\n\t\t}\n\t}\n\n\treturn dr.Value(), true\n}\n\nfunc (m *upsideDownMerge) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) {\n\tleft := int64(binary.LittleEndian.Uint64(leftOperand))\n\tright := int64(binary.LittleEndian.Uint64(rightOperand))\n\trv := make([]byte, 8)\n\tbinary.LittleEndian.PutUint64(rv, uint64(left+right))\n\treturn rv, true\n}\n\nfunc (m *upsideDownMerge) Name() string {\n\treturn \"upsideDownMerge\"\n}\n"
  },
  {
    "path": "index/upsidedown/row_merge_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"testing\"\n)\n\nfunc TestPartialMerge(t *testing.T) {\n\n\ttests := []struct {\n\t\tin  [][]byte\n\t\tout uint64\n\t}{\n\t\t{\n\t\t\tin:  [][]byte{dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr},\n\t\t\tout: 5,\n\t\t},\n\t}\n\n\tmo := &upsideDownMerge{}\n\tfor _, test := range tests {\n\t\tcurr := test.in[0]\n\t\tfor _, next := range test.in[1:] {\n\t\t\tvar ok bool\n\t\t\tcurr, ok = mo.PartialMerge([]byte(\"key\"), curr, next)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"expected partial merge ok\")\n\t\t\t}\n\t\t}\n\t\tactual := decodeCount(curr)\n\t\tif actual != test.out {\n\t\t\tt.Errorf(\"expected %d, got %d\", test.out, actual)\n\t\t}\n\t}\n\n}\n\nfunc decodeCount(in []byte) uint64 {\n\tbuf := bytes.NewBuffer(in)\n\tcount, _ := binary.ReadUvarint(buf)\n\treturn count\n}\n"
  },
  {
    "path": "index/upsidedown/row_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"math\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"google.golang.org/protobuf/proto\"\n)\n\nfunc TestRows(t *testing.T) {\n\ttests := []struct {\n\t\tinput  UpsideDownCouchRow\n\t\toutKey []byte\n\t\toutVal []byte\n\t}{\n\t\t{\n\t\t\tNewVersionRow(1),\n\t\t\t[]byte{'v'},\n\t\t\t[]byte{0x1},\n\t\t},\n\t\t{\n\t\t\tNewFieldRow(0, \"name\"),\n\t\t\t[]byte{'f', 0, 0},\n\t\t\t[]byte{'n', 'a', 'm', 'e', ByteSeparator},\n\t\t},\n\t\t{\n\t\t\tNewFieldRow(1, \"desc\"),\n\t\t\t[]byte{'f', 1, 0},\n\t\t\t[]byte{'d', 'e', 's', 'c', ByteSeparator},\n\t\t},\n\t\t{\n\t\t\tNewFieldRow(513, \"style\"),\n\t\t\t[]byte{'f', 1, 2},\n\t\t\t[]byte{'s', 't', 'y', 'l', 'e', ByteSeparator},\n\t\t},\n\t\t{\n\t\t\tNewDictionaryRow([]byte{'b', 'e', 'e', 'r'}, 0, 27),\n\t\t\t[]byte{'d', 0, 0, 'b', 'e', 'e', 'r'},\n\t\t\t[]byte{27},\n\t\t},\n\t\t{\n\t\t\tNewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, []byte(\"catz\"), 3, 3.14),\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'c', 'a', 't', 'z'},\n\t\t\t[]byte{3, 195, 235, 163, 130, 4},\n\t\t},\n\t\t{\n\t\t\tNewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, []byte(\"budweiser\"), 3, 3.14),\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{3, 195, 235, 163, 130, 4},\n\t\t},\n\t\t{\n\t\t\tNewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte(\"budweiser\"), 3, 3.14, []*TermVector{{field: 0, pos: 1, start: 3, end: 11}, {field: 0, pos: 2, start: 23, end: 31}, {field: 0, pos: 3, start: 43, end: 51}}),\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 0, 2, 23, 31, 0, 0, 3, 43, 51, 0},\n\t\t},\n\t\t// test larger varints\n\t\t{\n\t\t\tNewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte(\"budweiser\"), 25896, 3.14, []*TermVector{{field: 255, pos: 1, start: 3, end: 11}, {field: 0, pos: 2198, start: 23, end: 31}, {field: 0, pos: 3, start: 43, end: 51}}),\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{168, 202, 1, 195, 235, 163, 130, 4, 255, 1, 1, 3, 11, 0, 0, 150, 17, 23, 31, 0, 0, 3, 43, 51, 0},\n\t\t},\n\t\t// test vectors with arrayPositions\n\t\t{\n\t\t\tNewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte(\"budweiser\"), 25896, 3.14, []*TermVector{{field: 255, pos: 1, start: 3, end: 11, arrayPositions: []uint64{0}}, {field: 0, pos: 2198, start: 23, end: 31, arrayPositions: []uint64{1, 2}}, {field: 0, pos: 3, start: 43, end: 51, arrayPositions: []uint64{3, 4, 5}}}),\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{168, 202, 1, 195, 235, 163, 130, 4, 255, 1, 1, 3, 11, 1, 0, 0, 150, 17, 23, 31, 2, 1, 2, 0, 3, 43, 51, 3, 3, 4, 5},\n\t\t},\n\t\t{\n\t\t\tNewBackIndexRow([]byte(\"budweiser\"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{\"beer\"}}}, nil),\n\t\t\t[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r'},\n\t\t},\n\t\t{\n\t\t\tNewBackIndexRow([]byte(\"budweiser\"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{\"beer\"}}, {Field: proto.Uint32(1), Terms: []string{\"beat\"}}}, nil),\n\t\t\t[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't'},\n\t\t},\n\t\t{\n\t\t\tNewBackIndexRow([]byte(\"budweiser\"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{\"beer\"}}, {Field: proto.Uint32(1), Terms: []string{\"beat\"}}}, []*BackIndexStoreEntry{{Field: proto.Uint32(3)}, {Field: proto.Uint32(4)}, {Field: proto.Uint32(5)}}),\n\t\t\t[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't', 18, 2, 8, 3, 18, 2, 8, 4, 18, 2, 8, 5},\n\t\t},\n\t\t{\n\t\t\tNewStoredRow([]byte(\"budweiser\"), 0, []uint64{}, byte('t'), []byte(\"an american beer\")),\n\t\t\t[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0},\n\t\t\t[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},\n\t\t},\n\t\t{\n\t\t\tNewStoredRow([]byte(\"budweiser\"), 0, []uint64{2, 294, 3078}, byte('t'), []byte(\"an american beer\")),\n\t\t\t[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0, 2, 166, 2, 134, 24},\n\t\t\t[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},\n\t\t},\n\t\t{\n\t\t\tNewInternalRow([]byte(\"mapping\"), []byte(`{\"mapping\":\"json content\"}`)),\n\t\t\t[]byte{'i', 'm', 'a', 'p', 'p', 'i', 'n', 'g'},\n\t\t\t[]byte{'{', '\"', 'm', 'a', 'p', 'p', 'i', 'n', 'g', '\"', ':', '\"', 'j', 's', 'o', 'n', ' ', 'c', 'o', 'n', 't', 'e', 'n', 't', '\"', '}'},\n\t\t},\n\t}\n\n\t// test going from struct to k/v bytes\n\tfor i, test := range tests {\n\t\trk := test.input.Key()\n\t\tif !reflect.DeepEqual(rk, test.outKey) {\n\t\t\tt.Errorf(\"Expected key to be %v got: %v\", test.outKey, rk)\n\t\t}\n\t\trv := test.input.Value()\n\t\tif !reflect.DeepEqual(rv, test.outVal) {\n\t\t\tt.Errorf(\"Expected value to be %v got: %v for %d\", test.outVal, rv, i)\n\t\t}\n\t}\n\n\t// now test going back from k/v bytes to struct\n\tfor i, test := range tests {\n\t\trow, err := ParseFromKeyValue(test.outKey, test.outVal)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"error parsking key/value: %v\", err)\n\t\t}\n\t\tif !reflect.DeepEqual(row, test.input) {\n\t\t\tt.Errorf(\"Expected: %#v got: %#v for %d\", test.input, row, i)\n\t\t}\n\t}\n\n}\n\nfunc TestInvalidRows(t *testing.T) {\n\ttests := []struct {\n\t\tkey []byte\n\t\tval []byte\n\t}{\n\t\t// empty key\n\t\t{\n\t\t\t[]byte{},\n\t\t\t[]byte{},\n\t\t},\n\t\t// no such type q\n\t\t{\n\t\t\t[]byte{'q'},\n\t\t\t[]byte{},\n\t\t},\n\t\t// type v, invalid empty value\n\t\t{\n\t\t\t[]byte{'v'},\n\t\t\t[]byte{},\n\t\t},\n\t\t// type f, invalid key\n\t\t{\n\t\t\t[]byte{'f'},\n\t\t\t[]byte{},\n\t\t},\n\t\t// type f, valid key, invalid value\n\t\t{\n\t\t\t[]byte{'f', 0, 0},\n\t\t\t[]byte{},\n\t\t},\n\t\t// type t, invalid key (missing field)\n\t\t{\n\t\t\t[]byte{'t'},\n\t\t\t[]byte{},\n\t\t},\n\t\t// type t, invalid key (missing term)\n\t\t{\n\t\t\t[]byte{'t', 0, 0},\n\t\t\t[]byte{},\n\t\t},\n\t\t// type t, invalid key (missing id)\n\t\t{\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator},\n\t\t\t[]byte{},\n\t\t},\n\t\t// type t, invalid val (missing freq)\n\t\t{\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{},\n\t\t},\n\t\t// type t, invalid val (missing norm)\n\t\t{\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{3},\n\t\t},\n\t\t// type t, invalid val (half missing tv field, full missing is valid (no term vectors))\n\t\t{\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{3, 25, 255},\n\t\t},\n\t\t// type t, invalid val (missing tv pos)\n\t\t{\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{3, 25, 0},\n\t\t},\n\t\t// type t, invalid val (missing tv start)\n\t\t{\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{3, 25, 0, 0},\n\t\t},\n\t\t// type t, invalid val (missing tv end)\n\t\t{\n\t\t\t[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{3, 25, 0, 0, 0},\n\t\t},\n\t\t// type b, invalid key (missing id)\n\t\t{\n\t\t\t[]byte{'b'},\n\t\t\t[]byte{'b', 'e', 'e', 'r', ByteSeparator, 0, 0},\n\t\t},\n\t\t// type b, invalid val (missing field)\n\t\t{\n\t\t\t[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},\n\t\t\t[]byte{'g', 'a', 'r', 'b', 'a', 'g', 'e'},\n\t\t},\n\t\t// type s, invalid key (missing id)\n\t\t{\n\t\t\t[]byte{'s'},\n\t\t\t[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},\n\t\t},\n\t\t// type b, invalid val (missing field)\n\t\t{\n\t\t\t[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator},\n\t\t\t[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\t_, err := ParseFromKeyValue(test.key, test.val)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"expected error, got nil\")\n\t\t}\n\t}\n}\n\nfunc TestDictionaryRowValueBug197(t *testing.T) {\n\t// this was the smallest value that would trigger a crash\n\tdr := &DictionaryRow{\n\t\tfield: 0,\n\t\tterm:  []byte(\"marty\"),\n\t\tcount: 72057594037927936,\n\t}\n\tdr.Value()\n\t// this is the maximum possible value\n\tdr = &DictionaryRow{\n\t\tfield: 0,\n\t\tterm:  []byte(\"marty\"),\n\t\tcount: math.MaxUint64,\n\t}\n\tdr.Value()\n\t// neither of these should panic\n}\n\nfunc BenchmarkTermFrequencyRowEncode(b *testing.B) {\n\trow := NewTermFrequencyRowWithTermVectors(\n\t\t[]byte{'b', 'e', 'e', 'r'},\n\t\t0,\n\t\t[]byte(\"budweiser\"),\n\t\t3,\n\t\t3.14,\n\t\t[]*TermVector{\n\t\t\t{\n\t\t\t\tfield: 0,\n\t\t\t\tpos:   1,\n\t\t\t\tstart: 3,\n\t\t\t\tend:   11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tfield: 0,\n\t\t\t\tpos:   2,\n\t\t\t\tstart: 23,\n\t\t\t\tend:   31,\n\t\t\t},\n\t\t\t{\n\t\t\t\tfield: 0,\n\t\t\t\tpos:   3,\n\t\t\t\tstart: 43,\n\t\t\t\tend:   51,\n\t\t\t},\n\t\t})\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\trow.Key()\n\t\trow.Value()\n\t}\n}\n\nfunc BenchmarkTermFrequencyRowDecode(b *testing.B) {\n\tk := []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}\n\tv := []byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 0, 2, 23, 31, 0, 0, 3, 43, 51, 0}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := NewTermFrequencyRowKV(k, v)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkBackIndexRowEncode(b *testing.B) {\n\tfield := uint32(1)\n\tt1 := \"term1\"\n\trow := NewBackIndexRow([]byte(\"beername\"),\n\t\t[]*BackIndexTermsEntry{\n\t\t\t{\n\t\t\t\tField: &field,\n\t\t\t\tTerms: []string{t1},\n\t\t\t},\n\t\t},\n\t\t[]*BackIndexStoreEntry{\n\t\t\t{\n\t\t\t\tField: &field,\n\t\t\t},\n\t\t})\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\trow.Key()\n\t\trow.Value()\n\t\tb.Logf(\"%#v\", row.Value())\n\t}\n}\n\nfunc BenchmarkBackIndexRowDecode(b *testing.B) {\n\tk := []byte{0x62, 0x62, 0x65, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65}\n\tv := []byte{0xa, 0x9, 0x8, 0x1, 0x12, 0x5, 0x74, 0x65, 0x72, 0x6d, 0x31, 0x12, 0x2, 0x8, 0x1}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := NewBackIndexRowKV(k, v)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkStoredRowEncode(b *testing.B) {\n\trow := NewStoredRow([]byte(\"budweiser\"), 0, []uint64{}, byte('t'), []byte(\"an american beer\"))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\trow.Key()\n\t\trow.Value()\n\t}\n}\n\nfunc BenchmarkStoredRowDecode(b *testing.B) {\n\tk := []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0}\n\tv := []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := NewStoredRowKV(k, v)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc TestVisitBackIndexRow(t *testing.T) {\n\texpected := map[uint32][]byte{\n\t\t0: []byte(\"beer\"),\n\t\t1: []byte(\"beat\"),\n\t}\n\tval := []byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't', 18, 2, 8, 3, 18, 2, 8, 4, 18, 2, 8, 5}\n\terr := visitBackIndexRow(val, func(field uint32, term []byte) {\n\t\tif reflect.DeepEqual(expected[field], term) {\n\t\t\tdelete(expected, field)\n\t\t}\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(expected) > 0 {\n\t\tt.Errorf(\"expected visitor to see these but did not %v\", expected)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/stats.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"sync/atomic\"\n\n\t\"github.com/blevesearch/bleve/v2/util\"\n\t\"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype indexStat struct {\n\tupdates, deletes, batches, errors uint64\n\tanalysisTime, indexTime           uint64\n\ttermSearchersStarted              uint64\n\ttermSearchersFinished             uint64\n\tnumPlainTextBytesIndexed          uint64\n\ti                                 *UpsideDownCouch\n}\n\nfunc (i *indexStat) statsMap() map[string]interface{} {\n\tm := map[string]interface{}{}\n\tm[\"updates\"] = atomic.LoadUint64(&i.updates)\n\tm[\"deletes\"] = atomic.LoadUint64(&i.deletes)\n\tm[\"batches\"] = atomic.LoadUint64(&i.batches)\n\tm[\"errors\"] = atomic.LoadUint64(&i.errors)\n\tm[\"analysis_time\"] = atomic.LoadUint64(&i.analysisTime)\n\tm[\"index_time\"] = atomic.LoadUint64(&i.indexTime)\n\tm[\"term_searchers_started\"] = atomic.LoadUint64(&i.termSearchersStarted)\n\tm[\"term_searchers_finished\"] = atomic.LoadUint64(&i.termSearchersFinished)\n\tm[\"num_plain_text_bytes_indexed\"] = atomic.LoadUint64(&i.numPlainTextBytesIndexed)\n\n\tif o, ok := i.i.store.(store.KVStoreStats); ok {\n\t\tm[\"kv\"] = o.StatsMap()\n\t}\n\n\treturn m\n}\n\nfunc (i *indexStat) MarshalJSON() ([]byte, error) {\n\tm := i.statsMap()\n\treturn util.MarshalJSON(m)\n}\n"
  },
  {
    "path": "index/upsidedown/store/boltdb/iterator.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage boltdb\n\nimport (\n\t\"bytes\"\n\n\tbolt \"go.etcd.io/bbolt\"\n)\n\ntype Iterator struct {\n\tstore  *Store\n\ttx     *bolt.Tx\n\tcursor *bolt.Cursor\n\tprefix []byte\n\tstart  []byte\n\tend    []byte\n\tvalid  bool\n\tkey    []byte\n\tval    []byte\n}\n\nfunc (i *Iterator) updateValid() {\n\ti.valid = (i.key != nil)\n\tif i.valid {\n\t\tif i.prefix != nil {\n\t\t\ti.valid = bytes.HasPrefix(i.key, i.prefix)\n\t\t} else if i.end != nil {\n\t\t\ti.valid = bytes.Compare(i.key, i.end) < 0\n\t\t}\n\t}\n}\n\nfunc (i *Iterator) Seek(k []byte) {\n\tif i.start != nil && bytes.Compare(k, i.start) < 0 {\n\t\tk = i.start\n\t}\n\tif i.prefix != nil && !bytes.HasPrefix(k, i.prefix) {\n\t\tif bytes.Compare(k, i.prefix) < 0 {\n\t\t\tk = i.prefix\n\t\t} else {\n\t\t\ti.valid = false\n\t\t\treturn\n\t\t}\n\t}\n\ti.key, i.val = i.cursor.Seek(k)\n\ti.updateValid()\n}\n\nfunc (i *Iterator) Next() {\n\ti.key, i.val = i.cursor.Next()\n\ti.updateValid()\n}\n\nfunc (i *Iterator) Current() ([]byte, []byte, bool) {\n\treturn i.key, i.val, i.valid\n}\n\nfunc (i *Iterator) Key() []byte {\n\treturn i.key\n}\n\nfunc (i *Iterator) Value() []byte {\n\treturn i.val\n}\n\nfunc (i *Iterator) Valid() bool {\n\treturn i.valid\n}\n\nfunc (i *Iterator) Close() error {\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/boltdb/reader.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage boltdb\n\nimport (\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\tbolt \"go.etcd.io/bbolt\"\n)\n\ntype Reader struct {\n\tstore  *Store\n\ttx     *bolt.Tx\n\tbucket *bolt.Bucket\n}\n\nfunc (r *Reader) Get(key []byte) ([]byte, error) {\n\tvar rv []byte\n\tv := r.bucket.Get(key)\n\tif v != nil {\n\t\trv = make([]byte, len(v))\n\t\tcopy(rv, v)\n\t}\n\treturn rv, nil\n}\n\nfunc (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {\n\treturn store.MultiGet(r, keys)\n}\n\nfunc (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {\n\tcursor := r.bucket.Cursor()\n\n\trv := &Iterator{\n\t\tstore:  r.store,\n\t\ttx:     r.tx,\n\t\tcursor: cursor,\n\t\tprefix: prefix,\n\t}\n\n\trv.Seek(prefix)\n\treturn rv\n}\n\nfunc (r *Reader) RangeIterator(start, end []byte) store.KVIterator {\n\tcursor := r.bucket.Cursor()\n\n\trv := &Iterator{\n\t\tstore:  r.store,\n\t\ttx:     r.tx,\n\t\tcursor: cursor,\n\t\tstart:  start,\n\t\tend:    end,\n\t}\n\n\trv.Seek(start)\n\treturn rv\n}\n\nfunc (r *Reader) Close() error {\n\treturn r.tx.Rollback()\n}\n"
  },
  {
    "path": "index/upsidedown/store/boltdb/stats.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage boltdb\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/util\"\n)\n\ntype stats struct {\n\ts *Store\n}\n\nfunc (s *stats) MarshalJSON() ([]byte, error) {\n\tbs := s.s.db.Stats()\n\treturn util.MarshalJSON(bs)\n}\n"
  },
  {
    "path": "index/upsidedown/store/boltdb/store.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package boltdb implements a store.KVStore on top of BoltDB. It supports the\n// following options:\n//\n// \"bucket\" (string): the name of BoltDB bucket to use, defaults to \"bleve\".\n//\n// \"nosync\" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index\n// operations in exchange of losing integrity guarantees if indexation aborts\n// without closing the index. Use it when rebuilding indexes from zero.\npackage boltdb\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\tbolt \"go.etcd.io/bbolt\"\n)\n\nconst (\n\tName                    = \"boltdb\"\n\tdefaultCompactBatchSize = 100\n)\n\ntype Store struct {\n\tpath        string\n\tbucket      string\n\tdb          *bolt.DB\n\tnoSync      bool\n\tfillPercent float64\n\tmo          store.MergeOperator\n}\n\nfunc New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {\n\tpath, ok := config[\"path\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify path\")\n\t}\n\tif path == \"\" {\n\t\treturn nil, os.ErrInvalid\n\t}\n\n\tbucket, ok := config[\"bucket\"].(string)\n\tif !ok {\n\t\tbucket = \"bleve\"\n\t}\n\n\tnoSync, _ := config[\"nosync\"].(bool)\n\n\tfillPercent, ok := config[\"fillPercent\"].(float64)\n\tif !ok {\n\t\tfillPercent = bolt.DefaultFillPercent\n\t}\n\n\tbo := &bolt.Options{}\n\tro, ok := config[\"read_only\"].(bool)\n\tif ok {\n\t\tbo.ReadOnly = ro\n\t}\n\n\tif initialMmapSize, ok := config[\"initialMmapSize\"].(int); ok {\n\t\tbo.InitialMmapSize = initialMmapSize\n\t} else if initialMmapSize, ok := config[\"initialMmapSize\"].(float64); ok {\n\t\tbo.InitialMmapSize = int(initialMmapSize)\n\t}\n\n\tdb, err := bolt.Open(path, 0600, bo)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdb.NoSync = noSync\n\n\tif !bo.ReadOnly {\n\t\terr = db.Update(func(tx *bolt.Tx) error {\n\t\t\t_, err := tx.CreateBucketIfNotExists([]byte(bucket))\n\n\t\t\treturn err\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\trv := Store{\n\t\tpath:        path,\n\t\tbucket:      bucket,\n\t\tdb:          db,\n\t\tmo:          mo,\n\t\tnoSync:      noSync,\n\t\tfillPercent: fillPercent,\n\t}\n\treturn &rv, nil\n}\n\nfunc (bs *Store) Close() error {\n\treturn bs.db.Close()\n}\n\nfunc (bs *Store) Reader() (store.KVReader, error) {\n\ttx, err := bs.db.Begin(false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Reader{\n\t\tstore:  bs,\n\t\ttx:     tx,\n\t\tbucket: tx.Bucket([]byte(bs.bucket)),\n\t}, nil\n}\n\nfunc (bs *Store) Writer() (store.KVWriter, error) {\n\treturn &Writer{\n\t\tstore: bs,\n\t}, nil\n}\n\nfunc (bs *Store) Stats() json.Marshaler {\n\treturn &stats{\n\t\ts: bs,\n\t}\n}\n\n// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches)\n// Removing entries is a workaround for github issue #374.\nfunc (bs *Store) CompactWithBatchSize(batchSize int) error {\n\tfor {\n\t\tcnt := 0\n\t\terr := bs.db.Batch(func(tx *bolt.Tx) error {\n\t\t\tc := tx.Bucket([]byte(bs.bucket)).Cursor()\n\t\t\tprefix := []byte(\"d\")\n\n\t\t\tfor k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {\n\t\t\t\tif bytes.Equal(v, []byte{0}) {\n\t\t\t\t\tcnt++\n\t\t\t\t\tif err := c.Delete(); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif cnt == batchSize {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif cnt == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn nil\n}\n\n// Compact calls CompactWithBatchSize with a default batch size of 100.  This is a workaround\n// for github issue #374.\nfunc (bs *Store) Compact() error {\n\treturn bs.CompactWithBatchSize(defaultCompactBatchSize)\n}\n\nfunc init() {\n\terr := registry.RegisterKVStore(Name, New)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/store/boltdb/store_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !darwin || !arm64\n\npackage boltdb\n\nimport (\n\t\"os\"\n\t\"testing\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\t\"github.com/blevesearch/upsidedown_store_api/test\"\n\tbolt \"go.etcd.io/bbolt\"\n)\n\nfunc open(t *testing.T, mo store.MergeOperator) store.KVStore {\n\trv, err := New(mo, map[string]interface{}{\"path\": \"test\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn rv\n}\n\nfunc cleanup(t *testing.T, s store.KVStore) {\n\terr := s.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = os.RemoveAll(\"test\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestBoltDBKVCrud(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestKVCrud(t, s)\n}\n\nfunc TestBoltDBReaderIsolation(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderIsolation(t, s)\n}\n\nfunc TestBoltDBReaderOwnsGetBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderOwnsGetBytes(t, s)\n}\n\nfunc TestBoltDBWriterOwnsBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestWriterOwnsBytes(t, s)\n}\n\nfunc TestBoltDBPrefixIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIterator(t, s)\n}\n\nfunc TestBoltDBPrefixIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIteratorSeek(t, s)\n}\n\nfunc TestBoltDBRangeIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIterator(t, s)\n}\n\nfunc TestBoltDBRangeIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIteratorSeek(t, s)\n}\n\nfunc TestBoltDBMerge(t *testing.T) {\n\ts := open(t, &test.TestMergeCounter{})\n\tdefer cleanup(t, s)\n\ttest.CommonTestMerge(t, s)\n}\n\nfunc TestBoltDBConfig(t *testing.T) {\n\tvar tests = []struct {\n\t\tin          map[string]interface{}\n\t\tpath        string\n\t\tbucket      string\n\t\tnoSync      bool\n\t\tfillPercent float64\n\t}{\n\t\t{\n\t\t\tmap[string]interface{}{\"path\": \"test\", \"bucket\": \"mybucket\", \"nosync\": true, \"fillPercent\": 0.75},\n\t\t\t\"test\",\n\t\t\t\"mybucket\",\n\t\t\ttrue,\n\t\t\t0.75,\n\t\t},\n\t\t{\n\t\t\tmap[string]interface{}{\"path\": \"test\"},\n\t\t\t\"test\",\n\t\t\t\"bleve\",\n\t\t\tfalse,\n\t\t\tbolt.DefaultFillPercent,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tkv, err := New(nil, test.in)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tbs, ok := kv.(*Store)\n\t\tif !ok {\n\t\t\tt.Fatal(\"failed type assertion to *boltdb.Store\")\n\t\t}\n\t\tif bs.path != test.path {\n\t\t\tt.Fatalf(\"path: expected %q, got %q\", test.path, bs.path)\n\t\t}\n\t\tif bs.bucket != test.bucket {\n\t\t\tt.Fatalf(\"bucket: expected %q, got %q\", test.bucket, bs.bucket)\n\t\t}\n\t\tif bs.noSync != test.noSync {\n\t\t\tt.Fatalf(\"noSync: expected %t, got %t\", test.noSync, bs.noSync)\n\t\t}\n\t\tif bs.fillPercent != test.fillPercent {\n\t\t\tt.Fatalf(\"fillPercent: expected %f, got %f\", test.fillPercent, bs.fillPercent)\n\t\t}\n\t\tcleanup(t, kv)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/store/boltdb/writer.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage boltdb\n\nimport (\n\t\"fmt\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype Writer struct {\n\tstore *Store\n}\n\nfunc (w *Writer) NewBatch() store.KVBatch {\n\treturn store.NewEmulatedBatch(w.store.mo)\n}\n\nfunc (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {\n\treturn make([]byte, options.TotalBytes), w.NewBatch(), nil\n}\n\nfunc (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) {\n\n\temulatedBatch, ok := batch.(*store.EmulatedBatch)\n\tif !ok {\n\t\treturn fmt.Errorf(\"wrong type of batch\")\n\t}\n\n\ttx, err := w.store.db.Begin(true)\n\tif err != nil {\n\t\treturn\n\t}\n\t// defer function to ensure that once started,\n\t// we either Commit tx or Rollback\n\tdefer func() {\n\t\t// if nothing went wrong, commit\n\t\tif err == nil {\n\t\t\t// careful to catch error here too\n\t\t\terr = tx.Commit()\n\t\t} else {\n\t\t\t// caller should see error that caused abort,\n\t\t\t// not success or failure of Rollback itself\n\t\t\t_ = tx.Rollback()\n\t\t}\n\t}()\n\n\tbucket := tx.Bucket([]byte(w.store.bucket))\n\tbucket.FillPercent = w.store.fillPercent\n\n\tfor k, mergeOps := range emulatedBatch.Merger.Merges {\n\t\tkb := []byte(k)\n\t\texistingVal := bucket.Get(kb)\n\t\tmergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)\n\t\tif !fullMergeOk {\n\t\t\terr = fmt.Errorf(\"merge operator returned failure\")\n\t\t\treturn\n\t\t}\n\t\terr = bucket.Put(kb, mergedVal)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\tfor _, op := range emulatedBatch.Ops {\n\t\tif op.V != nil {\n\t\t\terr = bucket.Put(op.K, op.V)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t} else {\n\t\t\terr = bucket.Delete(op.K)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n\nfunc (w *Writer) Close() error {\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/goleveldb/batch.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage goleveldb\n\nimport (\n\t\"github.com/blevesearch/goleveldb/leveldb\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype Batch struct {\n\tstore *Store\n\tmerge *store.EmulatedMerge\n\tbatch *leveldb.Batch\n}\n\nfunc (b *Batch) Set(key, val []byte) {\n\tb.batch.Put(key, val)\n}\n\nfunc (b *Batch) Delete(key []byte) {\n\tb.batch.Delete(key)\n}\n\nfunc (b *Batch) Merge(key, val []byte) {\n\tb.merge.Merge(key, val)\n}\n\nfunc (b *Batch) Reset() {\n\tb.batch.Reset()\n\tb.merge = store.NewEmulatedMerge(b.store.mo)\n}\n\nfunc (b *Batch) Close() error {\n\tb.batch.Reset()\n\tb.batch = nil\n\tb.merge = nil\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/goleveldb/config.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage goleveldb\n\nimport (\n\t\"github.com/blevesearch/goleveldb/leveldb/filter\"\n\t\"github.com/blevesearch/goleveldb/leveldb/opt\"\n)\n\nfunc applyConfig(o *opt.Options, config map[string]interface{}) (*opt.Options, error) {\n\n\tro, ok := config[\"read_only\"].(bool)\n\tif ok {\n\t\to.ReadOnly = ro\n\t}\n\n\tcim, ok := config[\"create_if_missing\"].(bool)\n\tif ok {\n\t\to.ErrorIfMissing = !cim\n\t}\n\n\teie, ok := config[\"error_if_exists\"].(bool)\n\tif ok {\n\t\to.ErrorIfExist = eie\n\t}\n\n\twbs, ok := config[\"write_buffer_size\"].(float64)\n\tif ok {\n\t\to.WriteBuffer = int(wbs)\n\t}\n\n\tbs, ok := config[\"block_size\"].(float64)\n\tif ok {\n\t\to.BlockSize = int(bs)\n\t}\n\n\tbri, ok := config[\"block_restart_interval\"].(float64)\n\tif ok {\n\t\to.BlockRestartInterval = int(bri)\n\t}\n\n\tlcc, ok := config[\"lru_cache_capacity\"].(float64)\n\tif ok {\n\t\to.BlockCacheCapacity = int(lcc)\n\t}\n\n\tbfbpk, ok := config[\"bloom_filter_bits_per_key\"].(float64)\n\tif ok {\n\t\tbf := filter.NewBloomFilter(int(bfbpk))\n\t\to.Filter = bf\n\t}\n\n\treturn o, nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/goleveldb/iterator.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage goleveldb\n\nimport \"github.com/blevesearch/goleveldb/leveldb/iterator\"\n\ntype Iterator struct {\n\tstore    *Store\n\titerator iterator.Iterator\n}\n\nfunc (ldi *Iterator) Seek(key []byte) {\n\tldi.iterator.Seek(key)\n}\n\nfunc (ldi *Iterator) Next() {\n\tldi.iterator.Next()\n}\n\nfunc (ldi *Iterator) Current() ([]byte, []byte, bool) {\n\tif ldi.Valid() {\n\t\treturn ldi.Key(), ldi.Value(), true\n\t}\n\treturn nil, nil, false\n}\n\nfunc (ldi *Iterator) Key() []byte {\n\treturn ldi.iterator.Key()\n}\n\nfunc (ldi *Iterator) Value() []byte {\n\treturn ldi.iterator.Value()\n}\n\nfunc (ldi *Iterator) Valid() bool {\n\treturn ldi.iterator.Valid()\n}\n\nfunc (ldi *Iterator) Close() error {\n\tldi.iterator.Release()\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/goleveldb/reader.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage goleveldb\n\nimport (\n\t\"github.com/blevesearch/goleveldb/leveldb\"\n\t\"github.com/blevesearch/goleveldb/leveldb/util\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype Reader struct {\n\tstore    *Store\n\tsnapshot *leveldb.Snapshot\n}\n\nfunc (r *Reader) Get(key []byte) ([]byte, error) {\n\tb, err := r.snapshot.Get(key, r.store.defaultReadOptions)\n\tif err == leveldb.ErrNotFound {\n\t\treturn nil, nil\n\t}\n\treturn b, err\n}\n\nfunc (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {\n\treturn store.MultiGet(r, keys)\n}\n\nfunc (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {\n\tbyteRange := util.BytesPrefix(prefix)\n\titer := r.snapshot.NewIterator(byteRange, r.store.defaultReadOptions)\n\titer.First()\n\trv := Iterator{\n\t\tstore:    r.store,\n\t\titerator: iter,\n\t}\n\treturn &rv\n}\n\nfunc (r *Reader) RangeIterator(start, end []byte) store.KVIterator {\n\tbyteRange := &util.Range{\n\t\tStart: start,\n\t\tLimit: end,\n\t}\n\titer := r.snapshot.NewIterator(byteRange, r.store.defaultReadOptions)\n\titer.First()\n\trv := Iterator{\n\t\tstore:    r.store,\n\t\titerator: iter,\n\t}\n\treturn &rv\n}\n\nfunc (r *Reader) Close() error {\n\tr.snapshot.Release()\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/goleveldb/store.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage goleveldb\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/goleveldb/leveldb\"\n\t\"github.com/blevesearch/goleveldb/leveldb/opt\"\n\t\"github.com/blevesearch/goleveldb/leveldb/util\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\nconst (\n\tName                    = \"goleveldb\"\n\tdefaultCompactBatchSize = 250\n)\n\ntype Store struct {\n\tpath string\n\topts *opt.Options\n\tdb   *leveldb.DB\n\tmo   store.MergeOperator\n\n\tdefaultWriteOptions *opt.WriteOptions\n\tdefaultReadOptions  *opt.ReadOptions\n}\n\nfunc New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {\n\n\tpath, ok := config[\"path\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify path\")\n\t}\n\tif path == \"\" {\n\t\treturn nil, os.ErrInvalid\n\t}\n\n\topts, err := applyConfig(&opt.Options{}, config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdb, err := leveldb.OpenFile(path, opts)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trv := Store{\n\t\tpath:                path,\n\t\topts:                opts,\n\t\tdb:                  db,\n\t\tmo:                  mo,\n\t\tdefaultReadOptions:  &opt.ReadOptions{},\n\t\tdefaultWriteOptions: &opt.WriteOptions{},\n\t}\n\trv.defaultWriteOptions.Sync = true\n\treturn &rv, nil\n}\n\nfunc (ldbs *Store) Close() error {\n\treturn ldbs.db.Close()\n}\n\nfunc (ldbs *Store) Reader() (store.KVReader, error) {\n\tsnapshot, _ := ldbs.db.GetSnapshot()\n\treturn &Reader{\n\t\tstore:    ldbs,\n\t\tsnapshot: snapshot,\n\t}, nil\n}\n\nfunc (ldbs *Store) Writer() (store.KVWriter, error) {\n\treturn &Writer{\n\t\tstore: ldbs,\n\t}, nil\n}\n\n// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches), then\n// compacts the underlying goleveldb store.  Removing entries is a workaround for github issue #374.\nfunc (ldbs *Store) CompactWithBatchSize(batchSize int) error {\n\t// workaround for github issue #374 - remove DictionaryTerm keys with count=0\n\tbatch := &leveldb.Batch{}\n\tfor {\n\t\tt, err := ldbs.db.OpenTransaction()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\titer := t.NewIterator(util.BytesPrefix([]byte(\"d\")), ldbs.defaultReadOptions)\n\n\t\tfor iter.Next() {\n\t\t\tif bytes.Equal(iter.Value(), []byte{0}) {\n\t\t\t\tk := append([]byte{}, iter.Key()...)\n\t\t\t\tbatch.Delete(k)\n\t\t\t}\n\t\t\tif batch.Len() == batchSize {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\titer.Release()\n\t\tif iter.Error() != nil {\n\t\t\tt.Discard()\n\t\t\treturn iter.Error()\n\t\t}\n\n\t\tif batch.Len() > 0 {\n\t\t\terr := t.Write(batch, ldbs.defaultWriteOptions)\n\t\t\tif err != nil {\n\t\t\t\tt.Discard()\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terr = t.Commit()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tt.Discard()\n\t\t\tbreak\n\t\t}\n\t\tbatch.Reset()\n\t}\n\n\treturn ldbs.db.CompactRange(util.Range{Start: nil, Limit: nil})\n}\n\n// Compact compacts the underlying goleveldb store.  The current implementation includes a workaround\n// for github issue #374 (see CompactWithBatchSize).\nfunc (ldbs *Store) Compact() error {\n\treturn ldbs.CompactWithBatchSize(defaultCompactBatchSize)\n}\n\nfunc init() {\n\terr := registry.RegisterKVStore(Name, New)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/store/goleveldb/store_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage goleveldb\n\nimport (\n\t\"os\"\n\t\"testing\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\t\"github.com/blevesearch/upsidedown_store_api/test\"\n)\n\nfunc open(t *testing.T, mo store.MergeOperator) store.KVStore {\n\trv, err := New(mo, map[string]interface{}{\n\t\t\"path\":              \"test\",\n\t\t\"create_if_missing\": true,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn rv\n}\n\nfunc cleanup(t *testing.T, s store.KVStore) {\n\terr := s.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = os.RemoveAll(\"test\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestGoLevelDBKVCrud(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestKVCrud(t, s)\n}\n\nfunc TestGoLevelDBReaderIsolation(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderIsolation(t, s)\n}\n\nfunc TestGoLevelDBReaderOwnsGetBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderOwnsGetBytes(t, s)\n}\n\nfunc TestGoLevelDBWriterOwnsBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestWriterOwnsBytes(t, s)\n}\n\nfunc TestGoLevelDBPrefixIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIterator(t, s)\n}\n\nfunc TestGoLevelDBPrefixIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIteratorSeek(t, s)\n}\n\nfunc TestGoLevelDBRangeIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIterator(t, s)\n}\n\nfunc TestGoLevelDBRangeIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIteratorSeek(t, s)\n}\n\nfunc TestGoLevelDBMerge(t *testing.T) {\n\ts := open(t, &test.TestMergeCounter{})\n\tdefer cleanup(t, s)\n\ttest.CommonTestMerge(t, s)\n}\n"
  },
  {
    "path": "index/upsidedown/store/goleveldb/writer.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage goleveldb\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/goleveldb/leveldb\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype Writer struct {\n\tstore *Store\n}\n\nfunc (w *Writer) NewBatch() store.KVBatch {\n\trv := Batch{\n\t\tstore: w.store,\n\t\tmerge: store.NewEmulatedMerge(w.store.mo),\n\t\tbatch: new(leveldb.Batch),\n\t}\n\treturn &rv\n}\n\nfunc (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {\n\treturn make([]byte, options.TotalBytes), w.NewBatch(), nil\n}\n\nfunc (w *Writer) ExecuteBatch(b store.KVBatch) error {\n\tbatch, ok := b.(*Batch)\n\tif !ok {\n\t\treturn fmt.Errorf(\"wrong type of batch\")\n\t}\n\n\t// first process merges\n\tfor k, mergeOps := range batch.merge.Merges {\n\t\tkb := []byte(k)\n\t\texistingVal, err := w.store.db.Get(kb, w.store.defaultReadOptions)\n\t\tif err != nil && err != leveldb.ErrNotFound {\n\t\t\treturn err\n\t\t}\n\t\tmergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)\n\t\tif !fullMergeOk {\n\t\t\treturn fmt.Errorf(\"merge operator returned failure\")\n\t\t}\n\t\t// add the final merge to this batch\n\t\tbatch.batch.Put(kb, mergedVal)\n\t}\n\n\t// now execute the batch\n\treturn w.store.db.Write(batch.batch, w.store.defaultWriteOptions)\n}\n\nfunc (w *Writer) Close() error {\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/gtreap/iterator.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package gtreap provides an in-memory implementation of the\n// KVStore interfaces using the gtreap balanced-binary treap,\n// copy-on-write data structure.\npackage gtreap\n\nimport (\n\t\"bytes\"\n\t\"sync\"\n\n\t\"github.com/blevesearch/gtreap\"\n)\n\ntype Iterator struct {\n\tt *gtreap.Treap\n\n\tm        sync.Mutex\n\tcancelCh chan struct{}\n\tnextCh   chan *Item\n\tcurr     *Item\n\tcurrOk   bool\n\n\tprefix []byte\n\tstart  []byte\n\tend    []byte\n}\n\nfunc (w *Iterator) Seek(k []byte) {\n\tif w.start != nil && bytes.Compare(k, w.start) < 0 {\n\t\tk = w.start\n\t}\n\tif w.prefix != nil && !bytes.HasPrefix(k, w.prefix) {\n\t\tif bytes.Compare(k, w.prefix) < 0 {\n\t\t\tk = w.prefix\n\t\t} else {\n\t\t\tvar end []byte\n\t\t\tfor i := len(w.prefix) - 1; i >= 0; i-- {\n\t\t\t\tc := w.prefix[i]\n\t\t\t\tif c < 0xff {\n\t\t\t\t\tend = make([]byte, i+1)\n\t\t\t\t\tcopy(end, w.prefix)\n\t\t\t\t\tend[i] = c + 1\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tk = end\n\t\t}\n\t}\n\tw.restart(&Item{k: k})\n}\n\nfunc (w *Iterator) restart(start *Item) *Iterator {\n\tcancelCh := make(chan struct{})\n\tnextCh := make(chan *Item, 1)\n\n\tw.m.Lock()\n\tif w.cancelCh != nil {\n\t\tclose(w.cancelCh)\n\t}\n\tw.cancelCh = cancelCh\n\tw.nextCh = nextCh\n\tw.curr = nil\n\tw.currOk = false\n\tw.m.Unlock()\n\n\tgo func() {\n\t\tif start != nil {\n\t\t\tw.t.VisitAscend(start, func(itm gtreap.Item) bool {\n\t\t\t\tselect {\n\t\t\t\tcase <-cancelCh:\n\t\t\t\t\treturn false\n\t\t\t\tcase nextCh <- itm.(*Item):\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t\tclose(nextCh)\n\t}()\n\n\tw.Next()\n\n\treturn w\n}\n\nfunc (w *Iterator) Next() {\n\tw.m.Lock()\n\tnextCh := w.nextCh\n\tw.m.Unlock()\n\tw.curr, w.currOk = <-nextCh\n}\n\nfunc (w *Iterator) Current() ([]byte, []byte, bool) {\n\tw.m.Lock()\n\tdefer w.m.Unlock()\n\tif !w.currOk || w.curr == nil {\n\t\treturn nil, nil, false\n\t}\n\tif w.prefix != nil && !bytes.HasPrefix(w.curr.k, w.prefix) {\n\t\treturn nil, nil, false\n\t} else if w.end != nil && bytes.Compare(w.curr.k, w.end) >= 0 {\n\t\treturn nil, nil, false\n\t}\n\treturn w.curr.k, w.curr.v, w.currOk\n}\n\nfunc (w *Iterator) Key() []byte {\n\tk, _, ok := w.Current()\n\tif !ok {\n\t\treturn nil\n\t}\n\treturn k\n}\n\nfunc (w *Iterator) Value() []byte {\n\t_, v, ok := w.Current()\n\tif !ok {\n\t\treturn nil\n\t}\n\treturn v\n}\n\nfunc (w *Iterator) Valid() bool {\n\t_, _, ok := w.Current()\n\treturn ok\n}\n\nfunc (w *Iterator) Close() error {\n\tw.m.Lock()\n\tif w.cancelCh != nil {\n\t\tclose(w.cancelCh)\n\t}\n\tw.cancelCh = nil\n\tw.nextCh = nil\n\tw.curr = nil\n\tw.currOk = false\n\tw.m.Unlock()\n\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/gtreap/reader.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package gtreap provides an in-memory implementation of the\n// KVStore interfaces using the gtreap balanced-binary treap,\n// copy-on-write data structure.\npackage gtreap\n\nimport (\n\t\"github.com/blevesearch/upsidedown_store_api\"\n\n\t\"github.com/blevesearch/gtreap\"\n)\n\ntype Reader struct {\n\tt *gtreap.Treap\n}\n\nfunc (w *Reader) Get(k []byte) (v []byte, err error) {\n\tvar rv []byte\n\titm := w.t.Get(&Item{k: k})\n\tif itm != nil {\n\t\trv = make([]byte, len(itm.(*Item).v))\n\t\tcopy(rv, itm.(*Item).v)\n\t\treturn rv, nil\n\t}\n\treturn nil, nil\n}\n\nfunc (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {\n\treturn store.MultiGet(r, keys)\n}\n\nfunc (w *Reader) PrefixIterator(k []byte) store.KVIterator {\n\trv := Iterator{\n\t\tt:      w.t,\n\t\tprefix: k,\n\t}\n\trv.restart(&Item{k: k})\n\treturn &rv\n}\n\nfunc (w *Reader) RangeIterator(start, end []byte) store.KVIterator {\n\trv := Iterator{\n\t\tt:     w.t,\n\t\tstart: start,\n\t\tend:   end,\n\t}\n\trv.restart(&Item{k: start})\n\treturn &rv\n}\n\nfunc (w *Reader) Close() error {\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/gtreap/store.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package gtreap provides an in-memory implementation of the\n// KVStore interfaces using the gtreap balanced-binary treap,\n// copy-on-write data structure.\n\npackage gtreap\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/gtreap\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\nconst Name = \"gtreap\"\n\ntype Store struct {\n\tm  sync.Mutex\n\tt  *gtreap.Treap\n\tmo store.MergeOperator\n}\n\ntype Item struct {\n\tk []byte\n\tv []byte\n}\n\nfunc itemCompare(a, b interface{}) int {\n\treturn bytes.Compare(a.(*Item).k, b.(*Item).k)\n}\n\nfunc New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {\n\tpath, ok := config[\"path\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify path\")\n\t}\n\tif path != \"\" {\n\t\treturn nil, os.ErrInvalid\n\t}\n\n\trv := Store{\n\t\tt:  gtreap.NewTreap(itemCompare),\n\t\tmo: mo,\n\t}\n\treturn &rv, nil\n}\n\nfunc (s *Store) Close() error {\n\treturn nil\n}\n\nfunc (s *Store) Reader() (store.KVReader, error) {\n\ts.m.Lock()\n\tt := s.t\n\ts.m.Unlock()\n\treturn &Reader{t: t}, nil\n}\n\nfunc (s *Store) Writer() (store.KVWriter, error) {\n\treturn &Writer{s: s}, nil\n}\n\nfunc init() {\n\terr := registry.RegisterKVStore(Name, New)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/store/gtreap/store_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage gtreap\n\nimport (\n\t\"testing\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\t\"github.com/blevesearch/upsidedown_store_api/test\"\n)\n\nfunc open(t *testing.T, mo store.MergeOperator) store.KVStore {\n\trv, err := New(mo, map[string]interface{}{\n\t\t\"path\": \"\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn rv\n}\n\nfunc cleanup(t *testing.T, s store.KVStore) {\n\terr := s.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestGTreapKVCrud(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestKVCrud(t, s)\n}\n\nfunc TestGTreapReaderIsolation(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderIsolation(t, s)\n}\n\nfunc TestGTreapReaderOwnsGetBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderOwnsGetBytes(t, s)\n}\n\nfunc TestGTreapWriterOwnsBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestWriterOwnsBytes(t, s)\n}\n\nfunc TestGTreapPrefixIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIterator(t, s)\n}\n\nfunc TestGTreapPrefixIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIteratorSeek(t, s)\n}\n\nfunc TestGTreapRangeIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIterator(t, s)\n}\n\nfunc TestGTreapRangeIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIteratorSeek(t, s)\n}\n\nfunc TestGTreapMerge(t *testing.T) {\n\ts := open(t, &test.TestMergeCounter{})\n\tdefer cleanup(t, s)\n\ttest.CommonTestMerge(t, s)\n}\n"
  },
  {
    "path": "index/upsidedown/store/gtreap/writer.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package gtreap provides an in-memory implementation of the\n// KVStore interfaces using the gtreap balanced-binary treap,\n// copy-on-write data structure.\npackage gtreap\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\n\t\"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype Writer struct {\n\ts *Store\n}\n\nfunc (w *Writer) NewBatch() store.KVBatch {\n\treturn store.NewEmulatedBatch(w.s.mo)\n}\n\nfunc (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {\n\treturn make([]byte, options.TotalBytes), w.NewBatch(), nil\n}\n\nfunc (w *Writer) ExecuteBatch(batch store.KVBatch) error {\n\n\temulatedBatch, ok := batch.(*store.EmulatedBatch)\n\tif !ok {\n\t\treturn fmt.Errorf(\"wrong type of batch\")\n\t}\n\n\tw.s.m.Lock()\n\tfor k, mergeOps := range emulatedBatch.Merger.Merges {\n\t\tkb := []byte(k)\n\t\tvar existingVal []byte\n\t\texistingItem := w.s.t.Get(&Item{k: kb})\n\t\tif existingItem != nil {\n\t\t\texistingVal = w.s.t.Get(&Item{k: kb}).(*Item).v\n\t\t}\n\t\tmergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps)\n\t\tif !fullMergeOk {\n\t\t\treturn fmt.Errorf(\"merge operator returned failure\")\n\t\t}\n\t\tw.s.t = w.s.t.Upsert(&Item{k: kb, v: mergedVal}, rand.Int())\n\t}\n\n\tfor _, op := range emulatedBatch.Ops {\n\t\tif op.V != nil {\n\t\t\tw.s.t = w.s.t.Upsert(&Item{k: op.K, v: op.V}, rand.Int())\n\t\t} else {\n\t\t\tw.s.t = w.s.t.Delete(&Item{k: op.K})\n\t\t}\n\t}\n\tw.s.m.Unlock()\n\n\treturn nil\n}\n\nfunc (w *Writer) Close() error {\n\tw.s = nil\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/batch.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport store \"github.com/blevesearch/upsidedown_store_api\"\n\ntype Batch struct {\n\ts *Store\n\to store.KVBatch\n}\n\nfunc (b *Batch) Set(key, val []byte) {\n\tb.o.Set(key, val)\n}\n\nfunc (b *Batch) Delete(key []byte) {\n\tb.o.Delete(key)\n}\n\nfunc (b *Batch) Merge(key, val []byte) {\n\tb.s.timerBatchMerge.Time(func() {\n\t\tb.o.Merge(key, val)\n\t})\n}\n\nfunc (b *Batch) Reset() {\n\tb.o.Reset()\n}\n\nfunc (b *Batch) Close() error {\n\terr := b.o.Close()\n\tb.o = nil\n\treturn err\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/iterator.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport store \"github.com/blevesearch/upsidedown_store_api\"\n\ntype Iterator struct {\n\ts *Store\n\to store.KVIterator\n}\n\nfunc (i *Iterator) Seek(x []byte) {\n\ti.s.timerIteratorSeek.Time(func() {\n\t\ti.o.Seek(x)\n\t})\n}\n\nfunc (i *Iterator) Next() {\n\ti.s.timerIteratorNext.Time(func() {\n\t\ti.o.Next()\n\t})\n}\n\nfunc (i *Iterator) Current() ([]byte, []byte, bool) {\n\treturn i.o.Current()\n}\n\nfunc (i *Iterator) Key() []byte {\n\treturn i.o.Key()\n}\n\nfunc (i *Iterator) Value() []byte {\n\treturn i.o.Value()\n}\n\nfunc (i *Iterator) Valid() bool {\n\treturn i.o.Valid()\n}\n\nfunc (i *Iterator) Close() error {\n\terr := i.o.Close()\n\tif err != nil {\n\t\ti.s.AddError(\"Iterator.Close\", err, nil)\n\t}\n\treturn err\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/metrics_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n)\n\nfunc TestMetricsStore(t *testing.T) {\n\t_, err := New(nil, map[string]interface{}{})\n\tif err == nil {\n\t\tt.Errorf(\"expected err when bad config\")\n\t}\n\n\t_, err = New(nil, map[string]interface{}{\n\t\t\"kvStoreName_actual\": \"some-invalid-kvstore-name\",\n\t})\n\tif err == nil {\n\t\tt.Errorf(\"expected err when unknown kvStoreName_actual\")\n\t}\n\n\ts, err := New(nil, map[string]interface{}{\n\t\t\"kvStoreName_actual\": gtreap.Name,\n\t\t\"path\":               \"\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\terr = s.(*Store).WriteJSON(b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif b.Len() <= 0 {\n\t\tt.Errorf(\"expected some output from WriteJSON\")\n\t}\n\tvar m map[string]interface{}\n\terr = json.Unmarshal(b.Bytes(), &m)\n\tif err != nil {\n\t\tt.Errorf(\"expected WriteJSON to be unmarshallable\")\n\t}\n\tif len(m) == 0 {\n\t\tt.Errorf(\"expected some entries\")\n\t}\n\n\tb = bytes.NewBuffer(nil)\n\ts.(*Store).WriteCSVHeader(b)\n\tif b.Len() <= 0 {\n\t\tt.Errorf(\"expected some output from WriteCSVHeader\")\n\t}\n\n\tb = bytes.NewBuffer(nil)\n\ts.(*Store).WriteCSV(b)\n\tif b.Len() <= 0 {\n\t\tt.Errorf(\"expected some output from WriteCSV\")\n\t}\n}\n\nfunc TestErrors(t *testing.T) {\n\ts, err := New(nil, map[string]interface{}{\n\t\t\"kvStoreName_actual\": gtreap.Name,\n\t\t\"path\":               \"\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tx, ok := s.(*Store)\n\tif !ok {\n\t\tt.Errorf(\"expecting a Store\")\n\t}\n\n\tx.AddError(\"foo\", fmt.Errorf(\"Foo\"), []byte(\"fooKey\"))\n\tx.AddError(\"bar\", fmt.Errorf(\"Bar\"), nil)\n\tx.AddError(\"baz\", fmt.Errorf(\"Baz\"), []byte(\"bazKey\"))\n\n\tb := bytes.NewBuffer(nil)\n\terr = x.WriteJSON(b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar m map[string]interface{}\n\terr = json.Unmarshal(b.Bytes(), &m)\n\tif err != nil {\n\t\tt.Errorf(\"expected unmarshallable writeJSON, err: %v, b: %s\",\n\t\t\terr, b.Bytes())\n\t}\n\n\terrorsi, ok := m[\"Errors\"]\n\tif !ok || errorsi == nil {\n\t\tt.Errorf(\"expected errorsi\")\n\t}\n\terrors, ok := errorsi.([]interface{})\n\tif !ok || errors == nil {\n\t\tt.Errorf(\"expected errorsi is array\")\n\t}\n\tif len(errors) != 3 {\n\t\tt.Errorf(\"expected errors len 3\")\n\t}\n\n\te := errors[0].(map[string]interface{})\n\tif e[\"Op\"].(string) != \"foo\" ||\n\t\te[\"Err\"].(string) != \"Foo\" ||\n\t\tlen(e[\"Time\"].(string)) < 10 ||\n\t\te[\"Key\"].(string) != \"fooKey\" {\n\t\tt.Errorf(\"expected foo, %#v\", e)\n\t}\n\te = errors[1].(map[string]interface{})\n\tif e[\"Op\"].(string) != \"bar\" ||\n\t\te[\"Err\"].(string) != \"Bar\" ||\n\t\tlen(e[\"Time\"].(string)) < 10 ||\n\t\te[\"Key\"].(string) != \"\" {\n\t\tt.Errorf(\"expected bar, %#v\", e)\n\t}\n\te = errors[2].(map[string]interface{})\n\tif e[\"Op\"].(string) != \"baz\" ||\n\t\te[\"Err\"].(string) != \"Baz\" ||\n\t\tlen(e[\"Time\"].(string)) < 10 ||\n\t\te[\"Key\"].(string) != \"bazKey\" {\n\t\tt.Errorf(\"expected baz, %#v\", e)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/reader.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport store \"github.com/blevesearch/upsidedown_store_api\"\n\ntype Reader struct {\n\ts *Store\n\to store.KVReader\n}\n\nfunc (r *Reader) Get(key []byte) (v []byte, err error) {\n\tr.s.timerReaderGet.Time(func() {\n\t\tv, err = r.o.Get(key)\n\t\tif err != nil {\n\t\t\tr.s.AddError(\"Reader.Get\", err, key)\n\t\t}\n\t})\n\treturn\n}\n\nfunc (r *Reader) MultiGet(keys [][]byte) (vals [][]byte, err error) {\n\tr.s.timerReaderMultiGet.Time(func() {\n\t\tvals, err = r.o.MultiGet(keys)\n\t\tif err != nil {\n\t\t\tr.s.AddError(\"Reader.MultiGet\", err, nil)\n\t\t}\n\t})\n\treturn\n}\n\nfunc (r *Reader) PrefixIterator(prefix []byte) (i store.KVIterator) {\n\tr.s.timerReaderPrefixIterator.Time(func() {\n\t\ti = &Iterator{s: r.s, o: r.o.PrefixIterator(prefix)}\n\t})\n\treturn\n}\n\nfunc (r *Reader) RangeIterator(start, end []byte) (i store.KVIterator) {\n\tr.s.timerReaderRangeIterator.Time(func() {\n\t\ti = &Iterator{s: r.s, o: r.o.RangeIterator(start, end)}\n\t})\n\treturn\n}\n\nfunc (r *Reader) Close() error {\n\terr := r.o.Close()\n\tif err != nil {\n\t\tr.s.AddError(\"Reader.Close\", err, nil)\n\t}\n\treturn err\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/stats.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype stats struct {\n\ts *Store\n}\n\nfunc (s *stats) statsMap() map[string]interface{} {\n\tms := map[string]interface{}{}\n\n\tms[\"metrics\"] = map[string]interface{}{\n\t\t\"reader_get\":             TimerMap(s.s.timerReaderGet),\n\t\t\"reader_multi_get\":       TimerMap(s.s.timerReaderMultiGet),\n\t\t\"reader_prefix_iterator\": TimerMap(s.s.timerReaderPrefixIterator),\n\t\t\"reader_range_iterator\":  TimerMap(s.s.timerReaderRangeIterator),\n\t\t\"writer_execute_batch\":   TimerMap(s.s.timerWriterExecuteBatch),\n\t\t\"iterator_seek\":          TimerMap(s.s.timerIteratorSeek),\n\t\t\"iterator_next\":          TimerMap(s.s.timerIteratorNext),\n\t\t\"batch_merge\":            TimerMap(s.s.timerBatchMerge),\n\t}\n\n\tif o, ok := s.s.o.(store.KVStoreStats); ok {\n\t\tms[\"kv\"] = o.StatsMap()\n\t}\n\n\treturn ms\n}\n\nfunc (s *stats) MarshalJSON() ([]byte, error) {\n\tm := s.statsMap()\n\treturn util.MarshalJSON(m)\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/store.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package metrics provides a bleve.store.KVStore implementation that\n// wraps another, real KVStore implementation, and uses go-metrics to\n// track runtime performance metrics.\npackage metrics\n\nimport (\n\t\"container/list\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\t\"github.com/blevesearch/go-metrics\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\nconst Name = \"metrics\"\n\ntype Store struct {\n\to store.KVStore\n\n\ttimerReaderGet            metrics.Timer\n\ttimerReaderMultiGet       metrics.Timer\n\ttimerReaderPrefixIterator metrics.Timer\n\ttimerReaderRangeIterator  metrics.Timer\n\ttimerWriterExecuteBatch   metrics.Timer\n\ttimerIteratorSeek         metrics.Timer\n\ttimerIteratorNext         metrics.Timer\n\ttimerBatchMerge           metrics.Timer\n\n\tm      sync.Mutex // Protects the fields that follow.\n\terrors *list.List // Capped list of StoreError's.\n\n\ts *stats\n}\n\nfunc New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {\n\n\tname, ok := config[\"kvStoreName_actual\"].(string)\n\tif !ok || name == \"\" {\n\t\treturn nil, fmt.Errorf(\"metrics: missing kvStoreName_actual,\"+\n\t\t\t\" config: %#v\", config)\n\t}\n\n\tif name == Name {\n\t\treturn nil, fmt.Errorf(\"metrics: circular kvStoreName_actual\")\n\t}\n\n\tctr := registry.KVStoreConstructorByName(name)\n\tif ctr == nil {\n\t\treturn nil, fmt.Errorf(\"metrics: no kv store constructor,\"+\n\t\t\t\" kvStoreName_actual: %s\", name)\n\t}\n\n\tkvs, err := ctr(mo, config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trv := &Store{\n\t\to: kvs,\n\n\t\ttimerReaderGet:            metrics.NewTimer(),\n\t\ttimerReaderMultiGet:       metrics.NewTimer(),\n\t\ttimerReaderPrefixIterator: metrics.NewTimer(),\n\t\ttimerReaderRangeIterator:  metrics.NewTimer(),\n\t\ttimerWriterExecuteBatch:   metrics.NewTimer(),\n\t\ttimerIteratorSeek:         metrics.NewTimer(),\n\t\ttimerIteratorNext:         metrics.NewTimer(),\n\t\ttimerBatchMerge:           metrics.NewTimer(),\n\n\t\terrors: list.New(),\n\t}\n\n\trv.s = &stats{s: rv}\n\n\treturn rv, nil\n}\n\nfunc init() {\n\terr := registry.RegisterKVStore(Name, New)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc (s *Store) Close() error {\n\treturn s.o.Close()\n}\n\nfunc (s *Store) Reader() (store.KVReader, error) {\n\to, err := s.o.Reader()\n\tif err != nil {\n\t\ts.AddError(\"Reader\", err, nil)\n\t\treturn nil, err\n\t}\n\treturn &Reader{s: s, o: o}, nil\n}\n\nfunc (s *Store) Writer() (store.KVWriter, error) {\n\to, err := s.o.Writer()\n\tif err != nil {\n\t\ts.AddError(\"Writer\", err, nil)\n\t\treturn nil, err\n\t}\n\treturn &Writer{s: s, o: o}, nil\n}\n\n// Metric specific code below:\n\nconst MaxErrors = 100\n\ntype StoreError struct {\n\tTime string\n\tOp   string\n\tErr  string\n\tKey  string\n}\n\nfunc (s *Store) AddError(op string, err error, key []byte) {\n\te := &StoreError{\n\t\tTime: time.Now().Format(time.RFC3339Nano),\n\t\tOp:   op,\n\t\tErr:  fmt.Sprintf(\"%v\", err),\n\t\tKey:  string(key),\n\t}\n\n\ts.m.Lock()\n\tfor s.errors.Len() >= MaxErrors {\n\t\ts.errors.Remove(s.errors.Front())\n\t}\n\ts.errors.PushBack(e)\n\ts.m.Unlock()\n}\n\nfunc (s *Store) WriteJSON(w io.Writer) (err error) {\n\t_, err = w.Write([]byte(`{\"TimerReaderGet\":`))\n\tif err != nil {\n\t\treturn\n\t}\n\tWriteTimerJSON(w, s.timerReaderGet)\n\t_, err = w.Write([]byte(`,\"TimerReaderMultiGet\":`))\n\tif err != nil {\n\t\treturn\n\t}\n\tWriteTimerJSON(w, s.timerReaderMultiGet)\n\t_, err = w.Write([]byte(`,\"TimerReaderPrefixIterator\":`))\n\tif err != nil {\n\t\treturn\n\t}\n\tWriteTimerJSON(w, s.timerReaderPrefixIterator)\n\t_, err = w.Write([]byte(`,\"TimerReaderRangeIterator\":`))\n\tif err != nil {\n\t\treturn\n\t}\n\tWriteTimerJSON(w, s.timerReaderRangeIterator)\n\t_, err = w.Write([]byte(`,\"TimerWriterExecuteBatch\":`))\n\tif err != nil {\n\t\treturn\n\t}\n\tWriteTimerJSON(w, s.timerWriterExecuteBatch)\n\t_, err = w.Write([]byte(`,\"TimerIteratorSeek\":`))\n\tif err != nil {\n\t\treturn\n\t}\n\tWriteTimerJSON(w, s.timerIteratorSeek)\n\t_, err = w.Write([]byte(`,\"TimerIteratorNext\":`))\n\tif err != nil {\n\t\treturn\n\t}\n\tWriteTimerJSON(w, s.timerIteratorNext)\n\t_, err = w.Write([]byte(`,\"TimerBatchMerge\":`))\n\tif err != nil {\n\t\treturn\n\t}\n\tWriteTimerJSON(w, s.timerBatchMerge)\n\n\t_, err = w.Write([]byte(`,\"Errors\":[`))\n\tif err != nil {\n\t\treturn\n\t}\n\ts.m.Lock()\n\tdefer s.m.Unlock()\n\te := s.errors.Front()\n\ti := 0\n\tfor e != nil {\n\t\tse, ok := e.Value.(*StoreError)\n\t\tif ok && se != nil {\n\t\t\tif i > 0 {\n\t\t\t\t_, err = w.Write([]byte(\",\"))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tvar buf []byte\n\t\t\tbuf, err = util.MarshalJSON(se)\n\t\t\tif err == nil {\n\t\t\t\t_, err = w.Write(buf)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\te = e.Next()\n\t\ti = i + 1\n\t}\n\t_, err = w.Write([]byte(`]`))\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// see if the underlying implementation has its own stats\n\tif o, ok := s.o.(store.KVStoreStats); ok {\n\t\tstoreStats := o.Stats()\n\t\tvar storeBytes []byte\n\t\tstoreBytes, err = util.MarshalJSON(storeStats)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\t_, err = fmt.Fprintf(w, `, \"store\": %s`, string(storeBytes))\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\t_, err = w.Write([]byte(`}`))\n\tif err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n\nfunc (s *Store) WriteCSVHeader(w io.Writer) {\n\tWriteTimerCSVHeader(w, \"TimerReaderGet\")\n\tWriteTimerCSVHeader(w, \"TimerReaderPrefixIterator\")\n\tWriteTimerCSVHeader(w, \"TimerReaderRangeIterator\")\n\tWriteTimerCSVHeader(w, \"TimerWtierExecuteBatch\")\n\tWriteTimerCSVHeader(w, \"TimerIteratorSeek\")\n\tWriteTimerCSVHeader(w, \"TimerIteratorNext\")\n\tWriteTimerCSVHeader(w, \"TimerBatchMerge\")\n}\n\nfunc (s *Store) WriteCSV(w io.Writer) {\n\tWriteTimerCSV(w, s.timerReaderGet)\n\tWriteTimerCSV(w, s.timerReaderPrefixIterator)\n\tWriteTimerCSV(w, s.timerReaderRangeIterator)\n\tWriteTimerCSV(w, s.timerWriterExecuteBatch)\n\tWriteTimerCSV(w, s.timerIteratorSeek)\n\tWriteTimerCSV(w, s.timerIteratorNext)\n\tWriteTimerCSV(w, s.timerBatchMerge)\n}\n\nfunc (s *Store) Stats() json.Marshaler {\n\treturn s.s\n}\n\nfunc (s *Store) StatsMap() map[string]interface{} {\n\treturn s.s.statsMap()\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/store_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\t\"github.com/blevesearch/upsidedown_store_api/test\"\n)\n\nfunc open(t *testing.T, mo store.MergeOperator) store.KVStore {\n\trv, err := New(mo, map[string]interface{}{\n\t\t\"kvStoreName_actual\": gtreap.Name,\n\t\t\"path\":               \"\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn rv\n}\n\nfunc cleanup(t *testing.T, s store.KVStore) {\n\terr := s.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestMetricsKVCrud(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestKVCrud(t, s)\n}\n\nfunc TestMetricsReaderIsolation(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderIsolation(t, s)\n}\n\nfunc TestMetricsReaderOwnsGetBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderOwnsGetBytes(t, s)\n}\n\nfunc TestMetricsWriterOwnsBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestWriterOwnsBytes(t, s)\n}\n\nfunc TestMetricsPrefixIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIterator(t, s)\n}\n\nfunc TestMetricsPrefixIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIteratorSeek(t, s)\n}\n\nfunc TestMetricsRangeIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIterator(t, s)\n}\n\nfunc TestMetricsRangeIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIteratorSeek(t, s)\n}\n\nfunc TestMetricsMerge(t *testing.T) {\n\ts := open(t, &test.TestMergeCounter{})\n\tdefer cleanup(t, s)\n\ttest.CommonTestMerge(t, s)\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/util.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\n\t\"github.com/blevesearch/go-metrics\"\n)\n\n// NOTE: This is copy & pasted from cbft as otherwise there\n// would be an import cycle.\n\nvar timerPercentiles = []float64{0.5, 0.75, 0.95, 0.99, 0.999}\n\nfunc TimerMap(timer metrics.Timer) map[string]interface{} {\n\n\trv := make(map[string]interface{})\n\tt := timer.Snapshot()\n\tp := t.Percentiles(timerPercentiles)\n\n\tpercentileKeys := []string{\"median\", \"75%\", \"95%\", \"99%\", \"99.9%\"}\n\tpercentiles := make(map[string]interface{})\n\tfor i, pi := range p {\n\t\tif !isNanOrInf(pi) {\n\t\t\tpercentileKey := percentileKeys[i]\n\t\t\tpercentiles[percentileKey] = pi\n\t\t}\n\t}\n\n\trateKeys := []string{\"1-min\", \"5-min\", \"15-min\", \"mean\"}\n\trates := make(map[string]interface{})\n\tfor i, ri := range []float64{t.Rate1(), t.Rate5(), t.Rate15(), t.RateMean()} {\n\t\tif !isNanOrInf(ri) {\n\t\t\trateKey := rateKeys[i]\n\t\t\trates[rateKey] = ri\n\t\t}\n\t}\n\n\trv[\"count\"] = t.Count()\n\trv[\"min\"] = t.Min()\n\trv[\"max\"] = t.Max()\n\tmean := t.Mean()\n\tif !isNanOrInf(mean) {\n\t\trv[\"mean\"] = mean\n\t}\n\tstddev := t.StdDev()\n\tif !isNanOrInf(stddev) {\n\t\trv[\"stddev\"] = stddev\n\t}\n\trv[\"percentiles\"] = percentiles\n\trv[\"rates\"] = rates\n\n\treturn rv\n}\n\nfunc isNanOrInf(v float64) bool {\n\tif math.IsNaN(v) || math.IsInf(v, 0) {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc WriteTimerJSON(w io.Writer, timer metrics.Timer) {\n\tt := timer.Snapshot()\n\tp := t.Percentiles(timerPercentiles)\n\n\tfmt.Fprintf(w, `{\"count\":%9d,`, t.Count())\n\tfmt.Fprintf(w, `\"min\":%9d,`, t.Min())\n\tfmt.Fprintf(w, `\"max\":%9d,`, t.Max())\n\tfmt.Fprintf(w, `\"mean\":%12.2f,`, t.Mean())\n\tfmt.Fprintf(w, `\"stddev\":%12.2f,`, t.StdDev())\n\tfmt.Fprintf(w, `\"percentiles\":{`)\n\tfmt.Fprintf(w, `\"median\":%12.2f,`, p[0])\n\tfmt.Fprintf(w, `\"75%%\":%12.2f,`, p[1])\n\tfmt.Fprintf(w, `\"95%%\":%12.2f,`, p[2])\n\tfmt.Fprintf(w, `\"99%%\":%12.2f,`, p[3])\n\tfmt.Fprintf(w, `\"99.9%%\":%12.2f},`, p[4])\n\tfmt.Fprintf(w, `\"rates\":{`)\n\tfmt.Fprintf(w, `\"1-min\":%12.2f,`, t.Rate1())\n\tfmt.Fprintf(w, `\"5-min\":%12.2f,`, t.Rate5())\n\tfmt.Fprintf(w, `\"15-min\":%12.2f,`, t.Rate15())\n\tfmt.Fprintf(w, `\"mean\":%12.2f}}`, t.RateMean())\n}\n\nfunc WriteTimerCSVHeader(w io.Writer, prefix string) {\n\tfmt.Fprintf(w, \"%s-count,\", prefix)\n\tfmt.Fprintf(w, \"%s-min,\", prefix)\n\tfmt.Fprintf(w, \"%s-max,\", prefix)\n\tfmt.Fprintf(w, \"%s-mean,\", prefix)\n\tfmt.Fprintf(w, \"%s-stddev,\", prefix)\n\tfmt.Fprintf(w, \"%s-percentile-50%%,\", prefix)\n\tfmt.Fprintf(w, \"%s-percentile-75%%,\", prefix)\n\tfmt.Fprintf(w, \"%s-percentile-95%%,\", prefix)\n\tfmt.Fprintf(w, \"%s-percentile-99%%,\", prefix)\n\tfmt.Fprintf(w, \"%s-percentile-99.9%%,\", prefix)\n\tfmt.Fprintf(w, \"%s-rate-1-min,\", prefix)\n\tfmt.Fprintf(w, \"%s-rate-5-min,\", prefix)\n\tfmt.Fprintf(w, \"%s-rate-15-min,\", prefix)\n\tfmt.Fprintf(w, \"%s-rate-mean\", prefix)\n}\n\nfunc WriteTimerCSV(w io.Writer, timer metrics.Timer) {\n\tt := timer.Snapshot()\n\tp := t.Percentiles(timerPercentiles)\n\n\tfmt.Fprintf(w, `%d,`, t.Count())\n\tfmt.Fprintf(w, `%d,`, t.Min())\n\tfmt.Fprintf(w, `%d,`, t.Max())\n\tfmt.Fprintf(w, `%f,`, t.Mean())\n\tfmt.Fprintf(w, `%f,`, t.StdDev())\n\tfmt.Fprintf(w, `%f,`, p[0])\n\tfmt.Fprintf(w, `%f,`, p[1])\n\tfmt.Fprintf(w, `%f,`, p[2])\n\tfmt.Fprintf(w, `%f,`, p[3])\n\tfmt.Fprintf(w, `%f,`, p[4])\n\tfmt.Fprintf(w, `%f,`, t.Rate1())\n\tfmt.Fprintf(w, `%f,`, t.Rate5())\n\tfmt.Fprintf(w, `%f,`, t.Rate15())\n\tfmt.Fprintf(w, `%f`, t.RateMean())\n}\n"
  },
  {
    "path": "index/upsidedown/store/metrics/writer.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"fmt\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype Writer struct {\n\ts *Store\n\to store.KVWriter\n}\n\nfunc (w *Writer) Close() error {\n\terr := w.o.Close()\n\tif err != nil {\n\t\tw.s.AddError(\"Writer.Close\", err, nil)\n\t}\n\treturn err\n}\n\nfunc (w *Writer) NewBatch() store.KVBatch {\n\treturn &Batch{s: w.s, o: w.o.NewBatch()}\n}\n\nfunc (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {\n\tbuf, b, err := w.o.NewBatchEx(options)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\treturn buf, &Batch{s: w.s, o: b}, nil\n}\n\nfunc (w *Writer) ExecuteBatch(b store.KVBatch) (err error) {\n\tbatch, ok := b.(*Batch)\n\tif !ok {\n\t\treturn fmt.Errorf(\"wrong type of batch\")\n\t}\n\tw.s.timerWriterExecuteBatch.Time(func() {\n\t\terr = w.o.ExecuteBatch(batch.o)\n\t\tif err != nil {\n\t\t\tw.s.AddError(\"Writer.ExecuteBatch\", err, nil)\n\t\t}\n\t})\n\treturn\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/batch.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage moss\n\nimport (\n\t\"github.com/couchbase/moss\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype Batch struct {\n\tstore   *Store\n\tmerge   *store.EmulatedMerge\n\tbatch   moss.Batch\n\tbuf     []byte // Non-nil when using pre-alloc'ed / NewBatchEx().\n\tbufUsed int\n}\n\nfunc (b *Batch) Set(key, val []byte) {\n\tvar err error\n\tif b.buf != nil {\n\t\tb.bufUsed += len(key) + len(val)\n\t\terr = b.batch.AllocSet(key, val)\n\t} else {\n\t\terr = b.batch.Set(key, val)\n\t}\n\n\tif err != nil {\n\t\tb.store.Logf(\"bleve moss batch.Set err: %v\", err)\n\t}\n}\n\nfunc (b *Batch) Delete(key []byte) {\n\tvar err error\n\tif b.buf != nil {\n\t\tb.bufUsed += len(key)\n\t\terr = b.batch.AllocDel(key)\n\t} else {\n\t\terr = b.batch.Del(key)\n\t}\n\n\tif err != nil {\n\t\tb.store.Logf(\"bleve moss batch.Delete err: %v\", err)\n\t}\n}\n\nfunc (b *Batch) Merge(key, val []byte) {\n\tif b.buf != nil {\n\t\tb.bufUsed += len(key) + len(val)\n\t}\n\tb.merge.Merge(key, val)\n}\n\nfunc (b *Batch) Reset() {\n\terr := b.Close()\n\tif err != nil {\n\t\tb.store.Logf(\"bleve moss batch.Close err: %v\", err)\n\t\treturn\n\t}\n\n\tbatch, err := b.store.ms.NewBatch(0, 0)\n\tif err == nil {\n\t\tb.batch = batch\n\t\tb.merge = store.NewEmulatedMerge(b.store.mo)\n\t\tb.buf = nil\n\t\tb.bufUsed = 0\n\t}\n}\n\nfunc (b *Batch) Close() error {\n\tb.merge = nil\n\terr := b.batch.Close()\n\tb.batch = nil\n\treturn err\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/iterator.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage moss\n\nimport (\n\t\"github.com/couchbase/moss\"\n)\n\ntype Iterator struct {\n\tstore *Store\n\tss    moss.Snapshot\n\titer  moss.Iterator\n\tstart []byte\n\tend   []byte\n\tk     []byte\n\tv     []byte\n\terr   error\n}\n\nfunc (x *Iterator) Seek(seekToKey []byte) {\n\t_ = x.iter.SeekTo(seekToKey)\n\n\tx.k, x.v, x.err = x.iter.Current()\n}\n\nfunc (x *Iterator) Next() {\n\t_ = x.iter.Next()\n\n\tx.k, x.v, x.err = x.iter.Current()\n}\n\nfunc (x *Iterator) Current() ([]byte, []byte, bool) {\n\treturn x.k, x.v, x.err == nil\n}\n\nfunc (x *Iterator) Key() []byte {\n\tif x.err != nil {\n\t\treturn nil\n\t}\n\n\treturn x.k\n}\n\nfunc (x *Iterator) Value() []byte {\n\tif x.err != nil {\n\t\treturn nil\n\t}\n\n\treturn x.v\n}\n\nfunc (x *Iterator) Valid() bool {\n\treturn x.err == nil\n}\n\nfunc (x *Iterator) Close() error {\n\tvar err error\n\n\tx.ss = nil\n\n\tif x.iter != nil {\n\t\terr = x.iter.Close()\n\t\tx.iter = nil\n\t}\n\n\tx.k = nil\n\tx.v = nil\n\tx.err = moss.ErrIteratorDone\n\n\treturn err\n}\n\nfunc (x *Iterator) current() {\n\tx.k, x.v, x.err = x.iter.Current()\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/lower.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package moss provides a KVStore implementation based on the\n// github.com/couchbase/moss library.\n\npackage moss\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/couchbase/moss\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\nfunc initLowerLevelStore(\n\tconfig map[string]interface{},\n\tlowerLevelStoreName string,\n\tlowerLevelStoreConfig map[string]interface{},\n\tlowerLevelMaxBatchSize uint64,\n\toptions moss.CollectionOptions,\n) (moss.Snapshot, moss.LowerLevelUpdate, store.KVStore, statsFunc, error) {\n\tif lowerLevelStoreConfig == nil {\n\t\tlowerLevelStoreConfig = map[string]interface{}{}\n\t}\n\n\tfor k, v := range config {\n\t\t_, exists := lowerLevelStoreConfig[k]\n\t\tif !exists {\n\t\t\tlowerLevelStoreConfig[k] = v\n\t\t}\n\t}\n\n\tif lowerLevelStoreName == \"mossStore\" {\n\t\treturn InitMossStore(lowerLevelStoreConfig, options)\n\t}\n\n\tconstructor := registry.KVStoreConstructorByName(lowerLevelStoreName)\n\tif constructor == nil {\n\t\treturn nil, nil, nil, nil, fmt.Errorf(\"moss store, initLowerLevelStore,\"+\n\t\t\t\" could not find lower level store: %s\", lowerLevelStoreName)\n\t}\n\n\tkvStore, err := constructor(options.MergeOperator, lowerLevelStoreConfig)\n\tif err != nil {\n\t\treturn nil, nil, nil, nil, err\n\t}\n\n\tllStore := &llStore{\n\t\trefs:     0,\n\t\tconfig:   config,\n\t\tllConfig: lowerLevelStoreConfig,\n\t\tkvStore:  kvStore,\n\t\tlogf:     options.Log,\n\t}\n\n\tllUpdate := func(ssHigher moss.Snapshot) (ssLower moss.Snapshot, err error) {\n\t\treturn llStore.update(ssHigher, lowerLevelMaxBatchSize)\n\t}\n\n\tllSnapshot, err := llUpdate(nil)\n\tif err != nil {\n\t\t_ = kvStore.Close()\n\t\treturn nil, nil, nil, nil, err\n\t}\n\n\treturn llSnapshot, llUpdate, kvStore, nil, nil // llStore.refs is now 1.\n}\n\n// ------------------------------------------------\n\n// llStore is a lower level store and provides ref-counting around a\n// bleve store.KVStore.\ntype llStore struct {\n\tkvStore store.KVStore\n\n\tconfig   map[string]interface{}\n\tllConfig map[string]interface{}\n\n\tlogf func(format string, a ...interface{})\n\n\tm    sync.Mutex // Protects fields that follow.\n\trefs int\n}\n\n// llSnapshot represents a lower-level snapshot, wrapping a bleve\n// store.KVReader, and implements the moss.Snapshot interface.\ntype llSnapshot struct {\n\tllStore        *llStore // Holds 1 refs on the llStore.\n\tkvReader       store.KVReader\n\tchildSnapshots map[string]*llSnapshot\n\n\tm    sync.Mutex // Protects fields that follow.\n\trefs int\n}\n\n// llIterator represents a lower-level iterator, wrapping a bleve\n// store.KVIterator, and implements the moss.Iterator interface.\ntype llIterator struct {\n\tllSnapshot *llSnapshot // Holds 1 refs on the llSnapshot.\n\n\t// Some lower-level KVReader implementations need a separate\n\t// KVReader clone, due to KVReader single-threaded'ness.\n\tkvReader store.KVReader\n\n\tkvIterator store.KVIterator\n}\n\ntype readerSource interface {\n\tReader() (store.KVReader, error)\n}\n\n// ------------------------------------------------\n\nfunc (s *llStore) addRef() *llStore {\n\ts.m.Lock()\n\ts.refs += 1\n\ts.m.Unlock()\n\n\treturn s\n}\n\nfunc (s *llStore) decRef() {\n\ts.m.Lock()\n\ts.refs -= 1\n\tif s.refs <= 0 {\n\t\terr := s.kvStore.Close()\n\t\tif err != nil {\n\t\t\ts.logf(\"llStore kvStore.Close err: %v\", err)\n\t\t}\n\t}\n\ts.m.Unlock()\n}\n\n// update() mutates this lower level store with latest data from the\n// given higher level moss.Snapshot and returns a new moss.Snapshot\n// that the higher level can use which represents this lower level\n// store.\nfunc (s *llStore) update(ssHigher moss.Snapshot, maxBatchSize uint64) (\n\tssLower moss.Snapshot, err error,\n) {\n\tif ssHigher != nil {\n\t\titer, err := ssHigher.StartIterator(nil, nil, moss.IteratorOptions{\n\t\t\tIncludeDeletions: true,\n\t\t\tSkipLowerLevel:   true,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdefer func() {\n\t\t\terr = iter.Close()\n\t\t\tif err != nil {\n\t\t\t\ts.logf(\"llStore iter.Close err: %v\", err)\n\t\t\t}\n\t\t}()\n\n\t\tkvWriter, err := s.kvStore.Writer()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdefer func() {\n\t\t\terr = kvWriter.Close()\n\t\t\tif err != nil {\n\t\t\t\ts.logf(\"llStore kvWriter.Close err: %v\", err)\n\t\t\t}\n\t\t}()\n\n\t\tbatch := kvWriter.NewBatch()\n\n\t\tdefer func() {\n\t\t\tif batch != nil {\n\t\t\t\terr = batch.Close()\n\t\t\t\tif err != nil {\n\t\t\t\t\ts.logf(\"llStore batch.Close err: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\n\t\tvar readOptions moss.ReadOptions\n\n\t\ti := uint64(0)\n\t\tfor {\n\t\t\tif i%1000000 == 0 {\n\t\t\t\ts.logf(\"llStore.update, i: %d\", i)\n\t\t\t}\n\n\t\t\tex, key, val, err := iter.CurrentEx()\n\t\t\tif err == moss.ErrIteratorDone {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tswitch ex.Operation {\n\t\t\tcase moss.OperationSet:\n\t\t\t\tbatch.Set(key, val)\n\n\t\t\tcase moss.OperationDel:\n\t\t\t\tbatch.Delete(key)\n\n\t\t\tcase moss.OperationMerge:\n\t\t\t\tval, err = ssHigher.Get(key, readOptions)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\tif val != nil {\n\t\t\t\t\tbatch.Set(key, val)\n\t\t\t\t} else {\n\t\t\t\t\tbatch.Delete(key)\n\t\t\t\t}\n\n\t\t\tdefault:\n\t\t\t\treturn nil, fmt.Errorf(\"moss store, update,\"+\n\t\t\t\t\t\" unexpected operation, ex: %v\", ex)\n\t\t\t}\n\n\t\t\ti++\n\n\t\t\terr = iter.Next()\n\t\t\tif err == moss.ErrIteratorDone {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tif maxBatchSize > 0 && i%maxBatchSize == 0 {\n\t\t\t\terr = kvWriter.ExecuteBatch(batch)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\terr = batch.Close()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\tbatch = kvWriter.NewBatch()\n\t\t\t}\n\t\t}\n\n\t\tif i > 0 {\n\t\t\ts.logf(\"llStore.update, ExecuteBatch,\"+\n\t\t\t\t\" path: %s, total: %d, start\", s.llConfig[\"path\"], i)\n\n\t\t\terr = kvWriter.ExecuteBatch(batch)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\ts.logf(\"llStore.update, ExecuteBatch,\"+\n\t\t\t\t\" path: %s: total: %d, done\", s.llConfig[\"path\"], i)\n\t\t}\n\t}\n\n\tkvReader, err := s.kvStore.Reader()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\ts.logf(\"llStore.update, new reader\")\n\n\treturn &llSnapshot{\n\t\tllStore:  s.addRef(),\n\t\tkvReader: kvReader,\n\t\trefs:     1,\n\t}, nil\n}\n\n// ------------------------------------------------\n\nfunc (llss *llSnapshot) addRef() *llSnapshot {\n\tllss.m.Lock()\n\tllss.refs += 1\n\tllss.m.Unlock()\n\n\treturn llss\n}\n\nfunc (llss *llSnapshot) decRef() {\n\tllss.m.Lock()\n\tllss.refs -= 1\n\tif llss.refs <= 0 {\n\t\tif llss.kvReader != nil {\n\t\t\terr := llss.kvReader.Close()\n\t\t\tif err != nil {\n\t\t\t\tllss.llStore.logf(\"llSnapshot kvReader.Close err: %v\", err)\n\t\t\t}\n\n\t\t\tllss.kvReader = nil\n\t\t}\n\n\t\tif llss.llStore != nil {\n\t\t\tllss.llStore.decRef()\n\t\t\tllss.llStore = nil\n\t\t}\n\t}\n\tllss.m.Unlock()\n}\n\n// ChildCollectionNames returns an array of child collection name strings.\nfunc (llss *llSnapshot) ChildCollectionNames() ([]string, error) {\n\tchildCollections := make([]string, len(llss.childSnapshots))\n\tidx := 0\n\tfor name := range llss.childSnapshots {\n\t\tchildCollections[idx] = name\n\t\tidx++\n\t}\n\treturn childCollections, nil\n}\n\n// ChildCollectionSnapshot returns a Snapshot on a given child\n// collection by its name.\nfunc (llss *llSnapshot) ChildCollectionSnapshot(childCollectionName string) (\n\tmoss.Snapshot, error,\n) {\n\tchildSnapshot, exists := llss.childSnapshots[childCollectionName]\n\tif !exists {\n\t\treturn nil, nil\n\t}\n\tchildSnapshot.addRef()\n\treturn childSnapshot, nil\n}\n\nfunc (llss *llSnapshot) Close() error {\n\tllss.decRef()\n\n\treturn nil\n}\n\nfunc (llss *llSnapshot) Get(key []byte,\n\treadOptions moss.ReadOptions,\n) ([]byte, error) {\n\trs, ok := llss.kvReader.(readerSource)\n\tif ok {\n\t\tr2, err := rs.Reader()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tval, err := r2.Get(key)\n\n\t\t_ = r2.Close()\n\n\t\treturn val, err\n\t}\n\n\treturn llss.kvReader.Get(key)\n}\n\nfunc (llss *llSnapshot) StartIterator(\n\tstartKeyInclusive, endKeyExclusive []byte,\n\titeratorOptions moss.IteratorOptions,\n) (moss.Iterator, error) {\n\trs, ok := llss.kvReader.(readerSource)\n\tif ok {\n\t\tr2, err := rs.Reader()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\ti2 := r2.RangeIterator(startKeyInclusive, endKeyExclusive)\n\n\t\treturn &llIterator{llSnapshot: llss.addRef(), kvReader: r2, kvIterator: i2}, nil\n\t}\n\n\ti := llss.kvReader.RangeIterator(startKeyInclusive, endKeyExclusive)\n\n\treturn &llIterator{llSnapshot: llss.addRef(), kvReader: nil, kvIterator: i}, nil\n}\n\n// ------------------------------------------------\n\nfunc (lli *llIterator) Close() error {\n\tvar err0 error\n\tif lli.kvIterator != nil {\n\t\terr0 = lli.kvIterator.Close()\n\t\tlli.kvIterator = nil\n\t}\n\n\tvar err1 error\n\tif lli.kvReader != nil {\n\t\terr1 = lli.kvReader.Close()\n\t\tlli.kvReader = nil\n\t}\n\n\tlli.llSnapshot.decRef()\n\tlli.llSnapshot = nil\n\n\tif err0 != nil {\n\t\treturn err0\n\t}\n\n\tif err1 != nil {\n\t\treturn err1\n\t}\n\n\treturn nil\n}\n\nfunc (lli *llIterator) Next() error {\n\tlli.kvIterator.Next()\n\n\treturn nil\n}\n\nfunc (lli *llIterator) SeekTo(k []byte) error {\n\tlli.kvIterator.Seek(k)\n\n\treturn nil\n}\n\nfunc (lli *llIterator) Current() (key, val []byte, err error) {\n\tkey, val, ok := lli.kvIterator.Current()\n\tif !ok {\n\t\treturn nil, nil, moss.ErrIteratorDone\n\t}\n\n\treturn key, val, nil\n}\n\nfunc (lli *llIterator) CurrentEx() (\n\tentryEx moss.EntryEx, key, val []byte, err error,\n) {\n\treturn moss.EntryEx{}, nil, nil, moss.ErrUnimplemented\n}\n\n// ------------------------------------------------\n\nfunc InitMossStore(config map[string]interface{}, options moss.CollectionOptions) (\n\tmoss.Snapshot, moss.LowerLevelUpdate, store.KVStore, statsFunc, error,\n) {\n\tpath, ok := config[\"path\"].(string)\n\tif !ok {\n\t\treturn nil, nil, nil, nil, fmt.Errorf(\"lower: missing path for InitMossStore config\")\n\t}\n\tif path == \"\" {\n\t\treturn nil, nil, nil, nil, os.ErrInvalid\n\t}\n\n\terr := os.MkdirAll(path, 0o700)\n\tif err != nil {\n\t\treturn nil, nil, nil, nil, fmt.Errorf(\"lower: InitMossStore mkdir,\"+\n\t\t\t\" path: %s, err: %v\", path, err)\n\t}\n\n\tstoreOptions := moss.StoreOptions{\n\t\tCollectionOptions: options,\n\t}\n\tv, ok := config[\"mossStoreOptions\"]\n\tif ok {\n\t\tb, err := util.MarshalJSON(v) // Convert from map[string]interface{}.\n\t\tif err != nil {\n\t\t\treturn nil, nil, nil, nil, err\n\t\t}\n\n\t\terr = util.UnmarshalJSON(b, &storeOptions)\n\t\tif err != nil {\n\t\t\treturn nil, nil, nil, nil, err\n\t\t}\n\t}\n\n\ts, err := moss.OpenStore(path, storeOptions)\n\tif err != nil {\n\t\treturn nil, nil, nil, nil, fmt.Errorf(\"lower: moss.OpenStore,\"+\n\t\t\t\" path: %s, err: %v\", path, err)\n\t}\n\n\tsw := &mossStoreWrapper{s: s}\n\n\tllUpdate := func(ssHigher moss.Snapshot) (moss.Snapshot, error) {\n\t\tss, err := sw.s.Persist(ssHigher, moss.StorePersistOptions{\n\t\t\tCompactionConcern: moss.CompactionAllow,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tsw.AddRef() // Ref-count to be owned by snapshot wrapper.\n\n\t\treturn moss.NewSnapshotWrapper(ss, sw), nil\n\t}\n\n\tllSnapshot, err := llUpdate(nil)\n\tif err != nil {\n\t\t_ = s.Close()\n\t\treturn nil, nil, nil, nil, err\n\t}\n\n\tllStats := func() map[string]interface{} {\n\t\tstats, err := s.Stats()\n\t\tif err != nil {\n\t\t\treturn nil\n\t\t}\n\t\treturn stats\n\t}\n\n\treturn llSnapshot, llUpdate, sw, llStats, nil\n}\n\n// mossStoreWrapper implements the bleve.index.store.KVStore\n// interface, but only barely enough to allow it to be passed around\n// as a lower-level store.  Advanced apps will likely cast the\n// mossStoreWrapper to access the Actual() method.\ntype mossStoreWrapper struct {\n\tm    sync.Mutex\n\trefs int\n\ts    *moss.Store\n}\n\nfunc (w *mossStoreWrapper) AddRef() {\n\tw.m.Lock()\n\tw.refs++\n\tw.m.Unlock()\n}\n\nfunc (w *mossStoreWrapper) Close() (err error) {\n\tw.m.Lock()\n\tw.refs--\n\tif w.refs <= 0 {\n\t\terr = w.s.Close()\n\t\tw.s = nil\n\t}\n\tw.m.Unlock()\n\treturn err\n}\n\nfunc (w *mossStoreWrapper) Reader() (store.KVReader, error) {\n\treturn nil, fmt.Errorf(\"unexpected\")\n}\n\nfunc (w *mossStoreWrapper) Writer() (store.KVWriter, error) {\n\treturn nil, fmt.Errorf(\"unexpected\")\n}\n\nfunc (w *mossStoreWrapper) Actual() *moss.Store {\n\tw.m.Lock()\n\trv := w.s\n\tw.m.Unlock()\n\treturn rv\n}\n\nfunc (w *mossStoreWrapper) histograms() string {\n\tvar rv string\n\tw.m.Lock()\n\tif w.s != nil {\n\t\trv = w.s.Histograms().String()\n\t}\n\tw.m.Unlock()\n\treturn rv\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/lower_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage moss\n\nimport (\n\t\"os\"\n\t\"testing\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\t\"github.com/blevesearch/upsidedown_store_api/test\"\n)\n\nfunc openWithLower(t *testing.T, mo store.MergeOperator) (string, store.KVStore) {\n\ttmpDir, _ := os.MkdirTemp(\"\", \"mossStore\")\n\n\tconfig := map[string]interface{}{\n\t\t\"path\":                    tmpDir,\n\t\t\"mossLowerLevelStoreName\": \"mossStore\",\n\t}\n\n\trv, err := New(mo, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn tmpDir, rv\n}\n\nfunc cleanupWithLower(t *testing.T, s store.KVStore, tmpDir string) {\n\terr := s.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = os.RemoveAll(tmpDir)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestMossWithLowerKVCrud(t *testing.T) {\n\ttmpDir, s := openWithLower(t, nil)\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestKVCrud(t, s)\n}\n\nfunc TestMossWithLowerReaderIsolation(t *testing.T) {\n\ttmpDir, s := openWithLower(t, nil)\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestReaderIsolation(t, s)\n}\n\nfunc TestMossWithLowerReaderOwnsGetBytes(t *testing.T) {\n\ttmpDir, s := openWithLower(t, nil)\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestReaderOwnsGetBytes(t, s)\n}\n\nfunc TestMossWithLowerWriterOwnsBytes(t *testing.T) {\n\ttmpDir, s := openWithLower(t, nil)\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestWriterOwnsBytes(t, s)\n}\n\nfunc TestMossWithLowerPrefixIterator(t *testing.T) {\n\ttmpDir, s := openWithLower(t, nil)\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestPrefixIterator(t, s)\n}\n\nfunc TestMossWithLowerPrefixIteratorSeek(t *testing.T) {\n\ttmpDir, s := openWithLower(t, nil)\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestPrefixIteratorSeek(t, s)\n}\n\nfunc TestMossWithLowerRangeIterator(t *testing.T) {\n\ttmpDir, s := openWithLower(t, nil)\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestRangeIterator(t, s)\n}\n\nfunc TestMossWithLowerRangeIteratorSeek(t *testing.T) {\n\ttmpDir, s := openWithLower(t, nil)\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestRangeIteratorSeek(t, s)\n}\n\nfunc TestMossWithLowerMerge(t *testing.T) {\n\ttmpDir, s := openWithLower(t, &test.TestMergeCounter{})\n\tdefer cleanupWithLower(t, s, tmpDir)\n\ttest.CommonTestMerge(t, s)\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/reader.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage moss\n\nimport (\n\t\"github.com/couchbase/moss\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype Reader struct {\n\tstore *Store\n\tss    moss.Snapshot\n}\n\nfunc (r *Reader) Get(k []byte) (v []byte, err error) {\n\tv, err = r.ss.Get(k, moss.ReadOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif v != nil {\n\t\treturn append(make([]byte, 0, len(v)), v...), nil\n\t}\n\treturn nil, nil\n}\n\nfunc (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {\n\treturn store.MultiGet(r, keys)\n}\n\nfunc (r *Reader) PrefixIterator(k []byte) store.KVIterator {\n\tkEnd := incrementBytes(k)\n\n\titer, err := r.ss.StartIterator(k, kEnd, moss.IteratorOptions{})\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\trv := &Iterator{\n\t\tstore: r.store,\n\t\tss:    r.ss,\n\t\titer:  iter,\n\t\tstart: k,\n\t\tend:   kEnd,\n\t}\n\n\trv.current()\n\n\treturn rv\n}\n\nfunc (r *Reader) RangeIterator(start, end []byte) store.KVIterator {\n\titer, err := r.ss.StartIterator(start, end, moss.IteratorOptions{})\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\trv := &Iterator{\n\t\tstore: r.store,\n\t\tss:    r.ss,\n\t\titer:  iter,\n\t\tstart: start,\n\t\tend:   end,\n\t}\n\n\trv.current()\n\n\treturn rv\n}\n\nfunc (r *Reader) Close() error {\n\treturn r.ss.Close()\n}\n\nfunc incrementBytes(in []byte) []byte {\n\trv := make([]byte, len(in))\n\tcopy(rv, in)\n\tfor i := len(rv) - 1; i >= 0; i-- {\n\t\trv[i] = rv[i] + 1\n\t\tif rv[i] != 0 {\n\t\t\treturn rv // didn't overflow, so stop\n\t\t}\n\t}\n\treturn nil // overflowed\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/stats.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage moss\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\ntype stats struct {\n\ts *Store\n}\n\nfunc (s *stats) statsMap() map[string]interface{} {\n\tms := map[string]interface{}{}\n\n\tvar err error\n\tms[\"moss\"], err = s.s.ms.Stats()\n\tif err != nil {\n\t\treturn ms\n\t}\n\n\tif s.s.llstore != nil {\n\t\tif o, ok := s.s.llstore.(store.KVStoreStats); ok {\n\t\t\tms[\"kv\"] = o.StatsMap()\n\t\t}\n\t}\n\n\t_, exists := ms[\"kv\"]\n\tif !exists && s.s.llstats != nil {\n\t\tms[\"kv\"] = s.s.llstats()\n\t}\n\n\tif msw, ok := s.s.llstore.(*mossStoreWrapper); ok {\n\t\tms[\"store_histograms\"] = msw.histograms()\n\t}\n\n\tms[\"coll_histograms\"] = s.s.ms.Histograms().String()\n\n\treturn ms\n}\n\nfunc (s *stats) MarshalJSON() ([]byte, error) {\n\tm := s.statsMap()\n\treturn util.MarshalJSON(m)\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/store.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package moss provides a KVStore implementation based on the\n// github.com/couchbase/moss library.\n\npackage moss\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sync\"\n\n\t\"github.com/couchbase/moss\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\n// RegistryCollectionOptions should be treated as read-only after\n// process init()'ialization.\nvar RegistryCollectionOptions = map[string]moss.CollectionOptions{}\n\nconst Name = \"moss\"\n\ntype Store struct {\n\tm       sync.Mutex\n\tms      moss.Collection\n\tmo      store.MergeOperator\n\tllstore store.KVStore // May be nil.\n\tllstats statsFunc     // May be nil.\n\n\ts      *stats\n\tconfig map[string]interface{}\n}\n\ntype statsFunc func() map[string]interface{}\n\n// New initializes a moss storage with values from the optional\n// config[\"mossCollectionOptions\"] (a JSON moss.CollectionOptions).\n// Next, values from the RegistryCollectionOptions, named by the\n// optional config[\"mossCollectionOptionsName\"], take precedence.\n// Finally, base case defaults are taken from\n// moss.DefaultCollectionOptions.\nfunc New(mo store.MergeOperator, config map[string]interface{}) (\n\tstore.KVStore, error) {\n\toptions := moss.DefaultCollectionOptions // Copy.\n\n\tv, ok := config[\"mossCollectionOptionsName\"]\n\tif ok {\n\t\tname, ok := v.(string)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"moss store,\"+\n\t\t\t\t\" could not parse config[mossCollectionOptionsName]: %v\", v)\n\t\t}\n\n\t\toptions, ok = RegistryCollectionOptions[name] // Copy.\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"moss store,\"+\n\t\t\t\t\" could not find RegistryCollectionOptions, name: %s\", name)\n\t\t}\n\t}\n\n\toptions.MergeOperator = mo\n\toptions.DeferredSort = true\n\n\tv, ok = config[\"mossCollectionOptions\"]\n\tif ok {\n\t\tb, err := util.MarshalJSON(v) // Convert from map[string]interface{}.\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"moss store,\"+\n\t\t\t\t\" could not marshal config[mossCollectionOptions]: %v, err: %v\", v, err)\n\t\t}\n\n\t\terr = util.UnmarshalJSON(b, &options)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"moss store,\"+\n\t\t\t\t\" could not unmarshal config[mossCollectionOptions]: %v, err: %v\", v, err)\n\t\t}\n\t}\n\n\t// --------------------------------------------------\n\n\tif options.Log == nil || options.Debug <= 0 {\n\t\toptions.Log = func(format string, a ...interface{}) {}\n\t}\n\n\t// --------------------------------------------------\n\n\tmossLowerLevelStoreName := \"\"\n\tv, ok = config[\"mossLowerLevelStoreName\"]\n\tif ok {\n\t\tmossLowerLevelStoreName, ok = v.(string)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"moss store,\"+\n\t\t\t\t\" could not parse config[mossLowerLevelStoreName]: %v\", v)\n\t\t}\n\t}\n\n\tvar llStore store.KVStore\n\tvar llStats statsFunc\n\n\tif options.LowerLevelInit == nil &&\n\t\toptions.LowerLevelUpdate == nil &&\n\t\tmossLowerLevelStoreName != \"\" {\n\t\tmossLowerLevelStoreConfig := map[string]interface{}{}\n\t\tv, ok := config[\"mossLowerLevelStoreConfig\"]\n\t\tif ok {\n\t\t\tmossLowerLevelStoreConfig, ok = v.(map[string]interface{})\n\t\t\tif !ok {\n\t\t\t\treturn nil, fmt.Errorf(\"moss store, initLowerLevelStore,\"+\n\t\t\t\t\t\" could parse mossLowerLevelStoreConfig: %v\", v)\n\t\t\t}\n\t\t}\n\n\t\tmossLowerLevelMaxBatchSize := uint64(0)\n\t\tv, ok = config[\"mossLowerLevelMaxBatchSize\"]\n\t\tif ok {\n\t\t\tmossLowerLevelMaxBatchSizeF, ok := v.(float64)\n\t\t\tif !ok {\n\t\t\t\treturn nil, fmt.Errorf(\"moss store,\"+\n\t\t\t\t\t\" could not parse config[mossLowerLevelMaxBatchSize]: %v\", v)\n\t\t\t}\n\n\t\t\tmossLowerLevelMaxBatchSize = uint64(mossLowerLevelMaxBatchSizeF)\n\t\t}\n\n\t\tlowerLevelInit, lowerLevelUpdate, lowerLevelStore, lowerLevelStats, err :=\n\t\t\tinitLowerLevelStore(config,\n\t\t\t\tmossLowerLevelStoreName,\n\t\t\t\tmossLowerLevelStoreConfig,\n\t\t\t\tmossLowerLevelMaxBatchSize,\n\t\t\t\toptions)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\toptions.LowerLevelInit = lowerLevelInit\n\t\toptions.LowerLevelUpdate = lowerLevelUpdate\n\n\t\tllStore = lowerLevelStore\n\t\tllStats = lowerLevelStats\n\t}\n\n\t// --------------------------------------------------\n\n\tms, err := moss.NewCollection(options)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\terr = ms.Start()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trv := Store{\n\t\tms:      ms,\n\t\tmo:      mo,\n\t\tllstore: llStore,\n\t\tllstats: llStats,\n\t\tconfig:  config,\n\t}\n\trv.s = &stats{s: &rv}\n\treturn &rv, nil\n}\n\nfunc (s *Store) Close() error {\n\tif val, ok := s.config[\"mossAbortCloseEnabled\"]; ok {\n\t\tif v, ok := val.(bool); ok && v {\n\t\t\tif msw, ok := s.llstore.(*mossStoreWrapper); ok {\n\t\t\t\tif s := msw.Actual(); s != nil {\n\t\t\t\t\t_ = s.CloseEx(moss.StoreCloseExOptions{Abort: true})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn s.ms.Close()\n}\n\nfunc (s *Store) Reader() (store.KVReader, error) {\n\tss, err := s.ms.Snapshot()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Reader{ss: ss}, nil\n}\n\nfunc (s *Store) Writer() (store.KVWriter, error) {\n\treturn &Writer{s: s}, nil\n}\n\nfunc (s *Store) Logf(fmt string, args ...interface{}) {\n\toptions := s.ms.Options()\n\tif options.Log != nil {\n\t\toptions.Log(fmt, args...)\n\t}\n}\n\nfunc (s *Store) Stats() json.Marshaler {\n\treturn s.s\n}\n\nfunc (s *Store) StatsMap() map[string]interface{} {\n\treturn s.s.statsMap()\n}\n\nfunc (s *Store) LowerLevelStore() store.KVStore {\n\treturn s.llstore\n}\n\nfunc (s *Store) Collection() moss.Collection {\n\treturn s.ms\n}\n\nfunc init() {\n\terr := registry.RegisterKVStore(Name, New)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/store_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage moss\n\nimport (\n\t\"testing\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\t\"github.com/blevesearch/upsidedown_store_api/test\"\n)\n\nfunc open(t *testing.T, mo store.MergeOperator) store.KVStore {\n\trv, err := New(mo, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn rv\n}\n\nfunc cleanup(t *testing.T, s store.KVStore) {\n\terr := s.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestMossKVCrud(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestKVCrud(t, s)\n}\n\nfunc TestMossReaderIsolation(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderIsolation(t, s)\n}\n\nfunc TestMossReaderOwnsGetBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestReaderOwnsGetBytes(t, s)\n}\n\nfunc TestMossWriterOwnsBytes(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestWriterOwnsBytes(t, s)\n}\n\nfunc TestMossPrefixIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIterator(t, s)\n}\n\nfunc TestMossPrefixIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestPrefixIteratorSeek(t, s)\n}\n\nfunc TestMossRangeIterator(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIterator(t, s)\n}\n\nfunc TestMossRangeIteratorSeek(t *testing.T) {\n\ts := open(t, nil)\n\tdefer cleanup(t, s)\n\ttest.CommonTestRangeIteratorSeek(t, s)\n}\n\nfunc TestMossMerge(t *testing.T) {\n\ts := open(t, &test.TestMergeCounter{})\n\tdefer cleanup(t, s)\n\ttest.CommonTestMerge(t, s)\n}\n"
  },
  {
    "path": "index/upsidedown/store/moss/writer.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage moss\n\nimport (\n\t\"fmt\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\n\t\"github.com/couchbase/moss\"\n)\n\ntype Writer struct {\n\ts *Store\n}\n\nfunc (w *Writer) NewBatch() store.KVBatch {\n\tb, err := w.s.ms.NewBatch(0, 0)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn &Batch{\n\t\tstore: w.s,\n\t\tmerge: store.NewEmulatedMerge(w.s.mo),\n\t\tbatch: b,\n\t}\n}\n\nfunc (w *Writer) NewBatchEx(options store.KVBatchOptions) (\n\t[]byte, store.KVBatch, error) {\n\tnumOps := options.NumSets + options.NumDeletes + options.NumMerges\n\n\tb, err := w.s.ms.NewBatch(numOps, options.TotalBytes)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tbuf, err := b.Alloc(options.TotalBytes)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\treturn buf, &Batch{\n\t\tstore:   w.s,\n\t\tmerge:   store.NewEmulatedMerge(w.s.mo),\n\t\tbatch:   b,\n\t\tbuf:     buf,\n\t\tbufUsed: 0,\n\t}, nil\n}\n\nfunc (w *Writer) ExecuteBatch(b store.KVBatch) (err error) {\n\tbatch, ok := b.(*Batch)\n\tif !ok {\n\t\treturn fmt.Errorf(\"wrong type of batch\")\n\t}\n\n\tfor kStr, mergeOps := range batch.merge.Merges {\n\t\tfor _, v := range mergeOps {\n\t\t\tif batch.buf != nil {\n\t\t\t\tkLen := len(kStr)\n\t\t\t\tvLen := len(v)\n\t\t\t\tkBuf := batch.buf[batch.bufUsed : batch.bufUsed+kLen]\n\t\t\t\tvBuf := batch.buf[batch.bufUsed+kLen : batch.bufUsed+kLen+vLen]\n\t\t\t\tcopy(kBuf, kStr)\n\t\t\t\tcopy(vBuf, v)\n\t\t\t\tbatch.bufUsed += kLen + vLen\n\t\t\t\terr = batch.batch.AllocMerge(kBuf, vBuf)\n\t\t\t} else {\n\t\t\t\terr = batch.batch.Merge([]byte(kStr), v)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn w.s.ms.ExecuteBatch(batch.batch, moss.WriteOptions{})\n}\n\nfunc (w *Writer) Close() error {\n\tw.s = nil\n\treturn nil\n}\n"
  },
  {
    "path": "index/upsidedown/store/null/null.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage null\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\nconst Name = \"null\"\n\ntype Store struct{}\n\nfunc New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {\n\treturn &Store{}, nil\n}\n\nfunc (i *Store) Close() error {\n\treturn nil\n}\n\nfunc (i *Store) Reader() (store.KVReader, error) {\n\treturn &reader{}, nil\n}\n\nfunc (i *Store) Writer() (store.KVWriter, error) {\n\treturn &writer{}, nil\n}\n\ntype reader struct{}\n\nfunc (r *reader) Get(key []byte) ([]byte, error) {\n\treturn nil, nil\n}\n\nfunc (r *reader) MultiGet(keys [][]byte) ([][]byte, error) {\n\treturn make([][]byte, len(keys)), nil\n}\n\nfunc (r *reader) PrefixIterator(prefix []byte) store.KVIterator {\n\treturn &iterator{}\n}\n\nfunc (r *reader) RangeIterator(start, end []byte) store.KVIterator {\n\treturn &iterator{}\n}\n\nfunc (r *reader) Close() error {\n\treturn nil\n}\n\ntype iterator struct{}\n\nfunc (i *iterator) SeekFirst()    {}\nfunc (i *iterator) Seek(k []byte) {}\nfunc (i *iterator) Next()         {}\n\nfunc (i *iterator) Current() ([]byte, []byte, bool) {\n\treturn nil, nil, false\n}\n\nfunc (i *iterator) Key() []byte {\n\treturn nil\n}\n\nfunc (i *iterator) Value() []byte {\n\treturn nil\n}\n\nfunc (i *iterator) Valid() bool {\n\treturn false\n}\n\nfunc (i *iterator) Close() error {\n\treturn nil\n}\n\ntype batch struct{}\n\nfunc (i *batch) Set(key, val []byte)   {}\nfunc (i *batch) Delete(key []byte)     {}\nfunc (i *batch) Merge(key, val []byte) {}\nfunc (i *batch) Reset()                {}\nfunc (i *batch) Close() error          { return nil }\n\ntype writer struct{}\n\nfunc (w *writer) NewBatch() store.KVBatch {\n\treturn &batch{}\n}\n\nfunc (w *writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {\n\treturn make([]byte, options.TotalBytes), w.NewBatch(), nil\n}\n\nfunc (w *writer) ExecuteBatch(store.KVBatch) error {\n\treturn nil\n}\n\nfunc (w *writer) Close() error {\n\treturn nil\n}\n\nfunc init() {\n\terr := registry.RegisterKVStore(Name, New)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/store/null/null_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage null\n\nimport (\n\t\"testing\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\nfunc TestStore(t *testing.T) {\n\ts, err := New(nil, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tNullTestKVStore(t, s)\n}\n\n// NullTestKVStore has very different expectations\n// compared to CommonTestKVStore\nfunc NullTestKVStore(t *testing.T, s store.KVStore) {\n\n\twriter, err := s.Writer()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tbatch := writer.NewBatch()\n\tbatch.Set([]byte(\"b\"), []byte(\"val-b\"))\n\tbatch.Set([]byte(\"c\"), []byte(\"val-c\"))\n\tbatch.Set([]byte(\"d\"), []byte(\"val-d\"))\n\tbatch.Set([]byte(\"e\"), []byte(\"val-e\"))\n\tbatch.Set([]byte(\"f\"), []byte(\"val-f\"))\n\tbatch.Set([]byte(\"g\"), []byte(\"val-g\"))\n\tbatch.Set([]byte(\"h\"), []byte(\"val-h\"))\n\tbatch.Set([]byte(\"i\"), []byte(\"val-i\"))\n\tbatch.Set([]byte(\"j\"), []byte(\"val-j\"))\n\n\terr = writer.ExecuteBatch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = writer.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err := s.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\tit := reader.RangeIterator([]byte(\"b\"), nil)\n\tkey, val, valid := it.Current()\n\tif valid {\n\t\tt.Fatalf(\"valid true, expected false\")\n\t}\n\tif key != nil {\n\t\tt.Fatalf(\"expected key nil, got %s\", key)\n\t}\n\tif val != nil {\n\t\tt.Fatalf(\"expected value nil, got %s\", val)\n\t}\n\n\terr = it.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = s.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n"
  },
  {
    "path": "index/upsidedown/upsidedown.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:generate protoc --gofast_out=. upsidedown.proto\n\npackage upsidedown\n\nimport (\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n\n\t\"google.golang.org/protobuf/proto\"\n)\n\nconst Name = \"upside_down\"\n\n// RowBufferSize should ideally this is sized to be the smallest\n// size that can contain an index row key and its corresponding\n// value.  It is not a limit, if need be a larger buffer is\n// allocated, but performance will be more optimal if *most*\n// rows fit this size.\nconst RowBufferSize = 4 * 1024\n\nvar VersionKey = []byte{'v'}\n\nconst Version uint8 = 7\n\nvar IncompatibleVersion = fmt.Errorf(\"incompatible version, %d is supported\", Version)\n\nvar ErrorUnknownStorageType = fmt.Errorf(\"unknown storage type\")\n\ntype UpsideDownCouch struct {\n\tversion       uint8\n\tpath          string\n\tstoreName     string\n\tstoreConfig   map[string]interface{}\n\tstore         store.KVStore\n\tfieldCache    *FieldCache\n\tanalysisQueue *index.AnalysisQueue\n\tstats         *indexStat\n\n\tm sync.RWMutex\n\t// fields protected by m\n\tdocCount uint64\n\n\twriteMutex sync.Mutex\n}\n\ntype docBackIndexRow struct {\n\tdocID        string\n\tdoc          index.Document // If deletion, doc will be nil.\n\tbackIndexRow *BackIndexRow\n}\n\nfunc NewUpsideDownCouch(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error) {\n\trv := &UpsideDownCouch{\n\t\tversion:       Version,\n\t\tfieldCache:    NewFieldCache(),\n\t\tstoreName:     storeName,\n\t\tstoreConfig:   storeConfig,\n\t\tanalysisQueue: analysisQueue,\n\t}\n\trv.stats = &indexStat{i: rv}\n\treturn rv, nil\n}\n\nfunc (udc *UpsideDownCouch) init(kvwriter store.KVWriter) (err error) {\n\t// version marker\n\trowsAll := [][]UpsideDownCouchRow{\n\t\t{NewVersionRow(udc.version)},\n\t}\n\n\terr = udc.batchRows(kvwriter, nil, rowsAll, nil)\n\treturn\n}\n\nfunc (udc *UpsideDownCouch) loadSchema(kvreader store.KVReader) (err error) {\n\n\tit := kvreader.PrefixIterator([]byte{'f'})\n\tdefer func() {\n\t\tif cerr := it.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tkey, val, valid := it.Current()\n\tfor valid {\n\t\tvar fieldRow *FieldRow\n\t\tfieldRow, err = NewFieldRowKV(key, val)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tudc.fieldCache.AddExisting(fieldRow.name, fieldRow.index)\n\n\t\tit.Next()\n\t\tkey, val, valid = it.Current()\n\t}\n\n\tval, err = kvreader.Get([]byte{'v'})\n\tif err != nil {\n\t\treturn\n\t}\n\tvar vr *VersionRow\n\tvr, err = NewVersionRowKV([]byte{'v'}, val)\n\tif err != nil {\n\t\treturn\n\t}\n\tif vr.version != Version {\n\t\terr = IncompatibleVersion\n\t\treturn\n\t}\n\n\treturn\n}\n\ntype rowBuffer struct {\n\tbuf []byte\n}\n\nvar rowBufferPool sync.Pool\n\nfunc GetRowBuffer() *rowBuffer {\n\tif rb, ok := rowBufferPool.Get().(*rowBuffer); ok {\n\t\treturn rb\n\t} else {\n\t\tbuf := make([]byte, RowBufferSize)\n\t\treturn &rowBuffer{buf: buf}\n\t}\n}\n\nfunc PutRowBuffer(rb *rowBuffer) {\n\trowBufferPool.Put(rb)\n}\n\nfunc (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRowsAll [][]UpsideDownCouchRow, updateRowsAll [][]UpsideDownCouchRow, deleteRowsAll [][]UpsideDownCouchRow) (err error) {\n\tdictionaryDeltas := make(map[string]int64)\n\n\t// count up bytes needed for buffering.\n\taddNum := 0\n\taddKeyBytes := 0\n\taddValBytes := 0\n\n\tupdateNum := 0\n\tupdateKeyBytes := 0\n\tupdateValBytes := 0\n\n\tdeleteNum := 0\n\tdeleteKeyBytes := 0\n\n\trowBuf := GetRowBuffer()\n\n\tfor _, addRows := range addRowsAll {\n\t\tfor _, row := range addRows {\n\t\t\ttfr, ok := row.(*TermFrequencyRow)\n\t\t\tif ok {\n\t\t\t\tif tfr.DictionaryRowKeySize() > len(rowBuf.buf) {\n\t\t\t\t\trowBuf.buf = make([]byte, tfr.DictionaryRowKeySize())\n\t\t\t\t}\n\t\t\t\tdictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf.buf)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tdictionaryDeltas[string(rowBuf.buf[:dictKeySize])] += 1\n\t\t\t}\n\t\t\taddKeyBytes += row.KeySize()\n\t\t\taddValBytes += row.ValueSize()\n\t\t}\n\t\taddNum += len(addRows)\n\t}\n\n\tfor _, updateRows := range updateRowsAll {\n\t\tfor _, row := range updateRows {\n\t\t\tupdateKeyBytes += row.KeySize()\n\t\t\tupdateValBytes += row.ValueSize()\n\t\t}\n\t\tupdateNum += len(updateRows)\n\t}\n\n\tfor _, deleteRows := range deleteRowsAll {\n\t\tfor _, row := range deleteRows {\n\t\t\ttfr, ok := row.(*TermFrequencyRow)\n\t\t\tif ok {\n\t\t\t\t// need to decrement counter\n\t\t\t\tif tfr.DictionaryRowKeySize() > len(rowBuf.buf) {\n\t\t\t\t\trowBuf.buf = make([]byte, tfr.DictionaryRowKeySize())\n\t\t\t\t}\n\t\t\t\tdictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf.buf)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tdictionaryDeltas[string(rowBuf.buf[:dictKeySize])] -= 1\n\t\t\t}\n\t\t\tdeleteKeyBytes += row.KeySize()\n\t\t}\n\t\tdeleteNum += len(deleteRows)\n\t}\n\n\tPutRowBuffer(rowBuf)\n\n\tmergeNum := len(dictionaryDeltas)\n\tmergeKeyBytes := 0\n\tmergeValBytes := mergeNum * DictionaryRowMaxValueSize\n\n\tfor dictRowKey := range dictionaryDeltas {\n\t\tmergeKeyBytes += len(dictRowKey)\n\t}\n\n\t// prepare batch\n\ttotBytes := addKeyBytes + addValBytes +\n\t\tupdateKeyBytes + updateValBytes +\n\t\tdeleteKeyBytes +\n\t\t2*(mergeKeyBytes+mergeValBytes)\n\n\tbuf, wb, err := writer.NewBatchEx(store.KVBatchOptions{\n\t\tTotalBytes: totBytes,\n\t\tNumSets:    addNum + updateNum,\n\t\tNumDeletes: deleteNum,\n\t\tNumMerges:  mergeNum,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\t_ = wb.Close()\n\t}()\n\n\t// fill the batch\n\tfor _, addRows := range addRowsAll {\n\t\tfor _, row := range addRows {\n\t\t\tkeySize, err := row.KeyTo(buf)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvalSize, err := row.ValueTo(buf[keySize:])\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\twb.Set(buf[:keySize], buf[keySize:keySize+valSize])\n\t\t\tbuf = buf[keySize+valSize:]\n\t\t}\n\t}\n\n\tfor _, updateRows := range updateRowsAll {\n\t\tfor _, row := range updateRows {\n\t\t\tkeySize, err := row.KeyTo(buf)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvalSize, err := row.ValueTo(buf[keySize:])\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\twb.Set(buf[:keySize], buf[keySize:keySize+valSize])\n\t\t\tbuf = buf[keySize+valSize:]\n\t\t}\n\t}\n\n\tfor _, deleteRows := range deleteRowsAll {\n\t\tfor _, row := range deleteRows {\n\t\t\tkeySize, err := row.KeyTo(buf)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\twb.Delete(buf[:keySize])\n\t\t\tbuf = buf[keySize:]\n\t\t}\n\t}\n\n\tfor dictRowKey, delta := range dictionaryDeltas {\n\t\tdictRowKeyLen := copy(buf, dictRowKey)\n\t\tbinary.LittleEndian.PutUint64(buf[dictRowKeyLen:], uint64(delta))\n\t\twb.Merge(buf[:dictRowKeyLen], buf[dictRowKeyLen:dictRowKeyLen+DictionaryRowMaxValueSize])\n\t\tbuf = buf[dictRowKeyLen+DictionaryRowMaxValueSize:]\n\t}\n\n\t// write out the batch\n\treturn writer.ExecuteBatch(wb)\n}\n\nfunc (udc *UpsideDownCouch) Open() (err error) {\n\t// acquire the write mutex for the duration of Open()\n\tudc.writeMutex.Lock()\n\tdefer udc.writeMutex.Unlock()\n\n\t// open the kv store\n\tstoreConstructor := registry.KVStoreConstructorByName(udc.storeName)\n\tif storeConstructor == nil {\n\t\terr = ErrorUnknownStorageType\n\t\treturn\n\t}\n\n\t// now open the store\n\tudc.store, err = storeConstructor(&mergeOperator, udc.storeConfig)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// start a reader to look at the index\n\tvar kvreader store.KVReader\n\tkvreader, err = udc.store.Reader()\n\tif err != nil {\n\t\treturn\n\t}\n\n\tvar value []byte\n\tvalue, err = kvreader.Get(VersionKey)\n\tif err != nil {\n\t\t_ = kvreader.Close()\n\t\treturn\n\t}\n\n\tif value != nil {\n\t\terr = udc.loadSchema(kvreader)\n\t\tif err != nil {\n\t\t\t_ = kvreader.Close()\n\t\t\treturn\n\t\t}\n\n\t\t// set doc count\n\t\tudc.m.Lock()\n\t\tudc.docCount, err = udc.countDocs(kvreader)\n\t\tudc.m.Unlock()\n\n\t\terr = kvreader.Close()\n\t} else {\n\t\t// new index, close the reader and open writer to init\n\t\terr = kvreader.Close()\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\n\t\tvar kvwriter store.KVWriter\n\t\tkvwriter, err = udc.store.Writer()\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tdefer func() {\n\t\t\tif cerr := kvwriter.Close(); err == nil && cerr != nil {\n\t\t\t\terr = cerr\n\t\t\t}\n\t\t}()\n\n\t\t// init the index\n\t\terr = udc.init(kvwriter)\n\t}\n\n\treturn\n}\n\nfunc (udc *UpsideDownCouch) countDocs(kvreader store.KVReader) (count uint64, err error) {\n\tit := kvreader.PrefixIterator([]byte{'b'})\n\tdefer func() {\n\t\tif cerr := it.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\t_, _, valid := it.Current()\n\tfor valid {\n\t\tcount++\n\t\tit.Next()\n\t\t_, _, valid = it.Current()\n\t}\n\n\treturn\n}\n\nfunc (udc *UpsideDownCouch) rowCount() (count uint64, err error) {\n\t// start an isolated reader for use during the rowcount\n\tkvreader, err := udc.store.Reader()\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer func() {\n\t\tif cerr := kvreader.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\tit := kvreader.RangeIterator(nil, nil)\n\tdefer func() {\n\t\tif cerr := it.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\t_, _, valid := it.Current()\n\tfor valid {\n\t\tcount++\n\t\tit.Next()\n\t\t_, _, valid = it.Current()\n\t}\n\n\treturn\n}\n\nfunc (udc *UpsideDownCouch) Close() error {\n\treturn udc.store.Close()\n}\n\nfunc (udc *UpsideDownCouch) Update(doc index.Document) (err error) {\n\t// do analysis before acquiring write lock\n\tanalysisStart := time.Now()\n\tresultChan := make(chan *AnalysisResult)\n\n\t// put the work on the queue\n\tudc.analysisQueue.Queue(func() {\n\t\tar := udc.analyze(doc)\n\t\tresultChan <- ar\n\t})\n\n\t// wait for the result\n\tresult := <-resultChan\n\tclose(resultChan)\n\tatomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart)))\n\n\tudc.writeMutex.Lock()\n\tdefer udc.writeMutex.Unlock()\n\n\t// open a reader for backindex lookup\n\tvar kvreader store.KVReader\n\tkvreader, err = udc.store.Reader()\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// first we lookup the backindex row for the doc id if it exists\n\t// lookup the back index row\n\tvar backIndexRow *BackIndexRow\n\tbackIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID()))\n\tif err != nil {\n\t\t_ = kvreader.Close()\n\t\tatomic.AddUint64(&udc.stats.errors, 1)\n\t\treturn\n\t}\n\n\terr = kvreader.Close()\n\tif err != nil {\n\t\treturn\n\t}\n\n\treturn udc.UpdateWithAnalysis(doc, result, backIndexRow)\n}\n\nfunc (udc *UpsideDownCouch) UpdateWithAnalysis(doc index.Document,\n\tresult *AnalysisResult, backIndexRow *BackIndexRow) (err error) {\n\t// start a writer for this update\n\tindexStart := time.Now()\n\tvar kvwriter store.KVWriter\n\tkvwriter, err = udc.store.Writer()\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer func() {\n\t\tif cerr := kvwriter.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\t// prepare a list of rows\n\tvar addRowsAll [][]UpsideDownCouchRow\n\tvar updateRowsAll [][]UpsideDownCouchRow\n\tvar deleteRowsAll [][]UpsideDownCouchRow\n\n\taddRows, updateRows, deleteRows := udc.mergeOldAndNew(backIndexRow, result.Rows)\n\tif len(addRows) > 0 {\n\t\taddRowsAll = append(addRowsAll, addRows)\n\t}\n\tif len(updateRows) > 0 {\n\t\tupdateRowsAll = append(updateRowsAll, updateRows)\n\t}\n\tif len(deleteRows) > 0 {\n\t\tdeleteRowsAll = append(deleteRowsAll, deleteRows)\n\t}\n\n\terr = udc.batchRows(kvwriter, addRowsAll, updateRowsAll, deleteRowsAll)\n\tif err == nil && backIndexRow == nil {\n\t\tudc.m.Lock()\n\t\tudc.docCount++\n\t\tudc.m.Unlock()\n\t}\n\tatomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))\n\tif err == nil {\n\t\tatomic.AddUint64(&udc.stats.updates, 1)\n\t\tatomic.AddUint64(&udc.stats.numPlainTextBytesIndexed, doc.NumPlainTextBytes())\n\t} else {\n\t\tatomic.AddUint64(&udc.stats.errors, 1)\n\t}\n\treturn\n}\n\nfunc (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []IndexRow) (addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) {\n\taddRows = make([]UpsideDownCouchRow, 0, len(rows))\n\n\tif backIndexRow == nil {\n\t\taddRows = addRows[0:len(rows)]\n\t\tfor i, row := range rows {\n\t\t\taddRows[i] = row\n\t\t}\n\t\treturn addRows, nil, nil\n\t}\n\n\tupdateRows = make([]UpsideDownCouchRow, 0, len(rows))\n\tdeleteRows = make([]UpsideDownCouchRow, 0, len(rows))\n\n\tvar existingTermKeys map[string]struct{}\n\tbackIndexTermKeys := backIndexRow.AllTermKeys()\n\tif len(backIndexTermKeys) > 0 {\n\t\texistingTermKeys = make(map[string]struct{}, len(backIndexTermKeys))\n\t\tfor _, key := range backIndexTermKeys {\n\t\t\texistingTermKeys[string(key)] = struct{}{}\n\t\t}\n\t}\n\n\tvar existingStoredKeys map[string]struct{}\n\tbackIndexStoredKeys := backIndexRow.AllStoredKeys()\n\tif len(backIndexStoredKeys) > 0 {\n\t\texistingStoredKeys = make(map[string]struct{}, len(backIndexStoredKeys))\n\t\tfor _, key := range backIndexStoredKeys {\n\t\t\texistingStoredKeys[string(key)] = struct{}{}\n\t\t}\n\t}\n\n\tkeyBuf := GetRowBuffer()\n\tfor _, row := range rows {\n\t\tswitch row := row.(type) {\n\t\tcase *TermFrequencyRow:\n\t\t\tif existingTermKeys != nil {\n\t\t\t\tif row.KeySize() > len(keyBuf.buf) {\n\t\t\t\t\tkeyBuf.buf = make([]byte, row.KeySize())\n\t\t\t\t}\n\t\t\t\tkeySize, _ := row.KeyTo(keyBuf.buf)\n\t\t\t\tif _, ok := existingTermKeys[string(keyBuf.buf[:keySize])]; ok {\n\t\t\t\t\tupdateRows = append(updateRows, row)\n\t\t\t\t\tdelete(existingTermKeys, string(keyBuf.buf[:keySize]))\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\taddRows = append(addRows, row)\n\t\tcase *StoredRow:\n\t\t\tif existingStoredKeys != nil {\n\t\t\t\tif row.KeySize() > len(keyBuf.buf) {\n\t\t\t\t\tkeyBuf.buf = make([]byte, row.KeySize())\n\t\t\t\t}\n\t\t\t\tkeySize, _ := row.KeyTo(keyBuf.buf)\n\t\t\t\tif _, ok := existingStoredKeys[string(keyBuf.buf[:keySize])]; ok {\n\t\t\t\t\tupdateRows = append(updateRows, row)\n\t\t\t\t\tdelete(existingStoredKeys, string(keyBuf.buf[:keySize]))\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\taddRows = append(addRows, row)\n\t\tdefault:\n\t\t\tupdateRows = append(updateRows, row)\n\t\t}\n\t}\n\tPutRowBuffer(keyBuf)\n\n\t// any of the existing rows that weren't updated need to be deleted\n\tfor existingTermKey := range existingTermKeys {\n\t\ttermFreqRow, err := NewTermFrequencyRowK([]byte(existingTermKey))\n\t\tif err == nil {\n\t\t\tdeleteRows = append(deleteRows, termFreqRow)\n\t\t}\n\t}\n\n\t// any of the existing stored fields that weren't updated need to be deleted\n\tfor existingStoredKey := range existingStoredKeys {\n\t\tstoredRow, err := NewStoredRowK([]byte(existingStoredKey))\n\t\tif err == nil {\n\t\t\tdeleteRows = append(deleteRows, storedRow)\n\t\t}\n\t}\n\n\treturn addRows, updateRows, deleteRows\n}\n\nfunc (udc *UpsideDownCouch) storeField(docID []byte, field index.Field, fieldIndex uint16, rows []IndexRow, backIndexStoredEntries []*BackIndexStoreEntry) ([]IndexRow, []*BackIndexStoreEntry) {\n\tfieldType := field.EncodedFieldType()\n\tstoredRow := NewStoredRow(docID, fieldIndex, field.ArrayPositions(), fieldType, field.Value())\n\n\t// record the back index entry\n\tbackIndexStoredEntry := BackIndexStoreEntry{Field: proto.Uint32(uint32(fieldIndex)), ArrayPositions: field.ArrayPositions()}\n\n\treturn append(rows, storedRow), append(backIndexStoredEntries, &backIndexStoredEntry)\n}\n\nfunc (udc *UpsideDownCouch) indexField(docID []byte, includeTermVectors bool, fieldIndex uint16, fieldLength int, tokenFreqs index.TokenFrequencies, rows []IndexRow, backIndexTermsEntries []*BackIndexTermsEntry) ([]IndexRow, []*BackIndexTermsEntry) {\n\tfieldNorm := float32(1.0 / math.Sqrt(float64(fieldLength)))\n\n\ttermFreqRows := make([]TermFrequencyRow, len(tokenFreqs))\n\ttermFreqRowsUsed := 0\n\n\tterms := make([]string, 0, len(tokenFreqs))\n\tfor k, tf := range tokenFreqs {\n\t\ttermFreqRow := &termFreqRows[termFreqRowsUsed]\n\t\ttermFreqRowsUsed++\n\n\t\tInitTermFrequencyRow(termFreqRow, tf.Term, fieldIndex, docID,\n\t\t\tuint64(frequencyFromTokenFreq(tf)), fieldNorm)\n\n\t\tif includeTermVectors {\n\t\t\ttermFreqRow.vectors, rows = udc.termVectorsFromTokenFreq(fieldIndex, tf, rows)\n\t\t}\n\n\t\t// record the back index entry\n\t\tterms = append(terms, k)\n\n\t\trows = append(rows, termFreqRow)\n\t}\n\tbackIndexTermsEntry := BackIndexTermsEntry{Field: proto.Uint32(uint32(fieldIndex)), Terms: terms}\n\tbackIndexTermsEntries = append(backIndexTermsEntries, &backIndexTermsEntry)\n\n\treturn rows, backIndexTermsEntries\n}\n\nfunc (udc *UpsideDownCouch) Delete(id string) (err error) {\n\tindexStart := time.Now()\n\n\tudc.writeMutex.Lock()\n\tdefer udc.writeMutex.Unlock()\n\n\t// open a reader for backindex lookup\n\tvar kvreader store.KVReader\n\tkvreader, err = udc.store.Reader()\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// first we lookup the backindex row for the doc id if it exists\n\t// lookup the back index row\n\tvar backIndexRow *BackIndexRow\n\tbackIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(id))\n\tif err != nil {\n\t\t_ = kvreader.Close()\n\t\tatomic.AddUint64(&udc.stats.errors, 1)\n\t\treturn\n\t}\n\n\terr = kvreader.Close()\n\tif err != nil {\n\t\treturn\n\t}\n\n\tif backIndexRow == nil {\n\t\tatomic.AddUint64(&udc.stats.deletes, 1)\n\t\treturn\n\t}\n\n\t// start a writer for this delete\n\tvar kvwriter store.KVWriter\n\tkvwriter, err = udc.store.Writer()\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer func() {\n\t\tif cerr := kvwriter.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tvar deleteRowsAll [][]UpsideDownCouchRow\n\n\tdeleteRows := udc.deleteSingle(id, backIndexRow, nil)\n\tif len(deleteRows) > 0 {\n\t\tdeleteRowsAll = append(deleteRowsAll, deleteRows)\n\t}\n\n\terr = udc.batchRows(kvwriter, nil, nil, deleteRowsAll)\n\tif err == nil {\n\t\tudc.m.Lock()\n\t\tudc.docCount--\n\t\tudc.m.Unlock()\n\t}\n\tatomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))\n\tif err == nil {\n\t\tatomic.AddUint64(&udc.stats.deletes, 1)\n\t} else {\n\t\tatomic.AddUint64(&udc.stats.errors, 1)\n\t}\n\treturn\n}\n\nfunc (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow, deleteRows []UpsideDownCouchRow) []UpsideDownCouchRow {\n\tidBytes := []byte(id)\n\n\tfor _, backIndexEntry := range backIndexRow.termsEntries {\n\t\tfor i := range backIndexEntry.Terms {\n\t\t\ttfr := NewTermFrequencyRow([]byte(backIndexEntry.Terms[i]), uint16(*backIndexEntry.Field), idBytes, 0, 0)\n\t\t\tdeleteRows = append(deleteRows, tfr)\n\t\t}\n\t}\n\tfor _, se := range backIndexRow.storedEntries {\n\t\tsf := NewStoredRow(idBytes, uint16(*se.Field), se.ArrayPositions, 'x', nil)\n\t\tdeleteRows = append(deleteRows, sf)\n\t}\n\n\t// also delete the back entry itself\n\tdeleteRows = append(deleteRows, backIndexRow)\n\treturn deleteRows\n}\n\nfunc decodeFieldType(typ byte, name string, pos []uint64, value []byte) document.Field {\n\tswitch typ {\n\tcase 't':\n\t\treturn document.NewTextField(name, pos, value)\n\tcase 'n':\n\t\treturn document.NewNumericFieldFromBytes(name, pos, value)\n\tcase 'd':\n\t\treturn document.NewDateTimeFieldFromBytes(name, pos, value)\n\tcase 'b':\n\t\treturn document.NewBooleanFieldFromBytes(name, pos, value)\n\tcase 'g':\n\t\treturn document.NewGeoPointFieldFromBytes(name, pos, value)\n\tcase 'i':\n\t\treturn document.NewIPFieldFromBytes(name, pos, value)\n\t}\n\treturn nil\n}\n\nfunc frequencyFromTokenFreq(tf *index.TokenFreq) int {\n\treturn tf.Frequency()\n}\n\nfunc (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *index.TokenFreq, rows []IndexRow) ([]*TermVector, []IndexRow) {\n\ta := make([]TermVector, len(tf.Locations))\n\trv := make([]*TermVector, len(tf.Locations))\n\n\tfor i, l := range tf.Locations {\n\t\tvar newFieldRow *FieldRow\n\t\tfieldIndex := field\n\t\tif l.Field != \"\" {\n\t\t\t// lookup correct field\n\t\t\tfieldIndex, newFieldRow = udc.fieldIndexOrNewRow(l.Field)\n\t\t\tif newFieldRow != nil {\n\t\t\t\trows = append(rows, newFieldRow)\n\t\t\t}\n\t\t}\n\t\ta[i] = TermVector{\n\t\t\tfield:          fieldIndex,\n\t\t\tarrayPositions: l.ArrayPositions,\n\t\t\tpos:            uint64(l.Position),\n\t\t\tstart:          uint64(l.Start),\n\t\t\tend:            uint64(l.End),\n\t\t}\n\t\trv[i] = &a[i]\n\t}\n\n\treturn rv, rows\n}\n\nfunc (udc *UpsideDownCouch) termFieldVectorsFromTermVectors(in []*TermVector) []*index.TermFieldVector {\n\tif len(in) == 0 {\n\t\treturn nil\n\t}\n\n\ta := make([]index.TermFieldVector, len(in))\n\trv := make([]*index.TermFieldVector, len(in))\n\n\tfor i, tv := range in {\n\t\tfieldName := udc.fieldCache.FieldIndexed(tv.field)\n\t\ta[i] = index.TermFieldVector{\n\t\t\tField:          fieldName,\n\t\t\tArrayPositions: tv.arrayPositions,\n\t\t\tPos:            tv.pos,\n\t\t\tStart:          tv.start,\n\t\t\tEnd:            tv.end,\n\t\t}\n\t\trv[i] = &a[i]\n\t}\n\treturn rv\n}\n\nfunc (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) {\n\tpersistedCallback := batch.PersistedCallback()\n\tif persistedCallback != nil {\n\t\tdefer persistedCallback(err)\n\t}\n\tanalysisStart := time.Now()\n\n\tresultChan := make(chan *AnalysisResult, len(batch.IndexOps))\n\n\tvar numUpdates uint64\n\tvar numPlainTextBytes uint64\n\tfor _, doc := range batch.IndexOps {\n\t\tif doc != nil {\n\t\t\tnumUpdates++\n\t\t\tnumPlainTextBytes += doc.NumPlainTextBytes()\n\t\t}\n\t}\n\n\tif numUpdates > 0 {\n\t\tgo func() {\n\t\t\tfor k := range batch.IndexOps {\n\t\t\t\tdoc := batch.IndexOps[k]\n\t\t\t\tif doc != nil {\n\t\t\t\t\t// put the work on the queue\n\t\t\t\t\tudc.analysisQueue.Queue(func() {\n\t\t\t\t\t\tar := udc.analyze(doc)\n\t\t\t\t\t\tresultChan <- ar\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\t// retrieve back index rows concurrent with analysis\n\tdocBackIndexRowErr := error(nil)\n\tdocBackIndexRowCh := make(chan *docBackIndexRow, len(batch.IndexOps))\n\n\tudc.writeMutex.Lock()\n\tdefer udc.writeMutex.Unlock()\n\n\tgo func() {\n\t\tdefer close(docBackIndexRowCh)\n\n\t\t// open a reader for backindex lookup\n\t\tvar kvreader store.KVReader\n\t\tkvreader, err = udc.store.Reader()\n\t\tif err != nil {\n\t\t\tdocBackIndexRowErr = err\n\t\t\treturn\n\t\t}\n\t\tdefer func() {\n\t\t\tif cerr := kvreader.Close(); err == nil && cerr != nil {\n\t\t\t\tdocBackIndexRowErr = cerr\n\t\t\t}\n\t\t}()\n\n\t\tfor docID, doc := range batch.IndexOps {\n\t\t\tbackIndexRow, err := backIndexRowForDoc(kvreader, index.IndexInternalID(docID))\n\t\t\tif err != nil {\n\t\t\t\tdocBackIndexRowErr = err\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tdocBackIndexRowCh <- &docBackIndexRow{docID, doc, backIndexRow}\n\t\t}\n\t}()\n\n\t// wait for analysis result\n\tnewRowsMap := make(map[string][]IndexRow)\n\tvar itemsDeQueued uint64\n\tfor itemsDeQueued < numUpdates {\n\t\tresult := <-resultChan\n\t\tnewRowsMap[result.DocID] = result.Rows\n\t\titemsDeQueued++\n\t}\n\tclose(resultChan)\n\n\tatomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart)))\n\n\tdocsAdded := uint64(0)\n\tdocsDeleted := uint64(0)\n\n\tindexStart := time.Now()\n\n\t// prepare a list of rows\n\tvar addRowsAll [][]UpsideDownCouchRow\n\tvar updateRowsAll [][]UpsideDownCouchRow\n\tvar deleteRowsAll [][]UpsideDownCouchRow\n\n\t// add the internal ops\n\tvar updateRows []UpsideDownCouchRow\n\tvar deleteRows []UpsideDownCouchRow\n\n\tfor internalKey, internalValue := range batch.InternalOps {\n\t\tif internalValue == nil {\n\t\t\t// delete\n\t\t\tdeleteInternalRow := NewInternalRow([]byte(internalKey), nil)\n\t\t\tdeleteRows = append(deleteRows, deleteInternalRow)\n\t\t} else {\n\t\t\tupdateInternalRow := NewInternalRow([]byte(internalKey), internalValue)\n\t\t\tupdateRows = append(updateRows, updateInternalRow)\n\t\t}\n\t}\n\n\tif len(updateRows) > 0 {\n\t\tupdateRowsAll = append(updateRowsAll, updateRows)\n\t}\n\tif len(deleteRows) > 0 {\n\t\tdeleteRowsAll = append(deleteRowsAll, deleteRows)\n\t}\n\n\t// process back index rows as they arrive\n\tfor dbir := range docBackIndexRowCh {\n\t\tif dbir.doc == nil && dbir.backIndexRow != nil {\n\t\t\t// delete\n\t\t\tdeleteRows := udc.deleteSingle(dbir.docID, dbir.backIndexRow, nil)\n\t\t\tif len(deleteRows) > 0 {\n\t\t\t\tdeleteRowsAll = append(deleteRowsAll, deleteRows)\n\t\t\t}\n\t\t\tdocsDeleted++\n\t\t} else if dbir.doc != nil {\n\t\t\taddRows, updateRows, deleteRows := udc.mergeOldAndNew(dbir.backIndexRow, newRowsMap[dbir.docID])\n\t\t\tif len(addRows) > 0 {\n\t\t\t\taddRowsAll = append(addRowsAll, addRows)\n\t\t\t}\n\t\t\tif len(updateRows) > 0 {\n\t\t\t\tupdateRowsAll = append(updateRowsAll, updateRows)\n\t\t\t}\n\t\t\tif len(deleteRows) > 0 {\n\t\t\t\tdeleteRowsAll = append(deleteRowsAll, deleteRows)\n\t\t\t}\n\t\t\tif dbir.backIndexRow == nil {\n\t\t\t\tdocsAdded++\n\t\t\t}\n\t\t}\n\t}\n\n\tif docBackIndexRowErr != nil {\n\t\treturn docBackIndexRowErr\n\t}\n\n\t// start a writer for this batch\n\tvar kvwriter store.KVWriter\n\tkvwriter, err = udc.store.Writer()\n\tif err != nil {\n\t\treturn\n\t}\n\n\terr = udc.batchRows(kvwriter, addRowsAll, updateRowsAll, deleteRowsAll)\n\tif err != nil {\n\t\t_ = kvwriter.Close()\n\t\tatomic.AddUint64(&udc.stats.errors, 1)\n\t\treturn\n\t}\n\n\terr = kvwriter.Close()\n\n\tatomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))\n\n\tif err == nil {\n\t\tudc.m.Lock()\n\t\tudc.docCount += docsAdded\n\t\tudc.docCount -= docsDeleted\n\t\tudc.m.Unlock()\n\t\tatomic.AddUint64(&udc.stats.updates, numUpdates)\n\t\tatomic.AddUint64(&udc.stats.deletes, docsDeleted)\n\t\tatomic.AddUint64(&udc.stats.batches, 1)\n\t\tatomic.AddUint64(&udc.stats.numPlainTextBytesIndexed, numPlainTextBytes)\n\t} else {\n\t\tatomic.AddUint64(&udc.stats.errors, 1)\n\t}\n\n\treturn\n}\n\nfunc (udc *UpsideDownCouch) SetInternal(key, val []byte) (err error) {\n\tinternalRow := NewInternalRow(key, val)\n\tudc.writeMutex.Lock()\n\tdefer udc.writeMutex.Unlock()\n\tvar writer store.KVWriter\n\twriter, err = udc.store.Writer()\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer func() {\n\t\tif cerr := writer.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tbatch := writer.NewBatch()\n\tbatch.Set(internalRow.Key(), internalRow.Value())\n\n\treturn writer.ExecuteBatch(batch)\n}\n\nfunc (udc *UpsideDownCouch) DeleteInternal(key []byte) (err error) {\n\tinternalRow := NewInternalRow(key, nil)\n\tudc.writeMutex.Lock()\n\tdefer udc.writeMutex.Unlock()\n\tvar writer store.KVWriter\n\twriter, err = udc.store.Writer()\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer func() {\n\t\tif cerr := writer.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tbatch := writer.NewBatch()\n\tbatch.Delete(internalRow.Key())\n\treturn writer.ExecuteBatch(batch)\n}\n\nfunc (udc *UpsideDownCouch) Reader() (index.IndexReader, error) {\n\tkvr, err := udc.store.Reader()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error opening store reader: %v\", err)\n\t}\n\tudc.m.RLock()\n\tdefer udc.m.RUnlock()\n\treturn &IndexReader{\n\t\tindex:    udc,\n\t\tkvreader: kvr,\n\t\tdocCount: udc.docCount,\n\t}, nil\n}\n\nfunc (udc *UpsideDownCouch) Stats() json.Marshaler {\n\treturn udc.stats\n}\n\nfunc (udc *UpsideDownCouch) StatsMap() map[string]interface{} {\n\treturn udc.stats.statsMap()\n}\n\nfunc (udc *UpsideDownCouch) Advanced() (store.KVStore, error) {\n\treturn udc.store, nil\n}\n\nfunc (udc *UpsideDownCouch) fieldIndexOrNewRow(name string) (uint16, *FieldRow) {\n\tindex, existed := udc.fieldCache.FieldNamed(name, true)\n\tif !existed {\n\t\treturn index, NewFieldRow(index, name)\n\t}\n\treturn index, nil\n}\n\nfunc init() {\n\terr := registry.RegisterIndexType(Name, NewUpsideDownCouch)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc backIndexRowForDoc(kvreader store.KVReader, docID index.IndexInternalID) (*BackIndexRow, error) {\n\t// use a temporary row structure to build key\n\ttempRow := BackIndexRow{\n\t\tdoc: docID,\n\t}\n\n\tkeyBuf := GetRowBuffer()\n\tif tempRow.KeySize() > len(keyBuf.buf) {\n\t\tkeyBuf.buf = make([]byte, 2*tempRow.KeySize())\n\t}\n\tdefer PutRowBuffer(keyBuf)\n\tkeySize, err := tempRow.KeyTo(keyBuf.buf)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvalue, err := kvreader.Get(keyBuf.buf[:keySize])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif value == nil {\n\t\treturn nil, nil\n\t}\n\tbackIndexRow, err := NewBackIndexRowKV(keyBuf.buf[:keySize], value)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn backIndexRow, nil\n}\n"
  },
  {
    "path": "index/upsidedown/upsidedown.pb.go",
    "content": "// Code generated by protoc-gen-go. DO NOT EDIT.\n// versions:\n// \tprotoc-gen-go v1.36.6\n// \tprotoc        v5.29.3\n// source: index/upsidedown/upsidedown.proto\n\npackage upsidedown\n\nimport (\n\tfmt \"fmt\"\n\tprotoreflect \"google.golang.org/protobuf/reflect/protoreflect\"\n\tprotoimpl \"google.golang.org/protobuf/runtime/protoimpl\"\n\tio \"io\"\n\treflect \"reflect\"\n\tsync \"sync\"\n\tunsafe \"unsafe\"\n)\n\nconst (\n\t// Verify that this generated code is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)\n\t// Verify that runtime/protoimpl is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)\n)\n\nvar (\n\tErrInvalidLengthUpsidedown = fmt.Errorf(\"proto: negative length found during unmarshaling\")\n)\n\ntype BackIndexTermsEntry struct {\n\tstate         protoimpl.MessageState `protogen:\"open.v1\"`\n\tField         *uint32                `protobuf:\"varint,1,req,name=field\" json:\"field,omitempty\"`\n\tTerms         []string               `protobuf:\"bytes,2,rep,name=terms\" json:\"terms,omitempty\"`\n\tunknownFields protoimpl.UnknownFields\n\tsizeCache     protoimpl.SizeCache\n}\n\nfunc (x *BackIndexTermsEntry) Reset() {\n\t*x = BackIndexTermsEntry{}\n\tmi := &file_index_upsidedown_upsidedown_proto_msgTypes[0]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *BackIndexTermsEntry) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*BackIndexTermsEntry) ProtoMessage() {}\n\nfunc (x *BackIndexTermsEntry) ProtoReflect() protoreflect.Message {\n\tmi := &file_index_upsidedown_upsidedown_proto_msgTypes[0]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use BackIndexTermsEntry.ProtoReflect.Descriptor instead.\nfunc (*BackIndexTermsEntry) Descriptor() ([]byte, []int) {\n\treturn file_index_upsidedown_upsidedown_proto_rawDescGZIP(), []int{0}\n}\n\nfunc (x *BackIndexTermsEntry) GetField() uint32 {\n\tif x != nil && x.Field != nil {\n\t\treturn *x.Field\n\t}\n\treturn 0\n}\n\nfunc (x *BackIndexTermsEntry) GetTerms() []string {\n\tif x != nil {\n\t\treturn x.Terms\n\t}\n\treturn nil\n}\n\nfunc (x *BackIndexTermsEntry) MarshalTo(data []byte) (n int, err error) {\n\tvar i int\n\t_ = i\n\tvar l int\n\t_ = l\n\tif x.Field == nil {\n\t\treturn 0, fmt.Errorf(\"missing required `Field`\")\n\t} else {\n\t\tdata[i] = 0x8\n\t\ti++\n\t\ti = encodeVarintUpsidedown(data, i, uint64(*x.Field))\n\t}\n\tif len(x.Terms) > 0 {\n\t\tfor _, s := range x.Terms {\n\t\t\tdata[i] = 0x12\n\t\t\ti++\n\t\t\tl = len(s)\n\t\t\tfor l >= 1<<7 {\n\t\t\t\tdata[i] = uint8(uint64(l)&0x7f | 0x80)\n\t\t\t\tl >>= 7\n\t\t\t\ti++\n\t\t\t}\n\t\t\tdata[i] = uint8(l)\n\t\t\ti++\n\t\t\ti += copy(data[i:], s)\n\t\t}\n\t}\n\treturn i, nil\n}\n\nfunc (x *BackIndexTermsEntry) Size() (n int) {\n\tvar l int\n\t_ = l\n\tif x.Field != nil {\n\t\tn += 1 + sovUpsidedown(uint64(*x.Field))\n\t}\n\tif len(x.Terms) > 0 {\n\t\tfor _, s := range x.Terms {\n\t\t\tl = len(s)\n\t\t\tn += 1 + l + sovUpsidedown(uint64(l))\n\t\t}\n\t}\n\treturn n\n}\n\ntype BackIndexStoreEntry struct {\n\tstate          protoimpl.MessageState `protogen:\"open.v1\"`\n\tField          *uint32                `protobuf:\"varint,1,req,name=field\" json:\"field,omitempty\"`\n\tArrayPositions []uint64               `protobuf:\"varint,2,rep,name=arrayPositions\" json:\"arrayPositions,omitempty\"`\n\tunknownFields  protoimpl.UnknownFields\n\tsizeCache      protoimpl.SizeCache\n}\n\nfunc (x *BackIndexStoreEntry) Reset() {\n\t*x = BackIndexStoreEntry{}\n\tmi := &file_index_upsidedown_upsidedown_proto_msgTypes[1]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *BackIndexStoreEntry) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*BackIndexStoreEntry) ProtoMessage() {}\n\nfunc (x *BackIndexStoreEntry) ProtoReflect() protoreflect.Message {\n\tmi := &file_index_upsidedown_upsidedown_proto_msgTypes[1]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use BackIndexStoreEntry.ProtoReflect.Descriptor instead.\nfunc (*BackIndexStoreEntry) Descriptor() ([]byte, []int) {\n\treturn file_index_upsidedown_upsidedown_proto_rawDescGZIP(), []int{1}\n}\n\nfunc (x *BackIndexStoreEntry) GetField() uint32 {\n\tif x != nil && x.Field != nil {\n\t\treturn *x.Field\n\t}\n\treturn 0\n}\n\nfunc (x *BackIndexStoreEntry) GetArrayPositions() []uint64 {\n\tif x != nil {\n\t\treturn x.ArrayPositions\n\t}\n\treturn nil\n}\n\nfunc (x *BackIndexStoreEntry) MarshalTo(data []byte) (n int, err error) {\n\tvar i int\n\t_ = i\n\tvar l int\n\t_ = l\n\tif x.Field == nil {\n\t\treturn 0, fmt.Errorf(\"missing required `Field`\")\n\t} else {\n\t\tdata[i] = 0x8\n\t\ti++\n\t\ti = encodeVarintUpsidedown(data, i, uint64(*x.Field))\n\t}\n\tif len(x.ArrayPositions) > 0 {\n\t\tfor _, num := range x.ArrayPositions {\n\t\t\tdata[i] = 0x10\n\t\t\ti++\n\t\t\ti = encodeVarintUpsidedown(data, i, uint64(num))\n\t\t}\n\t}\n\treturn i, nil\n}\n\nfunc (x *BackIndexStoreEntry) Size() (n int) {\n\tvar l int\n\t_ = l\n\tif x.Field != nil {\n\t\tn += 1 + sovUpsidedown(uint64(*x.Field))\n\t}\n\tif len(x.ArrayPositions) > 0 {\n\t\tfor _, e := range x.ArrayPositions {\n\t\t\tn += 1 + sovUpsidedown(uint64(e))\n\t\t}\n\t}\n\treturn n\n}\n\ntype BackIndexRowValue struct {\n\tstate         protoimpl.MessageState `protogen:\"open.v1\"`\n\tTermsEntries  []*BackIndexTermsEntry `protobuf:\"bytes,1,rep,name=termsEntries\" json:\"termsEntries,omitempty\"`\n\tStoredEntries []*BackIndexStoreEntry `protobuf:\"bytes,2,rep,name=storedEntries\" json:\"storedEntries,omitempty\"`\n\tunknownFields protoimpl.UnknownFields\n\tsizeCache     protoimpl.SizeCache\n}\n\nfunc (x *BackIndexRowValue) Reset() {\n\t*x = BackIndexRowValue{}\n\tmi := &file_index_upsidedown_upsidedown_proto_msgTypes[2]\n\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\tms.StoreMessageInfo(mi)\n}\n\nfunc (x *BackIndexRowValue) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*BackIndexRowValue) ProtoMessage() {}\n\nfunc (x *BackIndexRowValue) ProtoReflect() protoreflect.Message {\n\tmi := &file_index_upsidedown_upsidedown_proto_msgTypes[2]\n\tif x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use BackIndexRowValue.ProtoReflect.Descriptor instead.\nfunc (*BackIndexRowValue) Descriptor() ([]byte, []int) {\n\treturn file_index_upsidedown_upsidedown_proto_rawDescGZIP(), []int{2}\n}\n\nfunc (x *BackIndexRowValue) GetTermsEntries() []*BackIndexTermsEntry {\n\tif x != nil {\n\t\treturn x.TermsEntries\n\t}\n\treturn nil\n}\n\nfunc (x *BackIndexRowValue) GetStoredEntries() []*BackIndexStoreEntry {\n\tif x != nil {\n\t\treturn x.StoredEntries\n\t}\n\treturn nil\n}\n\nfunc (x *BackIndexRowValue) MarshalTo(data []byte) (n int, err error) {\n\tvar i int\n\t_ = i\n\tvar l int\n\t_ = l\n\tif len(x.TermsEntries) > 0 {\n\t\tfor _, msg := range x.TermsEntries {\n\t\t\tdata[i] = 0xa\n\t\t\ti++\n\t\t\ti = encodeVarintUpsidedown(data, i, uint64(msg.Size()))\n\t\t\tn, err := msg.MarshalTo(data[i:])\n\t\t\tif err != nil {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\t\ti += n\n\t\t}\n\t}\n\tif len(x.StoredEntries) > 0 {\n\t\tfor _, msg := range x.StoredEntries {\n\t\t\tdata[i] = 0x12\n\t\t\ti++\n\t\t\ti = encodeVarintUpsidedown(data, i, uint64(msg.Size()))\n\t\t\tn, err := msg.MarshalTo(data[i:])\n\t\t\tif err != nil {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\t\ti += n\n\t\t}\n\t}\n\treturn i, nil\n}\n\nfunc (x *BackIndexRowValue) Size() (n int) {\n\tvar l int\n\t_ = l\n\tif len(x.TermsEntries) > 0 {\n\t\tfor _, e := range x.TermsEntries {\n\t\t\tl = e.Size()\n\t\t\tn += 1 + l + sovUpsidedown(uint64(l))\n\t\t}\n\t}\n\tif len(x.StoredEntries) > 0 {\n\t\tfor _, e := range x.StoredEntries {\n\t\t\tl = e.Size()\n\t\t\tn += 1 + l + sovUpsidedown(uint64(l))\n\t\t}\n\t}\n\treturn n\n}\n\nfunc skipUpsidedown(data []byte) (n int, err error) {\n\tl := len(data)\n\tiNdEx := 0\n\tfor iNdEx < l {\n\t\tvar wire uint64\n\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\tif iNdEx >= l {\n\t\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t\t}\n\t\t\tb := data[iNdEx]\n\t\t\tiNdEx++\n\t\t\twire |= (uint64(b) & 0x7F) << shift\n\t\t\tif b < 0x80 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\twireType := int(wire & 0x7)\n\t\tswitch wireType {\n\t\tcase 0:\n\t\t\tfor {\n\t\t\t\tif iNdEx >= l {\n\t\t\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t\t\t}\n\t\t\t\tiNdEx++\n\t\t\t\tif data[iNdEx-1] < 0x80 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn iNdEx, nil\n\t\tcase 1:\n\t\t\tiNdEx += 8\n\t\t\treturn iNdEx, nil\n\t\tcase 2:\n\t\t\tvar length int\n\t\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\t\tif iNdEx >= l {\n\t\t\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t\t\t}\n\t\t\t\tb := data[iNdEx]\n\t\t\t\tiNdEx++\n\t\t\t\tlength |= (int(b) & 0x7F) << shift\n\t\t\t\tif b < 0x80 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tiNdEx += length\n\t\t\tif length < 0 {\n\t\t\t\treturn 0, ErrInvalidLengthUpsidedown\n\t\t\t}\n\t\t\treturn iNdEx, nil\n\t\tcase 3:\n\t\t\tfor {\n\t\t\t\tvar innerWire uint64\n\t\t\t\tvar start int = iNdEx\n\t\t\t\tfor shift := uint(0); ; shift += 7 {\n\t\t\t\t\tif iNdEx >= l {\n\t\t\t\t\t\treturn 0, io.ErrUnexpectedEOF\n\t\t\t\t\t}\n\t\t\t\t\tb := data[iNdEx]\n\t\t\t\t\tiNdEx++\n\t\t\t\t\tinnerWire |= (uint64(b) & 0x7F) << shift\n\t\t\t\t\tif b < 0x80 {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tinnerWireType := int(innerWire & 0x7)\n\t\t\t\tif innerWireType == 4 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tnext, err := skipUpsidedown(data[start:])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn 0, err\n\t\t\t\t}\n\t\t\t\tiNdEx = start + next\n\t\t\t}\n\t\t\treturn iNdEx, nil\n\t\tcase 4:\n\t\t\treturn iNdEx, nil\n\t\tcase 5:\n\t\t\tiNdEx += 4\n\t\t\treturn iNdEx, nil\n\t\tdefault:\n\t\t\treturn 0, fmt.Errorf(\"proto: illegal wireType %d\", wireType)\n\t\t}\n\t}\n\tpanic(\"unreachable\")\n}\n\nfunc sovUpsidedown(x uint64) (n int) {\n\tfor {\n\t\tn++\n\t\tx >>= 7\n\t\tif x == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn n\n}\n\nfunc encodeVarintUpsidedown(data []byte, offset int, v uint64) int {\n\tfor v >= 1<<7 {\n\t\tdata[offset] = uint8(v&0x7f | 0x80)\n\t\tv >>= 7\n\t\toffset++\n\t}\n\tdata[offset] = uint8(v)\n\treturn offset + 1\n}\n\nvar File_index_upsidedown_upsidedown_proto protoreflect.FileDescriptor\n\nconst file_index_upsidedown_upsidedown_proto_rawDesc = \"\" +\n\t\"\\n\" +\n\t\"!index/upsidedown/upsidedown.proto\\\"A\\n\" +\n\t\"\\x13BackIndexTermsEntry\\x12\\x14\\n\" +\n\t\"\\x05field\\x18\\x01 \\x02(\\rR\\x05field\\x12\\x14\\n\" +\n\t\"\\x05terms\\x18\\x02 \\x03(\\tR\\x05terms\\\"S\\n\" +\n\t\"\\x13BackIndexStoreEntry\\x12\\x14\\n\" +\n\t\"\\x05field\\x18\\x01 \\x02(\\rR\\x05field\\x12&\\n\" +\n\t\"\\x0earrayPositions\\x18\\x02 \\x03(\\x04R\\x0earrayPositions\\\"\\x89\\x01\\n\" +\n\t\"\\x11BackIndexRowValue\\x128\\n\" +\n\t\"\\ftermsEntries\\x18\\x01 \\x03(\\v2\\x14.BackIndexTermsEntryR\\ftermsEntries\\x12:\\n\" +\n\t\"\\rstoredEntries\\x18\\x02 \\x03(\\v2\\x14.BackIndexStoreEntryR\\rstoredEntries\"\n\nvar (\n\tfile_index_upsidedown_upsidedown_proto_rawDescOnce sync.Once\n\tfile_index_upsidedown_upsidedown_proto_rawDescData []byte\n)\n\nfunc file_index_upsidedown_upsidedown_proto_rawDescGZIP() []byte {\n\tfile_index_upsidedown_upsidedown_proto_rawDescOnce.Do(func() {\n\t\tfile_index_upsidedown_upsidedown_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_index_upsidedown_upsidedown_proto_rawDesc), len(file_index_upsidedown_upsidedown_proto_rawDesc)))\n\t})\n\treturn file_index_upsidedown_upsidedown_proto_rawDescData\n}\n\nvar file_index_upsidedown_upsidedown_proto_msgTypes = make([]protoimpl.MessageInfo, 3)\nvar file_index_upsidedown_upsidedown_proto_goTypes = []any{\n\t(*BackIndexTermsEntry)(nil), // 0: BackIndexTermsEntry\n\t(*BackIndexStoreEntry)(nil), // 1: BackIndexStoreEntry\n\t(*BackIndexRowValue)(nil),   // 2: BackIndexRowValue\n}\nvar file_index_upsidedown_upsidedown_proto_depIdxs = []int32{\n\t0, // 0: BackIndexRowValue.termsEntries:type_name -> BackIndexTermsEntry\n\t1, // 1: BackIndexRowValue.storedEntries:type_name -> BackIndexStoreEntry\n\t2, // [2:2] is the sub-list for method output_type\n\t2, // [2:2] is the sub-list for method input_type\n\t2, // [2:2] is the sub-list for extension type_name\n\t2, // [2:2] is the sub-list for extension extendee\n\t0, // [0:2] is the sub-list for field type_name\n}\n\nfunc init() { file_index_upsidedown_upsidedown_proto_init() }\nfunc file_index_upsidedown_upsidedown_proto_init() {\n\tif File_index_upsidedown_upsidedown_proto != nil {\n\t\treturn\n\t}\n\ttype x struct{}\n\tout := protoimpl.TypeBuilder{\n\t\tFile: protoimpl.DescBuilder{\n\t\t\tGoPackagePath: reflect.TypeOf(x{}).PkgPath(),\n\t\t\tRawDescriptor: unsafe.Slice(unsafe.StringData(file_index_upsidedown_upsidedown_proto_rawDesc), len(file_index_upsidedown_upsidedown_proto_rawDesc)),\n\t\t\tNumEnums:      0,\n\t\t\tNumMessages:   3,\n\t\t\tNumExtensions: 0,\n\t\t\tNumServices:   0,\n\t\t},\n\t\tGoTypes:           file_index_upsidedown_upsidedown_proto_goTypes,\n\t\tDependencyIndexes: file_index_upsidedown_upsidedown_proto_depIdxs,\n\t\tMessageInfos:      file_index_upsidedown_upsidedown_proto_msgTypes,\n\t}.Build()\n\tFile_index_upsidedown_upsidedown_proto = out.File\n\tfile_index_upsidedown_upsidedown_proto_goTypes = nil\n\tfile_index_upsidedown_upsidedown_proto_depIdxs = nil\n}\n"
  },
  {
    "path": "index/upsidedown/upsidedown.proto",
    "content": "message BackIndexTermsEntry {\n  required uint32 field = 1;\n\trepeated string terms = 2;\n}\n\nmessage BackIndexStoreEntry {\n\trequired uint32 field = 1;\n\trepeated uint64 arrayPositions = 2;\n}\n\nmessage BackIndexRowValue {\n\trepeated BackIndexTermsEntry termsEntries = 1;\n\trepeated BackIndexStoreEntry storedEntries = 2;\n}\n"
  },
  {
    "path": "index/upsidedown/upsidedown_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage upsidedown\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\tregexpTokenizer \"github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/null\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar testAnalyzer = &analysis.DefaultAnalyzer{\n\tTokenizer: regexpTokenizer.NewRegexpTokenizer(regexp.MustCompile(`\\w+`)),\n}\n\nfunc TestIndexOpenReopen(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// opening the database should have inserted a version\n\texpectedLength := uint64(1)\n\trowCount, err := idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n\n\t// now close it\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err = NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\n\t// now close it\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsert(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// should have 4 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the term count, and 1 for the back index entry)\n\texpectedLength := uint64(1 + 1 + 1 + 1 + 1)\n\trowCount, err := idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n}\n\nfunc TestIndexInsertThenDelete(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc2 := document.NewDocument(\"2\")\n\tdoc2.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc2)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Delete(\"1\")\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\texpectedCount--\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Delete(\"2\")\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\texpectedCount--\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// should have 2 rows (1 for version, 1 for schema field, 1 for dictionary row garbage)\n\texpectedLength := uint64(1 + 1 + 1)\n\trowCount, err := idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n}\n\nfunc TestIndexInsertThenUpdate(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\t// this update should overwrite one term, and introduce one new one\n\tdoc = document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithAnalyzer(\"name\", []uint64{}, []byte(\"test fail\"), testAnalyzer))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\n\t// should have 2 rows (1 for version, 1 for schema field, and 2 for the two term, and 2 for the term counts, and 1 for the back index entry)\n\texpectedLength := uint64(1 + 1 + 2 + 2 + 1)\n\trowCount, err := idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n\n\t// now do another update that should remove one of the terms\n\tdoc = document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"fail\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\n\t// should have 2 rows (1 for version, 1 for schema field, and 1 for the remaining term, and 2 for the term diciontary, and 1 for the back index entry)\n\texpectedLength = uint64(1 + 1 + 1 + 2 + 1)\n\trowCount, err = idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n}\n\nfunc TestIndexInsertMultiple(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\n\tvar expectedCount uint64\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\t// should have 4 rows (1 for version, 1 for schema field, and 2 for single term, and 1 for the term count, and 2 for the back index entries)\n\texpectedLength := uint64(1 + 1 + 2 + 1 + 2)\n\trowCount, err := idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n\n\t// close, reopen and add one more to test that counting works correctly\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err = NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Fatalf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc = document.NewDocument(\"3\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsertWithStore(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// should have 6 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the stored field and 1 for the term count, and 1 for the back index entry)\n\texpectedLength := uint64(1 + 1 + 1 + 1 + 1 + 1)\n\trowCount, err := idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tstoredDocInt, err := indexReader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstoredDoc := storedDocInt.(*document.Document)\n\n\tif len(storedDoc.Fields) != 1 {\n\t\tt.Errorf(\"expected 1 stored field, got %d\", len(storedDoc.Fields))\n\t}\n\ttextField, ok := storedDoc.Fields[0].(*document.TextField)\n\tif !ok {\n\t\tt.Errorf(\"expected text field\")\n\t}\n\tif string(textField.Value()) != \"test\" {\n\t\tt.Errorf(\"expected field content 'test', got '%s'\", string(textField.Value()))\n\t}\n}\n\nfunc TestIndexInternalCRUD(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\t// get something that doesn't exist yet\n\tval, err := indexReader.GetInternal([]byte(\"key\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got %s\", val)\n\t}\n\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// set\n\terr = idx.SetInternal([]byte(\"key\"), []byte(\"abc\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tindexReader2, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\t// get\n\tval, err = indexReader2.GetInternal([]byte(\"key\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif string(val) != \"abc\" {\n\t\tt.Errorf(\"expected %s, got '%s'\", \"abc\", val)\n\t}\n\n\terr = indexReader2.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// delete\n\terr = idx.DeleteInternal([]byte(\"key\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tindexReader3, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\t// get again\n\tval, err = indexReader3.GetInternal([]byte(\"key\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got %s\", val)\n\t}\n\n\terr = indexReader3.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexBatch(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\n\t// first create 2 docs the old fashioned way\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test2\")))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\t// now create a batch which does 3 things\n\t// insert new doc\n\t// update existing doc\n\t// delete existing doc\n\t// net document count change 0\n\n\tbatch := index.NewBatch()\n\tdoc = document.NewDocument(\"3\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test3\")))\n\tbatch.Update(doc)\n\tdoc = document.NewDocument(\"2\")\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test2updated\")))\n\tbatch.Update(doc)\n\tbatch.Delete(\"1\")\n\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocCount, err := indexReader.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\n\tdocIDReader, err := indexReader.DocIDReaderAll()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tvar docIds []index.IndexInternalID\n\tdocID, err := docIDReader.Next()\n\tfor docID != nil && err == nil {\n\t\tdocIds = append(docIds, docID)\n\t\tdocID, err = docIDReader.Next()\n\t}\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texpectedDocIds := []index.IndexInternalID{index.IndexInternalID(\"2\"), index.IndexInternalID(\"3\")}\n\tif !reflect.DeepEqual(docIds, expectedDocIds) {\n\t\tt.Errorf(\"expected ids: %v, got ids: %v\", expectedDocIds, docIds)\n\t}\n}\n\nfunc TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar expectedCount uint64\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err := reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewNumericFieldWithIndexingOptions(\"age\", []uint64{}, 35.99, index.IndexField|index.StoreField))\n\tdf, err := document.NewDateTimeFieldWithIndexingOptions(\"unixEpoch\", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdoc.AddField(df)\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\texpectedCount++\n\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// should have 72 rows\n\t// 1 for version\n\t// 3 for schema fields\n\t// 1 for text term\n\t// 16 for numeric terms\n\t// 16 for date terms\n\t// 3 for the stored field\n\t// 1 for the text term count\n\t// 16 for numeric term counts\n\t// 16 for date term counts\n\t// 1 for the back index entry\n\texpectedLength := uint64(1 + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 1)\n\trowCount, err := idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstoredDocInt, err := indexReader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstoredDoc := storedDocInt.(*document.Document)\n\n\terr = indexReader.Close()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(storedDoc.Fields) != 3 {\n\t\tt.Errorf(\"expected 3 stored field, got %d\", len(storedDoc.Fields))\n\t}\n\ttextField, ok := storedDoc.Fields[0].(*document.TextField)\n\tif !ok {\n\t\tt.Errorf(\"expected text field\")\n\t}\n\tif string(textField.Value()) != \"test\" {\n\t\tt.Errorf(\"expected field content 'test', got '%s'\", string(textField.Value()))\n\t}\n\tnumField, ok := storedDoc.Fields[1].(*document.NumericField)\n\tif !ok {\n\t\tt.Errorf(\"expected numeric field\")\n\t}\n\tnumFieldNumer, err := numField.Number()\n\tif err != nil {\n\t\tt.Error(err)\n\t} else {\n\t\tif numFieldNumer != 35.99 {\n\t\t\tt.Errorf(\"expected numeric value 35.99, got %f\", numFieldNumer)\n\t\t}\n\t}\n\tdateField, ok := storedDoc.Fields[2].(*document.DateTimeField)\n\tif !ok {\n\t\tt.Errorf(\"expected date field\")\n\t}\n\tdateFieldDate, _, err := dateField.DateTime()\n\tif err != nil {\n\t\tt.Error(err)\n\t} else {\n\t\tif dateFieldDate != time.Unix(0, 0).UTC() {\n\t\t\tt.Errorf(\"expected date value unix epoch, got %v\", dateFieldDate)\n\t\t}\n\t}\n\n\t// now update the document, but omit one of the fields\n\tdoc = document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"testup\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewNumericFieldWithIndexingOptions(\"age\", []uint64{}, 36.99, index.IndexField|index.StoreField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader2, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\t// expected doc count shouldn't have changed\n\tdocCount, err = indexReader2.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\n\t// should only get 2 fields back now though\n\tstoredDocInt, err = indexReader2.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstoredDoc = storedDocInt.(*document.Document)\n\n\terr = indexReader2.Close()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(storedDoc.Fields) != 2 {\n\t\tt.Errorf(\"expected 3 stored field, got %d\", len(storedDoc.Fields))\n\t}\n\ttextField, ok = storedDoc.Fields[0].(*document.TextField)\n\tif !ok {\n\t\tt.Errorf(\"expected text field\")\n\t}\n\tif string(textField.Value()) != \"testup\" {\n\t\tt.Errorf(\"expected field content 'testup', got '%s'\", string(textField.Value()))\n\t}\n\tnumField, ok = storedDoc.Fields[1].(*document.NumericField)\n\tif !ok {\n\t\tt.Errorf(\"expected numeric field\")\n\t}\n\tnumFieldNumer, err = numField.Number()\n\tif err != nil {\n\t\tt.Error(err)\n\t} else {\n\t\tif numFieldNumer != 36.99 {\n\t\t\tt.Errorf(\"expected numeric value 36.99, got %f\", numFieldNumer)\n\t\t}\n\t}\n\n\t// now delete the document\n\terr = idx.Delete(\"1\")\n\tif err != nil {\n\t\tt.Errorf(\"Error deleting entry from index: %v\", err)\n\t}\n\n\texpectedCount--\n\n\t// expected doc count shouldn't have changed\n\treader, err = idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocCount, err = reader.DocCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif docCount != expectedCount {\n\t\tt.Errorf(\"Expected document count to be %d got %d\", expectedCount, docCount)\n\t}\n\terr = reader.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexInsertFields(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewNumericFieldWithIndexingOptions(\"age\", []uint64{}, 35.99, index.IndexField|index.StoreField))\n\tdateField, err := document.NewDateTimeFieldWithIndexingOptions(\"unixEpoch\", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdoc.AddField(dateField)\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfields, err := indexReader.Fields()\n\tif err != nil {\n\t\tt.Error(err)\n\t} else {\n\t\texpectedFields := []string{\"name\", \"age\", \"unixEpoch\"}\n\t\tif !reflect.DeepEqual(fields, expectedFields) {\n\t\t\tt.Errorf(\"expected fields: %v, got %v\", expectedFields, fields)\n\t\t}\n\t}\n}\n\nfunc TestIndexUpdateComposites(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewCompositeFieldWithIndexingOptions(\"_all\", true, nil, nil, index.IndexField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\t// should have 72 rows\n\t// 1 for version\n\t// 3 for schema fields\n\t// 4 for text term\n\t// 2 for the stored field\n\t// 4 for the text term count\n\t// 1 for the back index entry\n\texpectedLength := uint64(1 + 3 + 4 + 2 + 4 + 1)\n\trowCount, err := idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n\n\t// now lets update it\n\tdoc = document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"testupdated\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"misterupdated\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewCompositeFieldWithIndexingOptions(\"_all\", true, nil, nil, index.IndexField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// make sure new values are in index\n\tstoredDocInt, err := indexReader.Document(\"1\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstoredDoc := storedDocInt.(*document.Document)\n\tif len(storedDoc.Fields) != 2 {\n\t\tt.Errorf(\"expected 2 stored field, got %d\", len(storedDoc.Fields))\n\t}\n\ttextField, ok := storedDoc.Fields[0].(*document.TextField)\n\tif !ok {\n\t\tt.Errorf(\"expected text field\")\n\t}\n\tif string(textField.Value()) != \"testupdated\" {\n\t\tt.Errorf(\"expected field content 'test', got '%s'\", string(textField.Value()))\n\t}\n\n\t// should have the same row count as before, plus 4 term dictionary garbage rows\n\texpectedLength += 4\n\trowCount, err = idx.(*UpsideDownCouch).rowCount()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif rowCount != expectedLength {\n\t\tt.Errorf(\"expected %d rows, got: %d\", expectedLength, rowCount)\n\t}\n}\n\nfunc TestIndexFieldsMisc(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister\"), index.IndexField|index.StoreField))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tfieldName1 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(0)\n\tif fieldName1 != \"name\" {\n\t\tt.Errorf(\"expected field named 'name', got '%s'\", fieldName1)\n\t}\n\tfieldName2 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(1)\n\tif fieldName2 != \"title\" {\n\t\tt.Errorf(\"expected field named 'title', got '%s'\", fieldName2)\n\t}\n\tfieldName3 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(2)\n\tif fieldName3 != \"\" {\n\t\tt.Errorf(\"expected field named '', got '%s'\", fieldName3)\n\t}\n}\n\nfunc TestIndexTermReaderCompositeFields(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc.AddField(document.NewCompositeFieldWithIndexingOptions(\"_all\", true, nil, nil, index.IndexField|index.IncludeTermVectors))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttermFieldReader, err := indexReader.TermFieldReader(context.TODO(), []byte(\"mister\"), \"_all\", true, true, true)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\ttfd, err := termFieldReader.Next(nil)\n\tfor tfd != nil && err == nil {\n\t\tif !tfd.ID.Equals(index.IndexInternalID(\"1\")) {\n\t\t\tt.Errorf(\"expected to find document id 1\")\n\t\t}\n\t\ttfd, err = termFieldReader.Next(nil)\n\t}\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n}\n\nfunc TestIndexDocValueReader(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := document.NewDocument(\"1\")\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"name\", []uint64{}, []byte(\"test\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\"title\", []uint64{}, []byte(\"mister\"), index.IndexField|index.StoreField|index.IncludeTermVectors))\n\terr = idx.Update(doc)\n\tif err != nil {\n\t\tt.Errorf(\"Error updating index: %v\", err)\n\t}\n\n\tindexReader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tactualFieldTerms := make(fieldTerms)\n\n\tdvr, err := indexReader.DocValueReader([]string{\"name\", \"title\"})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\terr = dvr.VisitDocValues(index.IndexInternalID(\"1\"), func(field string, term []byte) {\n\t\tactualFieldTerms[field] = append(actualFieldTerms[field], string(term))\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\texpectedFieldTerms := fieldTerms{\n\t\t\"name\":  []string{\"test\"},\n\t\t\"title\": []string{\"mister\"},\n\t}\n\tif !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) {\n\t\tt.Errorf(\"expected field terms: %#v, got: %#v\", expectedFieldTerms, actualFieldTerms)\n\t}\n}\n\nfunc BenchmarkBatch(b *testing.B) {\n\tcache := registry.NewCache()\n\tanalyzer, err := cache.AnalyzerNamed(standard.Name)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tbatch := index.NewBatch()\n\tfor i := 0; i < 100; i++ {\n\t\td := document.NewDocument(strconv.Itoa(i))\n\t\tf := document.NewTextFieldWithAnalyzer(\"desc\", nil, bleveWikiArticle1K, analyzer)\n\t\td.AddField(f)\n\t\tbatch.Update(d)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\terr = idx.Batch(batch)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc TestConcurrentUpdate(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// do some concurrent updates\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1)\n\t\tgo func(i int) {\n\t\t\tdoc := document.NewDocument(\"1\")\n\t\t\tdoc.AddField(document.NewTextFieldWithIndexingOptions(strconv.Itoa(i), []uint64{}, []byte(strconv.Itoa(i)), index.StoreField))\n\t\t\terr := idx.Update(doc)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Error updating index: %v\", err)\n\t\t\t}\n\t\t\twg.Done()\n\t\t}(i)\n\t}\n\twg.Wait()\n\n\t// now load the name field and see what we get\n\tr, err := idx.Reader()\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tdocInt, err := r.Document(\"1\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdoc := docInt.(*document.Document)\n\n\tif len(doc.Fields) > 1 {\n\t\tt.Errorf(\"expected single field, found %d\", len(doc.Fields))\n\t}\n\n\terr = r.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestLargeField(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar largeFieldValue []byte\n\tfor len(largeFieldValue) < RowBufferSize {\n\t\tlargeFieldValue = append(largeFieldValue, bleveWikiArticle1K...)\n\t}\n\tt.Logf(\"large field size: %d\", len(largeFieldValue))\n\n\td := document.NewDocument(\"large\")\n\tf := document.NewTextFieldWithIndexingOptions(\"desc\", nil, largeFieldValue, index.IndexField|index.StoreField)\n\td.AddField(f)\n\n\terr = idx.Update(d)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexBatchPersistedCallbackWithErrorUpsideDown(t *testing.T) {\n\tdefer func() {\n\t\terr := DestroyTest()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\tidx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Open()\n\tif err != nil {\n\t\tt.Errorf(\"error opening index: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tvar callbackExecuted bool\n\tbatch := index.NewBatch()\n\tbatch.SetPersistedCallback(func(e error) {\n\t\tcallbackExecuted = true\n\t})\n\t// By using a really large ID, we ensure that the batch will fail,\n\t// because the key generated by upside down will be too large for BoltDB\n\treallyBigId := strings.Repeat(\"x\", 32768+1)\n\tdoc := document.NewDocument(reallyBigId)\n\tdoc.AddField(document.NewTextField(\"name\", []uint64{}, []byte(\"test3\")))\n\tbatch.Update(doc)\n\n\t_ = idx.Batch(batch)\n\t// don't fail on this error, that isn't what we're testing\n\n\tif !callbackExecuted {\n\t\tt.Fatal(\"expected callback to fire, it did not\")\n\t}\n}\n\n// fieldTerms contains the terms used by a document, keyed by field\ntype fieldTerms map[string][]string\n\n// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields\nfunc (f fieldTerms) FieldsNotYetCached(fields []string) []string {\n\trv := make([]string, 0, len(fields))\n\tfor _, field := range fields {\n\t\tif _, ok := f[field]; !ok {\n\t\t\trv = append(rv, field)\n\t\t}\n\t}\n\treturn rv\n}\n\n// Merge will combine two fieldTerms\n// it assumes that the terms lists are complete (thus do not need to be merged)\n// field terms from the other list always replace the ones in the receiver\nfunc (f fieldTerms) Merge(other fieldTerms) {\n\tfor field, terms := range other {\n\t\tf[field] = terms\n\t}\n}\n"
  },
  {
    "path": "index.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\n// A Batch groups together multiple Index and Delete\n// operations you would like performed at the same\n// time.  The Batch structure is NOT thread-safe.\n// You should only perform operations on a batch\n// from a single thread at a time.  Once batch\n// execution has started, you may not modify it.\ntype Batch struct {\n\tindex    Index\n\tinternal *index.Batch\n\n\tlastDocSize uint64\n\ttotalSize   uint64\n}\n\n// Index adds the specified index operation to the\n// batch.  NOTE: the bleve Index is not updated\n// until the batch is executed.\nfunc (b *Batch) Index(id string, data interface{}) error {\n\tif id == \"\" {\n\t\treturn ErrorEmptyID\n\t}\n\tif eventIndex, ok := b.index.(index.EventIndex); ok {\n\t\teventIndex.FireIndexEvent()\n\t}\n\tdoc := document.NewDocument(id)\n\terr := b.index.Mapping().MapDocument(doc, data)\n\tif err != nil {\n\t\treturn err\n\t}\n\tb.internal.Update(doc)\n\n\tb.lastDocSize = uint64(doc.Size() +\n\t\tlen(id) + size.SizeOfString) // overhead from internal\n\tb.totalSize += b.lastDocSize\n\n\treturn nil\n}\n\nfunc (b *Batch) IndexSynonym(id string, collection string, definition *SynonymDefinition) error {\n\tif id == \"\" {\n\t\treturn ErrorEmptyID\n\t}\n\tif eventIndex, ok := b.index.(index.EventIndex); ok {\n\t\teventIndex.FireIndexEvent()\n\t}\n\tsynMap, ok := b.index.Mapping().(mapping.SynonymMapping)\n\tif !ok {\n\t\treturn ErrorSynonymSearchNotSupported\n\t}\n\n\tif err := definition.Validate(); err != nil {\n\t\treturn err\n\t}\n\n\tdoc := document.NewSynonymDocument(id)\n\terr := synMap.MapSynonymDocument(doc, collection, definition.Input, definition.Synonyms)\n\tif err != nil {\n\t\treturn err\n\t}\n\tb.internal.Update(doc)\n\n\tb.lastDocSize = uint64(doc.Size() +\n\t\tlen(id) + size.SizeOfString) // overhead from internal\n\tb.totalSize += b.lastDocSize\n\n\treturn nil\n}\n\nfunc (b *Batch) LastDocSize() uint64 {\n\treturn b.lastDocSize\n}\n\nfunc (b *Batch) TotalDocsSize() uint64 {\n\treturn b.totalSize\n}\n\n// IndexAdvanced adds the specified index operation to the\n// batch which skips the mapping.  NOTE: the bleve Index is not updated\n// until the batch is executed.\nfunc (b *Batch) IndexAdvanced(doc *document.Document) (err error) {\n\tif doc.ID() == \"\" {\n\t\treturn ErrorEmptyID\n\t}\n\tb.internal.Update(doc)\n\treturn nil\n}\n\n// Delete adds the specified delete operation to the\n// batch.  NOTE: the bleve Index is not updated until\n// the batch is executed.\nfunc (b *Batch) Delete(id string) {\n\tif id != \"\" {\n\t\tb.internal.Delete(id)\n\t}\n}\n\n// SetInternal adds the specified set internal\n// operation to the batch. NOTE: the bleve Index is\n// not updated until the batch is executed.\nfunc (b *Batch) SetInternal(key, val []byte) {\n\tb.internal.SetInternal(key, val)\n}\n\n// DeleteInternal adds the specified delete internal\n// operation to the batch. NOTE: the bleve Index is\n// not updated until the batch is executed.\nfunc (b *Batch) DeleteInternal(key []byte) {\n\tb.internal.DeleteInternal(key)\n}\n\n// Size returns the total number of operations inside the batch\n// including normal index operations and internal operations.\nfunc (b *Batch) Size() int {\n\treturn len(b.internal.IndexOps) + len(b.internal.InternalOps)\n}\n\n// String prints a user friendly string representation of what\n// is inside this batch.\nfunc (b *Batch) String() string {\n\treturn b.internal.String()\n}\n\n// Reset returns a Batch to the empty state so that it can\n// be reused in the future.\nfunc (b *Batch) Reset() {\n\tb.internal.Reset()\n\tb.lastDocSize = 0\n\tb.totalSize = 0\n}\n\nfunc (b *Batch) Merge(o *Batch) {\n\tif o != nil && o.internal != nil {\n\t\tb.internal.Merge(o.internal)\n\t\tif o.LastDocSize() > 0 {\n\t\t\tb.lastDocSize = o.LastDocSize()\n\t\t}\n\t\tb.totalSize = uint64(b.internal.TotalDocSize())\n\t}\n}\n\nfunc (b *Batch) SetPersistedCallback(f index.BatchCallback) {\n\tb.internal.SetPersistedCallback(f)\n}\n\nfunc (b *Batch) PersistedCallback() index.BatchCallback {\n\treturn b.internal.PersistedCallback()\n}\n\n// An Index implements all the indexing and searching\n// capabilities of bleve.  An Index can be created\n// using the New() and Open() methods.\n//\n// Index() takes an input value, deduces a DocumentMapping for its type,\n// assigns string paths to its fields or values then applies field mappings on\n// them.\n//\n// The DocumentMapping used to index a value is deduced by the following rules:\n//  1. If value implements mapping.bleveClassifier interface, resolve the mapping\n//     from BleveType().\n//  2. If value implements mapping.Classifier interface, resolve the mapping\n//     from Type().\n//  3. If value has a string field or value at IndexMapping.TypeField.\n//\n// (defaulting to \"_type\"), use it to resolve the mapping. Fields addressing\n// is described below.\n// 4) If IndexMapping.DefaultType is registered, return it.\n// 5) Return IndexMapping.DefaultMapping.\n//\n// Each field or nested field of the value is identified by a string path, then\n// mapped to one or several FieldMappings which extract the result for analysis.\n//\n// Struct values fields are identified by their \"json:\" tag, or by their name.\n// Nested fields are identified by prefixing with their parent identifier,\n// separated by a dot.\n//\n// Map values entries are identified by their string key. Entries not indexed\n// by strings are ignored. Entry values are identified recursively like struct\n// fields.\n//\n// Slice and array values are identified by their field name. Their elements\n// are processed sequentially with the same FieldMapping.\n//\n// String, float64 and time.Time values are identified by their field name.\n// Other types are ignored.\n//\n// Each value identifier is decomposed in its parts and recursively address\n// SubDocumentMappings in the tree starting at the root DocumentMapping.  If a\n// mapping is found, all its FieldMappings are applied to the value. If no\n// mapping is found and the root DocumentMapping is dynamic, default mappings\n// are used based on value type and IndexMapping default configurations.\n//\n// Finally, mapped values are analyzed, indexed or stored. See\n// FieldMapping.Analyzer to know how an analyzer is resolved for a given field.\n//\n// Examples:\n//\n//\ttype Date struct {\n//\t  Day string `json:\"day\"`\n//\t  Month string\n//\t  Year string\n//\t}\n//\n//\ttype Person struct {\n//\t  FirstName string `json:\"first_name\"`\n//\t  LastName string\n//\t  BirthDate Date `json:\"birth_date\"`\n//\t}\n//\n// A Person value FirstName is mapped by the SubDocumentMapping at\n// \"first_name\". Its LastName is mapped by the one at \"LastName\". The day of\n// BirthDate is mapped to the SubDocumentMapping \"day\" of the root\n// SubDocumentMapping \"birth_date\". It will appear as the \"birth_date.day\"\n// field in the index. The month is mapped to \"birth_date.Month\".\ntype Index interface {\n\t// Index analyzes, indexes or stores mapped data fields. Supplied\n\t// identifier is bound to analyzed data and will be retrieved by search\n\t// requests. See Index interface documentation for details about mapping\n\t// rules.\n\tIndex(id string, data interface{}) error\n\tDelete(id string) error\n\n\tNewBatch() *Batch\n\tBatch(b *Batch) error\n\n\t// Document returns specified document or nil if the document is not\n\t// indexed or stored.\n\tDocument(id string) (index.Document, error)\n\t// DocCount returns the number of documents in the index.\n\tDocCount() (uint64, error)\n\n\tSearch(req *SearchRequest) (*SearchResult, error)\n\tSearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error)\n\n\tFields() ([]string, error)\n\n\tFieldDict(field string) (index.FieldDict, error)\n\tFieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error)\n\tFieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error)\n\n\tClose() error\n\n\tMapping() mapping.IndexMapping\n\n\tStats() *IndexStat\n\tStatsMap() map[string]interface{}\n\n\tGetInternal(key []byte) ([]byte, error)\n\tSetInternal(key, val []byte) error\n\tDeleteInternal(key []byte) error\n\n\t// Name returns the name of the index (by default this is the path)\n\tName() string\n\t// SetName lets you assign your own logical name to this index\n\tSetName(string)\n\n\t// Advanced returns the internal index implementation\n\tAdvanced() (index.Index, error)\n}\n\n// New index at the specified path, must not exist.\n// The provided mapping will be used for all\n// Index/Search operations.\nfunc New(path string, mapping mapping.IndexMapping) (Index, error) {\n\treturn newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)\n}\n\n// NewMemOnly creates a memory-only index.\n// The contents of the index is NOT persisted,\n// and will be lost once closed.\n// The provided mapping will be used for all\n// Index/Search operations.\nfunc NewMemOnly(mapping mapping.IndexMapping) (Index, error) {\n\treturn newIndexUsing(\"\", mapping, upsidedown.Name, Config.DefaultMemKVStore, nil)\n}\n\n// NewUsing creates index at the specified path,\n// which must not already exist.\n// The provided mapping will be used for all\n// Index/Search operations.\n// The specified index type will be used.\n// The specified kvstore implementation will be used\n// and the provided kvconfig will be passed to its\n// constructor. Note that currently the values of kvconfig must\n// be able to be marshaled and unmarshaled using the encoding/json library (used\n// when reading/writing the index metadata file).\nfunc NewUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) {\n\treturn newIndexUsing(path, mapping, indexType, kvstore, kvconfig)\n}\n\n// Open index at the specified path, must exist.\n// The mapping used when it was created will be used for all Index/Search operations.\nfunc Open(path string) (Index, error) {\n\treturn openIndexUsing(path, nil)\n}\n\n// OpenUsing opens index at the specified path, must exist.\n// The mapping used when it was created will be used for all Index/Search operations.\n// The provided runtimeConfig can override settings\n// persisted when the kvstore was created.\n// If runtimeConfig has updated mapping, then an index update is attempted\n// Throws an error without any changes to the index if an unupdatable mapping is provided\nfunc OpenUsing(path string, runtimeConfig map[string]interface{}) (Index, error) {\n\treturn openIndexUsing(path, runtimeConfig)\n}\n\n// Builder is a limited interface, used to build indexes in an offline mode.\n// Items cannot be updated or deleted, and the caller MUST ensure a document is\n// indexed only once.\ntype Builder interface {\n\tIndex(id string, data interface{}) error\n\tClose() error\n}\n\n// NewBuilder creates a builder, which will build an index at the specified path,\n// using the specified mapping and options.\nfunc NewBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) {\n\treturn newBuilder(path, mapping, config)\n}\n\n// IndexCopyable is an index which supports an online copy operation\n// of the index.\ntype IndexCopyable interface {\n\t// CopyTo creates a fully functional copy of the index at the\n\t// specified destination directory implementation.\n\tCopyTo(d index.Directory) error\n}\n\n// FileSystemDirectory is the default implementation for the\n// index.Directory interface.\ntype FileSystemDirectory string\n\n// SynonymDefinition represents a synonym mapping in Bleve.\n// Each instance associates one or more input terms with a list of synonyms,\n// defining how terms are treated as equivalent in searches.\ntype SynonymDefinition struct {\n\t// Input is an optional list of terms for unidirectional synonym mapping.\n\t// When terms are specified in Input, they will map to the terms in Synonyms,\n\t// making the relationship unidirectional (each Input maps to all Synonyms).\n\t// If Input is omitted, the relationship is bidirectional among all Synonyms.\n\tInput []string `json:\"input,omitempty\"`\n\n\t// Synonyms is a list of terms that are considered equivalent.\n\t// If Input is specified, each term in Input will map to each term in Synonyms.\n\t// If Input is not specified, the Synonyms list will be treated bidirectionally,\n\t// meaning each term in Synonyms is treated as synonymous with all others.\n\tSynonyms []string `json:\"synonyms\"`\n}\n\nfunc (sd *SynonymDefinition) Validate() error {\n\tif len(sd.Synonyms) == 0 {\n\t\treturn fmt.Errorf(\"synonym definition must have at least one synonym\")\n\t}\n\treturn nil\n}\n\n// SynonymIndex supports indexing synonym definitions alongside regular documents.\n// Synonyms, grouped by collection name, define term relationships for query expansion in searches.\ntype SynonymIndex interface {\n\tIndex\n\t// IndexSynonym indexes a synonym definition, with the specified id and belonging to the specified collection.\n\tIndexSynonym(id string, collection string, definition *SynonymDefinition) error\n}\n\ntype InsightsIndex interface {\n\tIndex\n\t// TermFrequencies returns the tokens ordered by frequencies for the field index.\n\tTermFrequencies(field string, limit int, descending bool) ([]index.TermFreq, error)\n\t// CentroidCardinalities returns the centroids (clusters) from IVF indexes ordered by data density.\n\tCentroidCardinalities(field string, limit int, desceding bool) ([]index.CentroidCardinality, error)\n}\n"
  },
  {
    "path": "index_alias.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\n// An IndexAlias is a wrapper around one or more\n// Index objects.  It has two distinct modes of\n// operation.\n// 1.  When it points to a single index, ALL index\n// operations are valid and will be passed through\n// to the underlying index.\n// 2.  When it points to more than one index, the only\n// valid operation is Search.  In this case the\n// search will be performed across all the\n// underlying indexes and the results merged.\n// Calls to Add/Remove/Swap the underlying indexes\n// are atomic, so you can safely change the\n// underlying Index objects while other components\n// are performing operations.\ntype IndexAlias interface {\n\tIndex\n\n\tAdd(i ...Index)\n\tRemove(i ...Index)\n\tSwap(in, out []Index)\n}\n"
  },
  {
    "path": "index_alias_impl.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"sort\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/collector\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype indexAliasImpl struct {\n\tname    string\n\tindexes []Index\n\tmutex   sync.RWMutex\n\topen    bool\n\t// if all the indexes in that alias have the same mapping\n\t// then the user can set the mapping here to avoid\n\t// checking the mapping of each index in the alias\n\tmapping mapping.IndexMapping\n}\n\n// NewIndexAlias creates a new IndexAlias over the provided\n// Index objects.\nfunc NewIndexAlias(indexes ...Index) *indexAliasImpl {\n\treturn &indexAliasImpl{\n\t\tname:    \"alias\",\n\t\tindexes: indexes,\n\t\topen:    true,\n\t}\n}\n\n// VisitIndexes invokes the visit callback on every\n// indexes included in the index alias.\nfunc (i *indexAliasImpl) VisitIndexes(visit func(Index)) {\n\ti.mutex.RLock()\n\tfor _, idx := range i.indexes {\n\t\tvisit(idx)\n\t}\n\ti.mutex.RUnlock()\n}\n\nfunc (i *indexAliasImpl) isAliasToSingleIndex() error {\n\tif len(i.indexes) < 1 {\n\t\treturn ErrorAliasEmpty\n\t} else if len(i.indexes) > 1 {\n\t\treturn ErrorAliasMulti\n\t}\n\treturn nil\n}\n\nfunc (i *indexAliasImpl) Index(id string, data interface{}) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn i.indexes[0].Index(id, data)\n}\n\nfunc (i *indexAliasImpl) IndexSynonym(id string, collection string, definition *SynonymDefinition) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif si, ok := i.indexes[0].(SynonymIndex); ok {\n\t\treturn si.IndexSynonym(id, collection, definition)\n\t}\n\treturn ErrorSynonymSearchNotSupported\n}\n\nfunc (i *indexAliasImpl) Delete(id string) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn i.indexes[0].Delete(id)\n}\n\nfunc (i *indexAliasImpl) Batch(b *Batch) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn i.indexes[0].Batch(b)\n}\n\nfunc (i *indexAliasImpl) Document(id string) (index.Document, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn i.indexes[0].Document(id)\n}\n\nfunc (i *indexAliasImpl) DocCount() (uint64, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\trv := uint64(0)\n\n\tif !i.open {\n\t\treturn 0, ErrorIndexClosed\n\t}\n\n\tfor _, index := range i.indexes {\n\t\totherCount, err := index.DocCount()\n\t\tif err == nil {\n\t\t\trv += otherCount\n\t\t}\n\t\t// tolerate errors to produce partial counts\n\t}\n\n\treturn rv, nil\n}\n\nfunc (i *indexAliasImpl) Search(req *SearchRequest) (*SearchResult, error) {\n\treturn i.SearchInContext(context.Background(), req)\n}\n\nfunc (i *indexAliasImpl) SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\tif len(i.indexes) < 1 {\n\t\treturn nil, ErrorAliasEmpty\n\t}\n\n\tif _, ok := ctx.Value(search.PreSearchKey).(bool); ok {\n\t\t// since preSearchKey is set, it means that the request\n\t\t// is being executed as part of a preSearch, which\n\t\t// indicates that this index alias is set as an Index\n\t\t// in another alias, so we need to do a preSearch search\n\t\t// and NOT a real search\n\t\tbm25PreSearch := isBM25Enabled(i.mapping)\n\t\tflags := &preSearchFlags{\n\t\t\tknn:      requestHasKNN(req),\n\t\t\tsynonyms: !isMatchNoneQuery(req.Query),\n\t\t\tbm25:     bm25PreSearch,\n\t\t}\n\t\treturn preSearchDataSearch(ctx, req, flags, i.indexes...)\n\t}\n\n\t// at this point we know we are doing a real search\n\t// either after a preSearch is done, or directly\n\t// on the alias\n\n\t// check if request has preSearchData which would indicate that the\n\t// request has already been preSearched and we can skip the\n\t// preSearch step now, we call an optional function to\n\t// redistribute the preSearchData to the individual indexes\n\t// if necessary\n\tvar preSearchData map[string]map[string]interface{}\n\tif req.PreSearchData != nil {\n\t\tvar err error\n\t\tpreSearchData, err = redistributePreSearchData(req, i.indexes)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// short circuit the simple case\n\tif len(i.indexes) == 1 {\n\t\tif preSearchData != nil {\n\t\t\treq.PreSearchData = preSearchData[i.indexes[0].Name()]\n\t\t}\n\t\treturn i.indexes[0].SearchInContext(ctx, req)\n\t}\n\n\t// rescorer will be set if score fusion is supposed to happen\n\t// at this alias (root alias), else will be nil\n\tvar rescorer *rescorer\n\tif _, ok := ctx.Value(search.ScoreFusionKey).(bool); !ok {\n\t\t// new context will be used in internal functions to collect data\n\t\t// as suitable for fusion. Rescorer is used for rescoring\n\t\t// using fusion algorithms.\n\t\tif IsScoreFusionRequested(req) {\n\t\t\tctx = context.WithValue(ctx, search.ScoreFusionKey, true)\n\t\t\trescorer = newRescorer(req)\n\t\t\trescorer.prepareSearchRequest()\n\t\t\tdefer rescorer.restoreSearchRequest()\n\t\t}\n\t}\n\n\t// at this stage we know we have multiple indexes\n\t// check if preSearchData needs to be gathered from all indexes\n\t// before executing the query\n\tvar err error\n\t// only perform preSearch if\n\t//  - the request does not already have preSearchData\n\t//  - the request requires preSearch\n\tvar preSearchDuration time.Duration\n\tvar sr *SearchResult\n\n\t// fusionKnnHits stores the KnnHits at the root alias.\n\t// This is used with score fusion in case there is no need to\n\t// send the knn hits to the leaf indexes in search phase.\n\t// Refer to constructPreSearchDataAndFusionKnnHits for more info.\n\t// This variable is left nil if we have to send the knn hits to leaf\n\t// indexes again, else contains the knn hits if not required.\n\tvar fusionKnnHits search.DocumentMatchCollection\n\tflags, err := preSearchRequired(ctx, req, i.mapping)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif req.PreSearchData == nil && flags != nil {\n\t\tsearchStart := time.Now()\n\t\tpreSearchResult, err := preSearch(ctx, req, flags, i.indexes...)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// check if the preSearch result has any errors and if so\n\t\t// return the search result as is without executing the query\n\t\t// so that the errors are not lost\n\t\tif preSearchResult.Status.Failed > 0 || len(preSearchResult.Status.Errors) > 0 {\n\t\t\treturn preSearchResult, nil\n\t\t}\n\t\t// finalize the preSearch result now\n\t\tfinalizePreSearchResult(req, flags, preSearchResult)\n\n\t\t// if there are no errors, then merge the data in the preSearch result\n\t\t// and construct the preSearchData to be used in the actual search\n\t\t// if the request is satisfied by the preSearch result, then we can\n\t\t// directly return the preSearch result as the final result\n\t\tif requestSatisfiedByPreSearch(req, flags) {\n\t\t\tsr = finalizeSearchResult(ctx, req, preSearchResult, rescorer)\n\t\t\t// no need to run the 2nd phase MultiSearch(..)\n\t\t} else {\n\t\t\tpreSearchData, fusionKnnHits, err = constructPreSearchDataAndFusionKnnHits(req, flags, preSearchResult, rescorer, i.indexes)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tpreSearchDuration = time.Since(searchStart)\n\t}\n\n\t// check if search result was generated as part of preSearch itself\n\tif sr == nil {\n\t\tmultiSearchParams := &multiSearchParams{preSearchData, rescorer, fusionKnnHits}\n\t\tsr, err = MultiSearch(ctx, req, multiSearchParams, i.indexes...)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tsr.Took += preSearchDuration\n\treturn sr, nil\n}\n\nfunc (i *indexAliasImpl) Fields() ([]string, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn i.indexes[0].Fields()\n}\n\nfunc (i *indexAliasImpl) FieldDict(field string) (index.FieldDict, error) {\n\ti.mutex.RLock()\n\n\tif !i.open {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\tfieldDict, err := i.indexes[0].FieldDict(field)\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\treturn &indexAliasImplFieldDict{\n\t\tindex:     i,\n\t\tfieldDict: fieldDict,\n\t}, nil\n}\n\nfunc (i *indexAliasImpl) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {\n\ti.mutex.RLock()\n\n\tif !i.open {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\tfieldDict, err := i.indexes[0].FieldDictRange(field, startTerm, endTerm)\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\treturn &indexAliasImplFieldDict{\n\t\tindex:     i,\n\t\tfieldDict: fieldDict,\n\t}, nil\n}\n\nfunc (i *indexAliasImpl) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {\n\ti.mutex.RLock()\n\n\tif !i.open {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\tfieldDict, err := i.indexes[0].FieldDictPrefix(field, termPrefix)\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\treturn &indexAliasImplFieldDict{\n\t\tindex:     i,\n\t\tfieldDict: fieldDict,\n\t}, nil\n}\n\nfunc (i *indexAliasImpl) Close() error {\n\ti.mutex.Lock()\n\tdefer i.mutex.Unlock()\n\n\ti.open = false\n\treturn nil\n}\n\n// SetIndexMapping sets the mapping for the alias and must be used\n// ONLY when all the indexes in the alias have the same mapping.\n// This is to avoid checking the mapping of each index in the alias\n// when executing a search request.\nfunc (i *indexAliasImpl) SetIndexMapping(m mapping.IndexMapping) error {\n\ti.mutex.Lock()\n\tdefer i.mutex.Unlock()\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\ti.mapping = m\n\treturn nil\n}\n\nfunc (i *indexAliasImpl) Mapping() mapping.IndexMapping {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil\n\t}\n\n\t// if the mapping is already set, return it\n\tif i.mapping != nil {\n\t\treturn i.mapping\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn i.indexes[0].Mapping()\n}\n\nfunc (i *indexAliasImpl) Stats() *IndexStat {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn i.indexes[0].Stats()\n}\n\nfunc (i *indexAliasImpl) StatsMap() map[string]interface{} {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn i.indexes[0].StatsMap()\n}\n\nfunc (i *indexAliasImpl) GetInternal(key []byte) ([]byte, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn i.indexes[0].GetInternal(key)\n}\n\nfunc (i *indexAliasImpl) SetInternal(key, val []byte) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn i.indexes[0].SetInternal(key, val)\n}\n\nfunc (i *indexAliasImpl) DeleteInternal(key []byte) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn i.indexes[0].DeleteInternal(key)\n}\n\nfunc (i *indexAliasImpl) Advanced() (index.Index, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn i.indexes[0].Advanced()\n}\n\nfunc (i *indexAliasImpl) Add(indexes ...Index) {\n\ti.mutex.Lock()\n\tdefer i.mutex.Unlock()\n\n\ti.indexes = append(i.indexes, indexes...)\n}\n\nfunc (i *indexAliasImpl) removeSingle(index Index) {\n\tfor pos, in := range i.indexes {\n\t\tif in == index {\n\t\t\ti.indexes = append(i.indexes[:pos], i.indexes[pos+1:]...)\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (i *indexAliasImpl) Remove(indexes ...Index) {\n\ti.mutex.Lock()\n\tdefer i.mutex.Unlock()\n\n\tfor _, in := range indexes {\n\t\ti.removeSingle(in)\n\t}\n}\n\nfunc (i *indexAliasImpl) Swap(in, out []Index) {\n\ti.mutex.Lock()\n\tdefer i.mutex.Unlock()\n\n\t// add\n\ti.indexes = append(i.indexes, in...)\n\n\t// delete\n\tfor _, ind := range out {\n\t\ti.removeSingle(ind)\n\t}\n}\n\n// createChildSearchRequest creates a separate\n// request from the original\n// For now, avoid data race on req structure.\n// TODO disable highlight/field load on child\n// requests, and add code to do this only on\n// the actual final results.\n// Perhaps that part needs to be optional,\n// could be slower in remote usages.\nfunc createChildSearchRequest(req *SearchRequest, preSearchData map[string]interface{}) *SearchRequest {\n\treturn copySearchRequest(req, preSearchData)\n}\n\ntype asyncSearchResult struct {\n\tName   string\n\tResult *SearchResult\n\tErr    error\n}\n\n// preSearchFlags is a struct to hold flags indicating why preSearch is required\ntype preSearchFlags struct {\n\tknn      bool\n\tsynonyms bool\n\tbm25     bool // needs presearch for this too\n}\n\nfunc isBM25Enabled(m mapping.IndexMapping) bool {\n\tvar rv bool\n\tif m, ok := m.(*mapping.IndexMappingImpl); ok {\n\t\trv = m.ScoringModel == index.BM25Scoring\n\t}\n\treturn rv\n}\n\n// preSearchRequired checks if preSearch is required and returns the presearch flags struct\n// indicating which preSearch is required\nfunc preSearchRequired(ctx context.Context, req *SearchRequest, m mapping.IndexMapping) (*preSearchFlags, error) {\n\t// Check for KNN query\n\tknn := requestHasKNN(req)\n\tvar synonyms bool\n\tif !isMatchNoneQuery(req.Query) {\n\t\t// Check if synonyms are defined in the mapping\n\t\tif sm, ok := m.(mapping.SynonymMapping); ok && sm.SynonymCount() > 0 {\n\t\t\t// check if any of the fields queried have a synonym source\n\t\t\t// in the index mapping, to prevent unnecessary preSearch\n\t\t\tfs, err := query.ExtractFields(req.Query, m, nil)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tfor field := range fs {\n\t\t\t\tif sm.SynonymSourceForPath(field) != \"\" {\n\t\t\t\t\tsynonyms = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tvar bm25 bool\n\tif !isMatchNoneQuery(req.Query) {\n\t\tif ctx != nil {\n\t\t\tif searchType := ctx.Value(search.SearchTypeKey); searchType != nil {\n\t\t\t\tif searchType.(string) == search.GlobalScoring {\n\t\t\t\t\tbm25 = isBM25Enabled(m)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif knn || synonyms || bm25 {\n\t\treturn &preSearchFlags{\n\t\t\tknn:      knn,\n\t\t\tsynonyms: synonyms,\n\t\t\tbm25:     bm25,\n\t\t}, nil\n\t}\n\treturn nil, nil\n}\n\nfunc preSearch(ctx context.Context, req *SearchRequest, flags *preSearchFlags, indexes ...Index) (*SearchResult, error) {\n\t// create a dummy request with a match none query\n\t// since we only care about the preSearchData in PreSearch\n\tdummyQuery := req.Query\n\tif !flags.bm25 && !flags.synonyms {\n\t\t// create a dummy request with a match none query\n\t\t// since we only care about the preSearchData in PreSearch\n\t\tdummyQuery = query.NewMatchNoneQuery()\n\t}\n\tdummyRequest := &SearchRequest{\n\t\tQuery: dummyQuery,\n\t}\n\tnewCtx := context.WithValue(ctx, search.PreSearchKey, true)\n\tif flags.knn {\n\t\taddKnnToDummyRequest(dummyRequest, req)\n\t}\n\treturn preSearchDataSearch(newCtx, dummyRequest, flags, indexes...)\n}\n\n// if the request is satisfied by just the preSearch result,\n// finalize the result and return it directly without\n// performing multi search\nfunc finalizeSearchResult(ctx context.Context, req *SearchRequest, preSearchResult *SearchResult, rescorer *rescorer) *SearchResult {\n\tif preSearchResult == nil {\n\t\treturn nil\n\t}\n\n\t// global values across all hits irrespective of pagination settings\n\tpreSearchResult.Total = uint64(preSearchResult.Hits.Len())\n\tmaxScore := float64(0)\n\tfor i, hit := range preSearchResult.Hits {\n\t\t// since we are now using the preSearch result as the final result\n\t\t// we can discard the indexNames from the hits as they are no longer\n\t\t// relevant.\n\t\thit.IndexNames = nil\n\t\tif hit.Score > maxScore {\n\t\t\tmaxScore = hit.Score\n\t\t}\n\t\thit.HitNumber = uint64(i)\n\t}\n\tpreSearchResult.MaxScore = maxScore\n\t// now apply pagination settings\n\tvar reverseQueryExecution bool\n\tif req.SearchBefore != nil {\n\t\treverseQueryExecution = true\n\t\treq.Sort.Reverse()\n\t\treq.SearchAfter = req.SearchBefore\n\t}\n\tif req.SearchAfter != nil {\n\t\tpreSearchResult.Hits = collector.FilterHitsBySearchAfter(preSearchResult.Hits, req.Sort, req.SearchAfter)\n\t}\n\n\tif rescorer != nil {\n\t\t// rescore takes ftsHits and knnHits as first and second argument respectively\n\t\t// since this is pure knn, set ftsHits to nil. preSearchResult.Hits contains knn results\n\t\tpreSearchResult.Hits, preSearchResult.Total, preSearchResult.MaxScore = rescorer.rescore(nil, preSearchResult.Hits)\n\t\trescorer.restoreSearchRequest()\n\t}\n\n\tpreSearchResult.Hits = hitsInCurrentPage(req, preSearchResult.Hits)\n\n\tif reverseQueryExecution {\n\t\t// reverse the sort back to the original\n\t\treq.Sort.Reverse()\n\t\t// resort using the original order\n\t\tmhs := newSearchHitSorter(req.Sort, preSearchResult.Hits)\n\t\treq.SortFunc()(mhs)\n\t\treq.SearchAfter = nil\n\t}\n\n\tif req.Explain {\n\t\tpreSearchResult.Request = req\n\t}\n\treturn preSearchResult\n}\n\nfunc requestSatisfiedByPreSearch(req *SearchRequest, flags *preSearchFlags) bool {\n\tif flags == nil {\n\t\treturn false\n\t}\n\t// if the synonyms presearch flag is set the request can never be satisfied by\n\t// the preSearch result as synonyms are not part of the preSearch result\n\tif flags.synonyms {\n\t\treturn false\n\t}\n\tif flags.knn && isKNNrequestSatisfiedByPreSearch(req) {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc constructSynonymPreSearchData(rv map[string]map[string]interface{}, sr *SearchResult, indexes []Index) map[string]map[string]interface{} {\n\tfor _, index := range indexes {\n\t\trv[index.Name()][search.SynonymPreSearchDataKey] = sr.SynonymResult\n\t}\n\treturn rv\n}\n\nfunc constructBM25PreSearchData(rv map[string]map[string]interface{}, sr *SearchResult, indexes []Index) map[string]map[string]interface{} {\n\tbmStats := sr.BM25Stats\n\tif bmStats != nil {\n\t\tfor _, index := range indexes {\n\t\t\trv[index.Name()][search.BM25PreSearchDataKey] = &search.BM25Stats{\n\t\t\t\tDocCount:         bmStats.DocCount,\n\t\t\t\tFieldCardinality: bmStats.FieldCardinality,\n\t\t\t}\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc constructPreSearchData(req *SearchRequest, flags *preSearchFlags,\n\tpreSearchResult *SearchResult, indexes []Index,\n) (map[string]map[string]interface{}, error) {\n\tif flags == nil || preSearchResult == nil {\n\t\treturn nil, fmt.Errorf(\"invalid input, flags: %v, preSearchResult: %v\", flags, preSearchResult)\n\t}\n\tmergedOut := make(map[string]map[string]interface{}, len(indexes))\n\tfor _, index := range indexes {\n\t\tmergedOut[index.Name()] = make(map[string]interface{})\n\t}\n\tvar err error\n\tif flags.knn {\n\t\tmergedOut, err = constructKnnPreSearchData(mergedOut, preSearchResult, indexes)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tif flags.synonyms {\n\t\tmergedOut = constructSynonymPreSearchData(mergedOut, preSearchResult, indexes)\n\t}\n\tif flags.bm25 {\n\t\tmergedOut = constructBM25PreSearchData(mergedOut, preSearchResult, indexes)\n\t}\n\treturn mergedOut, nil\n}\n\n// Constructs the presearch data if required during the search phase.\n// Also if we need to store knn hits at alias.\n// If we need to store knn hits at alias: returns all the knn hits\n// If we should send it to leaf indexes: includes in presearch data\nfunc constructPreSearchDataAndFusionKnnHits(req *SearchRequest, flags *preSearchFlags,\n\tpreSearchResult *SearchResult, rescorer *rescorer, indexes []Index,\n) (map[string]map[string]interface{}, search.DocumentMatchCollection, error) {\n\tvar fusionknnhits search.DocumentMatchCollection\n\n\t// Checks if we need to send the KNN hits to the indexes in the\n\t// search phase. If there is score fusion enabled, we do not\n\t// send the KNN hits to the indexes.\n\tif rescorer != nil && flags.knn {\n\t\tfusionknnhits = preSearchResult.Hits\n\t\tpreSearchResult.Hits = nil\n\t}\n\n\tpreSearchData, err := constructPreSearchData(req, flags, preSearchResult, indexes)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\treturn preSearchData, fusionknnhits, nil\n}\n\nfunc preSearchDataSearch(ctx context.Context, req *SearchRequest, flags *preSearchFlags, indexes ...Index) (*SearchResult, error) {\n\tasyncResults := make(chan *asyncSearchResult, len(indexes))\n\t// run search on each index in separate go routine\n\tvar waitGroup sync.WaitGroup\n\tsearchChildIndex := func(in Index, childReq *SearchRequest) {\n\t\trv := asyncSearchResult{Name: in.Name()}\n\t\trv.Result, rv.Err = in.SearchInContext(ctx, childReq)\n\t\tasyncResults <- &rv\n\t\twaitGroup.Done()\n\t}\n\twaitGroup.Add(len(indexes))\n\tfor _, in := range indexes {\n\t\tgo searchChildIndex(in, createChildSearchRequest(req, nil))\n\t}\n\t// on another go routine, close after finished\n\tgo func() {\n\t\twaitGroup.Wait()\n\t\tclose(asyncResults)\n\t}()\n\t// the final search result to be returned after combining the preSearch results\n\tvar sr *SearchResult\n\t// the preSearch result processor\n\tvar prp preSearchResultProcessor\n\t// error map\n\tindexErrors := make(map[string]error)\n\tfor asr := range asyncResults {\n\t\tif asr.Err == nil {\n\t\t\t// a valid preSearch result\n\t\t\tif prp == nil {\n\t\t\t\t// first valid preSearch result\n\t\t\t\t// create a new preSearch result processor\n\t\t\t\tprp = createPreSearchResultProcessor(req, flags)\n\t\t\t}\n\t\t\tprp.add(asr.Result, asr.Name)\n\t\t\tif sr == nil {\n\t\t\t\t// first result\n\t\t\t\tsr = &SearchResult{\n\t\t\t\t\tStatus: asr.Result.Status,\n\t\t\t\t\tCost:   asr.Result.Cost,\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// merge with previous\n\t\t\t\tsr.Status.Merge(asr.Result.Status)\n\t\t\t\tsr.Cost += asr.Result.Cost\n\t\t\t}\n\t\t} else {\n\t\t\tindexErrors[asr.Name] = asr.Err\n\t\t}\n\t}\n\t// handle case where no results were successful\n\tif sr == nil {\n\t\tsr = &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tErrors: make(map[string]error),\n\t\t\t},\n\t\t}\n\t}\n\t// in preSearch, partial results are not allowed as it can lead to\n\t// the real search giving incorrect results, and hence the search\n\t// result is not populated with any of the processed data from\n\t// the preSearch result processor if there are any errors\n\t// or the preSearch result status has any failures\n\tif len(indexErrors) > 0 || sr.Status.Failed > 0 {\n\t\tif sr.Status.Errors == nil {\n\t\t\tsr.Status.Errors = make(map[string]error)\n\t\t}\n\t\tfor indexName, indexErr := range indexErrors {\n\t\t\tsr.Status.Errors[indexName] = indexErr\n\t\t\tsr.Status.Total++\n\t\t}\n\t\t// At this point, all errors have been recorded—either from the preSearch phase\n\t\t// (via status.Merge) or from individual index search failures (indexErrors).\n\t\t// Since partial results are not allowed, mark the entire request as failed.\n\t\tsr.Status.Successful = 0\n\t\tsr.Status.Failed = sr.Status.Total\n\t} else {\n\t\tprp.finalize(sr)\n\t}\n\treturn sr, nil\n}\n\n// redistributePreSearchData redistributes the preSearchData sent in the search request to an index alias\n// which would happen in the case of an alias tree and depending on the level of the tree, the preSearchData\n// needs to be redistributed to the indexes at that level\nfunc redistributePreSearchData(req *SearchRequest, indexes []Index) (map[string]map[string]interface{}, error) {\n\trv := make(map[string]map[string]interface{}, len(indexes))\n\tfor _, index := range indexes {\n\t\trv[index.Name()] = make(map[string]interface{})\n\t}\n\tif knnHits, ok := req.PreSearchData[search.KnnPreSearchDataKey].([]*search.DocumentMatch); ok {\n\t\t// the preSearchData for KNN is a list of DocumentMatch objects\n\t\t// that need to be redistributed to the right index.\n\t\t// This is used only in the case of an alias tree, where the indexes\n\t\t// are at the leaves of the tree, and the master alias is at the root.\n\t\t// At each level of the tree, the preSearchData needs to be redistributed\n\t\t// to the indexes/aliases at that level. Because the preSearchData is\n\t\t// specific to each final index at the leaf.\n\t\tsegregatedKnnHits, err := validateAndDistributeKNNHits(knnHits, indexes)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfor _, index := range indexes {\n\t\t\trv[index.Name()][search.KnnPreSearchDataKey] = segregatedKnnHits[index.Name()]\n\t\t}\n\t}\n\tif fts, ok := req.PreSearchData[search.SynonymPreSearchDataKey].(search.FieldTermSynonymMap); ok {\n\t\tfor _, index := range indexes {\n\t\t\trv[index.Name()][search.SynonymPreSearchDataKey] = fts\n\t\t}\n\t}\n\n\tif bm25Data, ok := req.PreSearchData[search.BM25PreSearchDataKey].(*search.BM25Stats); ok {\n\t\tfor _, index := range indexes {\n\t\t\trv[index.Name()][search.BM25PreSearchDataKey] = bm25Data\n\t\t}\n\t}\n\treturn rv, nil\n}\n\n// finalizePreSearchResult finalizes the preSearch result by applying the finalization steps\n// specific to the preSearch flags\nfunc finalizePreSearchResult(req *SearchRequest, flags *preSearchFlags, preSearchResult *SearchResult) {\n\t// if flags is nil then return\n\tif flags == nil {\n\t\treturn\n\t}\n\tif flags.knn {\n\t\tpreSearchResult.Hits = finalizeKNNResults(req, preSearchResult.Hits)\n\t}\n}\n\n// hitsInCurrentPage returns the hits in the current page\n// using the From and Size parameters in the request\nfunc hitsInCurrentPage(req *SearchRequest, hits []*search.DocumentMatch) []*search.DocumentMatch {\n\tsortFunc := req.SortFunc()\n\t// sort all hits with the requested order\n\tif len(req.Sort) > 0 {\n\t\tsorter := newSearchHitSorter(req.Sort, hits)\n\t\tsortFunc(sorter)\n\t}\n\t// now skip over the correct From\n\tif req.From > 0 && len(hits) > req.From {\n\t\thits = hits[req.From:]\n\t} else if req.From > 0 {\n\t\thits = search.DocumentMatchCollection{}\n\t}\n\t// now trim to the correct size\n\tif req.Size > 0 && len(hits) > req.Size {\n\t\thits = hits[0:req.Size]\n\t}\n\treturn hits\n}\n\n// Extra parameters for MultiSearch\ntype multiSearchParams struct {\n\tpreSearchData map[string]map[string]interface{}\n\trescorer      *rescorer\n\tfusionKnnHits search.DocumentMatchCollection\n}\n\n// MultiSearch executes a SearchRequest across multiple Index objects,\n// then merges the results.  The indexes must honor any ctx deadline.\nfunc MultiSearch(ctx context.Context, req *SearchRequest, params *multiSearchParams, indexes ...Index) (*SearchResult, error) {\n\tsearchStart := time.Now()\n\tasyncResults := make(chan *asyncSearchResult, len(indexes))\n\n\tvar preSearchData map[string]map[string]interface{}\n\tvar rescorer *rescorer\n\tvar fusionKnnHits search.DocumentMatchCollection\n\tif params != nil {\n\t\tpreSearchData = params.preSearchData\n\t\trescorer = params.rescorer\n\t\tfusionKnnHits = params.fusionKnnHits\n\t}\n\n\tvar reverseQueryExecution bool\n\tif req.SearchBefore != nil {\n\t\treverseQueryExecution = true\n\t\treq.Sort.Reverse()\n\t\treq.SearchAfter = req.SearchBefore\n\t\treq.SearchBefore = nil\n\t}\n\n\t// run search on each index in separate go routine\n\tvar waitGroup sync.WaitGroup\n\n\tsearchChildIndex := func(in Index, childReq *SearchRequest) {\n\t\trv := asyncSearchResult{Name: in.Name()}\n\t\trv.Result, rv.Err = in.SearchInContext(ctx, childReq)\n\t\tasyncResults <- &rv\n\t\twaitGroup.Done()\n\t}\n\n\twaitGroup.Add(len(indexes))\n\tfor _, in := range indexes {\n\t\tvar payload map[string]interface{}\n\t\tif preSearchData != nil {\n\t\t\tpayload = preSearchData[in.Name()]\n\t\t}\n\t\tgo searchChildIndex(in, createChildSearchRequest(req, payload))\n\t}\n\n\t// on another go routine, close after finished\n\tgo func() {\n\t\twaitGroup.Wait()\n\t\tclose(asyncResults)\n\t}()\n\n\tvar sr *SearchResult\n\tindexErrors := make(map[string]error)\n\n\tfor asr := range asyncResults {\n\t\tif asr.Err == nil {\n\t\t\tif sr == nil {\n\t\t\t\t// first result\n\t\t\t\tsr = asr.Result\n\t\t\t} else {\n\t\t\t\t// merge with previous\n\t\t\t\tsr.Merge(asr.Result)\n\t\t\t}\n\t\t} else {\n\t\t\tindexErrors[asr.Name] = asr.Err\n\t\t}\n\t}\n\n\t// merge just concatenated all the hits\n\t// now lets clean it up\n\n\t// handle case where no results were successful\n\tif sr == nil {\n\t\tsr = &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tErrors: make(map[string]error),\n\t\t\t},\n\t\t}\n\t}\n\n\tif rescorer != nil {\n\t\tsr.Hits, sr.Total, sr.MaxScore = rescorer.rescore(sr.Hits, fusionKnnHits)\n\t\trescorer.restoreSearchRequest()\n\t}\n\n\tsr.Hits = hitsInCurrentPage(req, sr.Hits)\n\n\t// fix up facets\n\tfor name, fr := range req.Facets {\n\t\tsr.Facets.Fixup(name, fr.Size)\n\t}\n\n\tif reverseQueryExecution {\n\t\t// reverse the sort back to the original\n\t\treq.Sort.Reverse()\n\t\t// resort using the original order\n\t\tmhs := newSearchHitSorter(req.Sort, sr.Hits)\n\t\treq.SortFunc()(mhs)\n\t\t// reset request\n\t\treq.SearchBefore = req.SearchAfter\n\t\treq.SearchAfter = nil\n\t}\n\n\t// fix up original request\n\tif req.Explain {\n\t\tsr.Request = req\n\t}\n\tsearchDuration := time.Since(searchStart)\n\tsr.Took = searchDuration\n\n\t// fix up errors\n\tif len(indexErrors) > 0 {\n\t\tif sr.Status.Errors == nil {\n\t\t\tsr.Status.Errors = make(map[string]error)\n\t\t}\n\t\tfor indexName, indexErr := range indexErrors {\n\t\t\tsr.Status.Errors[indexName] = indexErr\n\t\t\tsr.Status.Total++\n\t\t\tsr.Status.Failed++\n\t\t}\n\t}\n\n\treturn sr, nil\n}\n\nfunc (i *indexAliasImpl) NewBatch() *Batch {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil\n\t}\n\n\terr := i.isAliasToSingleIndex()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn i.indexes[0].NewBatch()\n}\n\nfunc (i *indexAliasImpl) Name() string {\n\treturn i.name\n}\n\nfunc (i *indexAliasImpl) SetName(name string) {\n\ti.name = name\n}\n\ntype indexAliasImplFieldDict struct {\n\tindex     *indexAliasImpl\n\tfieldDict index.FieldDict\n}\n\nfunc (f *indexAliasImplFieldDict) BytesRead() uint64 {\n\treturn f.fieldDict.BytesRead()\n}\n\nfunc (f *indexAliasImplFieldDict) Next() (*index.DictEntry, error) {\n\treturn f.fieldDict.Next()\n}\n\nfunc (f *indexAliasImplFieldDict) Close() error {\n\tdefer f.index.mutex.RUnlock()\n\treturn f.fieldDict.Close()\n}\n\nfunc (f *indexAliasImplFieldDict) Cardinality() int {\n\treturn f.fieldDict.Cardinality()\n}\n\n// -----------------------------------------------------------------------------\n\nfunc (i *indexAliasImpl) TermFrequencies(field string, limit int, descending bool) (\n\t[]index.TermFreq, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\tif len(i.indexes) < 1 {\n\t\treturn nil, ErrorAliasEmpty\n\t}\n\n\t// short circuit the simple case\n\tif len(i.indexes) == 1 {\n\t\tif idx, ok := i.indexes[0].(InsightsIndex); ok {\n\t\t\treturn idx.TermFrequencies(field, limit, descending)\n\t\t}\n\t\treturn nil, nil\n\t}\n\n\t// run search on each index in separate go routine\n\tvar waitGroup sync.WaitGroup\n\tasyncResults := make(chan []index.TermFreq, len(i.indexes))\n\n\tsearchChildIndex := func(in Index, field string, limit int, descending bool) {\n\t\tvar rv []index.TermFreq\n\t\tif idx, ok := in.(InsightsIndex); ok {\n\t\t\t// over sample for higher accuracy\n\t\t\trv, _ = idx.TermFrequencies(field, limit*5, descending)\n\t\t}\n\t\tasyncResults <- rv\n\t\twaitGroup.Done()\n\t}\n\n\twaitGroup.Add(len(i.indexes))\n\tfor _, in := range i.indexes {\n\t\tgo searchChildIndex(in, field, limit, descending)\n\t}\n\n\t// on another go routine, close after finished\n\tgo func() {\n\t\twaitGroup.Wait()\n\t\tclose(asyncResults)\n\t}()\n\n\trvTermFreqsMap := make(map[string]uint64)\n\tfor asr := range asyncResults {\n\t\tfor _, entry := range asr {\n\t\t\trvTermFreqsMap[entry.Term] += entry.Frequency\n\t\t}\n\t}\n\n\trvTermFreqs := make([]index.TermFreq, 0, len(rvTermFreqsMap))\n\tfor term, freq := range rvTermFreqsMap {\n\t\trvTermFreqs = append(rvTermFreqs, index.TermFreq{\n\t\t\tTerm:      term,\n\t\t\tFrequency: freq,\n\t\t})\n\t}\n\n\tsort.Slice(rvTermFreqs, func(i, j int) bool {\n\t\tif rvTermFreqs[i].Frequency == rvTermFreqs[j].Frequency {\n\t\t\t// If frequencies are equal, sort by term lexicographically\n\t\t\treturn rvTermFreqs[i].Term < rvTermFreqs[j].Term\n\t\t}\n\t\tif descending {\n\t\t\treturn rvTermFreqs[i].Frequency > rvTermFreqs[j].Frequency\n\t\t}\n\t\treturn rvTermFreqs[i].Frequency < rvTermFreqs[j].Frequency\n\t})\n\n\tif limit > len(rvTermFreqs) {\n\t\tlimit = len(rvTermFreqs)\n\t}\n\n\treturn rvTermFreqs[:limit], nil\n}\n\nfunc (i *indexAliasImpl) CentroidCardinalities(field string, limit int, descending bool) (\n\t[]index.CentroidCardinality, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\tif len(i.indexes) < 1 {\n\t\treturn nil, ErrorAliasEmpty\n\t}\n\n\t// short circuit the simple case\n\tif len(i.indexes) == 1 {\n\t\tif idx, ok := i.indexes[0].(InsightsIndex); ok {\n\t\t\treturn idx.CentroidCardinalities(field, limit, descending)\n\t\t}\n\t\treturn nil, nil\n\t}\n\n\t// run search on each index in separate go routine\n\tvar waitGroup sync.WaitGroup\n\tasyncResults := make(chan []index.CentroidCardinality, len(i.indexes))\n\n\tsearchChildIndex := func(in Index, field string, limit int, descending bool) {\n\t\tvar rv []index.CentroidCardinality\n\t\tif idx, ok := in.(InsightsIndex); ok {\n\t\t\trv, _ = idx.CentroidCardinalities(field, limit, descending)\n\t\t}\n\t\tasyncResults <- rv\n\t\twaitGroup.Done()\n\t}\n\n\twaitGroup.Add(len(i.indexes))\n\tfor _, in := range i.indexes {\n\t\tgo searchChildIndex(in, field, limit, descending)\n\t}\n\n\t// on another go routine, close after finished\n\tgo func() {\n\t\twaitGroup.Wait()\n\t\tclose(asyncResults)\n\t}()\n\n\trvCentroidCardinalities := make([]index.CentroidCardinality, 0, limit*len(i.indexes))\n\tfor asr := range asyncResults {\n\t\trvCentroidCardinalities = append(rvCentroidCardinalities, asr...)\n\t}\n\n\tsort.Slice(rvCentroidCardinalities, func(i, j int) bool {\n\t\tif descending {\n\t\t\treturn rvCentroidCardinalities[i].Cardinality > rvCentroidCardinalities[j].Cardinality\n\t\t} else {\n\t\t\treturn rvCentroidCardinalities[i].Cardinality < rvCentroidCardinalities[j].Cardinality\n\t\t}\n\t})\n\n\tif limit > len(rvCentroidCardinalities) {\n\t\tlimit = len(rvCentroidCardinalities)\n\t}\n\n\treturn rvCentroidCardinalities[:limit], nil\n}\n"
  },
  {
    "path": "index_alias_impl_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestIndexAliasSingle(t *testing.T) {\n\texpectedError := fmt.Errorf(\"expected\")\n\tei1 := &stubIndex{\n\t\terr: expectedError,\n\t}\n\n\talias := NewIndexAlias(ei1)\n\n\terr := alias.Index(\"a\", \"a\")\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\terr = alias.Delete(\"a\")\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\tbatch := alias.NewBatch()\n\terr = alias.Batch(batch)\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\t_, err = alias.Document(\"a\")\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\t_, err = alias.Fields()\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\t_, err = alias.GetInternal([]byte(\"a\"))\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\terr = alias.SetInternal([]byte(\"a\"), []byte(\"a\"))\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\terr = alias.DeleteInternal([]byte(\"a\"))\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\tmapping := alias.Mapping()\n\tif mapping != nil {\n\t\tt.Errorf(\"expected nil, got %v\", mapping)\n\t}\n\n\tindexStat := alias.Stats()\n\tif indexStat != nil {\n\t\tt.Errorf(\"expected nil, got %v\", indexStat)\n\t}\n\n\t// now a few things that should work\n\tsr := NewSearchRequest(NewTermQuery(\"test\"))\n\t_, err = alias.Search(sr)\n\tif err != expectedError {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError, err)\n\t}\n\n\tcount, err := alias.DocCount()\n\tif err != nil {\n\t\tt.Errorf(\"expected no error, got %v\", err)\n\t}\n\tif count != 0 {\n\t\tt.Errorf(\"expected count 0, got %d\", count)\n\t}\n\n\t// now change the def using add/remove\n\texpectedError2 := fmt.Errorf(\"expected2\")\n\tei2 := &stubIndex{\n\t\terr: expectedError2,\n\t}\n\n\talias.Add(ei2)\n\talias.Remove(ei1)\n\n\terr = alias.Index(\"a\", \"a\")\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\terr = alias.Delete(\"a\")\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\terr = alias.Batch(batch)\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\t_, err = alias.Document(\"a\")\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\t_, err = alias.Fields()\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\t_, err = alias.GetInternal([]byte(\"a\"))\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\terr = alias.SetInternal([]byte(\"a\"), []byte(\"a\"))\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\terr = alias.DeleteInternal([]byte(\"a\"))\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\tmapping = alias.Mapping()\n\tif mapping != nil {\n\t\tt.Errorf(\"expected nil, got %v\", mapping)\n\t}\n\n\tindexStat = alias.Stats()\n\tif indexStat != nil {\n\t\tt.Errorf(\"expected nil, got %v\", indexStat)\n\t}\n\n\t// now a few things that should work\n\t_, err = alias.Search(sr)\n\tif err != expectedError2 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError2, err)\n\t}\n\n\tcount, err = alias.DocCount()\n\tif err != nil {\n\t\tt.Errorf(\"expected no error, got %v\", err)\n\t}\n\tif count != 0 {\n\t\tt.Errorf(\"expected count 0, got %d\", count)\n\t}\n\n\t// now change the def using swap\n\texpectedError3 := fmt.Errorf(\"expected3\")\n\tei3 := &stubIndex{\n\t\terr: expectedError3,\n\t}\n\n\talias.Swap([]Index{ei3}, []Index{ei2})\n\n\terr = alias.Index(\"a\", \"a\")\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\terr = alias.Delete(\"a\")\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\terr = alias.Batch(batch)\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\t_, err = alias.Document(\"a\")\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\t_, err = alias.Fields()\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\t_, err = alias.GetInternal([]byte(\"a\"))\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\terr = alias.SetInternal([]byte(\"a\"), []byte(\"a\"))\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\terr = alias.DeleteInternal([]byte(\"a\"))\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\tmapping = alias.Mapping()\n\tif mapping != nil {\n\t\tt.Errorf(\"expected nil, got %v\", mapping)\n\t}\n\n\tindexStat = alias.Stats()\n\tif indexStat != nil {\n\t\tt.Errorf(\"expected nil, got %v\", indexStat)\n\t}\n\n\t// now a few things that should work\n\t_, err = alias.Search(sr)\n\tif err != expectedError3 {\n\t\tt.Errorf(\"expected %v, got %v\", expectedError3, err)\n\t}\n\n\tcount, err = alias.DocCount()\n\tif err != nil {\n\t\tt.Errorf(\"expected no error, got %v\", err)\n\t}\n\tif count != 0 {\n\t\tt.Errorf(\"expected count 0, got %d\", count)\n\t}\n}\n\nfunc TestIndexAliasClosed(t *testing.T) {\n\talias := NewIndexAlias()\n\terr := alias.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = alias.Index(\"a\", \"a\")\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\terr = alias.Delete(\"a\")\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\tbatch := alias.NewBatch()\n\terr = alias.Batch(batch)\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\t_, err = alias.Document(\"a\")\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\t_, err = alias.Fields()\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\t_, err = alias.GetInternal([]byte(\"a\"))\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\terr = alias.SetInternal([]byte(\"a\"), []byte(\"a\"))\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\terr = alias.DeleteInternal([]byte(\"a\"))\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\tmapping := alias.Mapping()\n\tif mapping != nil {\n\t\tt.Errorf(\"expected nil, got %v\", mapping)\n\t}\n\n\tindexStat := alias.Stats()\n\tif indexStat != nil {\n\t\tt.Errorf(\"expected nil, got %v\", indexStat)\n\t}\n\n\t// now a few things that should work\n\tsr := NewSearchRequest(NewTermQuery(\"test\"))\n\t_, err = alias.Search(sr)\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n\n\t_, err = alias.DocCount()\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorIndexClosed, err)\n\t}\n}\n\nfunc TestIndexAliasEmpty(t *testing.T) {\n\talias := NewIndexAlias()\n\n\terr := alias.Index(\"a\", \"a\")\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\terr = alias.Delete(\"a\")\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\tbatch := alias.NewBatch()\n\terr = alias.Batch(batch)\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\t_, err = alias.Document(\"a\")\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\t_, err = alias.Fields()\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\t_, err = alias.GetInternal([]byte(\"a\"))\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\terr = alias.SetInternal([]byte(\"a\"), []byte(\"a\"))\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\terr = alias.DeleteInternal([]byte(\"a\"))\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\tmapping := alias.Mapping()\n\tif mapping != nil {\n\t\tt.Errorf(\"expected nil, got %v\", mapping)\n\t}\n\n\tindexStat := alias.Stats()\n\tif indexStat != nil {\n\t\tt.Errorf(\"expected nil, got %v\", indexStat)\n\t}\n\n\t// now a few things that should work\n\tsr := NewSearchRequest(NewTermQuery(\"test\"))\n\t_, err = alias.Search(sr)\n\tif err != ErrorAliasEmpty {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasEmpty, err)\n\t}\n\n\tcount, err := alias.DocCount()\n\tif err != nil {\n\t\tt.Errorf(\"error getting alias doc count: %v\", err)\n\t}\n\tif count != 0 {\n\t\tt.Errorf(\"expected %d, got %d\", 0, count)\n\t}\n}\n\nfunc TestIndexAliasMulti(t *testing.T) {\n\tscore1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0)\n\tscore2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0)\n\tei1Count := uint64(7)\n\tei1 := &stubIndex{\n\t\terr:            nil,\n\t\tdocCountResult: &ei1Count,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t{\n\t\t\t\t\tID:    \"a\",\n\t\t\t\t\tScore: 1.0,\n\t\t\t\t\tSort:  []string{string(score1)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 1.0,\n\t\t},\n\t}\n\tei2Count := uint64(8)\n\tei2 := &stubIndex{\n\t\terr:            nil,\n\t\tdocCountResult: &ei2Count,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t{\n\t\t\t\t\tID:    \"b\",\n\t\t\t\t\tScore: 2.0,\n\t\t\t\t\tSort:  []string{string(score2)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 2.0,\n\t\t},\n\t}\n\n\talias := NewIndexAlias(ei1, ei2)\n\n\terr := alias.Index(\"a\", \"a\")\n\tif err != ErrorAliasMulti {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasMulti, err)\n\t}\n\n\terr = alias.Delete(\"a\")\n\tif err != ErrorAliasMulti {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasMulti, err)\n\t}\n\n\tbatch := alias.NewBatch()\n\terr = alias.Batch(batch)\n\tif err != ErrorAliasMulti {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasMulti, err)\n\t}\n\n\t_, err = alias.Document(\"a\")\n\tif err != ErrorAliasMulti {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasMulti, err)\n\t}\n\n\t_, err = alias.Fields()\n\tif err != ErrorAliasMulti {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasMulti, err)\n\t}\n\n\t_, err = alias.GetInternal([]byte(\"a\"))\n\tif err != ErrorAliasMulti {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasMulti, err)\n\t}\n\n\terr = alias.SetInternal([]byte(\"a\"), []byte(\"a\"))\n\tif err != ErrorAliasMulti {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasMulti, err)\n\t}\n\n\terr = alias.DeleteInternal([]byte(\"a\"))\n\tif err != ErrorAliasMulti {\n\t\tt.Errorf(\"expected %v, got %v\", ErrorAliasMulti, err)\n\t}\n\n\tmapping := alias.Mapping()\n\tif mapping != nil {\n\t\tt.Errorf(\"expected nil, got %v\", mapping)\n\t}\n\n\tindexStat := alias.Stats()\n\tif indexStat != nil {\n\t\tt.Errorf(\"expected nil, got %v\", indexStat)\n\t}\n\n\t// now a few things that should work\n\tsr := NewSearchRequest(NewTermQuery(\"test\"))\n\texpected := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      2,\n\t\t\tSuccessful: 2,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal: 2,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tID:    \"b\",\n\t\t\t\tScore: 2.0,\n\t\t\t\tSort:  []string{string(score2)},\n\t\t\t},\n\t\t\t{\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1.0,\n\t\t\t\tSort:  []string{string(score1)},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 2.0,\n\t}\n\tresults, err := alias.Search(sr)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\t// cheat and ensure that Took field matches since it involves time\n\texpected.Took = results.Took\n\tif !reflect.DeepEqual(results, expected) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expected, results)\n\t}\n\n\tcount, err := alias.DocCount()\n\tif err != nil {\n\t\tt.Errorf(\"error getting alias doc count: %v\", err)\n\t}\n\tif count != (*ei1.docCountResult + *ei2.docCountResult) {\n\t\tt.Errorf(\"expected %d, got %d\", (*ei1.docCountResult + *ei2.docCountResult), count)\n\t}\n}\n\n// TestMultiSearchNoError\nfunc TestMultiSearchNoError(t *testing.T) {\n\tscore1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0)\n\tscore2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0)\n\tei1 := &stubIndex{err: nil, searchResult: &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal: 1,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tIndex: \"1\",\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1.0,\n\t\t\t\tSort:  []string{string(score1)},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 1.0,\n\t}}\n\tei2 := &stubIndex{err: nil, searchResult: &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal: 1,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tIndex: \"2\",\n\t\t\t\tID:    \"b\",\n\t\t\t\tScore: 2.0,\n\t\t\t\tSort:  []string{string(score2)},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 2.0,\n\t}}\n\n\tsr := NewSearchRequest(NewTermQuery(\"test\"))\n\texpected := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      2,\n\t\t\tSuccessful: 2,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal: 2,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tIndex: \"2\",\n\t\t\t\tID:    \"b\",\n\t\t\t\tScore: 2.0,\n\t\t\t\tSort:  []string{string(score2)},\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndex: \"1\",\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1.0,\n\t\t\t\tSort:  []string{string(score1)},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 2.0,\n\t}\n\n\tresults, err := MultiSearch(context.Background(), sr, nil, ei1, ei2)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\t// cheat and ensure that Took field matches since it involves time\n\texpected.Took = results.Took\n\tif !reflect.DeepEqual(results, expected) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expected, results)\n\t}\n}\n\n// TestMultiSearchSomeError\nfunc TestMultiSearchSomeError(t *testing.T) {\n\tei1 := &stubIndex{name: \"ei1\", err: nil, searchResult: &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal: 1,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1.0,\n\t\t\t},\n\t\t},\n\t\tTook:     1 * time.Second,\n\t\tMaxScore: 1.0,\n\t}}\n\tei2 := &stubIndex{name: \"ei2\", err: fmt.Errorf(\"deliberate error\")}\n\tsr := NewSearchRequest(NewTermQuery(\"test\"))\n\tres, err := MultiSearch(context.Background(), sr, nil, ei1, ei2)\n\tif err != nil {\n\t\tt.Errorf(\"expected no error, got %v\", err)\n\t}\n\tif res.Status.Total != 2 {\n\t\tt.Errorf(\"expected 2 indexes to be queried, got %d\", res.Status.Total)\n\t}\n\tif res.Status.Failed != 1 {\n\t\tt.Errorf(\"expected 1 index to fail, got %d\", res.Status.Failed)\n\t}\n\tif res.Status.Successful != 1 {\n\t\tt.Errorf(\"expected 1 index to be successful, got %d\", res.Status.Successful)\n\t}\n\tif len(res.Status.Errors) != 1 {\n\t\tt.Fatalf(\"expected 1 status error message, got %d\", len(res.Status.Errors))\n\t}\n\tif res.Status.Errors[\"ei2\"].Error() != \"deliberate error\" {\n\t\tt.Errorf(\"expected ei2 index error message 'deliberate error', got '%s'\", res.Status.Errors[\"ei2\"])\n\t}\n}\n\n// TestMultiSearchAllError\n// reproduces https://github.com/blevesearch/bleve/issues/126\nfunc TestMultiSearchAllError(t *testing.T) {\n\tei1 := &stubIndex{name: \"ei1\", err: fmt.Errorf(\"deliberate error\")}\n\tei2 := &stubIndex{name: \"ei2\", err: fmt.Errorf(\"deliberate error\")}\n\tsr := NewSearchRequest(NewTermQuery(\"test\"))\n\tres, err := MultiSearch(context.Background(), sr, nil, ei1, ei2)\n\tif err != nil {\n\t\tt.Errorf(\"expected no error, got %v\", err)\n\t}\n\tif res.Status.Total != 2 {\n\t\tt.Errorf(\"expected 2 indexes to be queried, got %d\", res.Status.Total)\n\t}\n\tif res.Status.Failed != 2 {\n\t\tt.Errorf(\"expected 2 indexes to fail, got %d\", res.Status.Failed)\n\t}\n\tif res.Status.Successful != 0 {\n\t\tt.Errorf(\"expected 0 indexes to be successful, got %d\", res.Status.Successful)\n\t}\n\tif len(res.Status.Errors) != 2 {\n\t\tt.Fatalf(\"expected 2 status error messages, got %d\", len(res.Status.Errors))\n\t}\n\tif res.Status.Errors[\"ei1\"].Error() != \"deliberate error\" {\n\t\tt.Errorf(\"expected ei1 index error message 'deliberate error', got '%s'\", res.Status.Errors[\"ei1\"])\n\t}\n\tif res.Status.Errors[\"ei2\"].Error() != \"deliberate error\" {\n\t\tt.Errorf(\"expected ei2 index error message 'deliberate error', got '%s'\", res.Status.Errors[\"ei2\"])\n\t}\n}\n\nfunc TestMultiSearchSecondPage(t *testing.T) {\n\tcheckRequest := func(sr *SearchRequest) error {\n\t\tif sr.From != 0 {\n\t\t\treturn fmt.Errorf(\"child request from should be 0\")\n\t\t}\n\t\tif sr.Size != 20 {\n\t\t\treturn fmt.Errorf(\"child request size should be 20\")\n\t\t}\n\t\treturn nil\n\t}\n\n\tei1 := &stubIndex{\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t},\n\t\tcheckRequest: checkRequest,\n\t}\n\tei2 := &stubIndex{\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t},\n\t\tcheckRequest: checkRequest,\n\t}\n\tsr := NewSearchRequestOptions(NewTermQuery(\"test\"), 10, 10, false)\n\t_, err := MultiSearch(context.Background(), sr, nil, ei1, ei2)\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error %v\", err)\n\t}\n}\n\n// TestMultiSearchTimeout tests simple timeout cases\n// 1. all searches finish successfully before timeout\n// 2. no searchers finish before the timeout\n// 3. no searches finish before cancellation\nfunc TestMultiSearchTimeout(t *testing.T) {\n\tscore1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0)\n\tscore2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0)\n\tvar ctx context.Context\n\tei1 := &stubIndex{\n\t\tname: \"ei1\",\n\t\tcheckRequest: func(req *SearchRequest) error {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn ctx.Err()\n\t\t\tcase <-time.After(50 * time.Millisecond):\n\t\t\t\treturn nil\n\t\t\t}\n\t\t},\n\t\terr: nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"1\",\n\t\t\t\t\tID:    \"a\",\n\t\t\t\t\tScore: 1.0,\n\t\t\t\t\tSort:  []string{string(score1)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 1.0,\n\t\t},\n\t}\n\tei2 := &stubIndex{\n\t\tname: \"ei2\",\n\t\tcheckRequest: func(req *SearchRequest) error {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn ctx.Err()\n\t\t\tcase <-time.After(50 * time.Millisecond):\n\t\t\t\treturn nil\n\t\t\t}\n\t\t},\n\t\terr: nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"2\",\n\t\t\t\t\tID:    \"b\",\n\t\t\t\t\tScore: 2.0,\n\t\t\t\t\tSort:  []string{string(score2)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 2.0,\n\t\t},\n\t}\n\n\t// first run with absurdly long time out, should succeed\n\tvar cancel context.CancelFunc\n\tctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\tquery := NewTermQuery(\"test\")\n\tsr := NewSearchRequest(query)\n\tres, err := MultiSearch(ctx, sr, nil, ei1, ei2)\n\tif err != nil {\n\t\tt.Errorf(\"expected no error, got %v\", err)\n\t}\n\tif res.Status.Total != 2 {\n\t\tt.Errorf(\"expected 2 total, got %d\", res.Status.Failed)\n\t}\n\tif res.Status.Successful != 2 {\n\t\tt.Errorf(\"expected 0 success, got %d\", res.Status.Successful)\n\t}\n\tif res.Status.Failed != 0 {\n\t\tt.Errorf(\"expected 2 failed, got %d\", res.Status.Failed)\n\t}\n\tif len(res.Status.Errors) != 0 {\n\t\tt.Errorf(\"expected 0 errors, got %v\", res.Status.Errors)\n\t}\n\n\t// now run a search again with an absurdly low timeout (should timeout)\n\tctx, cancel = context.WithTimeout(context.Background(), 1*time.Microsecond)\n\tdefer cancel()\n\tres, err = MultiSearch(ctx, sr, nil, ei1, ei2)\n\tif err != nil {\n\t\tt.Errorf(\"expected no error, got %v\", err)\n\t}\n\tif res.Status.Total != 2 {\n\t\tt.Errorf(\"expected 2 failed, got %d\", res.Status.Failed)\n\t}\n\tif res.Status.Successful != 0 {\n\t\tt.Errorf(\"expected 0 success, got %d\", res.Status.Successful)\n\t}\n\tif res.Status.Failed != 2 {\n\t\tt.Errorf(\"expected 2 failed, got %d\", res.Status.Failed)\n\t}\n\tif len(res.Status.Errors) != 2 {\n\t\tt.Errorf(\"expected 2 errors, got %v\", res.Status.Errors)\n\t} else {\n\t\tif res.Status.Errors[\"ei1\"].Error() != context.DeadlineExceeded.Error() {\n\t\t\tt.Errorf(\"expected err for 'ei1' to be '%s' got '%s'\", context.DeadlineExceeded.Error(), res.Status.Errors[\"ei1\"])\n\t\t}\n\t\tif res.Status.Errors[\"ei2\"].Error() != context.DeadlineExceeded.Error() {\n\t\t\tt.Errorf(\"expected err for 'ei2' to be '%s' got '%s'\", context.DeadlineExceeded.Error(), res.Status.Errors[\"ei2\"])\n\t\t}\n\t}\n\n\t// now run a search again with a normal timeout, but cancel it first\n\tctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)\n\tcancel()\n\tres, err = MultiSearch(ctx, sr, nil, ei1, ei2)\n\tif err != nil {\n\t\tt.Errorf(\"expected no error, got %v\", err)\n\t}\n\tif res.Status.Total != 2 {\n\t\tt.Errorf(\"expected 2 failed, got %d\", res.Status.Failed)\n\t}\n\tif res.Status.Successful != 0 {\n\t\tt.Errorf(\"expected 0 success, got %d\", res.Status.Successful)\n\t}\n\tif res.Status.Failed != 2 {\n\t\tt.Errorf(\"expected 2 failed, got %d\", res.Status.Failed)\n\t}\n\tif len(res.Status.Errors) != 2 {\n\t\tt.Errorf(\"expected 2 errors, got %v\", res.Status.Errors)\n\t} else {\n\t\tif res.Status.Errors[\"ei1\"].Error() != context.Canceled.Error() {\n\t\t\tt.Errorf(\"expected err for 'ei1' to be '%s' got '%s'\", context.Canceled.Error(), res.Status.Errors[\"ei1\"])\n\t\t}\n\t\tif res.Status.Errors[\"ei2\"].Error() != context.Canceled.Error() {\n\t\t\tt.Errorf(\"expected err for 'ei2' to be '%s' got '%s'\", context.Canceled.Error(), res.Status.Errors[\"ei2\"])\n\t\t}\n\t}\n}\n\n// TestMultiSearchTimeoutPartial tests the case where some indexes exceed\n// the timeout, while others complete successfully\nfunc TestMultiSearchTimeoutPartial(t *testing.T) {\n\tscore1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0)\n\tscore2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0)\n\tscore3, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(3.0), 0)\n\tvar ctx context.Context\n\tei1 := &stubIndex{\n\t\tname: \"ei1\",\n\t\terr:  nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"1\",\n\t\t\t\t\tID:    \"a\",\n\t\t\t\t\tScore: 1.0,\n\t\t\t\t\tSort:  []string{string(score1)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 1.0,\n\t\t},\n\t}\n\tei2 := &stubIndex{\n\t\tname: \"ei2\",\n\t\terr:  nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"2\",\n\t\t\t\t\tID:    \"b\",\n\t\t\t\t\tScore: 2.0,\n\t\t\t\t\tSort:  []string{string(score2)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 2.0,\n\t\t},\n\t}\n\n\tei3 := &stubIndex{\n\t\tname: \"ei3\",\n\t\tcheckRequest: func(req *SearchRequest) error {\n\t\t\t<-ctx.Done()\n\t\t\treturn ctx.Err()\n\t\t},\n\t\terr: nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"3\",\n\t\t\t\t\tID:    \"c\",\n\t\t\t\t\tScore: 3.0,\n\t\t\t\t\tSort:  []string{string(score3)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 3.0,\n\t\t},\n\t}\n\n\t// ei3 is set to take >50ms, so run search with timeout less than\n\t// this, this should return partial results\n\tvar cancel context.CancelFunc\n\tctx, cancel = context.WithTimeout(context.Background(), 25*time.Millisecond)\n\tdefer cancel()\n\tquery := NewTermQuery(\"test\")\n\tsr := NewSearchRequest(query)\n\texpected := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      3,\n\t\t\tSuccessful: 2,\n\t\t\tFailed:     1,\n\t\t\tErrors: map[string]error{\n\t\t\t\t\"ei3\": context.DeadlineExceeded,\n\t\t\t},\n\t\t},\n\t\tTotal: 2,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tIndex: \"2\",\n\t\t\t\tID:    \"b\",\n\t\t\t\tScore: 2.0,\n\t\t\t\tSort:  []string{string(score2)},\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndex: \"1\",\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1.0,\n\t\t\t\tSort:  []string{string(score1)},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 2.0,\n\t}\n\n\tres, err := MultiSearch(ctx, sr, nil, ei1, ei2, ei3)\n\tif err != nil {\n\t\tt.Fatalf(\"expected no err, got %v\", err)\n\t}\n\texpected.Took = res.Took\n\tif !reflect.DeepEqual(res, expected) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expected, res)\n\t}\n}\n\nfunc TestIndexAliasMultipleLayer(t *testing.T) {\n\tscore1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0)\n\tscore2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0)\n\tscore3, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(3.0), 0)\n\tscore4, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(4.0), 0)\n\tvar ctx context.Context\n\tei1 := &stubIndex{\n\t\tname: \"ei1\",\n\t\terr:  nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"1\",\n\t\t\t\t\tID:    \"a\",\n\t\t\t\t\tScore: 1.0,\n\t\t\t\t\tSort:  []string{string(score1)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 1.0,\n\t\t},\n\t}\n\tei2 := &stubIndex{\n\t\tname: \"ei2\",\n\t\tcheckRequest: func(req *SearchRequest) error {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn ctx.Err()\n\t\t\tcase <-time.After(250 * time.Millisecond):\n\t\t\t\treturn nil\n\t\t\t}\n\t\t},\n\t\terr: nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"2\",\n\t\t\t\t\tID:    \"b\",\n\t\t\t\t\tScore: 2.0,\n\t\t\t\t\tSort:  []string{string(score2)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 2.0,\n\t\t},\n\t}\n\n\tei3 := &stubIndex{\n\t\tname: \"ei3\",\n\t\tcheckRequest: func(req *SearchRequest) error {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn ctx.Err()\n\t\t\tcase <-time.After(250 * time.Millisecond):\n\t\t\t\treturn nil\n\t\t\t}\n\t\t},\n\t\terr: nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"3\",\n\t\t\t\t\tID:    \"c\",\n\t\t\t\t\tScore: 3.0,\n\t\t\t\t\tSort:  []string{string(score3)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 3.0,\n\t\t},\n\t}\n\n\tei4 := &stubIndex{\n\t\tname: \"ei4\",\n\t\terr:  nil,\n\t\tsearchResult: &SearchResult{\n\t\t\tStatus: &SearchStatus{\n\t\t\t\tTotal:      1,\n\t\t\t\tSuccessful: 1,\n\t\t\t\tErrors:     make(map[string]error),\n\t\t\t},\n\t\t\tTotal: 1,\n\t\t\tHits: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndex: \"4\",\n\t\t\t\t\tID:    \"d\",\n\t\t\t\t\tScore: 4.0,\n\t\t\t\t\tSort:  []string{string(score4)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tMaxScore: 4.0,\n\t\t},\n\t}\n\n\talias1 := NewIndexAlias(ei1, ei2)\n\talias2 := NewIndexAlias(ei3, ei4)\n\taliasTop := NewIndexAlias(alias1, alias2)\n\n\t// ei2 and ei3 have 50ms delay\n\t// search across aliasTop should still get results from ei1 and ei4\n\t// total should still be 4\n\tvar cancel context.CancelFunc\n\tctx, cancel = context.WithTimeout(context.Background(), 25*time.Millisecond)\n\tdefer cancel()\n\tquery := NewTermQuery(\"test\")\n\tsr := NewSearchRequest(query)\n\texpected := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      4,\n\t\t\tSuccessful: 2,\n\t\t\tFailed:     2,\n\t\t\tErrors: map[string]error{\n\t\t\t\t\"ei2\": context.DeadlineExceeded,\n\t\t\t\t\"ei3\": context.DeadlineExceeded,\n\t\t\t},\n\t\t},\n\t\tTotal: 2,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tIndex: \"4\",\n\t\t\t\tID:    \"d\",\n\t\t\t\tScore: 4.0,\n\t\t\t\tSort:  []string{string(score4)},\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndex: \"1\",\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1.0,\n\t\t\t\tSort:  []string{string(score1)},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 4.0,\n\t}\n\n\tres, err := aliasTop.SearchInContext(ctx, sr)\n\tif err != nil {\n\t\tt.Fatalf(\"expected no err, got %v\", err)\n\t}\n\texpected.Took = res.Took\n\tif !reflect.DeepEqual(res, expected) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expected, res)\n\t}\n}\n\n// TestMultiSearchCustomSort\nfunc TestMultiSearchCustomSort(t *testing.T) {\n\tei1 := &stubIndex{err: nil, searchResult: &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal: 2,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tIndex: \"1\",\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1.0,\n\t\t\t\tSort:  []string{\"albert\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndex: \"1\",\n\t\t\t\tID:    \"b\",\n\t\t\t\tScore: 2.0,\n\t\t\t\tSort:  []string{\"crown\"},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 2.0,\n\t}}\n\tei2 := &stubIndex{err: nil, searchResult: &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal: 2,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tIndex: \"2\",\n\t\t\t\tID:    \"c\",\n\t\t\t\tScore: 2.5,\n\t\t\t\tSort:  []string{\"frank\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndex: \"2\",\n\t\t\t\tID:    \"d\",\n\t\t\t\tScore: 3.0,\n\t\t\t\tSort:  []string{\"zombie\"},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 3.0,\n\t}}\n\n\tsr := NewSearchRequest(NewTermQuery(\"test\"))\n\tsr.Explain = true\n\tsr.SortBy([]string{\"name\"})\n\texpected := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      2,\n\t\t\tSuccessful: 2,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tRequest: sr,\n\t\tTotal:   4,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t{\n\t\t\t\tIndex: \"1\",\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1.0,\n\t\t\t\tSort:  []string{\"albert\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndex: \"1\",\n\t\t\t\tID:    \"b\",\n\t\t\t\tScore: 2.0,\n\t\t\t\tSort:  []string{\"crown\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndex: \"2\",\n\t\t\t\tID:    \"c\",\n\t\t\t\tScore: 2.5,\n\t\t\t\tSort:  []string{\"frank\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndex: \"2\",\n\t\t\t\tID:    \"d\",\n\t\t\t\tScore: 3.0,\n\t\t\t\tSort:  []string{\"zombie\"},\n\t\t\t},\n\t\t},\n\t\tMaxScore: 3.0,\n\t}\n\n\tresults, err := MultiSearch(context.Background(), sr, nil, ei1, ei2)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\t// cheat and ensure that Took field matches since it involves time\n\texpected.Took = results.Took\n\tif !reflect.DeepEqual(results, expected) {\n\t\tt.Errorf(\"expected %v, got %v\", expected, results)\n\t}\n}\n\n// stubIndex is an Index impl for which all operations\n// return the configured error value, unless the\n// corresponding operation result value has been\n// set, in which case that is returned instead\ntype stubIndex struct {\n\tname           string\n\terr            error\n\tsearchResult   *SearchResult\n\tdocumentResult *document.Document\n\tdocCountResult *uint64\n\tcheckRequest   func(*SearchRequest) error\n}\n\nfunc (i *stubIndex) Index(id string, data interface{}) error {\n\treturn i.err\n}\n\nfunc (i *stubIndex) Delete(id string) error {\n\treturn i.err\n}\n\nfunc (i *stubIndex) Batch(b *Batch) error {\n\treturn i.err\n}\n\nfunc (i *stubIndex) Document(id string) (index.Document, error) {\n\tif i.documentResult != nil {\n\t\treturn i.documentResult, nil\n\t}\n\treturn nil, i.err\n}\n\nfunc (i *stubIndex) DocCount() (uint64, error) {\n\tif i.docCountResult != nil {\n\t\treturn *i.docCountResult, nil\n\t}\n\treturn 0, i.err\n}\n\nfunc (i *stubIndex) Search(req *SearchRequest) (*SearchResult, error) {\n\treturn i.SearchInContext(context.Background(), req)\n}\n\nfunc (i *stubIndex) SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) {\n\tif i.checkRequest != nil {\n\t\terr := i.checkRequest(req)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tif i.searchResult != nil {\n\t\treturn i.searchResult, nil\n\t}\n\treturn nil, i.err\n}\n\nfunc (i *stubIndex) Fields() ([]string, error) {\n\treturn nil, i.err\n}\n\nfunc (i *stubIndex) FieldDict(field string) (index.FieldDict, error) {\n\treturn nil, i.err\n}\n\nfunc (i *stubIndex) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {\n\treturn nil, i.err\n}\n\nfunc (i *stubIndex) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {\n\treturn nil, i.err\n}\n\nfunc (i *stubIndex) Close() error {\n\treturn i.err\n}\n\nfunc (i *stubIndex) Mapping() mapping.IndexMapping {\n\treturn nil\n}\n\nfunc (i *stubIndex) Stats() *IndexStat {\n\treturn nil\n}\n\nfunc (i *stubIndex) StatsMap() map[string]interface{} {\n\treturn nil\n}\n\nfunc (i *stubIndex) GetInternal(key []byte) ([]byte, error) {\n\treturn nil, i.err\n}\n\nfunc (i *stubIndex) SetInternal(key, val []byte) error {\n\treturn i.err\n}\n\nfunc (i *stubIndex) DeleteInternal(key []byte) error {\n\treturn i.err\n}\n\nfunc (i *stubIndex) Advanced() (index.Index, error) {\n\treturn nil, nil\n}\n\nfunc (i *stubIndex) NewBatch() *Batch {\n\treturn &Batch{}\n}\n\nfunc (i *stubIndex) Name() string {\n\treturn i.name\n}\n\nfunc (i *stubIndex) SetName(name string) {\n\ti.name = name\n}\n"
  },
  {
    "path": "index_impl.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/collector\"\n\t\"github.com/blevesearch/bleve/v2/search/facet\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"github.com/blevesearch/geo/s2\"\n)\n\ntype indexImpl struct {\n\tpath  string\n\tname  string\n\tmeta  *indexMeta\n\ti     index.Index\n\tm     mapping.IndexMapping\n\tmutex sync.RWMutex\n\topen  bool\n\tstats *IndexStat\n}\n\nconst storePath = \"store\"\n\nconst (\n\tSearchQueryStartCallbackKey search.ContextKey = \"_search_query_start_callback_key\"\n\tSearchQueryEndCallbackKey   search.ContextKey = \"_search_query_end_callback_key\"\n)\n\ntype (\n\tSearchQueryStartCallbackFn func(size uint64) error\n\tSearchQueryEndCallbackFn   func(size uint64) error\n)\n\nfunc indexStorePath(path string) string {\n\treturn path + string(os.PathSeparator) + storePath\n}\n\nfunc newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {\n\t// first validate the mapping\n\terr := mapping.Validate()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif kvconfig == nil {\n\t\tkvconfig = map[string]interface{}{}\n\t}\n\n\tif kvstore == \"\" {\n\t\treturn nil, fmt.Errorf(\"bleve not configured for file based indexing\")\n\t}\n\n\trv := indexImpl{\n\t\tpath: path,\n\t\tname: path,\n\t\tm:    mapping,\n\t\tmeta: newIndexMeta(indexType, kvstore, kvconfig),\n\t}\n\trv.stats = &IndexStat{i: &rv}\n\t// at this point there is hope that we can be successful, so save index meta\n\tif path != \"\" {\n\t\terr = rv.meta.Save(path)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tkvconfig[\"create_if_missing\"] = true\n\t\tkvconfig[\"error_if_exists\"] = true\n\t\tkvconfig[\"path\"] = indexStorePath(path)\n\t} else {\n\t\tkvconfig[\"path\"] = \"\"\n\t}\n\n\t// open the index\n\tindexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)\n\tif indexTypeConstructor == nil {\n\t\treturn nil, ErrorUnknownIndexType\n\t}\n\n\trv.i, err = indexTypeConstructor(rv.meta.Storage, kvconfig, Config.analysisQueue)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\terr = rv.i.Open()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func(rv *indexImpl) {\n\t\tif !rv.open {\n\t\t\trv.i.Close()\n\t\t}\n\t}(&rv)\n\n\t// now persist the mapping\n\tmappingBytes, err := util.MarshalJSON(mapping)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\terr = rv.i.SetInternal(util.MappingInternalKey, mappingBytes)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// mark the index as open\n\trv.mutex.Lock()\n\tdefer rv.mutex.Unlock()\n\trv.open = true\n\tindexStats.Register(&rv)\n\treturn &rv, nil\n}\n\nfunc openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) {\n\trv = &indexImpl{\n\t\tpath: path,\n\t\tname: path,\n\t}\n\trv.stats = &IndexStat{i: rv}\n\n\trv.meta, err = openIndexMeta(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// backwards compatibility if index type is missing\n\tif rv.meta.IndexType == \"\" {\n\t\trv.meta.IndexType = upsidedown.Name\n\t}\n\n\tvar um *mapping.IndexMappingImpl\n\tvar umBytes []byte\n\n\tstoreConfig := rv.meta.Config\n\tif storeConfig == nil {\n\t\tstoreConfig = map[string]interface{}{}\n\t}\n\n\tstoreConfig[\"path\"] = indexStorePath(path)\n\tstoreConfig[\"create_if_missing\"] = false\n\tstoreConfig[\"error_if_exists\"] = false\n\tfor rck, rcv := range runtimeConfig {\n\t\tstoreConfig[rck] = rcv\n\t\tif rck == \"updated_mapping\" {\n\t\t\tif val, ok := rcv.(string); ok {\n\t\t\t\tif len(val) == 0 {\n\t\t\t\t\treturn nil, fmt.Errorf(\"updated_mapping is empty\")\n\t\t\t\t}\n\t\t\t\tumBytes = []byte(val)\n\n\t\t\t\terr = util.UnmarshalJSON(umBytes, &um)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"error parsing updated_mapping into JSON: %v\\nmapping contents:\\n%v\", err, rck)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn nil, fmt.Errorf(\"updated_mapping not of type string\")\n\t\t\t}\n\t\t}\n\t}\n\n\t// open the index\n\tindexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)\n\tif indexTypeConstructor == nil {\n\t\treturn nil, ErrorUnknownIndexType\n\t}\n\n\trv.i, err = indexTypeConstructor(rv.meta.Storage, storeConfig, Config.analysisQueue)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar ui index.UpdateIndex\n\tif um != nil {\n\t\tvar ok bool\n\t\tui, ok = rv.i.(index.UpdateIndex)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"updated mapping present for unupdatable index\")\n\t\t}\n\n\t\t// Load the meta data from bolt so that we can read the current index\n\t\t// mapping to compare with\n\t\terr = ui.OpenMeta()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\terr = rv.i.Open()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer func(rv *indexImpl) {\n\t\t\tif !rv.open {\n\t\t\t\trv.i.Close()\n\t\t\t}\n\t\t}(rv)\n\t}\n\n\t// now load the mapping\n\tindexReader, err := rv.i.Reader()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := indexReader.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tmappingBytes, err := indexReader.GetInternal(util.MappingInternalKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar im *mapping.IndexMappingImpl\n\terr = util.UnmarshalJSON(mappingBytes, &im)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error parsing mapping JSON: %v\\nmapping contents:\\n%s\", err, string(mappingBytes))\n\t}\n\n\t// validate the mapping\n\terr = im.Validate()\n\tif err != nil {\n\t\t// no longer return usable index on error because there\n\t\t// is a chance the index is not open at this stage\n\t\treturn nil, err\n\t}\n\n\t// Validate and update the index with the new mapping\n\tif um != nil && ui != nil {\n\t\terr = um.Validate()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tfieldInfo, err := DeletedFields(im, um)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\terr = ui.UpdateFields(fieldInfo, umBytes)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tim = um\n\n\t\terr = rv.i.Open()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer func(rv *indexImpl) {\n\t\t\tif !rv.open {\n\t\t\t\trv.i.Close()\n\t\t\t}\n\t\t}(rv)\n\t}\n\n\t// mark the index as open\n\trv.mutex.Lock()\n\tdefer rv.mutex.Unlock()\n\trv.open = true\n\n\trv.m = im\n\tindexStats.Register(rv)\n\treturn rv, err\n}\n\n// Advanced returns internal index implementation\nfunc (i *indexImpl) Advanced() (index.Index, error) {\n\treturn i.i, nil\n}\n\n// Mapping returns the IndexMapping in use by this\n// Index.\nfunc (i *indexImpl) Mapping() mapping.IndexMapping {\n\treturn i.m\n}\n\n// Index the object with the specified identifier.\n// The IndexMapping for this index will determine\n// how the object is indexed.\nfunc (i *indexImpl) Index(id string, data interface{}) (err error) {\n\tif id == \"\" {\n\t\treturn ErrorEmptyID\n\t}\n\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\ti.FireIndexEvent()\n\n\tdoc := document.NewDocument(id)\n\terr = i.m.MapDocument(doc, data)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = i.i.Update(doc)\n\treturn\n}\n\n// IndexSynonym indexes a synonym definition, with the specified id and belonging to the specified collection.\n// Synonym definition defines term relationships for query expansion in searches.\nfunc (i *indexImpl) IndexSynonym(id string, collection string, definition *SynonymDefinition) error {\n\tif id == \"\" {\n\t\treturn ErrorEmptyID\n\t}\n\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\ti.FireIndexEvent()\n\n\tsynMap, ok := i.m.(mapping.SynonymMapping)\n\tif !ok {\n\t\treturn ErrorSynonymSearchNotSupported\n\t}\n\n\tif err := definition.Validate(); err != nil {\n\t\treturn err\n\t}\n\n\tdoc := document.NewSynonymDocument(id)\n\terr := synMap.MapSynonymDocument(doc, collection, definition.Input, definition.Synonyms)\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = i.i.Update(doc)\n\treturn err\n}\n\n// IndexAdvanced takes a document.Document object\n// skips the mapping and indexes it.\nfunc (i *indexImpl) IndexAdvanced(doc *document.Document) (err error) {\n\tif doc.ID() == \"\" {\n\t\treturn ErrorEmptyID\n\t}\n\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\terr = i.i.Update(doc)\n\treturn\n}\n\n// Delete entries for the specified identifier from\n// the index.\nfunc (i *indexImpl) Delete(id string) (err error) {\n\tif id == \"\" {\n\t\treturn ErrorEmptyID\n\t}\n\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\terr = i.i.Delete(id)\n\treturn\n}\n\n// Batch executes multiple Index and Delete\n// operations at the same time.  There are often\n// significant performance benefits when performing\n// operations in a batch.\nfunc (i *indexImpl) Batch(b *Batch) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\treturn i.i.Batch(b.internal)\n}\n\n// Document is used to find the values of all the\n// stored fields for a document in the index.  These\n// stored fields are put back into a Document object\n// and returned.\nfunc (i *indexImpl) Document(id string) (doc index.Document, err error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\tindexReader, err := i.i.Reader()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := indexReader.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tdoc, err = indexReader.Document(id)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn doc, nil\n}\n\n// DocCount returns the number of documents in the\n// index.\nfunc (i *indexImpl) DocCount() (count uint64, err error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn 0, ErrorIndexClosed\n\t}\n\n\t// open a reader for this search\n\tindexReader, err := i.i.Reader()\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"error opening index reader %v\", err)\n\t}\n\tdefer func() {\n\t\tif cerr := indexReader.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tcount, err = indexReader.DocCount()\n\treturn\n}\n\n// Search executes a search request operation.\n// Returns a SearchResult object or an error.\nfunc (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) {\n\treturn i.SearchInContext(context.Background(), req)\n}\n\nvar (\n\tdocumentMatchEmptySize int\n\tsearchContextEmptySize int\n\tfacetResultEmptySize   int\n\tdocumentEmptySize      int\n)\n\nfunc init() {\n\tvar dm search.DocumentMatch\n\tdocumentMatchEmptySize = dm.Size()\n\n\tvar sc search.SearchContext\n\tsearchContextEmptySize = sc.Size()\n\n\tvar fr search.FacetResult\n\tfacetResultEmptySize = fr.Size()\n\n\tvar d document.Document\n\tdocumentEmptySize = d.Size()\n}\n\n// memNeededForSearch is a helper function that returns an estimate of RAM\n// needed to execute a search request.\nfunc memNeededForSearch(req *SearchRequest,\n\tsearcher search.Searcher,\n\ttopnCollector *collector.TopNCollector,\n) uint64 {\n\tbackingSize := req.Size + req.From + 1\n\tif req.Size+req.From > collector.PreAllocSizeSkipCap {\n\t\tbackingSize = collector.PreAllocSizeSkipCap + 1\n\t}\n\tnumDocMatches := backingSize + searcher.DocumentMatchPoolSize()\n\n\testimate := 0\n\n\t// overhead, size in bytes from collector\n\testimate += topnCollector.Size()\n\n\t// pre-allocing DocumentMatchPool\n\testimate += searchContextEmptySize + numDocMatches*documentMatchEmptySize\n\n\t// searcher overhead\n\testimate += searcher.Size()\n\n\t// overhead from results, lowestMatchOutsideResults\n\testimate += (numDocMatches + 1) * documentMatchEmptySize\n\n\t// additional overhead from SearchResult\n\testimate += reflectStaticSizeSearchResult + reflectStaticSizeSearchStatus\n\n\t// overhead from facet results\n\tif req.Facets != nil {\n\t\testimate += len(req.Facets) * facetResultEmptySize\n\t}\n\n\t// highlighting, store\n\tif len(req.Fields) > 0 || req.Highlight != nil {\n\t\t// Size + From => number of hits\n\t\testimate += (req.Size + req.From) * documentEmptySize\n\t}\n\n\treturn uint64(estimate)\n}\n\nfunc (i *indexImpl) preSearch(ctx context.Context, req *SearchRequest, reader index.IndexReader) (*SearchResult, error) {\n\tvar knnHits []*search.DocumentMatch\n\tvar err error\n\tif requestHasKNN(req) {\n\t\tknnHits, err = i.runKnnCollector(ctx, req, reader, true)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tvar fts search.FieldTermSynonymMap\n\tvar count uint64\n\tvar fieldCardinality map[string]int\n\tif !isMatchNoneQuery(req.Query) {\n\t\tif synMap, ok := i.m.(mapping.SynonymMapping); ok {\n\t\t\tif synReader, ok := reader.(index.ThesaurusReader); ok {\n\t\t\t\tfts, err = query.ExtractSynonyms(ctx, synMap, synReader, req.Query, fts)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif ok := isBM25Enabled(i.m); ok {\n\t\t\tfieldCardinality = make(map[string]int)\n\t\t\tcount, err = reader.DocCount()\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tfs, err := query.ExtractFields(req.Query, i.m, search.NewFieldSet())\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tfor field := range fs {\n\t\t\t\tif bm25Reader, ok := reader.(index.BM25Reader); ok {\n\t\t\t\t\tfieldCardinality[field], err = bm25Reader.FieldCardinality(field)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t},\n\t\tHits:          knnHits,\n\t\tSynonymResult: fts,\n\t\tBM25Stats: &search.BM25Stats{\n\t\t\tDocCount:         float64(count),\n\t\t\tFieldCardinality: fieldCardinality,\n\t\t},\n\t}, nil\n}\n\n// SearchInContext executes a search request operation within the provided\n// Context. Returns a SearchResult object or an error.\nfunc (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr *SearchResult, err error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tsearchStart := time.Now()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\t// open a reader for this search\n\tindexReader, err := i.i.Reader()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error opening index reader %v\", err)\n\t}\n\tdefer func() {\n\t\tif cerr := indexReader.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\t// rescorer will be set if score fusion is supposed to happen\n\t// at this alias (root alias), else will be nil\n\tvar rescorer *rescorer\n\tif _, ok := ctx.Value(search.ScoreFusionKey).(bool); !ok {\n\t\t// new context will be used in internal functions to collect data\n\t\t// as suitable for hybrid search. Rescorer is used for rescoring\n\t\t// using fusion algorithms.\n\t\tif IsScoreFusionRequested(req) {\n\t\t\tctx = context.WithValue(ctx, search.ScoreFusionKey, true)\n\t\t\trescorer = newRescorer(req)\n\t\t\trescorer.prepareSearchRequest()\n\t\t\tdefer rescorer.restoreSearchRequest()\n\t\t}\n\t}\n\n\t// ------------------------------------------------------------------------------------------\n\t// set up additional contexts for any search operation that will proceed from\n\t// here, such as presearch, knn collector, topn collector etc.\n\n\t// Scoring model callback to be used to get scoring model\n\tscoringModelCallback := func() string {\n\t\tif isBM25Enabled(i.m) {\n\t\t\treturn index.BM25Scoring\n\t\t}\n\t\treturn index.DefaultScoringModel\n\t}\n\tctx = context.WithValue(ctx, search.GetScoringModelCallbackKey,\n\t\tsearch.GetScoringModelCallbackFn(scoringModelCallback))\n\n\t// This callback and variable handles the tracking of bytes read\n\t//  1. as part of creation of tfr and its Next() calls which is\n\t//     accounted by invoking this callback when the TFR is closed.\n\t//  2. the docvalues portion (accounted in collector) and the retrieval\n\t//     of stored fields bytes (by LoadAndHighlightFields)\n\tvar totalSearchCost uint64\n\tsendBytesRead := func(bytesRead uint64) {\n\t\ttotalSearchCost += bytesRead\n\t}\n\t// Ensure IO cost accounting and result cost assignment happen on all return paths\n\tdefer func() {\n\t\tif sr != nil {\n\t\t\tsr.Cost = totalSearchCost\n\t\t}\n\t\tif is, ok := indexReader.(*scorch.IndexSnapshot); ok {\n\t\t\tis.UpdateIOStats(totalSearchCost)\n\t\t}\n\t\tsearch.RecordSearchCost(ctx, search.DoneM, 0)\n\t}()\n\n\tctx = context.WithValue(ctx, search.SearchIOStatsCallbackKey, search.SearchIOStatsCallbackFunc(sendBytesRead))\n\n\t// Geo buffer pool callback to be used for getting geo buffer pool\n\tvar bufPool *s2.GeoBufferPool\n\tgetBufferPool := func() *s2.GeoBufferPool {\n\t\tif bufPool == nil {\n\t\t\tbufPool = s2.NewGeoBufferPool(search.MaxGeoBufPoolSize, search.MinGeoBufPoolSize)\n\t\t}\n\n\t\treturn bufPool\n\t}\n\n\tctx = context.WithValue(ctx, search.GeoBufferPoolCallbackKey, search.GeoBufferPoolCallbackFunc(getBufferPool))\n\t// check if the index mapping has any nested fields, which should force\n\t// all collectors and searchers to be run in nested mode\n\tif nm, ok := i.m.(mapping.NestedMapping); ok {\n\t\tif nm.CountNested() > 0 {\n\t\t\tctx = context.WithValue(ctx, search.NestedSearchKey, true)\n\t\t}\n\t}\n\t// ------------------------------------------------------------------------------------------\n\n\tif _, ok := ctx.Value(search.PreSearchKey).(bool); ok {\n\t\tsr, err = i.preSearch(ctx, req, indexReader)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// increment the search count here itself,\n\t\t// since the presearch may already satisfy\n\t\t// the search request\n\t\tatomic.AddUint64(&i.stats.searches, 1)\n\t\t// increment the search time stat here as well,\n\t\t// since presearch is part of the overall search\n\t\t// operation and should be included in the search\n\t\t// time stat\n\t\tsearchDuration := time.Since(searchStart)\n\t\tatomic.AddUint64(&i.stats.searchTime, uint64(searchDuration))\n\n\t\treturn sr, nil\n\t}\n\n\tvar reverseQueryExecution bool\n\tif req.SearchBefore != nil {\n\t\treverseQueryExecution = true\n\t\treq.Sort.Reverse()\n\t\treq.SearchAfter = req.SearchBefore\n\t\treq.SearchBefore = nil\n\t}\n\n\tcoll, err := i.buildTopNCollector(ctx, req, indexReader)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar knnHits []*search.DocumentMatch\n\tvar skipKNNCollector bool\n\n\tvar fts search.FieldTermSynonymMap\n\tvar skipSynonymCollector bool\n\n\tvar bm25Stats *search.BM25Stats\n\tvar ok bool\n\tif req.PreSearchData != nil {\n\t\tfor k, v := range req.PreSearchData {\n\t\t\tswitch k {\n\t\t\tcase search.KnnPreSearchDataKey:\n\t\t\t\tif v != nil {\n\t\t\t\t\tknnHits, ok = v.([]*search.DocumentMatch)\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"knn preSearchData must be of type []*search.DocumentMatch\")\n\t\t\t\t\t}\n\t\t\t\t\tskipKNNCollector = true\n\t\t\t\t}\n\t\t\tcase search.SynonymPreSearchDataKey:\n\t\t\t\tif v != nil {\n\t\t\t\t\tfts, ok = v.(search.FieldTermSynonymMap)\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"synonym preSearchData must be of type search.FieldTermSynonymMap\")\n\t\t\t\t\t}\n\t\t\t\t\tskipSynonymCollector = true\n\t\t\t\t}\n\t\t\tcase search.BM25PreSearchDataKey:\n\t\t\t\tif v != nil {\n\t\t\t\t\tbm25Stats, ok = v.(*search.BM25Stats)\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"bm25 preSearchData must be of type *search.BM25Stats\")\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t_, contextScoreFusionKeyExists := ctx.Value(search.ScoreFusionKey).(bool)\n\n\tif !contextScoreFusionKeyExists {\n\t\t// if no score fusion, default behaviour\n\t\tif !skipKNNCollector && requestHasKNN(req) {\n\t\t\tknnHits, err = i.runKnnCollector(ctx, req, indexReader, false)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// if score fusion, run collect if rescorer is defined\n\t\tif rescorer != nil && requestHasKNN(req) {\n\t\t\tknnHits, err = i.runKnnCollector(ctx, req, indexReader, false)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\tif !skipSynonymCollector {\n\t\tif synMap, ok := i.m.(mapping.SynonymMapping); ok && synMap.SynonymCount() > 0 {\n\t\t\tif synReader, ok := indexReader.(index.ThesaurusReader); ok {\n\t\t\t\tfts, err = query.ExtractSynonyms(ctx, synMap, synReader, req.Query, fts)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// if score fusion, no faceting for knn hits is done\n\t// hence we can skip setting the knn hits in the collector\n\tif !contextScoreFusionKeyExists {\n\t\tsetKnnHitsInCollector(knnHits, coll)\n\t}\n\n\tif fts != nil {\n\t\tif is, ok := indexReader.(*scorch.IndexSnapshot); ok {\n\t\t\tis.UpdateSynonymSearchCount(1)\n\t\t}\n\t\tctx = context.WithValue(ctx, search.FieldTermSynonymMapKey, fts)\n\t}\n\n\t// set the bm25Stats (stats important for consistent scoring) in\n\t// the context object\n\tif bm25Stats != nil {\n\t\tctx = context.WithValue(ctx, search.BM25StatsKey, bm25Stats)\n\t}\n\n\tsearcher, err := req.Query.Searcher(ctx, indexReader, i.m, search.SearcherOptions{\n\t\tExplain:            req.Explain,\n\t\tIncludeTermVectors: req.IncludeLocations || req.Highlight != nil,\n\t\tScore:              req.Score,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif serr := searcher.Close(); err == nil && serr != nil {\n\t\t\terr = serr\n\t\t}\n\t}()\n\n\tif req.Facets != nil {\n\t\tfacetsBuilder := search.NewFacetsBuilder(indexReader)\n\t\tfor facetName, facetRequest := range req.Facets {\n\t\t\tif facetRequest.NumericRanges != nil {\n\t\t\t\t// build numeric range facet\n\t\t\t\tfacetBuilder := facet.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size)\n\t\t\t\tfor _, nr := range facetRequest.NumericRanges {\n\t\t\t\t\tfacetBuilder.AddRange(nr.Name, nr.Min, nr.Max)\n\t\t\t\t}\n\t\t\t\tfacetsBuilder.Add(facetName, facetBuilder)\n\t\t\t} else if facetRequest.DateTimeRanges != nil {\n\t\t\t\t// build date range facet\n\t\t\t\tfacetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size)\n\t\t\t\tfor _, dr := range facetRequest.DateTimeRanges {\n\t\t\t\t\tdateTimeParserName := defaultDateTimeParser\n\t\t\t\t\tif dr.DateTimeParser != \"\" {\n\t\t\t\t\t\tdateTimeParserName = dr.DateTimeParser\n\t\t\t\t\t}\n\t\t\t\t\tdateTimeParser := i.m.DateTimeParserNamed(dateTimeParserName)\n\t\t\t\t\tif dateTimeParser == nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"no date time parser named `%s` registered\", dateTimeParserName)\n\t\t\t\t\t}\n\t\t\t\t\tstart, end, err := dr.ParseDates(dateTimeParser)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"ParseDates err: %v, using date time parser named %s\", err, dateTimeParserName)\n\t\t\t\t\t}\n\t\t\t\t\tif start.IsZero() && end.IsZero() {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"date range query must specify either start, end or both for date range name '%s'\", dr.Name)\n\t\t\t\t\t}\n\t\t\t\t\tfacetBuilder.AddRange(dr.Name, start, end)\n\t\t\t\t}\n\t\t\t\tfacetsBuilder.Add(facetName, facetBuilder)\n\t\t\t} else {\n\t\t\t\t// build terms facet\n\t\t\t\tfacetBuilder := facet.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size)\n\n\t\t\t\t// Set prefix filter if provided\n\t\t\t\tif facetRequest.TermPrefix != \"\" {\n\t\t\t\t\tfacetBuilder.SetPrefixFilter(facetRequest.TermPrefix)\n\t\t\t\t}\n\n\t\t\t\t// Set regex filter if provided\n\t\t\t\tif facetRequest.TermPattern != \"\" {\n\t\t\t\t\t// Use cached compiled pattern if available, otherwise compile it now\n\t\t\t\t\tif facetRequest.compiledPattern != nil {\n\t\t\t\t\t\tfacetBuilder.SetRegexFilter(facetRequest.compiledPattern)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tregex, err := regexp.Compile(facetRequest.TermPattern)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn nil, fmt.Errorf(\"error compiling regex pattern for facet '%s': %v\", facetName, err)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tfacetBuilder.SetRegexFilter(regex)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tfacetsBuilder.Add(facetName, facetBuilder)\n\t\t\t}\n\t\t}\n\t\tcoll.SetFacetsBuilder(facetsBuilder)\n\t}\n\n\tmemNeeded := memNeededForSearch(req, searcher, coll)\n\tif cb := ctx.Value(SearchQueryStartCallbackKey); cb != nil {\n\t\tif cbF, ok := cb.(SearchQueryStartCallbackFn); ok {\n\t\t\terr = cbF(memNeeded)\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif cb := ctx.Value(SearchQueryEndCallbackKey); cb != nil {\n\t\tif cbF, ok := cb.(SearchQueryEndCallbackFn); ok {\n\t\t\tdefer func() {\n\t\t\t\t_ = cbF(memNeeded)\n\t\t\t}()\n\t\t}\n\t}\n\n\terr = coll.Collect(ctx, searcher, indexReader)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\thits := coll.Results()\n\n\tvar highlighter highlight.Highlighter\n\n\tif req.Highlight != nil {\n\t\t// get the right highlighter\n\t\thighlighter, err = Config.Cache.HighlighterNamed(Config.DefaultHighlighter)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif req.Highlight.Style != nil {\n\t\t\thighlighter, err = Config.Cache.HighlighterNamed(*req.Highlight.Style)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tif highlighter == nil {\n\t\t\treturn nil, fmt.Errorf(\"no highlighter named `%s` registered\", *req.Highlight.Style)\n\t\t}\n\t}\n\n\tvar storedFieldsCost uint64\n\tfor _, hit := range hits {\n\t\t// KNN documents will already have their Index value set as part of the knn collector output\n\t\t// so check if the index is empty and set it to the current index name\n\t\tif i.name != \"\" && hit.Index == \"\" {\n\t\t\thit.Index = i.name\n\t\t}\n\t\terr, storedFieldsBytes := LoadAndHighlightAllFields(hit, req, i.name, indexReader, highlighter)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tstoredFieldsCost += storedFieldsBytes\n\t}\n\n\ttotalSearchCost += storedFieldsCost\n\tsearch.RecordSearchCost(ctx, search.AddM, storedFieldsCost)\n\n\tif req.PreSearchData == nil {\n\t\t// increment the search count only if this is not a second-phase search\n\t\t// (e.g., for Hybrid Search), since the first-phase search already increments it\n\t\tatomic.AddUint64(&i.stats.searches, 1)\n\t}\n\t// increment the search time stat, as the first-phase search is part of\n\t// the overall operation; adding second-phase time later keeps it accurate\n\tsearchDuration := time.Since(searchStart)\n\tatomic.AddUint64(&i.stats.searchTime, uint64(searchDuration))\n\n\tif Config.SlowSearchLogThreshold > 0 &&\n\t\tsearchDuration > Config.SlowSearchLogThreshold {\n\t\tlogger.Printf(\"slow search took %s - %v\", searchDuration, req)\n\t}\n\n\tif reverseQueryExecution {\n\t\t// reverse the sort back to the original\n\t\treq.Sort.Reverse()\n\t\t// resort using the original order\n\t\tmhs := newSearchHitSorter(req.Sort, hits)\n\t\treq.SortFunc()(mhs)\n\t\t// reset request\n\t\treq.SearchBefore = req.SearchAfter\n\t\treq.SearchAfter = nil\n\t}\n\n\trv := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t},\n\t\tHits:     hits,\n\t\tTotal:    coll.Total(),\n\t\tMaxScore: coll.MaxScore(),\n\t\tTook:     searchDuration,\n\t\tFacets:   coll.FacetResults(),\n\t}\n\n\t// rescore if fusion flag is set\n\tif rescorer != nil {\n\t\trv.Hits, rv.Total, rv.MaxScore = rescorer.rescore(rv.Hits, knnHits)\n\t\trescorer.restoreSearchRequest()\n\t\trv.Hits = hitsInCurrentPage(req, rv.Hits)\n\t}\n\n\tif req.Explain {\n\t\trv.Request = req\n\t}\n\n\treturn rv, nil\n}\n\nfunc LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,\n\tindexName string, r index.IndexReader,\n\thighlighter highlight.Highlighter,\n) (error, uint64) {\n\tvar totalStoredFieldsBytes uint64\n\tif len(req.Fields) > 0 || highlighter != nil {\n\t\tdoc, err := r.Document(hit.ID)\n\t\tif err == nil && doc != nil {\n\t\t\tif len(req.Fields) > 0 && hit.Fields == nil {\n\t\t\t\ttotalStoredFieldsBytes = doc.StoredFieldsBytes()\n\t\t\t\tfieldsToLoad := deDuplicate(req.Fields)\n\t\t\t\tfor _, f := range fieldsToLoad {\n\t\t\t\t\tdoc.VisitFields(func(docF index.Field) {\n\t\t\t\t\t\tif f == \"*\" || docF.Name() == f {\n\t\t\t\t\t\t\tvar value interface{}\n\t\t\t\t\t\t\tswitch docF := docF.(type) {\n\t\t\t\t\t\t\tcase index.TextField:\n\t\t\t\t\t\t\t\tvalue = docF.Text()\n\t\t\t\t\t\t\tcase index.NumericField:\n\t\t\t\t\t\t\t\tnum, err := docF.Number()\n\t\t\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\t\t\tvalue = num\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcase index.DateTimeField:\n\t\t\t\t\t\t\t\tdatetime, layout, err := docF.DateTime()\n\t\t\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\t\t\tif layout == \"\" {\n\t\t\t\t\t\t\t\t\t\t// missing layout means we fallback to\n\t\t\t\t\t\t\t\t\t\t// the default layout which is RFC3339\n\t\t\t\t\t\t\t\t\t\tvalue = datetime.Format(time.RFC3339)\n\t\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\t\t// the layout here can now either be representative\n\t\t\t\t\t\t\t\t\t\t// of an actual datetime layout or a timestamp\n\t\t\t\t\t\t\t\t\t\tswitch layout {\n\t\t\t\t\t\t\t\t\t\tcase seconds.Name:\n\t\t\t\t\t\t\t\t\t\t\tvalue = strconv.FormatInt(datetime.Unix(), 10)\n\t\t\t\t\t\t\t\t\t\tcase milliseconds.Name:\n\t\t\t\t\t\t\t\t\t\t\tvalue = strconv.FormatInt(datetime.UnixMilli(), 10)\n\t\t\t\t\t\t\t\t\t\tcase microseconds.Name:\n\t\t\t\t\t\t\t\t\t\t\tvalue = strconv.FormatInt(datetime.UnixMicro(), 10)\n\t\t\t\t\t\t\t\t\t\tcase nanoseconds.Name:\n\t\t\t\t\t\t\t\t\t\t\tvalue = strconv.FormatInt(datetime.UnixNano(), 10)\n\t\t\t\t\t\t\t\t\t\tdefault:\n\t\t\t\t\t\t\t\t\t\t\t// the layout for formatting the date to a string\n\t\t\t\t\t\t\t\t\t\t\t// is provided by a datetime parser which is not\n\t\t\t\t\t\t\t\t\t\t\t// handling the timestamp case, hence the layout\n\t\t\t\t\t\t\t\t\t\t\t// can be directly used to format the date\n\t\t\t\t\t\t\t\t\t\t\tvalue = datetime.Format(layout)\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcase index.BooleanField:\n\t\t\t\t\t\t\t\tboolean, err := docF.Boolean()\n\t\t\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\t\t\tvalue = boolean\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcase index.GeoPointField:\n\t\t\t\t\t\t\t\tlon, err := docF.Lon()\n\t\t\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\t\t\tlat, err := docF.Lat()\n\t\t\t\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\t\t\t\tvalue = []float64{lon, lat}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcase index.GeoShapeField:\n\t\t\t\t\t\t\t\tv, err := docF.GeoShape()\n\t\t\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\t\t\tvalue = v\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcase index.IPField:\n\t\t\t\t\t\t\t\tip, err := docF.IP()\n\t\t\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\t\t\tvalue = ip.String()\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif value != nil {\n\t\t\t\t\t\t\t\thit.AddFieldValue(docF.Name(), value)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t\tif highlighter != nil {\n\t\t\t\thighlightFields := req.Highlight.Fields\n\t\t\t\tif highlightFields == nil {\n\t\t\t\t\t// add all fields with matches\n\t\t\t\t\thighlightFields = make([]string, 0, len(hit.Locations))\n\t\t\t\t\tfor k := range hit.Locations {\n\t\t\t\t\t\thighlightFields = append(highlightFields, k)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfor _, hf := range highlightFields {\n\t\t\t\t\thighlighter.BestFragmentsInField(hit, doc, hf, 1)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if doc == nil {\n\t\t\t// unexpected case, a doc ID that was found as a search hit\n\t\t\t// was unable to be found during document lookup\n\t\t\treturn ErrorIndexReadInconsistency, 0\n\t\t}\n\t}\n\n\treturn nil, totalStoredFieldsBytes\n}\n\nconst NestedDocumentKey = \"_$nested\"\n\n// LoadAndHighlightAllFields loads stored fields + highlights for root and its descendants.\n// All descendant documents are collected into a _$nested array in the root DocumentMatch.\nfunc LoadAndHighlightAllFields(\n\troot *search.DocumentMatch,\n\treq *SearchRequest,\n\tindexName string,\n\tr index.IndexReader,\n\thighlighter highlight.Highlighter,\n) (error, uint64) {\n\tvar totalStoredFieldsBytes uint64\n\t// load root fields/highlights\n\terr, bytes := LoadAndHighlightFields(root, req, indexName, r, highlighter)\n\ttotalStoredFieldsBytes += bytes\n\tif err != nil {\n\t\treturn err, totalStoredFieldsBytes\n\t}\n\t// collect all descendant documents\n\tnestedDocs := make([]*search.NestedDocumentMatch, 0, len(root.Descendants))\n\t// create a dummy desc DocumentMatch to reuse LoadAndHighlightFields\n\tdesc := &search.DocumentMatch{}\n\tfor _, descID := range root.Descendants {\n\t\textID, err := r.ExternalID(descID)\n\t\tif err != nil {\n\t\t\treturn err, totalStoredFieldsBytes\n\t\t}\n\t\t// reset desc for reuse\n\t\tdesc.ID = extID\n\t\tdesc.IndexInternalID = descID\n\t\tdesc.Locations = root.Locations\n\t\terr, bytes := LoadAndHighlightFields(desc, req, indexName, r, highlighter)\n\t\ttotalStoredFieldsBytes += bytes\n\t\tif err != nil {\n\t\t\treturn err, totalStoredFieldsBytes\n\t\t}\n\t\t// copy fields to nested doc and append\n\t\tif len(desc.Fields) != 0 || len(desc.Fragments) != 0 {\n\t\t\tnestedDocs = append(nestedDocs, search.NewNestedDocumentMatch(desc.Fields, desc.Fragments))\n\t\t}\n\t\tdesc.Fields = nil\n\t\tdesc.Fragments = nil\n\t}\n\t// add nested documents to root under _$nested key\n\tif len(nestedDocs) > 0 {\n\t\troot.AddFieldValue(NestedDocumentKey, nestedDocs)\n\t}\n\treturn nil, totalStoredFieldsBytes\n}\n\n// Fields returns the name of all the fields this\n// Index has operated on.\nfunc (i *indexImpl) Fields() (fields []string, err error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\tindexReader, err := i.i.Reader()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := indexReader.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tfields, err = indexReader.Fields()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn fields, nil\n}\n\nfunc (i *indexImpl) FieldDict(field string) (index.FieldDict, error) {\n\ti.mutex.RLock()\n\n\tif !i.open {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\tindexReader, err := i.i.Reader()\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\tfieldDict, err := indexReader.FieldDict(field)\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\treturn &indexImplFieldDict{\n\t\tindex:       i,\n\t\tindexReader: indexReader,\n\t\tfieldDict:   fieldDict,\n\t}, nil\n}\n\nfunc (i *indexImpl) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {\n\ti.mutex.RLock()\n\n\tif !i.open {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\tindexReader, err := i.i.Reader()\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\tfieldDict, err := indexReader.FieldDictRange(field, startTerm, endTerm)\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\treturn &indexImplFieldDict{\n\t\tindex:       i,\n\t\tindexReader: indexReader,\n\t\tfieldDict:   fieldDict,\n\t}, nil\n}\n\nfunc (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {\n\ti.mutex.RLock()\n\n\tif !i.open {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\tindexReader, err := i.i.Reader()\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\tfieldDict, err := indexReader.FieldDictPrefix(field, termPrefix)\n\tif err != nil {\n\t\ti.mutex.RUnlock()\n\t\treturn nil, err\n\t}\n\n\treturn &indexImplFieldDict{\n\t\tindex:       i,\n\t\tindexReader: indexReader,\n\t\tfieldDict:   fieldDict,\n\t}, nil\n}\n\nfunc (i *indexImpl) Close() error {\n\ti.mutex.Lock()\n\tdefer i.mutex.Unlock()\n\n\tindexStats.UnRegister(i)\n\n\ti.open = false\n\treturn i.i.Close()\n}\n\nfunc (i *indexImpl) Stats() *IndexStat {\n\treturn i.stats\n}\n\nfunc (i *indexImpl) StatsMap() map[string]interface{} {\n\treturn i.stats.statsMap()\n}\n\nfunc (i *indexImpl) GetInternal(key []byte) (val []byte, err error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\treader, err := i.i.Reader()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := reader.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tval, err = reader.GetInternal(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn val, nil\n}\n\nfunc (i *indexImpl) SetInternal(key, val []byte) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\treturn i.i.SetInternal(key, val)\n}\n\nfunc (i *indexImpl) DeleteInternal(key []byte) error {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\treturn i.i.DeleteInternal(key)\n}\n\n// NewBatch creates a new empty batch.\nfunc (i *indexImpl) NewBatch() *Batch {\n\treturn &Batch{\n\t\tindex:    i,\n\t\tinternal: index.NewBatch(),\n\t}\n}\n\nfunc (i *indexImpl) Name() string {\n\treturn i.name\n}\n\nfunc (i *indexImpl) SetName(name string) {\n\tindexStats.UnRegister(i)\n\ti.name = name\n\tindexStats.Register(i)\n}\n\ntype indexImplFieldDict struct {\n\tindex       *indexImpl\n\tindexReader index.IndexReader\n\tfieldDict   index.FieldDict\n}\n\nfunc (f *indexImplFieldDict) BytesRead() uint64 {\n\treturn f.fieldDict.BytesRead()\n}\n\nfunc (f *indexImplFieldDict) Next() (*index.DictEntry, error) {\n\treturn f.fieldDict.Next()\n}\n\nfunc (f *indexImplFieldDict) Close() error {\n\tdefer f.index.mutex.RUnlock()\n\terr := f.fieldDict.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn f.indexReader.Close()\n}\n\nfunc (f *indexImplFieldDict) Cardinality() int {\n\treturn f.fieldDict.Cardinality()\n}\n\n// helper function to remove duplicate entries from slice of strings\nfunc deDuplicate(fields []string) []string {\n\tif len(fields) == 0 {\n\t\treturn fields\n\t}\n\tentries := make(map[string]struct{})\n\tret := []string{}\n\tfor _, entry := range fields {\n\t\tif _, exists := entries[entry]; !exists {\n\t\t\tentries[entry] = struct{}{}\n\t\t\tret = append(ret, entry)\n\t\t}\n\t}\n\treturn ret\n}\n\ntype searchHitSorter struct {\n\thits          search.DocumentMatchCollection\n\tsort          search.SortOrder\n\tcachedScoring []bool\n\tcachedDesc    []bool\n}\n\nfunc newSearchHitSorter(sort search.SortOrder, hits search.DocumentMatchCollection) *searchHitSorter {\n\treturn &searchHitSorter{\n\t\tsort:          sort,\n\t\thits:          hits,\n\t\tcachedScoring: sort.CacheIsScore(),\n\t\tcachedDesc:    sort.CacheDescending(),\n\t}\n}\n\nfunc (m *searchHitSorter) Len() int      { return len(m.hits) }\nfunc (m *searchHitSorter) Swap(i, j int) { m.hits[i], m.hits[j] = m.hits[j], m.hits[i] }\nfunc (m *searchHitSorter) Less(i, j int) bool {\n\tc := m.sort.Compare(m.cachedScoring, m.cachedDesc, m.hits[i], m.hits[j])\n\treturn c < 0\n}\n\nfunc (i *indexImpl) CopyTo(d index.Directory) (err error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn ErrorIndexClosed\n\t}\n\n\tcopyIndex, ok := i.i.(index.CopyIndex)\n\tif !ok {\n\t\treturn fmt.Errorf(\"index implementation does not support copy reader\")\n\t}\n\n\tcopyReader := copyIndex.CopyReader()\n\tif copyReader == nil {\n\t\treturn fmt.Errorf(\"index's copyReader is nil\")\n\t}\n\n\tdefer func() {\n\t\tif cerr := copyReader.CloseCopyReader(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\terr = copyReader.CopyTo(d)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error copying index metadata: %v\", err)\n\t}\n\n\t// copy the metadata\n\treturn i.meta.CopyTo(d)\n}\n\nfunc (f FileSystemDirectory) GetWriter(filePath string) (io.WriteCloser,\n\terror,\n) {\n\tdir, file := filepath.Split(filePath)\n\tif dir != \"\" {\n\t\terr := os.MkdirAll(filepath.Join(string(f), dir), os.ModePerm)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn os.OpenFile(filepath.Join(string(f), dir, file),\n\t\tos.O_RDWR|os.O_CREATE, 0o600)\n}\n\nfunc (i *indexImpl) FireIndexEvent() {\n\t// get the internal index implementation\n\tinternalIndex, err := i.Advanced()\n\tif err != nil {\n\t\treturn\n\t}\n\t// check if the internal index implementation supports events\n\tif internalEventIndex, ok := internalIndex.(index.EventIndex); ok {\n\t\t// fire the Index() event\n\t\tinternalEventIndex.FireIndexEvent()\n\t}\n}\n\n// -----------------------------------------------------------------------------\n\nfunc (i *indexImpl) TermFrequencies(field string, limit int, descending bool) (\n\t[]index.TermFreq, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\treader, err := i.i.Reader()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := reader.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tinsightsReader, ok := reader.(index.IndexInsightsReader)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"index reader does not support TermFrequencies\")\n\t}\n\n\treturn insightsReader.TermFrequencies(field, limit, descending)\n}\n\nfunc (i *indexImpl) CentroidCardinalities(field string, limit int, descending bool) (\n\t[]index.CentroidCardinality, error) {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\n\tif !i.open {\n\t\treturn nil, ErrorIndexClosed\n\t}\n\n\treader, err := i.i.Reader()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := reader.Close(); err == nil && cerr != nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tinsightsReader, ok := reader.(index.IndexInsightsReader)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"index reader does not support CentroidCardinalities\")\n\t}\n\n\tcentroidCardinalities, err := insightsReader.CentroidCardinalities(field, limit, descending)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor j := 0; j < len(centroidCardinalities); j++ {\n\t\tcentroidCardinalities[j].Index = i.name\n\t}\n\n\treturn centroidCardinalities, nil\n}\n\nfunc (i *indexImpl) buildTopNCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader) (*collector.TopNCollector, error) {\n\tnewCollector := func() *collector.TopNCollector {\n\t\tif req.SearchAfter != nil {\n\t\t\treturn collector.NewTopNCollectorAfter(req.Size, req.Sort, req.SearchAfter)\n\t\t}\n\t\treturn collector.NewTopNCollector(req.Size, req.From, req.Sort)\n\t}\n\n\tnewNestedCollector := func(nr index.NestedReader) *collector.TopNCollector {\n\t\tif req.SearchAfter != nil {\n\t\t\treturn collector.NewNestedTopNCollectorAfter(req.Size, req.Sort, req.SearchAfter, nr)\n\t\t}\n\t\treturn collector.NewNestedTopNCollector(req.Size, req.From, req.Sort, nr)\n\t}\n\n\t// check if we are in nested mode\n\tif nestedMode, ok := ctx.Value(search.NestedSearchKey).(bool); ok && nestedMode {\n\t\t// get the nested reader from the index reader\n\t\tif nr, ok := reader.(index.NestedReader); ok {\n\t\t\t// check if the mapping has any nested fields that intersect\n\t\t\tif nm, ok := i.m.(mapping.NestedMapping); ok {\n\t\t\t\tvar fs search.FieldSet\n\t\t\t\tvar err error\n\t\t\t\tfs, err = query.ExtractFields(req.Query, i.m, fs)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tif fs.HasID() || nm.IntersectsPrefix(fs) {\n\t\t\t\t\treturn newNestedCollector(nr), nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn newCollector(), nil\n}\n"
  },
  {
    "path": "index_meta.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nconst metaFilename = \"index_meta.json\"\n\ntype indexMeta struct {\n\tStorage   string                 `json:\"storage\"`\n\tIndexType string                 `json:\"index_type\"`\n\tConfig    map[string]interface{} `json:\"config,omitempty\"`\n}\n\nfunc newIndexMeta(indexType string, storage string, config map[string]interface{}) *indexMeta {\n\treturn &indexMeta{\n\t\tIndexType: indexType,\n\t\tStorage:   storage,\n\t\tConfig:    config,\n\t}\n}\n\nfunc openIndexMeta(path string) (*indexMeta, error) {\n\tif _, err := os.Stat(path); os.IsNotExist(err) {\n\t\treturn nil, ErrorIndexPathDoesNotExist\n\t}\n\tindexMetaPath := indexMetaPath(path)\n\tmetaBytes, err := os.ReadFile(indexMetaPath)\n\tif err != nil {\n\t\treturn nil, ErrorIndexMetaMissing\n\t}\n\tvar im indexMeta\n\terr = util.UnmarshalJSON(metaBytes, &im)\n\tif err != nil {\n\t\treturn nil, ErrorIndexMetaCorrupt\n\t}\n\tif im.IndexType == \"\" {\n\t\tim.IndexType = upsidedown.Name\n\t}\n\treturn &im, nil\n}\n\nfunc (i *indexMeta) Save(path string) (err error) {\n\tindexMetaPath := indexMetaPath(path)\n\t// ensure any necessary parent directories exist\n\terr = os.MkdirAll(path, 0700)\n\tif err != nil {\n\t\tif os.IsExist(err) {\n\t\t\treturn ErrorIndexPathExists\n\t\t}\n\t\treturn err\n\t}\n\tmetaBytes, err := util.MarshalJSON(i)\n\tif err != nil {\n\t\treturn err\n\t}\n\tindexMetaFile, err := os.OpenFile(indexMetaPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)\n\tif err != nil {\n\t\tif os.IsExist(err) {\n\t\t\treturn ErrorIndexPathExists\n\t\t}\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tif ierr := indexMetaFile.Close(); err == nil && ierr != nil {\n\t\t\terr = ierr\n\t\t}\n\t}()\n\t_, err = indexMetaFile.Write(metaBytes)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (i *indexMeta) CopyTo(d index.Directory) (err error) {\n\tmetaBytes, err := util.MarshalJSON(i)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tw, err := d.GetWriter(metaFilename)\n\tif w == nil || err != nil {\n\t\treturn fmt.Errorf(\"invalid writer for file: %s, err: %v\",\n\t\t\tmetaFilename, err)\n\t}\n\tdefer w.Close()\n\n\t_, err = w.Write(metaBytes)\n\treturn err\n}\n\nfunc indexMetaPath(path string) string {\n\treturn filepath.Join(path, metaFilename)\n}\n"
  },
  {
    "path": "index_meta_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestIndexMeta(t *testing.T) {\n\tvar testIndexPath = \"doesnotexit.bleve\"\n\tdefer func() {\n\t\terr := os.RemoveAll(testIndexPath)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// open non-existent meta should give an error\n\t_, err := openIndexMeta(testIndexPath)\n\tif err == nil {\n\t\tt.Errorf(\"expected error, got nil\")\n\t}\n\n\t// create meta\n\tim := &indexMeta{Storage: \"boltdb\"}\n\terr = im.Save(testIndexPath)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tim = nil\n\n\t// open a meta that exists\n\tim, err = openIndexMeta(testIndexPath)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif im.Storage != \"boltdb\" {\n\t\tt.Errorf(\"expected storage 'boltdb', got '%s'\", im.Storage)\n\t}\n\n\t// save a meta that already exists\n\terr = im.Save(testIndexPath)\n\tif err == nil {\n\t\tt.Errorf(\"expected error, got nil\")\n\t}\n}\n"
  },
  {
    "path": "index_stats.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"encoding/json\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\ntype IndexStat struct {\n\tsearches   uint64\n\tsearchTime uint64\n\ti          *indexImpl\n}\n\nfunc (is *IndexStat) statsMap() map[string]interface{} {\n\tm := map[string]interface{}{}\n\tm[\"index\"] = is.i.i.StatsMap()\n\tm[\"searches\"] = atomic.LoadUint64(&is.searches)\n\tm[\"search_time\"] = atomic.LoadUint64(&is.searchTime)\n\treturn m\n}\n\nfunc (is *IndexStat) MarshalJSON() ([]byte, error) {\n\tm := is.statsMap()\n\treturn json.Marshal(m)\n}\n\ntype IndexStats struct {\n\tindexes map[string]*IndexStat\n\tmutex   sync.RWMutex\n}\n\nfunc NewIndexStats() *IndexStats {\n\treturn &IndexStats{\n\t\tindexes: make(map[string]*IndexStat),\n\t}\n}\n\nfunc (i *IndexStats) Register(index Index) {\n\ti.mutex.Lock()\n\tdefer i.mutex.Unlock()\n\ti.indexes[index.Name()] = index.Stats()\n}\n\nfunc (i *IndexStats) UnRegister(index Index) {\n\ti.mutex.Lock()\n\tdefer i.mutex.Unlock()\n\tdelete(i.indexes, index.Name())\n}\n\nfunc (i *IndexStats) String() string {\n\ti.mutex.RLock()\n\tdefer i.mutex.RUnlock()\n\tbytes, err := json.Marshal(i.indexes)\n\tif err != nil {\n\t\treturn \"error marshaling stats\"\n\t}\n\treturn string(bytes)\n}\n\nvar indexStats *IndexStats\n"
  },
  {
    "path": "index_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/null\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n)\n\ntype Fatalfable interface {\n\tFatalf(format string, args ...interface{})\n}\n\nfunc createTmpIndexPath(f Fatalfable) string {\n\ttmpIndexPath, err := os.MkdirTemp(\"\", \"bleve-testidx\")\n\tif err != nil {\n\t\tf.Fatalf(\"error creating temp dir: %v\", err)\n\t}\n\treturn tmpIndexPath\n}\n\nfunc cleanupTmpIndexPath(f Fatalfable, path string) {\n\terr := os.RemoveAll(path)\n\tif err != nil {\n\t\tf.Fatalf(\"error removing temp dir: %v\", err)\n\t}\n}\n\nfunc TestCrud(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoca := map[string]interface{}{\n\t\t\"name\": \"marty\",\n\t\t\"desc\": \"gophercon india\",\n\t}\n\terr = idx.Index(\"a\", doca)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocy := map[string]interface{}{\n\t\t\"name\": \"jasper\",\n\t\t\"desc\": \"clojure\",\n\t}\n\terr = idx.Index(\"y\", docy)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\terr = idx.Delete(\"y\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocx := map[string]interface{}{\n\t\t\"name\": \"rose\",\n\t\t\"desc\": \"googler\",\n\t}\n\terr = idx.Index(\"x\", docx)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\terr = idx.SetInternal([]byte(\"status\"), []byte(\"pending\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocb := map[string]interface{}{\n\t\t\"name\": \"steve\",\n\t\t\"desc\": \"cbft master\",\n\t}\n\tbatch := idx.NewBatch()\n\terr = batch.Index(\"b\", docb)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tbatch.Delete(\"x\")\n\tbatch.SetInternal([]byte(\"batchi\"), []byte(\"batchv\"))\n\tbatch.DeleteInternal([]byte(\"status\"))\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tval, err := idx.GetInternal([]byte(\"batchi\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif string(val) != \"batchv\" {\n\t\tt.Errorf(\"expected 'batchv', got '%s'\", val)\n\t}\n\tval, err = idx.GetInternal([]byte(\"status\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got '%s'\", val)\n\t}\n\n\terr = idx.SetInternal([]byte(\"seqno\"), []byte(\"7\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\terr = idx.SetInternal([]byte(\"status\"), []byte(\"ready\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\terr = idx.DeleteInternal([]byte(\"status\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tval, err = idx.GetInternal([]byte(\"status\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got '%s'\", val)\n\t}\n\n\tval, err = idx.GetInternal([]byte(\"seqno\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif string(val) != \"7\" {\n\t\tt.Errorf(\"expected '7', got '%s'\", val)\n\t}\n\n\t// close the index, open it again, and try some more things\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err = Open(tmpIndexPath)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tcount, err := idx.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif count != 2 {\n\t\tt.Errorf(\"expected doc count 2, got %d\", count)\n\t}\n\n\tdoc, err := idx.Document(\"a\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfoundNameField := false\n\tdoc.VisitFields(func(field index.Field) {\n\t\tif field.Name() == \"name\" && string(field.Value()) == \"marty\" {\n\t\t\tfoundNameField = true\n\t\t}\n\t})\n\tif !foundNameField {\n\t\tt.Errorf(\"expected to find field named 'name' with value 'marty'\")\n\t}\n\n\tfields, err := idx.Fields()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texpectedFields := map[string]bool{\n\t\t\"_all\": false,\n\t\t\"name\": false,\n\t\t\"desc\": false,\n\t}\n\tif len(fields) < len(expectedFields) {\n\t\tt.Fatalf(\"expected %d fields got %d\", len(expectedFields), len(fields))\n\t}\n\tfor _, f := range fields {\n\t\texpectedFields[f] = true\n\t}\n\tfor ef, efp := range expectedFields {\n\t\tif !efp {\n\t\t\tt.Errorf(\"field %s is missing\", ef)\n\t\t}\n\t}\n}\n\nfunc approxSame(actual, expected uint64) bool {\n\tmodulus := func(a, b uint64) uint64 {\n\t\tif a > b {\n\t\t\treturn a - b\n\t\t}\n\t\treturn b - a\n\t}\n\n\treturn float64(modulus(actual, expected))/float64(expected) < float64(0.30)\n}\n\nfunc checkStatsOnIndexedBatch(indexPath string, indexMapping mapping.IndexMapping,\n\texpectedVal uint64,\n) error {\n\tvar wg sync.WaitGroup\n\tvar statValError error\n\n\tidx, err := NewUsing(indexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbatch, err := getBatchFromData(idx, \"sample-data.json\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to form a batch %v\\n\", err)\n\t}\n\twg.Add(1)\n\tbatch.SetPersistedCallback(func(e error) {\n\t\tdefer wg.Done()\n\t\tstats, _ := idx.StatsMap()[\"index\"].(map[string]interface{})\n\t\tbytesWritten, _ := stats[\"num_bytes_written_at_index_time\"].(uint64)\n\t\tif !approxSame(bytesWritten, expectedVal) {\n\t\t\tstatValError = fmt.Errorf(\"expected bytes written is %d, got %v\", expectedVal,\n\t\t\t\tbytesWritten)\n\t\t}\n\t})\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to index batch %v\\n\", err)\n\t}\n\twg.Wait()\n\tidx.Close()\n\n\treturn statValError\n}\n\nfunc TestBytesWritten(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tdocumentMapping := NewDocumentMapping()\n\tindexMapping.AddDocumentMapping(\"hotel\", documentMapping)\n\n\tindexMapping.DocValuesDynamic = false\n\tindexMapping.StoreDynamic = false\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Index = true\n\tcontentFieldMapping.Store = false\n\tcontentFieldMapping.IncludeInAll = false\n\tcontentFieldMapping.IncludeTermVectors = false\n\tcontentFieldMapping.DocValues = false\n\n\treviewsMapping := NewDocumentMapping()\n\treviewsMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tdocumentMapping.AddSubDocumentMapping(\"reviews\", reviewsMapping)\n\n\ttypeFieldMapping := NewTextFieldMapping()\n\ttypeFieldMapping.Store = false\n\ttypeFieldMapping.IncludeInAll = false\n\ttypeFieldMapping.IncludeTermVectors = false\n\ttypeFieldMapping.DocValues = false\n\tdocumentMapping.AddFieldMappingsAt(\"type\", typeFieldMapping)\n\n\terr = checkStatsOnIndexedBatch(tmpIndexPath, indexMapping, 57273)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcleanupTmpIndexPath(t, tmpIndexPath)\n\n\tcontentFieldMapping.Store = true\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\n\terr := checkStatsOnIndexedBatch(tmpIndexPath1, indexMapping, 76069)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcleanupTmpIndexPath(t, tmpIndexPath1)\n\n\tcontentFieldMapping.Store = false\n\tcontentFieldMapping.IncludeInAll = true\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\n\terr = checkStatsOnIndexedBatch(tmpIndexPath2, indexMapping, 68875)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcleanupTmpIndexPath(t, tmpIndexPath2)\n\n\tcontentFieldMapping.IncludeInAll = false\n\tcontentFieldMapping.IncludeTermVectors = true\n\ttmpIndexPath3 := createTmpIndexPath(t)\n\n\terr = checkStatsOnIndexedBatch(tmpIndexPath3, indexMapping, 78985)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcleanupTmpIndexPath(t, tmpIndexPath3)\n\n\tcontentFieldMapping.IncludeTermVectors = false\n\tcontentFieldMapping.DocValues = true\n\ttmpIndexPath4 := createTmpIndexPath(t)\n\n\terr = checkStatsOnIndexedBatch(tmpIndexPath4, indexMapping, 64228)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcleanupTmpIndexPath(t, tmpIndexPath4)\n}\n\nfunc createIndexMappingOnSampleData() *mapping.IndexMappingImpl {\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tindexMapping.ScoringModel = index.DefaultScoringModel\n\tdocumentMapping := NewDocumentMapping()\n\tindexMapping.AddDocumentMapping(\"hotel\", documentMapping)\n\tindexMapping.StoreDynamic = false\n\tindexMapping.DocValuesDynamic = false\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Store = false\n\n\treviewsMapping := NewDocumentMapping()\n\treviewsMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tdocumentMapping.AddSubDocumentMapping(\"reviews\", reviewsMapping)\n\n\ttypeFieldMapping := NewTextFieldMapping()\n\ttypeFieldMapping.Store = false\n\tdocumentMapping.AddFieldMappingsAt(\"type\", typeFieldMapping)\n\n\treturn indexMapping\n}\n\nfunc TestBM25TFIDFScoring(t *testing.T) {\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath1)\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath2)\n\n\tindexMapping := createIndexMappingOnSampleData()\n\tindexMapping.ScoringModel = index.BM25Scoring\n\tindexBM25, err := NewUsing(tmpIndexPath1, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexMapping1 := createIndexMappingOnSampleData()\n\tindexTFIDF, err := NewUsing(tmpIndexPath2, indexMapping1, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := indexBM25.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\terr = indexTFIDF.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch, err := getBatchFromData(indexBM25, \"sample-data.json\")\n\tif err != nil {\n\t\tt.Fatalf(\"failed to form a batch\")\n\t}\n\terr = indexBM25.Batch(batch)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to index batch %v\\n\", err)\n\t}\n\tquery := NewMatchQuery(\"Hotel\")\n\tquery.FieldVal = \"name\"\n\tsearchRequest := NewSearchRequestOptions(query, int(10), 0, true)\n\n\tresBM25, err := indexBM25.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tbatch, err = getBatchFromData(indexTFIDF, \"sample-data.json\")\n\tif err != nil {\n\t\tt.Fatalf(\"failed to form a batch\")\n\t}\n\terr = indexTFIDF.Batch(batch)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to index batch %v\\n\", err)\n\t}\n\n\tresTFIDF, err := indexTFIDF.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tfor i, hit := range resTFIDF.Hits {\n\t\tif hit.Score < resBM25.Hits[i].Score {\n\t\t\tt.Fatalf(\"expected the score to be higher for BM25, got %v and %v\",\n\t\t\t\tresBM25.Hits[i].Score, hit.Score)\n\t\t}\n\t}\n}\n\nfunc TestBM25GlobalScoring(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindexMapping := createIndexMappingOnSampleData()\n\tindexMapping.ScoringModel = index.BM25Scoring\n\tidxSinglePartition, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := idxSinglePartition.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch, err := getBatchFromData(idxSinglePartition, \"sample-data.json\")\n\tif err != nil {\n\t\tt.Fatalf(\"failed to form a batch\")\n\t}\n\terr = idxSinglePartition.Batch(batch)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to index batch %v\\n\", err)\n\t}\n\tquery := NewMatchQuery(\"Hotel\")\n\tquery.FieldVal = \"name\"\n\tsearchRequest := NewSearchRequestOptions(query, int(10), 0, true)\n\n\tres, err := idxSinglePartition.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tsinglePartHits := res.Hits\n\n\tdataset, _ := readDataFromFile(\"sample-data.json\")\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath1)\n\n\tidxPart1, err := NewUsing(tmpIndexPath1, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := idxPart1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch1 := idxPart1.NewBatch()\n\tfor _, doc := range dataset[:len(dataset)/2] {\n\t\terr = batch1.Index(fmt.Sprintf(\"%d\", doc[\"id\"]), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idxPart1.Batch(batch1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath2)\n\n\tidxPart2, err := NewUsing(tmpIndexPath2, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := idxPart2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch2 := idxPart2.NewBatch()\n\tfor _, doc := range dataset[len(dataset)/2:] {\n\t\terr = batch2.Index(fmt.Sprintf(\"%d\", doc[\"id\"]), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idxPart2.Batch(batch2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultiPartIndex := NewIndexAlias(idxPart1, idxPart2)\n\terr = multiPartIndex.SetIndexMapping(indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tctx := context.Background()\n\t// this key is set to ensure that we have a consistent scoring at the index alias\n\t// level (it forces a pre search phase which can have a small overhead)\n\tctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring)\n\n\tres, err = multiPartIndex.SearchInContext(ctx, searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tfor i, hit := range res.Hits {\n\t\tif hit.Score != singlePartHits[i].Score {\n\t\t\tt.Fatalf(\"expected the scores to be the same, got %v and %v\",\n\t\t\t\thit.Score, singlePartHits[i].Score)\n\t\t}\n\t}\n}\n\nfunc TestBytesRead(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tdocumentMapping := NewDocumentMapping()\n\tindexMapping.AddDocumentMapping(\"hotel\", documentMapping)\n\tindexMapping.StoreDynamic = false\n\tindexMapping.DocValuesDynamic = false\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Store = false\n\n\treviewsMapping := NewDocumentMapping()\n\treviewsMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tdocumentMapping.AddSubDocumentMapping(\"reviews\", reviewsMapping)\n\n\ttypeFieldMapping := NewTextFieldMapping()\n\ttypeFieldMapping.Store = false\n\tdocumentMapping.AddFieldMappingsAt(\"type\", typeFieldMapping)\n\n\tidx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch, err := getBatchFromData(idx, \"sample-data.json\")\n\tif err != nil {\n\t\tt.Fatalf(\"failed to form a batch\")\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to index batch %v\\n\", err)\n\t}\n\tquery := NewQueryStringQuery(\"united\")\n\tsearchRequest := NewSearchRequestOptions(query, int(10), 0, true)\n\n\tres, err := idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstats, _ := idx.StatsMap()[\"index\"].(map[string]interface{})\n\tprevBytesRead, _ := stats[\"num_bytes_read_at_query_time\"].(uint64)\n\n\texpectedBytesRead := uint64(21164)\n\tif supportForVectorSearch {\n\t\texpectedBytesRead = 21574\n\t}\n\n\tif prevBytesRead != expectedBytesRead && res.Cost == prevBytesRead {\n\t\tt.Fatalf(\"expected bytes read for query string %v, got %v\",\n\t\t\texpectedBytesRead, prevBytesRead)\n\t}\n\n\t// subsequent queries on the same field results in lesser amount\n\t// of bytes read because the segment static and dictionary is reused and not\n\t// loaded from mmap'd filed\n\tres, err = idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstats, _ = idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ := stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif bytesRead-prevBytesRead != 66 && res.Cost == bytesRead-prevBytesRead {\n\t\tt.Fatalf(\"expected bytes read for query string 66, got %v\",\n\t\t\tbytesRead-prevBytesRead)\n\t}\n\tprevBytesRead = bytesRead\n\n\tfuzz := NewFuzzyQuery(\"hotel\")\n\tfuzz.FieldVal = \"reviews.content\"\n\tfuzz.Fuzziness = 2\n\tsearchRequest = NewSearchRequest(fuzz)\n\tres, err = idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstats, _ = idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif bytesRead-prevBytesRead != 8468 && res.Cost == bytesRead-prevBytesRead {\n\t\tt.Fatalf(\"expected bytes read for fuzzy query is 8468, got %v\",\n\t\t\tbytesRead-prevBytesRead)\n\t}\n\tprevBytesRead = bytesRead\n\n\ttypeFacet := NewFacetRequest(\"type\", 2)\n\tquery = NewQueryStringQuery(\"united\")\n\tsearchRequest = NewSearchRequestOptions(query, int(0), 0, true)\n\tsearchRequest.AddFacet(\"types\", typeFacet)\n\tres, err = idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstats, _ = idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif !approxSame(bytesRead-prevBytesRead, 196) && res.Cost == bytesRead-prevBytesRead {\n\t\tt.Fatalf(\"expected bytes read for faceted query is around 196, got %v\",\n\t\t\tbytesRead-prevBytesRead)\n\t}\n\tprevBytesRead = bytesRead\n\n\tmin := float64(8660)\n\tmax := float64(8665)\n\tnumRangeQuery := NewNumericRangeQuery(&min, &max)\n\tnumRangeQuery.FieldVal = \"id\"\n\tsearchRequest = NewSearchRequestOptions(numRangeQuery, int(10), 0, true)\n\tres, err = idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstats, _ = idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif bytesRead-prevBytesRead != 924 && res.Cost == bytesRead-prevBytesRead {\n\t\tt.Fatalf(\"expected bytes read for numeric range query is 924, got %v\",\n\t\t\tbytesRead-prevBytesRead)\n\t}\n\tprevBytesRead = bytesRead\n\n\tsearchRequest = NewSearchRequestOptions(query, int(10), 0, true)\n\tsearchRequest.Highlight = &HighlightRequest{}\n\tres, err = idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstats, _ = idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif bytesRead-prevBytesRead != 105 && res.Cost == bytesRead-prevBytesRead {\n\t\tt.Fatalf(\"expected bytes read for query with highlighter is 105, got %v\",\n\t\t\tbytesRead-prevBytesRead)\n\t}\n\tprevBytesRead = bytesRead\n\n\tdisQuery := NewDisjunctionQuery(NewMatchQuery(\"hotel\"), NewMatchQuery(\"united\"))\n\tsearchRequest = NewSearchRequestOptions(disQuery, int(10), 0, true)\n\tres, err = idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\t// expectation is that the bytes read is roughly equal to sum of sub queries in\n\t// the disjunction query plus the segment loading portion for the second subquery\n\t// since it's created afresh and not reused\n\tstats, _ = idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif bytesRead-prevBytesRead != 120 && res.Cost == bytesRead-prevBytesRead {\n\t\tt.Fatalf(\"expected bytes read for disjunction query is 120, got %v\",\n\t\t\tbytesRead-prevBytesRead)\n\t}\n}\n\nfunc TestBytesReadStored(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tdocumentMapping := NewDocumentMapping()\n\tindexMapping.AddDocumentMapping(\"hotel\", documentMapping)\n\n\tindexMapping.DocValuesDynamic = false\n\tindexMapping.StoreDynamic = false\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Store = true\n\tcontentFieldMapping.IncludeInAll = false\n\tcontentFieldMapping.IncludeTermVectors = false\n\n\treviewsMapping := NewDocumentMapping()\n\treviewsMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tdocumentMapping.AddSubDocumentMapping(\"reviews\", reviewsMapping)\n\n\ttypeFieldMapping := NewTextFieldMapping()\n\ttypeFieldMapping.Store = false\n\ttypeFieldMapping.IncludeInAll = false\n\ttypeFieldMapping.IncludeTermVectors = false\n\tdocumentMapping.AddFieldMappingsAt(\"type\", typeFieldMapping)\n\tidx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbatch, err := getBatchFromData(idx, \"sample-data.json\")\n\tif err != nil {\n\t\tt.Fatalf(\"failed to form a batch %v\\n\", err)\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to index batch %v\\n\", err)\n\t}\n\tquery := NewTermQuery(\"hotel\")\n\tquery.FieldVal = \"reviews.content\"\n\tsearchRequest := NewSearchRequestOptions(query, int(10), 0, true)\n\tres, err := idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstats, _ := idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ := stats[\"num_bytes_read_at_query_time\"].(uint64)\n\n\texpectedBytesRead := uint64(11025)\n\tif supportForVectorSearch {\n\t\texpectedBytesRead = 11435\n\t}\n\n\tif bytesRead != expectedBytesRead && bytesRead == res.Cost {\n\t\tt.Fatalf(\"expected the bytes read stat to be around %v, got %v\", expectedBytesRead, bytesRead)\n\t}\n\tprevBytesRead := bytesRead\n\n\tsearchRequest = NewSearchRequestOptions(query, int(10), 0, true)\n\tres, err = idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstats, _ = idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif bytesRead-prevBytesRead != 48 && bytesRead-prevBytesRead == res.Cost {\n\t\tt.Fatalf(\"expected the bytes read stat to be around 48, got %v\", bytesRead-prevBytesRead)\n\t}\n\tprevBytesRead = bytesRead\n\n\tsearchRequest = NewSearchRequestOptions(query, int(10), 0, true)\n\tsearchRequest.Fields = []string{\"*\"}\n\tres, err = idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstats, _ = idx.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\n\tif bytesRead-prevBytesRead != 26511 && bytesRead-prevBytesRead == res.Cost {\n\t\tt.Fatalf(\"expected the bytes read stat to be around 26511, got %v\",\n\t\t\tbytesRead-prevBytesRead)\n\t}\n\tidx.Close()\n\tcleanupTmpIndexPath(t, tmpIndexPath)\n\n\t// same type of querying but on field \"type\"\n\tcontentFieldMapping.Store = false\n\ttypeFieldMapping.Store = true\n\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath1)\n\n\tidx1, err := NewUsing(tmpIndexPath1, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := idx1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch, err = getBatchFromData(idx1, \"sample-data.json\")\n\tif err != nil {\n\t\tt.Fatalf(\"failed to form a batch %v\\n\", err)\n\t}\n\terr = idx1.Batch(batch)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to index batch %v\\n\", err)\n\t}\n\n\tquery = NewTermQuery(\"hotel\")\n\tquery.FieldVal = \"type\"\n\tsearchRequest = NewSearchRequestOptions(query, int(10), 0, true)\n\tres, err = idx1.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstats, _ = idx1.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\n\texpectedBytesRead = uint64(3212)\n\tif supportForVectorSearch {\n\t\texpectedBytesRead = 3622\n\t}\n\n\tif bytesRead != expectedBytesRead && bytesRead == res.Cost {\n\t\tt.Fatalf(\"expected the bytes read stat to be around %v, got %v\", expectedBytesRead, bytesRead)\n\t}\n\tprevBytesRead = bytesRead\n\n\tres, err = idx1.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tstats, _ = idx1.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif bytesRead-prevBytesRead != 47 && bytesRead-prevBytesRead == res.Cost {\n\t\tt.Fatalf(\"expected the bytes read stat to be around 47, got %v\", bytesRead-prevBytesRead)\n\t}\n\tprevBytesRead = bytesRead\n\n\tsearchRequest.Fields = []string{\"*\"}\n\tres, err = idx1.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tstats, _ = idx1.StatsMap()[\"index\"].(map[string]interface{})\n\tbytesRead, _ = stats[\"num_bytes_read_at_query_time\"].(uint64)\n\tif bytesRead-prevBytesRead != 77 && bytesRead-prevBytesRead == res.Cost {\n\t\tt.Fatalf(\"expected the bytes read stat to be around 77, got %v\", bytesRead-prevBytesRead)\n\t}\n}\n\nfunc readDataFromFile(fileName string) ([]map[string]interface{}, error) {\n\tpwd, err := os.Getwd()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tpath := filepath.Join(pwd, \"data\", \"test\", fileName)\n\n\tvar dataset []map[string]interface{}\n\tfileContent, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\terr = json.Unmarshal(fileContent, &dataset)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn dataset, nil\n}\n\nfunc getBatchFromData(idx Index, fileName string) (*Batch, error) {\n\tdataset, err := readDataFromFile(fileName)\n\tbatch := idx.NewBatch()\n\tfor _, doc := range dataset {\n\t\terr = batch.Index(fmt.Sprintf(\"%d\", doc[\"id\"]), doc)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn batch, err\n}\n\nfunc TestIndexCreateNewOverExisting(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t_, err = New(tmpIndexPath, NewIndexMapping())\n\tif err != ErrorIndexPathExists {\n\t\tt.Fatalf(\"expected error index path exists, got %v\", err)\n\t}\n}\n\nfunc TestIndexOpenNonExisting(t *testing.T) {\n\t_, err := Open(\"doesnotexist\")\n\tif err != ErrorIndexPathDoesNotExist {\n\t\tt.Fatalf(\"expected error index path does not exist, got %v\", err)\n\t}\n}\n\nfunc TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttmpIndexPathMeta := filepath.Join(tmpIndexPath, \"index_meta.json\")\n\n\t// now intentionally change the storage type\n\terr = os.WriteFile(tmpIndexPathMeta, []byte(`{\"storage\":\"mystery\"}`), 0o666)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t_, err = Open(tmpIndexPath)\n\tif err == nil {\n\t\tt.Fatalf(\"expected error for unknown storage type, got %v\", err)\n\t}\n\n\t// now intentionally corrupt the metadata\n\terr = os.WriteFile(tmpIndexPathMeta, []byte(\"corrupted\"), 0o666)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t_, err = Open(tmpIndexPath)\n\tif err != ErrorIndexMetaCorrupt {\n\t\tt.Fatalf(\"expected error index metadata corrupted, got %v\", err)\n\t}\n\n\t// now intentionally remove the metadata\n\terr = os.Remove(tmpIndexPathMeta)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t_, err = Open(tmpIndexPath)\n\tif err != ErrorIndexMetaMissing {\n\t\tt.Fatalf(\"expected error index metadata missing, got %v\", err)\n\t}\n}\n\nfunc TestInMemIndex(t *testing.T) {\n\tindex, err := NewMemOnly(NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestClosedIndex(t *testing.T) {\n\tindex, err := NewMemOnly(NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = index.Index(\"test\", \"test\")\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected error index closed, got %v\", err)\n\t}\n\n\terr = index.Delete(\"test\")\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected error index closed, got %v\", err)\n\t}\n\n\tb := index.NewBatch()\n\terr = index.Batch(b)\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected error index closed, got %v\", err)\n\t}\n\n\t_, err = index.Document(\"test\")\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected error index closed, got %v\", err)\n\t}\n\n\t_, err = index.DocCount()\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected error index closed, got %v\", err)\n\t}\n\n\t_, err = index.Search(NewSearchRequest(NewTermQuery(\"test\")))\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected error index closed, got %v\", err)\n\t}\n\n\t_, err = index.Fields()\n\tif err != ErrorIndexClosed {\n\t\tt.Errorf(\"expected error index closed, got %v\", err)\n\t}\n}\n\ntype slowQuery struct {\n\tactual query.Query\n\tdelay  time.Duration\n}\n\nfunc (s *slowQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\ttime.Sleep(s.delay)\n\treturn s.actual.Searcher(ctx, i, m, options)\n}\n\nfunc TestSlowSearch(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tdefer func() {\n\t\t// reset logger back to normal\n\t\tSetLog(log.New(io.Discard, \"bleve\", log.LstdFlags))\n\t}()\n\t// set custom logger\n\tvar sdw sawDataWriter\n\tSetLog(log.New(&sdw, \"bleve\", log.LstdFlags))\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tConfig.SlowSearchLogThreshold = 1 * time.Minute\n\n\tquery := NewTermQuery(\"water\")\n\treq := NewSearchRequest(query)\n\t_, err = index.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif sdw.sawData {\n\t\tt.Errorf(\"expected to not see slow query logged, but did\")\n\t}\n\n\tsq := &slowQuery{\n\t\tactual: query,\n\t\tdelay:  50 * time.Millisecond, // on Windows timer resolution is 15ms\n\t}\n\treq.Query = sq\n\tConfig.SlowSearchLogThreshold = 1 * time.Microsecond\n\t_, err = index.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif !sdw.sawData {\n\t\tt.Errorf(\"expected to see slow query logged, but didn't\")\n\t}\n}\n\ntype sawDataWriter struct {\n\tsawData bool\n}\n\nfunc (s *sawDataWriter) Write(p []byte) (n int, err error) {\n\ts.sawData = true\n\treturn len(p), nil\n}\n\nfunc TestStoredFieldPreserved(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoca := map[string]interface{}{\n\t\t\"name\": \"Marty\",\n\t\t\"desc\": \"GopherCON India\",\n\t\t\"bool\": true,\n\t\t\"num\":  float64(1),\n\t}\n\terr = index.Index(\"a\", doca)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tq := NewTermQuery(\"marty\")\n\treq := NewSearchRequest(q)\n\treq.Fields = []string{\"name\", \"desc\", \"bool\", \"num\"}\n\tres, err := index.Search(req)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].Fields[\"name\"] != \"Marty\" {\n\t\tt.Errorf(\"expected 'Marty' got '%s'\", res.Hits[0].Fields[\"name\"])\n\t}\n\tif res.Hits[0].Fields[\"desc\"] != \"GopherCON India\" {\n\t\tt.Errorf(\"expected 'GopherCON India' got '%s'\", res.Hits[0].Fields[\"desc\"])\n\t}\n\tif res.Hits[0].Fields[\"num\"] != float64(1) {\n\t\tt.Errorf(\"expected '1' got '%v'\", res.Hits[0].Fields[\"num\"])\n\t}\n\tif res.Hits[0].Fields[\"bool\"] != true {\n\t\tt.Errorf(\"expected 'true' got '%v'\", res.Hits[0].Fields[\"bool\"])\n\t}\n}\n\nfunc TestDict(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoca := map[string]interface{}{\n\t\t\"name\": \"marty\",\n\t\t\"desc\": \"gophercon india\",\n\t}\n\terr = index.Index(\"a\", doca)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocy := map[string]interface{}{\n\t\t\"name\": \"jasper\",\n\t\t\"desc\": \"clojure\",\n\t}\n\terr = index.Index(\"y\", docy)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocx := map[string]interface{}{\n\t\t\"name\": \"rose\",\n\t\t\"desc\": \"googler\",\n\t}\n\terr = index.Index(\"x\", docx)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdict, err := index.FieldDict(\"name\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tterms := []string{}\n\tde, err := dict.Next()\n\tfor err == nil && de != nil {\n\t\tterms = append(terms, string(de.Term))\n\t\tde, err = dict.Next()\n\t}\n\n\texpectedTerms := []string{\"jasper\", \"marty\", \"rose\"}\n\tif !reflect.DeepEqual(terms, expectedTerms) {\n\t\tt.Errorf(\"expected %v, got %v\", expectedTerms, terms)\n\t}\n\n\terr = dict.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test start and end range\n\tdict, err = index.FieldDictRange(\"name\", []byte(\"marty\"), []byte(\"rose\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tterms = []string{}\n\tde, err = dict.Next()\n\tfor err == nil && de != nil {\n\t\tterms = append(terms, string(de.Term))\n\t\tde, err = dict.Next()\n\t}\n\n\texpectedTerms = []string{\"marty\", \"rose\"}\n\tif !reflect.DeepEqual(terms, expectedTerms) {\n\t\tt.Errorf(\"expected %v, got %v\", expectedTerms, terms)\n\t}\n\n\terr = dict.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdocz := map[string]interface{}{\n\t\t\"name\": \"prefix\",\n\t\t\"desc\": \"bob cat cats catting dog doggy zoo\",\n\t}\n\terr = index.Index(\"z\", docz)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdict, err = index.FieldDictPrefix(\"desc\", []byte(\"cat\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tterms = []string{}\n\tde, err = dict.Next()\n\tfor err == nil && de != nil {\n\t\tterms = append(terms, string(de.Term))\n\t\tde, err = dict.Next()\n\t}\n\n\texpectedTerms = []string{\"cat\", \"cats\", \"catting\"}\n\tif !reflect.DeepEqual(terms, expectedTerms) {\n\t\tt.Errorf(\"expected %v, got %v\", expectedTerms, terms)\n\t}\n\n\tstats := index.Stats()\n\tif stats == nil {\n\t\tt.Errorf(\"expected IndexStat, got nil\")\n\t}\n\n\terr = dict.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestBatchString(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch := index.NewBatch()\n\terr = batch.Index(\"a\", []byte(\"{}\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbatch.Delete(\"b\")\n\tbatch.SetInternal([]byte(\"c\"), []byte{})\n\tbatch.DeleteInternal([]byte(\"d\"))\n\n\tbatchStr := batch.String()\n\tif !strings.HasPrefix(batchStr, \"Batch (2 ops, 2 internal ops)\") {\n\t\tt.Errorf(\"expected to start with Batch (2 ops, 2 internal ops), did not\")\n\t}\n\tif !strings.Contains(batchStr, \"INDEX - 'a'\") {\n\t\tt.Errorf(\"expected to contain INDEX - 'a', did not\")\n\t}\n\tif !strings.Contains(batchStr, \"DELETE - 'b'\") {\n\t\tt.Errorf(\"expected to contain DELETE - 'b', did not\")\n\t}\n\tif !strings.Contains(batchStr, \"SET INTERNAL - 'c'\") {\n\t\tt.Errorf(\"expected to contain SET INTERNAL - 'c', did not\")\n\t}\n\tif !strings.Contains(batchStr, \"DELETE INTERNAL - 'd'\") {\n\t\tt.Errorf(\"expected to contain DELETE INTERNAL - 'd', did not\")\n\t}\n}\n\nfunc TestIndexMetadataRaceBug198(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tdone := make(chan struct{})\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\t_, err2 := index.DocCount()\n\t\t\t\tif err2 != nil {\n\t\t\t\t\tt.Error(err2)\n\t\t\t\t\twg.Done()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 0; i < 100; i++ {\n\t\tbatch := index.NewBatch()\n\t\terr = batch.Index(\"a\", []byte(\"{}\"))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = index.Batch(batch)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tclose(done)\n\twg.Wait()\n}\n\nfunc TestSortMatchSearch(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tnames := []string{\"Noam\", \"Uri\", \"David\", \"Yosef\", \"Eitan\", \"Itay\", \"Ariel\", \"Daniel\", \"Omer\", \"Yogev\", \"Yehonatan\", \"Moshe\", \"Mohammed\", \"Yusuf\", \"Omar\"}\n\tdays := []string{\"Sunday\", \"Monday\", \"Tuesday\", \"Wednesday\", \"Thursday\", \"Friday\", \"Saturday\"}\n\tnumbers := []string{\"One\", \"Two\", \"Three\", \"Four\", \"Five\", \"Six\", \"Seven\", \"Eight\", \"Nine\", \"Ten\", \"Eleven\", \"Twelve\"}\n\tb := index.NewBatch()\n\tfor i := 0; i < 200; i++ {\n\t\tdoc := make(map[string]interface{})\n\t\tdoc[\"Name\"] = names[i%len(names)]\n\t\tdoc[\"Day\"] = days[i%len(days)]\n\t\tdoc[\"Number\"] = numbers[i%len(numbers)]\n\t\terr = b.Index(fmt.Sprintf(\"%d\", i), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index.Batch(b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treq := NewSearchRequest(NewMatchQuery(\"One\"))\n\treq.SortBy([]string{\"Day\", \"Name\"})\n\treq.Fields = []string{\"*\"}\n\tsr, err := index.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tprev := \"\"\n\tfor _, hit := range sr.Hits {\n\t\tval := hit.Fields[\"Day\"].(string)\n\t\tif prev > val {\n\t\t\tt.Errorf(\"Hits must be sorted by 'Day'. Found '%s' before '%s'\", prev, val)\n\t\t}\n\t\tprev = val\n\t}\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexCountMatchSearch(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1)\n\t\tgo func(i int) {\n\t\t\tb := index.NewBatch()\n\t\t\tfor j := 0; j < 200; j++ {\n\t\t\t\tid := fmt.Sprintf(\"%d\", (i*200)+j)\n\n\t\t\t\tdoc := struct {\n\t\t\t\t\tBody string\n\t\t\t\t}{\n\t\t\t\t\tBody: \"match\",\n\t\t\t\t}\n\n\t\t\t\terr := b.Index(id, doc)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Error(err)\n\t\t\t\t\twg.Done()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\terr := index.Batch(b)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t\twg.Done()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\twg.Done()\n\t\t}(i)\n\t}\n\twg.Wait()\n\n\t// search for something that should match all documents\n\tsr, err := index.Search(NewSearchRequest(NewMatchQuery(\"match\")))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// get the index document count\n\tdc, err := index.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// make sure test is working correctly, doc count should 2000\n\tif dc != 2000 {\n\t\tt.Errorf(\"expected doc count 2000, got %d\", dc)\n\t}\n\n\t// make sure our search found all the documents\n\tif dc != sr.Total {\n\t\tt.Errorf(\"expected search result total %d to match doc count %d\", sr.Total, dc)\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestBatchReset(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tbatch := index.NewBatch()\n\terr = batch.Index(\"k1\", struct {\n\t\tBody string\n\t}{\n\t\tBody: \"v1\",\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tbatch.Delete(\"k2\")\n\tbatch.SetInternal([]byte(\"k3\"), []byte(\"v3\"))\n\tbatch.DeleteInternal([]byte(\"k4\"))\n\n\tif batch.Size() != 4 {\n\t\tt.Logf(\"%v\", batch)\n\t\tt.Errorf(\"expected batch size 4, got %d\", batch.Size())\n\t}\n\n\tbatch.Reset()\n\n\tif batch.Size() != 0 {\n\t\tt.Errorf(\"expected batch size 0 after reset, got %d\", batch.Size())\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestDocumentFieldArrayPositions(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// index a document with an array of strings\n\terr = idx.Index(\"k\", struct {\n\t\tMessages []string\n\t}{\n\t\tMessages: []string{\n\t\t\t\"first\",\n\t\t\t\"second\",\n\t\t\t\"third\",\n\t\t\t\"last\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// load the document\n\tdoc, err := idx.Document(\"k\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc.VisitFields(func(f index.Field) {\n\t\tif reflect.DeepEqual(f.Value(), []byte(\"first\")) {\n\t\t\tap := f.ArrayPositions()\n\t\t\tif len(ap) < 1 {\n\t\t\t\tt.Errorf(\"expected an array position, got none\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif ap[0] != 0 {\n\t\t\t\tt.Errorf(\"expected 'first' in array position 0, got %d\", ap[0])\n\t\t\t}\n\t\t}\n\t\tif reflect.DeepEqual(f.Value(), []byte(\"second\")) {\n\t\t\tap := f.ArrayPositions()\n\t\t\tif len(ap) < 1 {\n\t\t\t\tt.Errorf(\"expected an array position, got none\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif ap[0] != 1 {\n\t\t\t\tt.Errorf(\"expected 'second' in array position 1, got %d\", ap[0])\n\t\t\t}\n\t\t}\n\t\tif reflect.DeepEqual(f.Value(), []byte(\"third\")) {\n\t\t\tap := f.ArrayPositions()\n\t\t\tif len(ap) < 1 {\n\t\t\t\tt.Errorf(\"expected an array position, got none\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif ap[0] != 2 {\n\t\t\t\tt.Errorf(\"expected 'third' in array position 2, got %d\", ap[0])\n\t\t\t}\n\t\t}\n\t\tif reflect.DeepEqual(f.Value(), []byte(\"last\")) {\n\t\t\tap := f.ArrayPositions()\n\t\t\tif len(ap) < 1 {\n\t\t\t\tt.Errorf(\"expected an array position, got none\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif ap[0] != 3 {\n\t\t\t\tt.Errorf(\"expected 'last' in array position 3, got %d\", ap[0])\n\t\t\t}\n\t\t}\n\t})\n\n\t// now index a document in the same field with a single string\n\terr = idx.Index(\"k2\", struct {\n\t\tMessages string\n\t}{\n\t\tMessages: \"only\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// load the document\n\tdoc, err = idx.Document(\"k2\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc.VisitFields(func(f index.Field) {\n\t\tif reflect.DeepEqual(f.Value(), []byte(\"only\")) {\n\t\t\tap := f.ArrayPositions()\n\t\t\tif len(ap) != 0 {\n\t\t\t\tt.Errorf(\"expected no array positions, got %d\", len(ap))\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t})\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestKeywordSearchBug207(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tf := NewTextFieldMapping()\n\tf.Analyzer = keyword.Name\n\n\tm := NewIndexMapping()\n\tm.DefaultMapping = NewDocumentMapping()\n\tm.DefaultMapping.AddFieldMappingsAt(\"Body\", f)\n\n\tindex, err := New(tmpIndexPath, m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc1 := struct {\n\t\tBody string\n\t}{\n\t\tBody: \"a555c3bb06f7a127cda000005\",\n\t}\n\n\terr = index.Index(\"a\", doc1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc2 := struct {\n\t\tBody string\n\t}{\n\t\tBody: \"555c3bb06f7a127cda000005\",\n\t}\n\n\terr = index.Index(\"b\", doc2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now search for these terms\n\tsreq := NewSearchRequest(NewTermQuery(\"a555c3bb06f7a127cda000005\"))\n\tsres, err := index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 result, got %d\", sres.Total)\n\t}\n\tif sres.Hits[0].ID != \"a\" {\n\t\tt.Errorf(\"expecated id 'a', got '%s'\", sres.Hits[0].ID)\n\t}\n\n\tsreq = NewSearchRequest(NewTermQuery(\"555c3bb06f7a127cda000005\"))\n\tsres, err = index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 result, got %d\", sres.Total)\n\t}\n\tif sres.Hits[0].ID != \"b\" {\n\t\tt.Errorf(\"expecated id 'b', got '%s'\", sres.Hits[0].ID)\n\t}\n\n\t// now do the same searches using query strings\n\tsreq = NewSearchRequest(NewQueryStringQuery(\"Body:a555c3bb06f7a127cda000005\"))\n\tsres, err = index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 result, got %d\", sres.Total)\n\t}\n\tif sres.Hits[0].ID != \"a\" {\n\t\tt.Errorf(\"expecated id 'a', got '%s'\", sres.Hits[0].ID)\n\t}\n\n\tsreq = NewSearchRequest(NewQueryStringQuery(`Body:555c3bb06f7a127cda000005`))\n\tsres, err = index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 result, got %d\", sres.Total)\n\t}\n\tif sres.Hits[0].ID != \"b\" {\n\t\tt.Errorf(\"expecated id 'b', got '%s'\", sres.Hits[0].ID)\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestTermVectorArrayPositions(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// index a document with an array of strings\n\terr = index.Index(\"k\", struct {\n\t\tMessages []string\n\t}{\n\t\tMessages: []string{\n\t\t\t\"first\",\n\t\t\t\"second\",\n\t\t\t\"third\",\n\t\t\t\"last\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// search for this document in all field\n\ttq := NewTermQuery(\"second\")\n\ttsr := NewSearchRequest(tq)\n\ttsr.IncludeLocations = true\n\tresults, err := index.Search(tsr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif results.Total != 1 {\n\t\tt.Fatalf(\"expected 1 result, got %d\", results.Total)\n\t}\n\tif len(results.Hits[0].Locations[\"Messages\"][\"second\"]) < 1 {\n\t\tt.Fatalf(\"expected at least one location\")\n\t}\n\tif len(results.Hits[0].Locations[\"Messages\"][\"second\"][0].ArrayPositions) < 1 {\n\t\tt.Fatalf(\"expected at least one location array position\")\n\t}\n\tif results.Hits[0].Locations[\"Messages\"][\"second\"][0].ArrayPositions[0] != 1 {\n\t\tt.Fatalf(\"expected array position 1, got %d\", results.Hits[0].Locations[\"Messages\"][\"second\"][0].ArrayPositions[0])\n\t}\n\n\t// repeat search for this document in Messages field\n\ttq2 := NewTermQuery(\"third\")\n\ttq2.SetField(\"Messages\")\n\ttsr = NewSearchRequest(tq2)\n\ttsr.IncludeLocations = true\n\tresults, err = index.Search(tsr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif results.Total != 1 {\n\t\tt.Fatalf(\"expected 1 result, got %d\", results.Total)\n\t}\n\tif len(results.Hits[0].Locations[\"Messages\"][\"third\"]) < 1 {\n\t\tt.Fatalf(\"expected at least one location\")\n\t}\n\tif len(results.Hits[0].Locations[\"Messages\"][\"third\"][0].ArrayPositions) < 1 {\n\t\tt.Fatalf(\"expected at least one location array position\")\n\t}\n\tif results.Hits[0].Locations[\"Messages\"][\"third\"][0].ArrayPositions[0] != 2 {\n\t\tt.Fatalf(\"expected array position 2, got %d\", results.Hits[0].Locations[\"Messages\"][\"third\"][0].ArrayPositions[0])\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestDocumentStaticMapping(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tm := NewIndexMapping()\n\tm.DefaultMapping = NewDocumentStaticMapping()\n\tm.DefaultMapping.AddFieldMappingsAt(\"Text\", NewTextFieldMapping())\n\tm.DefaultMapping.AddFieldMappingsAt(\"Date\", NewDateTimeFieldMapping())\n\tm.DefaultMapping.AddFieldMappingsAt(\"Numeric\", NewNumericFieldMapping())\n\n\tindex, err := New(tmpIndexPath, m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc1 := struct {\n\t\tText           string\n\t\tIgnoredText    string\n\t\tNumeric        float64\n\t\tIgnoredNumeric float64\n\t\tDate           time.Time\n\t\tIgnoredDate    time.Time\n\t}{\n\t\tText:           \"valid text\",\n\t\tIgnoredText:    \"ignored text\",\n\t\tNumeric:        10,\n\t\tIgnoredNumeric: 20,\n\t\tDate:           time.Unix(1, 0),\n\t\tIgnoredDate:    time.Unix(2, 0),\n\t}\n\n\terr = index.Index(\"a\", doc1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfields, err := index.Fields()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsort.Strings(fields)\n\texpectedFields := []string{\"Date\", \"Numeric\", \"Text\", \"_all\"}\n\tif len(fields) < len(expectedFields) {\n\t\tt.Fatalf(\"invalid field count: %d\", len(fields))\n\t}\n\tfor i, expected := range expectedFields {\n\t\tif expected != fields[i] {\n\t\t\tt.Fatalf(\"unexpected field[%d]: %s\", i, fields[i])\n\t\t}\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexEmptyDocId(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := map[string]interface{}{\n\t\t\"body\": \"nodocid\",\n\t}\n\n\terr = index.Index(\"\", doc)\n\tif err != ErrorEmptyID {\n\t\tt.Errorf(\"expect index empty doc id to fail\")\n\t}\n\n\terr = index.Delete(\"\")\n\tif err != ErrorEmptyID {\n\t\tt.Errorf(\"expect delete empty doc id to fail\")\n\t}\n\n\tbatch := index.NewBatch()\n\terr = batch.Index(\"\", doc)\n\tif err != ErrorEmptyID {\n\t\tt.Errorf(\"expect index empty doc id in batch to fail\")\n\t}\n\n\tbatch.Delete(\"\")\n\tif batch.Size() > 0 {\n\t\tt.Errorf(\"expect delete empty doc id in batch to be ignored\")\n\t}\n}\n\nfunc TestDateTimeFieldMappingIssue287(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tf := NewDateTimeFieldMapping()\n\n\tm := NewIndexMapping()\n\tm.DefaultMapping = NewDocumentMapping()\n\tm.DefaultMapping.AddFieldMappingsAt(\"Date\", f)\n\n\tindex, err := New(tmpIndexPath, m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttype doc struct {\n\t\tDate time.Time\n\t}\n\n\tnow := time.Now()\n\n\t// 3hr ago to 1hr ago\n\tfor i := 0; i < 3; i++ {\n\t\td := doc{now.Add(time.Duration((i - 3)) * time.Hour)}\n\n\t\terr = index.Index(strconv.FormatInt(int64(i), 10), d)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\t// search range across all docs\n\tstart := now.Add(-4 * time.Hour)\n\tend := now\n\tsreq := NewSearchRequest(NewDateRangeQuery(start, end))\n\tsres, err := index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 3 {\n\t\tt.Errorf(\"expected 3 results, got %d\", sres.Total)\n\t}\n\n\t// search range includes only oldest\n\tstart = now.Add(-4 * time.Hour)\n\tend = now.Add(-121 * time.Minute)\n\tsreq = NewSearchRequest(NewDateRangeQuery(start, end))\n\tsres, err = index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 results, got %d\", sres.Total)\n\t}\n\tif sres.Total > 0 && sres.Hits[0].ID != \"0\" {\n\t\tt.Errorf(\"expecated id '0', got '%s'\", sres.Hits[0].ID)\n\t}\n\n\t// search range includes only newest\n\tstart = now.Add(-61 * time.Minute)\n\tend = now\n\tsreq = NewSearchRequest(NewDateRangeQuery(start, end))\n\tsres, err = index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 results, got %d\", sres.Total)\n\t}\n\tif sres.Total > 0 && sres.Hits[0].ID != \"2\" {\n\t\tt.Errorf(\"expecated id '2', got '%s'\", sres.Hits[0].ID)\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestDocumentFieldArrayPositionsBug295(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// index a document with an array of strings\n\terr = index.Index(\"k\", struct {\n\t\tMessages []string\n\t\tAnother  string\n\t\tMoreData []string\n\t}{\n\t\tMessages: []string{\n\t\t\t\"bleve\",\n\t\t\t\"bleve\",\n\t\t},\n\t\tAnother: \"text\",\n\t\tMoreData: []string{\n\t\t\t\"a\",\n\t\t\t\"b\",\n\t\t\t\"c\",\n\t\t\t\"bleve\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// search for it in the messages field\n\ttq := NewTermQuery(\"bleve\")\n\ttq.SetField(\"Messages\")\n\ttsr := NewSearchRequest(tq)\n\ttsr.IncludeLocations = true\n\tresults, err := index.Search(tsr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif results.Total != 1 {\n\t\tt.Fatalf(\"expected 1 result, got %d\", results.Total)\n\t}\n\tif len(results.Hits[0].Locations[\"Messages\"][\"bleve\"]) != 2 {\n\t\tt.Fatalf(\"expected 2 locations of 'bleve', got %d\", len(results.Hits[0].Locations[\"Messages\"][\"bleve\"]))\n\t}\n\tif results.Hits[0].Locations[\"Messages\"][\"bleve\"][0].ArrayPositions[0] != 0 {\n\t\tt.Errorf(\"expected array position to be 0\")\n\t}\n\tif results.Hits[0].Locations[\"Messages\"][\"bleve\"][1].ArrayPositions[0] != 1 {\n\t\tt.Errorf(\"expected array position to be 1\")\n\t}\n\n\t// search for it in all\n\ttq = NewTermQuery(\"bleve\")\n\ttsr = NewSearchRequest(tq)\n\ttsr.IncludeLocations = true\n\tresults, err = index.Search(tsr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif results.Total != 1 {\n\t\tt.Fatalf(\"expected 1 result, got %d\", results.Total)\n\t}\n\tif len(results.Hits[0].Locations[\"Messages\"][\"bleve\"]) != 2 {\n\t\tt.Fatalf(\"expected 2 locations of 'bleve', got %d\", len(results.Hits[0].Locations[\"Messages\"][\"bleve\"]))\n\t}\n\tif results.Hits[0].Locations[\"Messages\"][\"bleve\"][0].ArrayPositions[0] != 0 {\n\t\tt.Errorf(\"expected array position to be 0\")\n\t}\n\tif results.Hits[0].Locations[\"Messages\"][\"bleve\"][1].ArrayPositions[0] != 1 {\n\t\tt.Errorf(\"expected array position to be 1\")\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestBooleanFieldMappingIssue109(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tm := NewIndexMapping()\n\tm.DefaultMapping = NewDocumentMapping()\n\tm.DefaultMapping.AddFieldMappingsAt(\"Bool\", NewBooleanFieldMapping())\n\n\tindex, err := New(tmpIndexPath, m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttype doc struct {\n\t\tBool bool\n\t}\n\terr = index.Index(\"true\", &doc{Bool: true})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Index(\"false\", &doc{Bool: false})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tq := NewBoolFieldQuery(true)\n\tq.SetField(\"Bool\")\n\tsreq := NewSearchRequest(q)\n\tsres, err := index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 results, got %d\", sres.Total)\n\t}\n\n\tq = NewBoolFieldQuery(false)\n\tq.SetField(\"Bool\")\n\tsreq = NewSearchRequest(q)\n\tsres, err = index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 results, got %d\", sres.Total)\n\t}\n\n\tsreq = NewSearchRequest(NewBoolFieldQuery(true))\n\tsres, err = index.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif sres.Total != 1 {\n\t\tt.Errorf(\"expected 1 results, got %d\", sres.Total)\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestSearchTimeout(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// first run a search with an absurdly long timeout (should succeed)\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\tquery := NewTermQuery(\"water\")\n\treq := NewSearchRequest(query)\n\t_, err = index.SearchInContext(ctx, req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now run a search again with an absurdly low timeout (should timeout)\n\tctx, cancel = context.WithTimeout(context.Background(), 1*time.Microsecond)\n\tdefer cancel()\n\tsq := &slowQuery{\n\t\tactual: query,\n\t\tdelay:  50 * time.Millisecond, // on Windows timer resolution is 15ms\n\t}\n\treq.Query = sq\n\t_, err = index.SearchInContext(ctx, req)\n\tif err != context.DeadlineExceeded {\n\t\tt.Fatalf(\"expected %v, got: %v\", context.DeadlineExceeded, err)\n\t}\n\n\t// now run a search with a long timeout, but with a long query, and cancel it\n\tctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)\n\tsq = &slowQuery{\n\t\tactual: query,\n\t\tdelay:  100 * time.Millisecond, // on Windows timer resolution is 15ms\n\t}\n\treq = NewSearchRequest(sq)\n\tcancel()\n\t_, err = index.SearchInContext(ctx, req)\n\tif err != context.Canceled {\n\t\tt.Fatalf(\"expected %v, got: %v\", context.Canceled, err)\n\t}\n}\n\n// TestConfigCache exposes a concurrent map write with go 1.6\nfunc TestConfigCache(t *testing.T) {\n\tfor i := 0; i < 100; i++ {\n\t\tgo func() {\n\t\t\t_, err := Config.Cache.HighlighterNamed(Config.DefaultHighlighter)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t}()\n\t}\n}\n\nfunc TestBatchRaceBug260(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\ti, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := i.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\tb := i.NewBatch()\n\terr = b.Index(\"1\", 1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Batch(b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tb.Reset()\n\terr = b.Index(\"2\", 2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Batch(b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tb.Reset()\n}\n\nfunc BenchmarkBatchOverhead(b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\tm := NewIndexMapping()\n\ti, err := NewUsing(tmpIndexPath, m, Config.DefaultIndexType, null.Name, nil)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tfor n := 0; n < b.N; n++ {\n\t\t// put 1000 items in a batch\n\t\tbatch := i.NewBatch()\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\terr = batch.Index(fmt.Sprintf(\"%d\", i), map[string]interface{}{\"name\": \"bleve\"})\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t}\n\t\terr = i.Batch(batch)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tbatch.Reset()\n\t}\n}\n\nfunc TestOpenReadonlyMultiple(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\t// build an index and close it\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoca := map[string]interface{}{\n\t\t\"name\": \"marty\",\n\t\t\"desc\": \"gophercon india\",\n\t}\n\terr = index.Index(\"a\", doca)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now open it read-only\n\tindex, err = OpenUsing(tmpIndexPath, map[string]interface{}{\n\t\t\"read_only\": true,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// now open it again\n\tindex2, err := OpenUsing(tmpIndexPath, map[string]interface{}{\n\t\t\"read_only\": true,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index2.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\n// TestBug408 tests for VERY large values of size, even though actual result\n// set may be reasonable size\nfunc TestBug408(t *testing.T) {\n\ttype TestStruct struct {\n\t\tID     string  `json:\"id\"`\n\t\tUserID *string `json:\"user_id\"`\n\t}\n\n\tdocMapping := NewDocumentMapping()\n\tdocMapping.AddFieldMappingsAt(\"id\", NewTextFieldMapping())\n\tdocMapping.AddFieldMappingsAt(\"user_id\", NewTextFieldMapping())\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.DefaultMapping = docMapping\n\n\tindex, err := NewMemOnly(indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tnumToTest := 10\n\tmatchUserID := \"match\"\n\tnoMatchUserID := \"no_match\"\n\tmatchingDocIds := make(map[string]struct{})\n\n\tfor i := 0; i < numToTest; i++ {\n\t\tds := &TestStruct{\"id_\" + strconv.Itoa(i), nil}\n\t\tif i%2 == 0 {\n\t\t\tds.UserID = &noMatchUserID\n\t\t} else {\n\t\t\tds.UserID = &matchUserID\n\t\t\tmatchingDocIds[ds.ID] = struct{}{}\n\t\t}\n\t\terr = index.Index(ds.ID, ds)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tcnt, err := index.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif int(cnt) != numToTest {\n\t\tt.Fatalf(\"expected %d documents in index, got %d\", numToTest, cnt)\n\t}\n\n\tq := NewTermQuery(matchUserID)\n\tq.SetField(\"user_id\")\n\tsearchReq := NewSearchRequestOptions(q, math.MaxInt32, 0, false)\n\tresults, err := index.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif int(results.Total) != numToTest/2 {\n\t\tt.Fatalf(\"expected %d search hits, got %d\", numToTest/2, results.Total)\n\t}\n\n\tfor _, result := range results.Hits {\n\t\tif _, found := matchingDocIds[result.ID]; !found {\n\t\t\tt.Fatalf(\"document with ID %s not in results as expected\", result.ID)\n\t\t}\n\t}\n}\n\nfunc TestIndexAdvancedCountMatchSearch(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\terrChan := make(chan error, 10)\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1)\n\t\tgo func(i int) {\n\t\t\tdefer wg.Done()\n\t\t\tb := index.NewBatch()\n\t\t\tfor j := 0; j < 200; j++ {\n\t\t\t\tid := fmt.Sprintf(\"%d\", (i*200)+j)\n\n\t\t\t\tdoc := document.NewDocument(id)\n\t\t\t\tdoc.AddField(document.NewTextField(\"body\", []uint64{}, []byte(\"match\")))\n\t\t\t\tdoc.AddField(document.NewCompositeField(\"_all\", true, []string{}, []string{}))\n\n\t\t\t\terr := b.IndexAdvanced(doc)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrChan <- err\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\terr := index.Batch(b)\n\t\t\tif err != nil {\n\t\t\t\terrChan <- err\n\t\t\t\treturn\n\t\t\t}\n\t\t}(i)\n\t}\n\twg.Wait()\n\n\tclose(errChan)\n\tfor err := range errChan {\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\t// search for something that should match all documents\n\tsr, err := index.Search(NewSearchRequest(NewMatchQuery(\"match\")))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// get the index document count\n\tdc, err := index.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// make sure test is working correctly, doc count should 2000\n\tif dc != 2000 {\n\t\tt.Errorf(\"expected doc count 2000, got %d\", dc)\n\t}\n\n\t// make sure our search found all the documents\n\tif dc != sr.Total {\n\t\tt.Errorf(\"expected search result total %d to match doc count %d\", sr.Total, dc)\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc benchmarkSearchOverhead(indexType string, b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tindex, err := NewUsing(tmpIndexPath, NewIndexMapping(),\n\t\tindexType, Config.DefaultKVStore, nil)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\telements := []string{\"air\", \"water\", \"fire\", \"earth\"}\n\tfor j := 0; j < 10000; j++ {\n\t\terr = index.Index(fmt.Sprintf(\"%d\", j),\n\t\t\tmap[string]interface{}{\"name\": elements[j%len(elements)]})\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tquery1 := NewTermQuery(\"water\")\n\tquery2 := NewTermQuery(\"fire\")\n\tquery := NewDisjunctionQuery(query1, query2)\n\treq := NewSearchRequest(query)\n\n\tb.ResetTimer()\n\n\tfor n := 0; n < b.N; n++ {\n\t\t_, err = index.Search(req)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkUpsidedownSearchOverhead(b *testing.B) {\n\tbenchmarkSearchOverhead(upsidedown.Name, b)\n}\n\nfunc BenchmarkScorchSearchOverhead(b *testing.B) {\n\tbenchmarkSearchOverhead(scorch.Name, b)\n}\n\nfunc TestSearchQueryCallback(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindex, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tquery := NewTermQuery(\"water\")\n\treq := NewSearchRequest(query)\n\n\texpErr := fmt.Errorf(\"MEM_LIMIT_EXCEEDED\")\n\tf := func(size uint64) error {\n\t\t// the intended usage of this callback is to see the estimated\n\t\t// memory usage before executing, and possibly abort early\n\t\t// in this test we simulate returning such an error\n\t\treturn expErr\n\t}\n\n\tctx := context.WithValue(context.Background(), SearchQueryStartCallbackKey, SearchQueryStartCallbackFn(f))\n\t_, err = index.SearchInContext(ctx, req)\n\tif err != expErr {\n\t\tt.Fatalf(\"Expected: %v, Got: %v\", expErr, err)\n\t}\n}\n\nfunc TestBatchMerge(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoca := map[string]interface{}{\n\t\t\"name\":   \"scorch\",\n\t\t\"desc\":   \"gophercon india\",\n\t\t\"nation\": \"india\",\n\t}\n\n\tbatchA := idx.NewBatch()\n\terr = batchA.Index(\"a\", doca)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tbatchA.SetInternal([]byte(\"batchkA\"), []byte(\"batchvA\"))\n\n\tdocb := map[string]interface{}{\n\t\t\"name\": \"moss\",\n\t\t\"desc\": \"gophercon MV\",\n\t}\n\n\tbatchB := idx.NewBatch()\n\terr = batchB.Index(\"b\", docb)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tbatchB.SetInternal([]byte(\"batchkB\"), []byte(\"batchvB\"))\n\n\tdocC := map[string]interface{}{\n\t\t\"name\":    \"blahblah\",\n\t\t\"desc\":    \"inProgress\",\n\t\t\"country\": \"usa\",\n\t}\n\n\tbatchC := idx.NewBatch()\n\terr = batchC.Index(\"c\", docC)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tbatchC.SetInternal([]byte(\"batchkC\"), []byte(\"batchvC\"))\n\tbatchC.SetInternal([]byte(\"batchkB\"), []byte(\"batchvBNew\"))\n\tbatchC.Delete(\"a\")\n\tbatchC.DeleteInternal([]byte(\"batchkA\"))\n\n\tbatchA.Merge(batchB)\n\n\tif batchA.Size() != 4 {\n\t\tt.Errorf(\"expected batch size 4, got %d\", batchA.Size())\n\t}\n\n\tbatchA.Merge(batchC)\n\n\tif batchA.Size() != 6 {\n\t\tt.Errorf(\"expected batch size 6, got %d\", batchA.Size())\n\t}\n\n\terr = idx.Batch(batchA)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// close the index, open it again, and try some more things\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err = Open(tmpIndexPath)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tcount, err := idx.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif count != 2 {\n\t\tt.Errorf(\"expected doc count 2, got %d\", count)\n\t}\n\n\tdoc, err := idx.Document(\"c\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tval, err := idx.GetInternal([]byte(\"batchkB\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif val == nil || string(val) != \"batchvBNew\" {\n\t\tt.Errorf(\"expected val: batchvBNew , got %s\", val)\n\t}\n\n\tval, err = idx.GetInternal([]byte(\"batchkA\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got %s\", val)\n\t}\n\n\tfoundNameField := false\n\tdoc.VisitFields(func(field index.Field) {\n\t\tif field.Name() == \"name\" && string(field.Value()) == \"blahblah\" {\n\t\t\tfoundNameField = true\n\t\t}\n\t})\n\tif !foundNameField {\n\t\tt.Errorf(\"expected to find field named 'name' with value 'blahblah'\")\n\t}\n\n\tfields, err := idx.Fields()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedFields := map[string]bool{\n\t\t\"_all\":    false,\n\t\t\"name\":    false,\n\t\t\"desc\":    false,\n\t\t\"country\": false,\n\t}\n\tif len(fields) < len(expectedFields) {\n\t\tt.Fatalf(\"expected %d fields got %d\", len(expectedFields), len(fields))\n\t}\n\n\tfor _, f := range fields {\n\t\texpectedFields[f] = true\n\t}\n\n\tfor ef, efp := range expectedFields {\n\t\tif !efp {\n\t\t\tt.Errorf(\"field %s is missing\", ef)\n\t\t}\n\t}\n}\n\nfunc TestBug1096(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\t// use default mapping\n\tmapping := NewIndexMapping()\n\n\t// create a scorch index with default SAFE batches\n\tvar idx Index\n\tidx, err = NewUsing(tmpIndexPath, mapping, \"scorch\", \"scorch\", nil)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// create a single batch instance that we will reuse\n\t// this should be safe because we have single goroutine\n\t// and we always wait for batch execution to finish\n\tbatch := idx.NewBatch()\n\n\t// number of batches to execute\n\tfor i := 0; i < 10; i++ {\n\n\t\t// number of documents to put into the batch\n\t\tfor j := 0; j < 91; j++ {\n\n\t\t\t// create a doc id 0-90 (important so that we get id's 9 and 90)\n\t\t\t// this could duplicate something already in the index\n\t\t\t//   this too should be OK and update the item in the index\n\t\t\tid := fmt.Sprintf(\"%d\", j)\n\n\t\t\terr = batch.Index(id, map[string]interface{}{\n\t\t\t\t\"name\":  id,\n\t\t\t\t\"batch\": fmt.Sprintf(\"%d\", i),\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatal(err)\n\t\t\t}\n\t\t}\n\n\t\t// execute the batch\n\t\terr = idx.Batch(batch)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\n\t\t// reset the batch before reusing it\n\t\tbatch.Reset()\n\t}\n\n\t// search for docs having name starting with the number 9\n\tq := NewWildcardQuery(\"9*\")\n\tq.SetField(\"name\")\n\treq := NewSearchRequestOptions(q, 1000, 0, false)\n\treq.Fields = []string{\"*\"}\n\tvar res *SearchResult\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\t// we expect only 2 hits, for docs 9 and 90\n\tif res.Total > 2 {\n\t\tt.Fatalf(\"expected only 2 hits '9' and '90', got %v\", res)\n\t}\n}\n\nfunc TestDataRaceBug1092(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\t// use default mapping\n\tmapping := NewIndexMapping()\n\n\tvar idx Index\n\tidx, err = NewUsing(tmpIndexPath, mapping, upsidedown.Name, boltdb.Name, nil)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer func() {\n\t\tcerr := idx.Close()\n\t\tif cerr != nil {\n\t\t\tt.Fatal(cerr)\n\t\t}\n\t}()\n\n\tbatch := idx.NewBatch()\n\tfor i := 0; i < 10; i++ {\n\t\terr = idx.Batch(batch)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\n\t\tbatch.Reset()\n\t}\n}\n\nfunc TestBatchRaceBug1149(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\ti, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := i.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\ttestBatchRaceBug1149(t, i)\n}\n\nfunc TestBatchRaceBug1149Scorch(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\ti, err := NewUsing(tmpIndexPath, NewIndexMapping(), \"scorch\", \"scorch\", nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := i.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\ttestBatchRaceBug1149(t, i)\n}\n\nfunc testBatchRaceBug1149(t *testing.T, i Index) {\n\tb := i.NewBatch()\n\tb.Delete(\"1\")\n\terr = i.Batch(b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tb.Reset()\n\terr = i.Batch(b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tb.Reset()\n}\n\nfunc TestOptimisedConjunctionSearchHits(t *testing.T) {\n\tscorch.OptimizeDisjunctionUnadorned = false\n\tdefer func() {\n\t\tscorch.OptimizeDisjunctionUnadorned = true\n\t}()\n\n\tdefer func() {\n\t\terr := os.RemoveAll(\"testidx\")\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tidx, err := NewUsing(\"testidx\", NewIndexMapping(), \"scorch\", \"scorch\", nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoca := map[string]interface{}{\n\t\t\"country\":    \"united\",\n\t\t\"name\":       \"Mercure Hotel\",\n\t\t\"directions\": \"B560 and B56 Follow signs to the M56\",\n\t}\n\tdocb := map[string]interface{}{\n\t\t\"country\":    \"united\",\n\t\t\"name\":       \"Mercure Altrincham Bowdon Hotel\",\n\t\t\"directions\": \"A570 and A57 Follow signs to the M56 Manchester Airport\",\n\t}\n\tdocc := map[string]interface{}{\n\t\t\"country\":    \"india united\",\n\t\t\"name\":       \"Sonoma Hotel\",\n\t\t\"directions\": \"Northwest\",\n\t}\n\tdocd := map[string]interface{}{\n\t\t\"country\":    \"United Kingdom\",\n\t\t\"name\":       \"Cresta Court Hotel\",\n\t\t\"directions\": \"junction of A560 and A56\",\n\t}\n\n\tb := idx.NewBatch()\n\terr = b.Index(\"a\", doca)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\terr = b.Index(\"b\", docb)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\terr = b.Index(\"c\", docc)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\terr = b.Index(\"d\", docd)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\t// execute the batch\n\terr = idx.Batch(b)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tmq := NewMatchQuery(\"united\")\n\tmq.SetField(\"country\")\n\n\tcq := NewConjunctionQuery(mq)\n\n\tmq1 := NewMatchQuery(\"hotel\")\n\tmq1.SetField(\"name\")\n\tcq.AddQuery(mq1)\n\n\tmq2 := NewMatchQuery(\"56\")\n\tmq2.SetField(\"directions\")\n\tmq2.SetFuzziness(1)\n\tcq.AddQuery(mq2)\n\n\treq := NewSearchRequest(cq)\n\treq.Score = \"none\"\n\n\tres, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\thitsWithOutScore := res.Total\n\n\treq = NewSearchRequest(cq)\n\treq.Score = \"\"\n\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\thitsWithScore := res.Total\n\n\tif hitsWithOutScore != hitsWithScore {\n\t\tt.Errorf(\"expected %d hits without score, got %d\", hitsWithScore, hitsWithOutScore)\n\t}\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestIndexMappingDocValuesDynamic(t *testing.T) {\n\tim := NewIndexMapping()\n\t// DocValuesDynamic's default is true\n\t// Now explicitly set it to false\n\tim.DocValuesDynamic = false\n\n\t// Next, retrieve the JSON dump of the index mapping\n\tvar data []byte\n\tdata, err = json.Marshal(im)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Now, edit an unrelated setting in the index mapping\n\tvar m map[string]interface{}\n\terr = json.Unmarshal(data, &m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tm[\"index_dynamic\"] = false\n\tdata, err = json.Marshal(m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Unmarshal back the changes into the index mapping struct\n\tif err = im.UnmarshalJSON(data); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Expect DocValuesDynamic to remain false!\n\tif im.DocValuesDynamic {\n\t\tt.Fatalf(\"Expected DocValuesDynamic to remain false after the index mapping edit\")\n\t}\n}\n\nfunc TestCopyIndex(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoca := map[string]interface{}{\n\t\t\"name\": \"tester\",\n\t\t\"desc\": \"gophercon india testing\",\n\t}\n\terr = idx.Index(\"a\", doca)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocy := map[string]interface{}{\n\t\t\"name\": \"jasper\",\n\t\t\"desc\": \"clojure\",\n\t}\n\terr = idx.Index(\"y\", docy)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\terr = idx.Delete(\"y\")\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocx := map[string]interface{}{\n\t\t\"name\": \"rose\",\n\t\t\"desc\": \"xoogler\",\n\t}\n\terr = idx.Index(\"x\", docx)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\terr = idx.SetInternal([]byte(\"status\"), []byte(\"pending\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdocb := map[string]interface{}{\n\t\t\"name\": \"sree\",\n\t\t\"desc\": \"cbft janitor\",\n\t}\n\tbatch := idx.NewBatch()\n\terr = batch.Index(\"b\", docb)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tbatch.Delete(\"x\")\n\tbatch.SetInternal([]byte(\"batchi\"), []byte(\"batchv\"))\n\tbatch.DeleteInternal([]byte(\"status\"))\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tval, err := idx.GetInternal([]byte(\"batchi\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif string(val) != \"batchv\" {\n\t\tt.Errorf(\"expected 'batchv', got '%s'\", val)\n\t}\n\tval, err = idx.GetInternal([]byte(\"status\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got '%s'\", val)\n\t}\n\n\terr = idx.SetInternal([]byte(\"seqno\"), []byte(\"7\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\terr = idx.SetInternal([]byte(\"status\"), []byte(\"ready\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\terr = idx.DeleteInternal([]byte(\"status\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tval, err = idx.GetInternal([]byte(\"status\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif val != nil {\n\t\tt.Errorf(\"expected nil, got '%s'\", val)\n\t}\n\n\tval, err = idx.GetInternal([]byte(\"seqno\"))\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif string(val) != \"7\" {\n\t\tt.Errorf(\"expected '7', got '%s'\", val)\n\t}\n\n\tcount, err := idx.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif count != 2 {\n\t\tt.Errorf(\"expected doc count 2, got %d\", count)\n\t}\n\n\tdoc, err := idx.Document(\"a\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfoundNameField := false\n\tdoc.VisitFields(func(field index.Field) {\n\t\tif field.Name() == \"name\" && string(field.Value()) == \"tester\" {\n\t\t\tfoundNameField = true\n\t\t}\n\t})\n\tif !foundNameField {\n\t\tt.Errorf(\"expected to find field named 'name' with value 'tester'\")\n\t}\n\n\tfields, err := idx.Fields()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texpectedFields := map[string]bool{\n\t\t\"_all\": false,\n\t\t\"name\": false,\n\t\t\"desc\": false,\n\t}\n\tif len(fields) < len(expectedFields) {\n\t\tt.Fatalf(\"expected %d fields got %d\", len(expectedFields), len(fields))\n\t}\n\tfor _, f := range fields {\n\t\texpectedFields[f] = true\n\t}\n\tfor ef, efp := range expectedFields {\n\t\tif !efp {\n\t\t\tt.Errorf(\"field %s is missing\", ef)\n\t\t}\n\t}\n\n\t// now create a copy of the index, and repeat assertions on it\n\tcopyableIndex, ok := idx.(IndexCopyable)\n\tif !ok {\n\t\tt.Fatal(\"index doesn't support copy\")\n\t}\n\tbackupIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, backupIndexPath)\n\n\terr = copyableIndex.CopyTo(FileSystemDirectory(backupIndexPath))\n\tif err != nil {\n\t\tt.Fatalf(\"error copying the index: %v\", err)\n\t}\n\n\t// open the copied  index\n\tidxCopied, err := Open(backupIndexPath)\n\tif err != nil {\n\t\tt.Fatalf(\"unable to open copy index\")\n\t}\n\tdefer func() {\n\t\terr := idxCopied.Close()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error closing copy index: %v\", err)\n\t\t}\n\t}()\n\n\t// assertions on copied index\n\tcopyCount, err := idxCopied.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif copyCount != 2 {\n\t\tt.Errorf(\"expected doc count 2, got %d\", copyCount)\n\t}\n\n\tcopyDoc, err := idxCopied.Document(\"a\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcopyFoundNameField := false\n\tcopyDoc.VisitFields(func(field index.Field) {\n\t\tif field.Name() == \"name\" && string(field.Value()) == \"tester\" {\n\t\t\tcopyFoundNameField = true\n\t\t}\n\t})\n\tif !copyFoundNameField {\n\t\tt.Errorf(\"expected copy index to find field named 'name' with value 'tester'\")\n\t}\n\n\tcopyFields, err := idx.Fields()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcopyExpectedFields := map[string]bool{\n\t\t\"_all\": false,\n\t\t\"name\": false,\n\t\t\"desc\": false,\n\t}\n\tif len(copyFields) < len(copyExpectedFields) {\n\t\tt.Fatalf(\"expected %d fields got %d\", len(copyExpectedFields), len(copyFields))\n\t}\n\tfor _, f := range copyFields {\n\t\tcopyExpectedFields[f] = true\n\t}\n\tfor ef, efp := range copyExpectedFields {\n\t\tif !efp {\n\t\t\tt.Errorf(\"copy field %s is missing\", ef)\n\t\t}\n\t}\n}\n\nfunc TestFuzzyScoring(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tmp := NewIndexMapping()\n\tmp.DefaultAnalyzer = \"simple\"\n\tidx, err := New(tmpIndexPath, mp)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tbatch := idx.NewBatch()\n\n\tdocs := []map[string]interface{}{\n\t\t{\n\t\t\t\"textField\": \"ab\",\n\t\t},\n\t\t{\n\t\t\t\"textField\": \"abc\",\n\t\t},\n\t\t{\n\t\t\t\"textField\": \"abcd\",\n\t\t},\n\t}\n\n\tfor _, doc := range docs {\n\t\terr := batch.Index(fmt.Sprintf(\"%v\", doc[\"textField\"]), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tquery := NewFuzzyQuery(\"ab\")\n\tquery.Fuzziness = 2\n\tsearchRequest := NewSearchRequestOptions(query, 10, 0, true)\n\tres, err := idx.Search(searchRequest)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tmaxScore := res.Hits[0].Score\n\n\tfor i, hit := range res.Hits {\n\t\tif maxScore/float64(i+1) != hit.Score {\n\t\t\tt.Errorf(\"expected score - %f, got score - %f\", maxScore/float64(i+1), hit.Score)\n\t\t}\n\t}\n\n\terr = idx.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n"
  },
  {
    "path": "index_update.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\n// Store all the fields that interact with the data\n// from a document path\ntype pathInfo struct {\n\tfieldMapInfo []*fieldMapInfo\n\tdynamic      bool\n\tpath         string\n\tanalyser     string\n\tparentPath   string\n}\n\n// Store the field information with respect to the\n// document paths\ntype fieldMapInfo struct {\n\tfieldMapping   *mapping.FieldMapping\n\tanalyzer       string\n\tdatetimeParser string\n\trootName       string\n\tparent         *pathInfo\n}\n\n// Compare two index mappings to identify all of the updatable changes\nfunc DeletedFields(ori, upd *mapping.IndexMappingImpl) (map[string]*index.UpdateFieldInfo, error) {\n\t// Compare all of the top level fields in an index mapping\n\terr := compareMappings(ori, upd)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Check for new mappings present in the type mappings\n\t// of the updated compared to the original\n\tfor name, updDMapping := range upd.TypeMapping {\n\t\terr = checkUpdatedMapping(ori.TypeMapping[name], updDMapping)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Check for new mappings present in the default mappings\n\t// of the updated compared to the original\n\terr = checkUpdatedMapping(ori.DefaultMapping, upd.DefaultMapping)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\toriPaths := make(map[string]*pathInfo)\n\tupdPaths := make(map[string]*pathInfo)\n\n\t// Go through each mapping present in the original\n\t// and consolidate according to the document paths\n\tfor name, oriDMapping := range ori.TypeMapping {\n\t\taddPathInfo(oriPaths, \"\", oriDMapping, ori, nil, name)\n\t}\n\taddPathInfo(oriPaths, \"\", ori.DefaultMapping, ori, nil, \"\")\n\n\t// Go through each mapping present in the updated\n\t// and consolidate according to the document paths\n\tfor name, updDMapping := range upd.TypeMapping {\n\t\taddPathInfo(updPaths, \"\", updDMapping, upd, nil, name)\n\t}\n\taddPathInfo(updPaths, \"\", upd.DefaultMapping, upd, nil, \"\")\n\n\t// Compare all components of custom analysis currently in use\n\terr = compareCustomComponents(oriPaths, updPaths, ori, upd)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Compare both the mappings based on the document paths\n\t// and create a list of index, docvalues, store differences\n\t// for every single field possible\n\tfieldInfo := make(map[string]*index.UpdateFieldInfo, len(oriPaths))\n\tfor path, info := range oriPaths {\n\t\terr = addFieldInfo(fieldInfo, info, updPaths[path])\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Remove entries from the list with no changes between the\n\t// original and the updated mapping\n\tfor name, info := range fieldInfo {\n\t\tif !info.Deleted && !info.Index && !info.DocValues && !info.Store {\n\t\t\tdelete(fieldInfo, name)\n\t\t}\n\t\t// A field cannot be completely deleted with any dynamic value turned on\n\t\tif info.Deleted {\n\t\t\tif upd.IndexDynamic {\n\t\t\t\treturn nil, fmt.Errorf(\"mapping cannot be removed when index dynamic is true\")\n\t\t\t}\n\t\t\tif upd.StoreDynamic {\n\t\t\t\treturn nil, fmt.Errorf(\"mapping cannot be removed when store dynamic is true\")\n\t\t\t}\n\t\t\tif upd.DocValuesDynamic {\n\t\t\t\treturn nil, fmt.Errorf(\"mapping cannot be removed when docvalues dynamic is true\")\n\t\t\t}\n\t\t}\n\t}\n\treturn fieldInfo, nil\n}\n\n// Ensures none of the top level index mapping fields have changed\nfunc compareMappings(ori, upd *mapping.IndexMappingImpl) error {\n\tif ori.TypeField != upd.TypeField &&\n\t\t(len(ori.TypeMapping) != 0 || len(upd.TypeMapping) != 0) {\n\t\treturn fmt.Errorf(\"type field cannot be changed when type mappings are present\")\n\t}\n\n\tif ori.DefaultType != upd.DefaultType {\n\t\treturn fmt.Errorf(\"default type cannot be changed\")\n\t}\n\n\tif ori.IndexDynamic != upd.IndexDynamic {\n\t\treturn fmt.Errorf(\"index dynamic cannot be changed\")\n\t}\n\n\tif ori.StoreDynamic != upd.StoreDynamic {\n\t\treturn fmt.Errorf(\"store dynamic cannot be changed\")\n\t}\n\n\tif ori.DocValuesDynamic != upd.DocValuesDynamic {\n\t\treturn fmt.Errorf(\"docvalues dynamic cannot be changed\")\n\t}\n\n\tif ori.DefaultAnalyzer != upd.DefaultAnalyzer && upd.IndexDynamic {\n\t\treturn fmt.Errorf(\"default analyser cannot be changed if index dynamic is true\")\n\t}\n\n\tif ori.DefaultDateTimeParser != upd.DefaultDateTimeParser && upd.IndexDynamic {\n\t\treturn fmt.Errorf(\"default datetime parser cannot be changed if index dynamic is true\")\n\t}\n\n\t// Scoring model changes between \"\", \"tf-idf\" and \"bm25\" require no index changes to be made\n\tif ori.ScoringModel != upd.ScoringModel {\n\t\tif ori.ScoringModel != \"\" && ori.ScoringModel != index.TFIDFScoring && ori.ScoringModel != index.BM25Scoring ||\n\t\t\tupd.ScoringModel != \"\" && upd.ScoringModel != index.TFIDFScoring && upd.ScoringModel != index.BM25Scoring {\n\t\t\treturn fmt.Errorf(\"scoring model can only be changed between \\\"\\\", %q and %q\", index.TFIDFScoring, index.BM25Scoring)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Ensures updated document mapping does not contain new\n// field mappings or document mappings\nfunc checkUpdatedMapping(ori, upd *mapping.DocumentMapping) error {\n\t// Check to verify both original and updated are not nil\n\t// and are enabled before proceeding\n\tif ori == nil {\n\t\tif upd == nil || !upd.Enabled {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"updated index mapping contains new properties\")\n\t}\n\n\tif upd == nil || !upd.Enabled {\n\t\treturn nil\n\t}\n\n\tif ori.Nested != upd.Nested {\n\t\treturn fmt.Errorf(\"nested property cannot be changed\")\n\t}\n\n\tvar err error\n\t// Recursively go through the child mappings\n\tfor name, updDMapping := range upd.Properties {\n\t\terr = checkUpdatedMapping(ori.Properties[name], updDMapping)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Simple checks to ensure no new field mappings present\n\t// in updated\n\t// Create a map of original field names for O(1) lookup\n\toriFieldNames := make(map[string]bool, len(ori.Fields))\n\tfor _, fMapping := range ori.Fields {\n\t\toriFieldNames[fMapping.Name] = true\n\t}\n\n\tfor _, updFMapping := range upd.Fields {\n\t\tif !oriFieldNames[updFMapping.Name] {\n\t\t\treturn fmt.Errorf(\"updated index mapping contains new fields\")\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Adds all of the field mappings while maintaining a tree of the document structure\n// to ensure traversal and verification is possible incase of multiple mappings defined\n// for a single field or multiple document fields' data getting written to a single zapx field\nfunc addPathInfo(paths map[string]*pathInfo, name string, mp *mapping.DocumentMapping,\n\tim *mapping.IndexMappingImpl, parent *pathInfo, rootName string) {\n\t// Early exit if mapping has been disabled\n\t// Comparisions later on will be done with a nil object\n\tif !mp.Enabled {\n\t\treturn\n\t}\n\n\t// Consolidate path information like index dynamic across multiple\n\t// mappings if path is the same\n\tvar pInfo *pathInfo\n\tif val, ok := paths[name]; ok {\n\t\tpInfo = val\n\t} else {\n\t\tpInfo = &pathInfo{\n\t\t\tfieldMapInfo: make([]*fieldMapInfo, 0),\n\t\t}\n\t\tpInfo.dynamic = mp.Dynamic && im.IndexDynamic\n\t\tpInfo.analyser = im.AnalyzerNameForPath(name)\n\t}\n\n\tpInfo.dynamic = (pInfo.dynamic || mp.Dynamic) && im.IndexDynamic\n\tpInfo.path = name\n\tif parent != nil {\n\t\tpInfo.parentPath = parent.path\n\t}\n\n\t// Recursively add path information for all child mappings\n\tfor cName, cMapping := range mp.Properties {\n\t\tpathName := cName\n\t\tif name != \"\" {\n\t\t\tpathName = name + \".\" + cName\n\t\t}\n\t\taddPathInfo(paths, pathName, cMapping, im, pInfo, rootName)\n\t}\n\n\t// Add field mapping information keeping the document structure intact\n\tfor _, fMap := range mp.Fields {\n\t\tfieldMapInfo := &fieldMapInfo{\n\t\t\tfieldMapping: fMap,\n\t\t\trootName:     rootName,\n\t\t\tparent:       pInfo,\n\t\t}\n\t\tpInfo.fieldMapInfo = append(pInfo.fieldMapInfo, fieldMapInfo)\n\t}\n\n\tpaths[name] = pInfo\n}\n\n// Compares all of the custom analysis components in use\nfunc compareCustomComponents(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error {\n\t// Compare all analysers currently in use\n\terr := compareAnalysers(oriPaths, updPaths, ori, upd)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Compare all datetime parsers currently in use\n\terr = compareDateTimeParsers(oriPaths, updPaths, ori, upd)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Compare all synonum sources\n\terr = compareSynonymSources(ori, upd)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Compare all char filters, tokenizers, token filters and token maps\n\terr = compareAnalyserSubcomponents(ori, upd)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// Compares all analysers currently in use\n// Standard analysers not in custom analysis are not compared\n// Analysers in custom analysis but not in use are not compared\nfunc compareAnalysers(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error {\n\toriAnalyzers := make(map[string]interface{})\n\tupdAnalyzers := make(map[string]interface{})\n\n\textractAnalyzers := func(paths map[string]*pathInfo, customAnalyzers map[string]map[string]interface{},\n\t\tanalyzers map[string]interface{}, indexMapping *mapping.IndexMappingImpl) {\n\t\tfor path, info := range paths {\n\t\t\tfor _, fInfo := range info.fieldMapInfo {\n\t\t\t\tif fInfo.fieldMapping.Type == \"text\" {\n\t\t\t\t\tanalyzerName := indexMapping.AnalyzerNameForPath(path)\n\t\t\t\t\tfInfo.analyzer = analyzerName\n\t\t\t\t\tif val, ok := customAnalyzers[analyzerName]; ok {\n\t\t\t\t\t\tanalyzers[analyzerName] = val\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\textractAnalyzers(oriPaths, ori.CustomAnalysis.Analyzers, oriAnalyzers, ori)\n\textractAnalyzers(updPaths, upd.CustomAnalysis.Analyzers, updAnalyzers, upd)\n\n\tfor name, anUpd := range updAnalyzers {\n\t\tif anOri, ok := oriAnalyzers[name]; ok {\n\t\t\tif !reflect.DeepEqual(anUpd, anOri) {\n\t\t\t\treturn fmt.Errorf(\"analyser %s changed while being used by fields\", name)\n\t\t\t}\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"analyser %s newly added to an existing field\", name)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Compares all date time parsers currently in use\n// Date time parsers in custom analysis but not in use are not compared\nfunc compareDateTimeParsers(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error {\n\toriDateTimeParsers := make(map[string]interface{})\n\tupdDateTimeParsers := make(map[string]interface{})\n\n\textractDateTimeParsers := func(paths map[string]*pathInfo, customParsers map[string]map[string]interface{},\n\t\tparsers map[string]interface{}, indexMapping *mapping.IndexMappingImpl) {\n\t\tfor _, info := range paths {\n\t\t\tfor _, fInfo := range info.fieldMapInfo {\n\t\t\t\tif fInfo.fieldMapping.Type == \"datetime\" {\n\t\t\t\t\tparserName := fInfo.fieldMapping.DateFormat\n\t\t\t\t\tif parserName == \"\" {\n\t\t\t\t\t\tparserName = indexMapping.DefaultDateTimeParser\n\t\t\t\t\t}\n\t\t\t\t\tfInfo.datetimeParser = parserName\n\t\t\t\t\tif val, ok := customParsers[parserName]; ok {\n\t\t\t\t\t\tparsers[parserName] = val\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\textractDateTimeParsers(oriPaths, ori.CustomAnalysis.DateTimeParsers, oriDateTimeParsers, ori)\n\textractDateTimeParsers(updPaths, upd.CustomAnalysis.DateTimeParsers, updDateTimeParsers, upd)\n\n\tfor name, dtUpd := range updDateTimeParsers {\n\t\tif dtOri, ok := oriDateTimeParsers[name]; ok {\n\t\t\tif !reflect.DeepEqual(dtUpd, dtOri) {\n\t\t\t\treturn fmt.Errorf(\"datetime parser %s changed while being used by fields\", name)\n\t\t\t}\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"datetime parser %s added to an existing field\", name)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Compares all synonym sources\n// Synonym sources currently not in use are also compared\nfunc compareSynonymSources(ori, upd *mapping.IndexMappingImpl) error {\n\tif !reflect.DeepEqual(ori.CustomAnalysis.SynonymSources, upd.CustomAnalysis.SynonymSources) {\n\t\treturn fmt.Errorf(\"synonym sources cannot be changed\")\n\t}\n\n\treturn nil\n}\n\n// Compares all char filters, tokenizers, token filters and token maps\n// Components not currently in use are also compared\nfunc compareAnalyserSubcomponents(ori, upd *mapping.IndexMappingImpl) error {\n\tif !reflect.DeepEqual(ori.CustomAnalysis.CharFilters, upd.CustomAnalysis.CharFilters) {\n\t\treturn fmt.Errorf(\"char filters cannot be changed\")\n\t}\n\n\tif !reflect.DeepEqual(ori.CustomAnalysis.TokenFilters, upd.CustomAnalysis.TokenFilters) {\n\t\treturn fmt.Errorf(\"token filters cannot be changed\")\n\t}\n\n\tif !reflect.DeepEqual(ori.CustomAnalysis.TokenMaps, upd.CustomAnalysis.TokenMaps) {\n\t\treturn fmt.Errorf(\"token maps cannot be changed\")\n\t}\n\n\tif !reflect.DeepEqual(ori.CustomAnalysis.Tokenizers, upd.CustomAnalysis.Tokenizers) {\n\t\treturn fmt.Errorf(\"tokenizers cannot be changed\")\n\t}\n\n\treturn nil\n}\n\n// Compare all of the fields at a particular document path and add its field information\nfunc addFieldInfo(fInfo map[string]*index.UpdateFieldInfo, ori, upd *pathInfo) error {\n\tvar info *index.UpdateFieldInfo\n\tvar err error\n\n\t// Assume deleted or disabled mapping if upd is nil. Checks for ori being nil\n\t// or upd having mappings not in orihave already been done before this stage\n\tif upd == nil {\n\t\tfor _, oriFMapInfo := range ori.fieldMapInfo {\n\t\t\tinfo, err = compareFieldMapping(oriFMapInfo.fieldMapping, nil)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terr = validateFieldInfo(info, fInfo, ori, oriFMapInfo)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t} else {\n\t\tif upd.dynamic && ori.analyser != upd.analyser {\n\t\t\treturn fmt.Errorf(\"analyser has been changed for a dynamic mapping\")\n\t\t}\n\t\tfor _, oriFMapInfo := range ori.fieldMapInfo {\n\t\t\tvar updFMap *mapping.FieldMapping\n\t\t\tvar updAnalyser string\n\t\t\tvar updDatetimeParser string\n\n\t\t\t// For multiple fields at a single document path, compare\n\t\t\t// only with the matching ones\n\t\t\tfor _, updFMapInfo := range upd.fieldMapInfo {\n\t\t\t\tif oriFMapInfo.rootName == updFMapInfo.rootName &&\n\t\t\t\t\toriFMapInfo.fieldMapping.Name == updFMapInfo.fieldMapping.Name {\n\t\t\t\t\tupdFMap = updFMapInfo.fieldMapping\n\t\t\t\t\tif updFMap.Type == \"text\" {\n\t\t\t\t\t\tupdAnalyser = updFMapInfo.analyzer\n\t\t\t\t\t} else if updFMap.Type == \"datetime\" {\n\t\t\t\t\t\tupdDatetimeParser = updFMapInfo.datetimeParser\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Compare analyser, datetime parser and synonym source before comparing\n\t\t\t// the field mapping as it might not have this information\n\t\t\tif updAnalyser != \"\" && oriFMapInfo.analyzer != updAnalyser {\n\t\t\t\treturn fmt.Errorf(\"analyser has been changed for a text field\")\n\t\t\t}\n\t\t\tif updDatetimeParser != \"\" && oriFMapInfo.datetimeParser != updDatetimeParser {\n\t\t\t\treturn fmt.Errorf(\"datetime parser has been changed for a date time field\")\n\t\t\t}\n\t\t\tinfo, err = compareFieldMapping(oriFMapInfo.fieldMapping, updFMap)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\t// Validate to ensure change is possible\n\t\t\t// Needed if multiple mappings are aliased to the same field\n\t\t\terr = validateFieldInfo(info, fInfo, ori, oriFMapInfo)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Compares two field mappings against each other, checking for changes in index, store, doc values\n// and complete deletiion of the mapping while noting that the changes made are doable based on\n// other values like includeInAll and dynamic\n// first return argument gives an empty fieldInfo if no changes detected\n// second return argument gives a flag indicating whether any changes, if detected, are doable or if\n// update is impossible\n// third argument is an error explaining exactly why the change is not possible\nfunc compareFieldMapping(original, updated *mapping.FieldMapping) (*index.UpdateFieldInfo, error) {\n\trv := &index.UpdateFieldInfo{}\n\n\tif updated == nil {\n\t\tif original != nil && !original.IncludeInAll {\n\t\t\trv.Deleted = true\n\t\t\treturn rv, nil\n\t\t} else if original == nil {\n\t\t\treturn nil, fmt.Errorf(\"both field mappings cannot be nil\")\n\t\t}\n\t\treturn nil, fmt.Errorf(\"deleted field present in '_all' field\")\n\t} else if original == nil {\n\t\treturn nil, fmt.Errorf(\"matching field not found in original index mapping\")\n\t}\n\n\tif original.Type != updated.Type {\n\t\treturn nil, fmt.Errorf(\"field type cannot be updated\")\n\t}\n\tif original.Type == \"text\" {\n\t\tif original.Analyzer != updated.Analyzer {\n\t\t\treturn nil, fmt.Errorf(\"analyzer cannot be updated for text fields\")\n\t\t}\n\t}\n\tif original.Type == \"datetime\" {\n\t\tif original.DateFormat != updated.DateFormat {\n\t\t\treturn nil, fmt.Errorf(\"dateFormat cannot be updated for datetime fields\")\n\t\t}\n\t}\n\tif original.Type == \"vector\" || original.Type == \"vector_base64\" {\n\t\tif original.Dims != updated.Dims {\n\t\t\treturn nil, fmt.Errorf(\"dimensions cannot be updated for vector and vector_base64 fields\")\n\t\t}\n\t\tif original.Similarity != updated.Similarity {\n\t\t\treturn nil, fmt.Errorf(\"similarity cannot be updated for vector and vector_base64 fields\")\n\t\t}\n\t\tif original.VectorIndexOptimizedFor != updated.VectorIndexOptimizedFor {\n\t\t\treturn nil, fmt.Errorf(\"vectorIndexOptimizedFor cannot be updated for vector and vector_base64 fields\")\n\t\t}\n\t}\n\tif original.IncludeInAll != updated.IncludeInAll {\n\t\treturn nil, fmt.Errorf(\"includeInAll cannot be changed\")\n\t}\n\tif original.IncludeTermVectors != updated.IncludeTermVectors {\n\t\treturn nil, fmt.Errorf(\"includeTermVectors cannot be changed\")\n\t}\n\tif original.SkipFreqNorm != updated.SkipFreqNorm {\n\t\treturn nil, fmt.Errorf(\"skipFreqNorm cannot be changed\")\n\t}\n\n\t// Updating is not possible if store changes from true\n\t// to false when the field is included in _all\n\tif original.Store != updated.Store {\n\t\tif updated.Store {\n\t\t\treturn nil, fmt.Errorf(\"store cannot be changed from false to true\")\n\t\t} else if updated.IncludeInAll {\n\t\t\treturn nil, fmt.Errorf(\"store cannot be changed if field present in `_all' field\")\n\t\t} else {\n\t\t\trv.Store = true\n\t\t}\n\t}\n\n\t// Updating is not possible if index changes from true\n\t// to false when the field is included in _all\n\tif original.Index != updated.Index {\n\t\tif updated.Index {\n\t\t\treturn nil, fmt.Errorf(\"index cannot be changed from false to true\")\n\t\t} else if updated.IncludeInAll {\n\t\t\treturn nil, fmt.Errorf(\"index cannot be changed if field present in `_all' field\")\n\t\t} else {\n\t\t\trv.Index = true\n\t\t\trv.DocValues = true\n\t\t}\n\t}\n\n\t// Updating is not possible if docvalues changes from true\n\t// to false when the field is included in _all\n\tif original.DocValues != updated.DocValues {\n\t\tif updated.DocValues {\n\t\t\treturn nil, fmt.Errorf(\"docvalues cannot be changed from false to true\")\n\t\t} else if updated.IncludeInAll {\n\t\t\treturn nil, fmt.Errorf(\"docvalues cannot be changed if field present in `_all' field\")\n\t\t} else {\n\t\t\trv.DocValues = true\n\t\t}\n\t}\n\n\treturn rv, nil\n}\n\n// After identifying changes, validate against the existing changes incase of duplicate fields.\n// In such a situation, any conflicting changes found will abort the update process\nfunc validateFieldInfo(newInfo *index.UpdateFieldInfo, fInfo map[string]*index.UpdateFieldInfo,\n\tori *pathInfo, oriFMapInfo *fieldMapInfo) error {\n\t// Determine field name\n\tfieldName := oriFMapInfo.fieldMapping.Name\n\tif fieldName == \"\" {\n\t\tfieldName = oriFMapInfo.parent.path\n\t}\n\n\t// Construct full name with parent path\n\tvar name string\n\tif oriFMapInfo.parent.parentPath == \"\" {\n\t\tname = fieldName\n\t} else {\n\t\tname = oriFMapInfo.parent.parentPath + \".\" + fieldName\n\t}\n\tif (newInfo.Deleted || newInfo.Index || newInfo.DocValues || newInfo.Store) && ori.dynamic {\n\t\treturn fmt.Errorf(\"updated field is under a dynamic property\")\n\t}\n\tif oldInfo, ok := fInfo[name]; ok {\n\t\tif !reflect.DeepEqual(oldInfo, newInfo) {\n\t\t\treturn fmt.Errorf(\"updated field impossible to verify because multiple mappings point to the same field name\")\n\t\t}\n\t} else {\n\t\tfInfo[name] = newInfo\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "index_update_test.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/custom\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/simple\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/percent\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/sanitized\"\n\t\"github.com/blevesearch/bleve/v2/analysis/lang/en\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch/mergeplan\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestCompareFieldMapping(t *testing.T) {\n\ttests := []struct {\n\t\toriginal       *mapping.FieldMapping\n\t\tupdated        *mapping.FieldMapping\n\t\tindexFieldInfo *index.UpdateFieldInfo\n\t\terr            bool\n\t}{\n\t\t{ // both nil => error\n\t\t\toriginal:       nil,\n\t\t\tupdated:        nil,\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ // updated nil => delete all\n\t\t\toriginal: &mapping.FieldMapping{},\n\t\t\tupdated:  nil,\n\t\t\tindexFieldInfo: &index.UpdateFieldInfo{\n\t\t\t\tDeleted: true,\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{ // type changed => not updatable\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType: \"text\",\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType: \"datetime\",\n\t\t\t},\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ // synonym source changed for text => updatable\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:          \"text\",\n\t\t\t\tSynonymSource: \"a\",\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:          \"text\",\n\t\t\t\tSynonymSource: \"b\",\n\t\t\t},\n\t\t\tindexFieldInfo: &index.UpdateFieldInfo{},\n\t\t\terr:            false,\n\t\t},\n\t\t{ // analyser changed for text => not updatable\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:     \"text\",\n\t\t\t\tAnalyzer: \"a\",\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:     \"text\",\n\t\t\t\tAnalyzer: \"b\",\n\t\t\t},\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ // dims changed for vector => not updatable\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:                    \"vector\",\n\t\t\t\tDims:                    128,\n\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\tVectorIndexOptimizedFor: \"memory-efficient\",\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:                    \"vector\",\n\t\t\t\tDims:                    1024,\n\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\tVectorIndexOptimizedFor: \"memory-efficient\",\n\t\t\t},\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ // similarity changed for vectorbase64 => not updatable\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:                    \"vector_base64\",\n\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\tDims:                    128,\n\t\t\t\tVectorIndexOptimizedFor: \"memory-efficient\",\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:                    \"vector_base64\",\n\t\t\t\tSimilarity:              \"dot_product\",\n\t\t\t\tDims:                    128,\n\t\t\t\tVectorIndexOptimizedFor: \"memory-efficient\",\n\t\t\t},\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ // vectorindexoptimizedfor chagned for vector => not updatable\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:                    \"vector\",\n\t\t\t\tSimilarity:              \"dot_product\",\n\t\t\t\tDims:                    128,\n\t\t\t\tVectorIndexOptimizedFor: \"memory-efficient\",\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:                    \"vector\",\n\t\t\t\tSimilarity:              \"dot_product\",\n\t\t\t\tDims:                    128,\n\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t},\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ // includeinall changed => not updatable\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:         \"numeric\",\n\t\t\t\tIncludeInAll: true,\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:         \"numeric\",\n\t\t\t\tIncludeInAll: false,\n\t\t\t},\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ //includetermvectors changed => not updatable\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:               \"numeric\",\n\t\t\t\tIncludeTermVectors: false,\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:               \"numeric\",\n\t\t\t\tIncludeTermVectors: true,\n\t\t\t},\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ // store changed after all checks => updatable with store delete\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:         \"numeric\",\n\t\t\t\tSkipFreqNorm: true,\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:         \"numeric\",\n\t\t\t\tSkipFreqNorm: false,\n\t\t\t},\n\t\t\tindexFieldInfo: nil,\n\t\t\terr:            true,\n\t\t},\n\t\t{ // index changed after all checks => updatable with index and docvalues delete\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:  \"geopoint\",\n\t\t\t\tIndex: true,\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:  \"geopoint\",\n\t\t\t\tIndex: false,\n\t\t\t},\n\t\t\tindexFieldInfo: &index.UpdateFieldInfo{\n\t\t\t\tIndex:     true,\n\t\t\t\tDocValues: true,\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{ // docvalues changed after all checks => docvalues delete\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tType:      \"numeric\",\n\t\t\t\tDocValues: true,\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tType:      \"numeric\",\n\t\t\t\tDocValues: false,\n\t\t\t},\n\t\t\tindexFieldInfo: &index.UpdateFieldInfo{\n\t\t\t\tDocValues: true,\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{ // no relavent changes => continue but no op\n\t\t\toriginal: &mapping.FieldMapping{\n\t\t\t\tName:                    \"\",\n\t\t\t\tType:                    \"datetime\",\n\t\t\t\tAnalyzer:                \"a\",\n\t\t\t\tStore:                   true,\n\t\t\t\tIndex:                   false,\n\t\t\t\tIncludeTermVectors:      true,\n\t\t\t\tIncludeInAll:            false,\n\t\t\t\tDateFormat:              \"a\",\n\t\t\t\tDocValues:               false,\n\t\t\t\tSkipFreqNorm:            true,\n\t\t\t\tDims:                    128,\n\t\t\t\tSimilarity:              \"dot_product\",\n\t\t\t\tVectorIndexOptimizedFor: \"memory-efficient\",\n\t\t\t\tSynonymSource:           \"a\",\n\t\t\t},\n\t\t\tupdated: &mapping.FieldMapping{\n\t\t\t\tName:                    \"\",\n\t\t\t\tType:                    \"datetime\",\n\t\t\t\tAnalyzer:                \"b\",\n\t\t\t\tStore:                   true,\n\t\t\t\tIndex:                   false,\n\t\t\t\tIncludeTermVectors:      true,\n\t\t\t\tIncludeInAll:            false,\n\t\t\t\tDateFormat:              \"a\",\n\t\t\t\tDocValues:               false,\n\t\t\t\tSkipFreqNorm:            true,\n\t\t\t\tDims:                    256,\n\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t\tSynonymSource:           \"b\",\n\t\t\t},\n\t\t\tindexFieldInfo: &index.UpdateFieldInfo{},\n\t\t\terr:            false,\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\trv, err := compareFieldMapping(test.original, test.updated)\n\n\t\tif err == nil && test.err || err != nil && !test.err {\n\t\t\tt.Errorf(\"Unexpected error value for test %d, expecting %t, got %v\\n\", i, test.err, err)\n\t\t}\n\t\tif rv == nil && test.indexFieldInfo != nil || rv != nil && test.indexFieldInfo == nil || !reflect.DeepEqual(rv, test.indexFieldInfo) {\n\t\t\tt.Errorf(\"Unexpected index field info value for test %d, expecting %+v, got %+v, err %v\", i, test.indexFieldInfo, rv, err)\n\t\t}\n\t}\n}\n\nfunc TestCompareMappings(t *testing.T) {\n\ttests := []struct {\n\t\toriginal *mapping.IndexMappingImpl\n\t\tupdated  *mapping.IndexMappingImpl\n\t\terr      bool\n\t}{\n\t\t{ // changed type field when non empty mappings are present => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeField: \"a\",\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"a\": {},\n\t\t\t\t\t\"b\": {},\n\t\t\t\t},\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeField: \"b\",\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"a\": {},\n\t\t\t\t\t\"b\": {},\n\t\t\t\t},\n\t\t\t},\n\t\t\terr: true,\n\t\t},\n\t\t{ // changed default type => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultType: \"a\",\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultType: \"b\",\n\t\t\t},\n\t\t\terr: true,\n\t\t},\n\t\t{ // changed default analyzer => analyser true\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultAnalyzer: \"a\",\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultAnalyzer: \"b\",\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{ // changed default datetimeparser => datetimeparser true\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultDateTimeParser: \"a\",\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultDateTimeParser: \"b\",\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{ // changed default synonym source => synonym source true\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultSynonymSource: \"a\",\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultSynonymSource: \"b\",\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{ // changed default field => false\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultField: \"a\",\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tDefaultField: \"b\",\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{ // changed index dynamic => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tIndexDynamic: true,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tIndexDynamic: false,\n\t\t\t},\n\t\t\terr: true,\n\t\t},\n\t\t{ // changed store dynamic => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tStoreDynamic: false,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tStoreDynamic: true,\n\t\t\t},\n\t\t\terr: true,\n\t\t},\n\t\t{ // changed docvalues dynamic => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tDocValuesDynamic: true,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t},\n\t\t\terr: true,\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\terr := compareMappings(test.original, test.updated)\n\n\t\tif err == nil && test.err || err != nil && !test.err {\n\t\t\tt.Errorf(\"Unexpected error value for test %d, expecting %t, got %v\\n\", i, test.err, err)\n\t\t}\n\t}\n}\n\nfunc TestCompareAnalysers(t *testing.T) {\n\n\tori := mapping.NewIndexMapping()\n\tori.DefaultMapping.AddFieldMappingsAt(\"a\", NewTextFieldMapping())\n\tori.DefaultMapping.AddFieldMappingsAt(\"b\", NewTextFieldMapping())\n\tori.DefaultMapping.AddFieldMappingsAt(\"c\", NewTextFieldMapping())\n\tori.DefaultMapping.Properties[\"b\"].DefaultAnalyzer = \"3xbla\"\n\tori.DefaultMapping.Properties[\"c\"].DefaultAnalyzer = simple.Name\n\n\tupd := mapping.NewIndexMapping()\n\tupd.DefaultMapping.AddFieldMappingsAt(\"a\", NewTextFieldMapping())\n\tupd.DefaultMapping.AddFieldMappingsAt(\"b\", NewTextFieldMapping())\n\tupd.DefaultMapping.AddFieldMappingsAt(\"c\", NewTextFieldMapping())\n\tupd.DefaultMapping.Properties[\"b\"].DefaultAnalyzer = \"3xbla\"\n\tupd.DefaultMapping.Properties[\"c\"].DefaultAnalyzer = simple.Name\n\n\tif err := ori.AddCustomAnalyzer(\"3xbla\", map[string]interface{}{\n\t\t\"type\":          custom.Name,\n\t\t\"tokenizer\":     whitespace.Name,\n\t\t\"token_filters\": []interface{}{lowercase.Name, \"stop_en\"},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := upd.AddCustomAnalyzer(\"3xbla\", map[string]interface{}{\n\t\t\"type\":          custom.Name,\n\t\t\"tokenizer\":     whitespace.Name,\n\t\t\"token_filters\": []interface{}{lowercase.Name, \"stop_en\"},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\toriPaths := map[string]*pathInfo{\n\t\t\"a\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"text\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"a\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t\t\"b\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"text\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"b\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t\t\"c\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"text\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"c\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t}\n\n\tupdPaths := map[string]*pathInfo{\n\t\t\"a\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"text\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"a\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t\t\"b\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"text\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"b\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t\t\"c\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"text\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"c\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t}\n\n\t// Test case has identical analysers for text fields\n\terr := compareAnalysers(oriPaths, updPaths, ori, upd)\n\tif err != nil {\n\t\tt.Errorf(\"Expected error to be nil, got %v\", err)\n\t}\n\n\tori2 := mapping.NewIndexMapping()\n\tori2.DefaultMapping.AddFieldMappingsAt(\"a\", NewTextFieldMapping())\n\tori2.DefaultMapping.AddFieldMappingsAt(\"b\", NewTextFieldMapping())\n\tori2.DefaultMapping.AddFieldMappingsAt(\"c\", NewTextFieldMapping())\n\tori2.DefaultMapping.Properties[\"b\"].DefaultAnalyzer = \"3xbla\"\n\tori2.DefaultMapping.Properties[\"c\"].DefaultAnalyzer = simple.Name\n\n\tupd2 := mapping.NewIndexMapping()\n\tupd2.DefaultMapping.AddFieldMappingsAt(\"a\", NewTextFieldMapping())\n\tupd2.DefaultMapping.AddFieldMappingsAt(\"b\", NewTextFieldMapping())\n\tupd2.DefaultMapping.AddFieldMappingsAt(\"c\", NewTextFieldMapping())\n\tupd2.DefaultMapping.Properties[\"b\"].DefaultAnalyzer = \"3xbla\"\n\tupd2.DefaultMapping.Properties[\"c\"].DefaultAnalyzer = simple.Name\n\n\tif err := ori2.AddCustomAnalyzer(\"3xbla\", map[string]interface{}{\n\t\t\"type\":          custom.Name,\n\t\t\"tokenizer\":     whitespace.Name,\n\t\t\"token_filters\": []interface{}{lowercase.Name, \"stop_en\"},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := upd2.AddCustomAnalyzer(\"3xbla\", map[string]interface{}{\n\t\t\"type\":          custom.Name,\n\t\t\"tokenizer\":     letter.Name,\n\t\t\"token_filters\": []interface{}{lowercase.Name, \"stop_en\"},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Test case has different custom analyser for field \"b\"\n\terr = compareAnalysers(oriPaths, updPaths, ori2, upd2)\n\tif err == nil {\n\t\tt.Errorf(\"Expected error, got nil\")\n\t}\n}\n\nfunc TestCompareDatetimeParsers(t *testing.T) {\n\n\tori := mapping.NewIndexMapping()\n\tori.DefaultMapping.AddFieldMappingsAt(\"a\", NewDateTimeFieldMapping())\n\tori.DefaultMapping.AddFieldMappingsAt(\"b\", NewDateTimeFieldMapping())\n\tori.DefaultMapping.AddFieldMappingsAt(\"c\", NewDateTimeFieldMapping())\n\tori.DefaultMapping.Properties[\"b\"].Fields[0].DateFormat = \"customDT\"\n\tori.DefaultMapping.Properties[\"c\"].Fields[0].DateFormat = percent.Name\n\n\tupd := mapping.NewIndexMapping()\n\tupd.DefaultMapping.AddFieldMappingsAt(\"a\", NewDateTimeFieldMapping())\n\tupd.DefaultMapping.AddFieldMappingsAt(\"b\", NewDateTimeFieldMapping())\n\tupd.DefaultMapping.AddFieldMappingsAt(\"c\", NewDateTimeFieldMapping())\n\tupd.DefaultMapping.Properties[\"b\"].Fields[0].DateFormat = \"customDT\"\n\tupd.DefaultMapping.Properties[\"c\"].Fields[0].DateFormat = percent.Name\n\n\terr := ori.AddCustomDateTimeParser(\"customDT\", map[string]interface{}{\n\t\t\"type\": sanitized.Name,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"02/01/2006 15:04:05\",\n\t\t\t\"2006/01/02 3:04PM\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = upd.AddCustomDateTimeParser(\"customDT\", map[string]interface{}{\n\t\t\"type\": sanitized.Name,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"02/01/2006 15:04:05\",\n\t\t\t\"2006/01/02 3:04PM\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\toriPaths := map[string]*pathInfo{\n\t\t\"a\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"datetime\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"a\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t\t\"b\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType:       \"datetime\",\n\t\t\t\t\t\tDateFormat: \"customDT\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"b\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t\t\"c\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"datetime\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"c\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t}\n\n\tupdPaths := map[string]*pathInfo{\n\t\t\"a\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"datetime\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"a\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t\t\"b\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType:       \"datetime\",\n\t\t\t\t\t\tDateFormat: \"customDT\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"b\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t\t\"c\": {\n\t\t\tfieldMapInfo: []*fieldMapInfo{\n\t\t\t\t{\n\t\t\t\t\tfieldMapping: &mapping.FieldMapping{\n\t\t\t\t\t\tType: \"datetime\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdynamic:    false,\n\t\t\tpath:       \"c\",\n\t\t\tparentPath: \"\",\n\t\t},\n\t}\n\n\t// Test case has identical datetime parsers for all fields\n\terr = compareDateTimeParsers(oriPaths, updPaths, ori, upd)\n\tif err != nil {\n\t\tt.Fatalf(\"Expected error to be nil, got %v\", err)\n\t}\n\n\tori2 := mapping.NewIndexMapping()\n\tori2.DefaultMapping.AddFieldMappingsAt(\"a\", NewDateTimeFieldMapping())\n\tori2.DefaultMapping.AddFieldMappingsAt(\"b\", NewDateTimeFieldMapping())\n\tori2.DefaultMapping.AddFieldMappingsAt(\"c\", NewDateTimeFieldMapping())\n\tori2.DefaultMapping.Properties[\"b\"].Fields[0].DateFormat = \"customDT\"\n\tori2.DefaultMapping.Properties[\"c\"].Fields[0].DateFormat = percent.Name\n\n\tupd2 := mapping.NewIndexMapping()\n\tupd2.DefaultMapping.AddFieldMappingsAt(\"a\", NewDateTimeFieldMapping())\n\tupd2.DefaultMapping.AddFieldMappingsAt(\"b\", NewDateTimeFieldMapping())\n\tupd2.DefaultMapping.AddFieldMappingsAt(\"c\", NewDateTimeFieldMapping())\n\tupd2.DefaultMapping.Properties[\"b\"].Fields[0].DateFormat = \"customDT\"\n\tupd2.DefaultMapping.Properties[\"c\"].Fields[0].DateFormat = percent.Name\n\n\terr = ori2.AddCustomDateTimeParser(\"customDT\", map[string]interface{}{\n\t\t\"type\": sanitized.Name,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"02/01/2006 15:04:05\",\n\t\t\t\"2006/01/02 3:04PM\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = upd2.AddCustomDateTimeParser(\"customDT\", map[string]interface{}{\n\t\t\"type\": sanitized.Name,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"02/01/2006 15:04:05\",\n\t\t\t\"2006/01/02\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test case has different custom datetime parser for field \"b\"\n\terr = compareDateTimeParsers(oriPaths, updPaths, ori2, upd2)\n\tif err == nil {\n\t\tt.Errorf(\"Expected error, got nil\")\n\t}\n}\n\nfunc TestCompareSynonymSources(t *testing.T) {\n\n\tori := mapping.NewIndexMapping()\n\tori.DefaultMapping.AddFieldMappingsAt(\"a\", NewTextFieldMapping())\n\tori.DefaultMapping.AddFieldMappingsAt(\"b\", NewTextFieldMapping())\n\tori.DefaultMapping.DefaultSynonymSource = \"syn1\"\n\tori.DefaultMapping.Properties[\"b\"].Fields[0].SynonymSource = \"syn2\"\n\n\tupd := mapping.NewIndexMapping()\n\tupd.DefaultMapping.AddFieldMappingsAt(\"a\", NewTextFieldMapping())\n\tupd.DefaultMapping.AddFieldMappingsAt(\"b\", NewTextFieldMapping())\n\tupd.DefaultMapping.DefaultSynonymSource = \"syn1\"\n\tupd.DefaultMapping.Properties[\"b\"].Fields[0].SynonymSource = \"syn2\"\n\n\terr := ori.AddSynonymSource(\"syn1\", map[string]interface{}{\n\t\t\"collection\": \"col1\",\n\t\t\"analyzer\":   simple.Name,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = ori.AddSynonymSource(\"syn2\", map[string]interface{}{\n\t\t\"collection\": \"col2\",\n\t\t\"analyzer\":   standard.Name,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = upd.AddSynonymSource(\"syn1\", map[string]interface{}{\n\t\t\"collection\": \"col1\",\n\t\t\"analyzer\":   simple.Name,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = upd.AddSynonymSource(\"syn2\", map[string]interface{}{\n\t\t\"collection\": \"col2\",\n\t\t\"analyzer\":   standard.Name,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Test case has identical synonym sources\n\terr = compareSynonymSources(ori, upd)\n\tif err != nil {\n\t\tt.Errorf(\"Expected error to be nil, got %v\", err)\n\t}\n\n\tori2 := mapping.NewIndexMapping()\n\tori2.DefaultMapping.AddFieldMappingsAt(\"a\", NewTextFieldMapping())\n\tori2.DefaultMapping.AddFieldMappingsAt(\"b\", NewTextFieldMapping())\n\tori2.DefaultMapping.DefaultSynonymSource = \"syn1\"\n\tori2.DefaultMapping.Properties[\"b\"].Fields[0].SynonymSource = \"syn2\"\n\n\tupd2 := mapping.NewIndexMapping()\n\tupd2.DefaultMapping.AddFieldMappingsAt(\"a\", NewTextFieldMapping())\n\tupd2.DefaultMapping.AddFieldMappingsAt(\"b\", NewTextFieldMapping())\n\tupd2.DefaultMapping.DefaultSynonymSource = \"syn1\"\n\tupd2.DefaultMapping.Properties[\"b\"].Fields[0].SynonymSource = \"syn2\"\n\n\terr = ori2.AddSynonymSource(\"syn1\", map[string]interface{}{\n\t\t\"collection\": \"col1\",\n\t\t\"analyzer\":   simple.Name,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = ori2.AddSynonymSource(\"syn2\", map[string]interface{}{\n\t\t\"collection\": \"col2\",\n\t\t\"analyzer\":   standard.Name,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = upd2.AddSynonymSource(\"syn1\", map[string]interface{}{\n\t\t\"collection\": \"col1\",\n\t\t\"analyzer\":   simple.Name,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = upd2.AddSynonymSource(\"syn2\", map[string]interface{}{\n\t\t\"collection\": \"col3\",\n\t\t\"analyzer\":   standard.Name,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test case has different synonym sources\n\terr = compareSynonymSources(ori2, upd2)\n\tif err == nil {\n\t\tt.Errorf(\"Expected error, got nil\")\n\t}\n}\n\nfunc TestDeletedFields(t *testing.T) {\n\ttests := []struct {\n\t\toriginal  *mapping.IndexMappingImpl\n\t\tupdated   *mapping.IndexMappingImpl\n\t\tfieldInfo map[string]*index.UpdateFieldInfo\n\t\terr       bool\n\t}{\n\t\t{\n\t\t\t// changed default analyzer with index dynamic\n\t\t\t// => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping:      map[string]*mapping.DocumentMapping{},\n\t\t\t\tDefaultMapping:   &mapping.DocumentMapping{},\n\t\t\t\tDefaultAnalyzer:  standard.Name,\n\t\t\t\tIndexDynamic:     true,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping:      map[string]*mapping.DocumentMapping{},\n\t\t\t\tDefaultMapping:   &mapping.DocumentMapping{},\n\t\t\t\tDefaultAnalyzer:  simple.Name,\n\t\t\t\tIndexDynamic:     true,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: nil,\n\t\t\terr:       true,\n\t\t},\n\t\t{\n\t\t\t// changed default analyzer within a mapping with index dynamic\n\t\t\t// => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled:         true,\n\t\t\t\t\tDynamic:         true,\n\t\t\t\t\tDefaultAnalyzer: standard.Name,\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:  \"\",\n\t\t\t\tIndexDynamic:     true,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled:         true,\n\t\t\t\t\tDynamic:         true,\n\t\t\t\t\tDefaultAnalyzer: simple.Name,\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     true,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: nil,\n\t\t\terr:       true,\n\t\t},\n\t\t{\n\t\t\t// changed default datetime parser with index dynamic\n\t\t\t// => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping:           map[string]*mapping.DocumentMapping{},\n\t\t\t\tDefaultMapping:        &mapping.DocumentMapping{},\n\t\t\t\tDefaultDateTimeParser: percent.Name,\n\t\t\t\tIndexDynamic:          true,\n\t\t\t\tStoreDynamic:          false,\n\t\t\t\tDocValuesDynamic:      false,\n\t\t\t\tCustomAnalysis:        NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping:           map[string]*mapping.DocumentMapping{},\n\t\t\t\tDefaultMapping:        &mapping.DocumentMapping{},\n\t\t\t\tDefaultDateTimeParser: sanitized.Name,\n\t\t\t\tIndexDynamic:          true,\n\t\t\t\tStoreDynamic:          false,\n\t\t\t\tDocValuesDynamic:      false,\n\t\t\t\tCustomAnalysis:        NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: nil,\n\t\t\terr:       true,\n\t\t},\n\t\t{\n\t\t\t// no change between original and updated having type and default mapping\n\t\t\t// => empty fieldInfo with no error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: map[string]*index.UpdateFieldInfo{},\n\t\t\terr:       false,\n\t\t},\n\t\t{\n\t\t\t// no changes in type mappings and default mapping disabled with changes\n\t\t\t// => empty fieldInfo with no error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: false,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: false,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"d\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: map[string]*index.UpdateFieldInfo{},\n\t\t\terr:       false,\n\t\t},\n\t\t{\n\t\t\t// new type mappings in updated => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled:              true,\n\t\t\t\t\tDynamic:              false,\n\t\t\t\t\tProperties:           map[string]*mapping.DocumentMapping{},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled:              true,\n\t\t\t\t\tDynamic:              false,\n\t\t\t\t\tProperties:           map[string]*mapping.DocumentMapping{},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: nil,\n\t\t\terr:       true,\n\t\t},\n\t\t{\n\t\t\t// new mappings in default mapping => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: nil,\n\t\t\terr:       true,\n\t\t},\n\t\t{\n\t\t\t// fully removed mapping in type with some dynamic => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     true,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: nil,\n\t\t\terr:       true,\n\t\t},\n\t\t{\n\t\t\t// semi removed mapping in default with some dynamic\n\t\t\t// proper fieldInfo with no errors\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: false,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: map[string]*index.UpdateFieldInfo{\n\t\t\t\t\"b\": {\n\t\t\t\t\tIndex:     true,\n\t\t\t\t\tDocValues: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{\n\t\t\t// two fields from diff paths with removed content matching\n\t\t\t// => relavent fieldInfo\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: false,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: false,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: map[string]*index.UpdateFieldInfo{\n\t\t\t\t\"a\": {\n\t\t\t\t\tIndex:     true,\n\t\t\t\t\tDocValues: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{\n\t\t\t// two fields from diff paths with removed content not matching\n\t\t\t// => error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: false,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: nil,\n\t\t\terr:       true,\n\t\t},\n\t\t{\n\t\t\t// two fields from the same path => relavent fieldInfo\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tName:  \"a\",\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t\tStore: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tName:  \"b\",\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t\tStore: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tName:  \"a\",\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: false,\n\t\t\t\t\t\t\t\t\t\tStore: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tName:  \"b\",\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t\tStore: false,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: map[string]*index.UpdateFieldInfo{\n\t\t\t\t\"a\": {\n\t\t\t\t\tIndex:     true,\n\t\t\t\t\tDocValues: true,\n\t\t\t\t},\n\t\t\t\t\"b\": {\n\t\t\t\t\tStore: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t\t{\n\t\t\t// one store, one index, one dynamic and one all removed in type and default\n\t\t\t// => relavent fieldInfo without error\n\t\t\toriginal: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tStore: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map3\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:      \"numeric\",\n\t\t\t\t\t\t\t\t\t\tDocValues: true,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled: true,\n\t\t\t\t\tDynamic: false,\n\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\"d\": {\n\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tType:      \"numeric\",\n\t\t\t\t\t\t\t\t\tIndex:     true,\n\t\t\t\t\t\t\t\t\tStore:     true,\n\t\t\t\t\t\t\t\t\tDocValues: true,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tupdated: &mapping.IndexMappingImpl{\n\t\t\t\tTypeMapping: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\"map1\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tIndex: false,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map2\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:  \"numeric\",\n\t\t\t\t\t\t\t\t\t\tStore: false,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t\t\"map3\": {\n\t\t\t\t\t\tEnabled: true,\n\t\t\t\t\t\tDynamic: false,\n\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\t\tEnabled:    true,\n\t\t\t\t\t\t\t\tDynamic:    false,\n\t\t\t\t\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\t\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tType:      \"numeric\",\n\t\t\t\t\t\t\t\t\t\tDocValues: false,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultMapping: &mapping.DocumentMapping{\n\t\t\t\t\tEnabled:              true,\n\t\t\t\t\tDynamic:              false,\n\t\t\t\t\tProperties:           map[string]*mapping.DocumentMapping{},\n\t\t\t\t\tFields:               []*mapping.FieldMapping{},\n\t\t\t\t\tDefaultAnalyzer:      \"\",\n\t\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t\t},\n\t\t\t\tIndexDynamic:     false,\n\t\t\t\tStoreDynamic:     false,\n\t\t\t\tDocValuesDynamic: false,\n\t\t\t\tCustomAnalysis:   NewIndexMapping().CustomAnalysis,\n\t\t\t},\n\t\t\tfieldInfo: map[string]*index.UpdateFieldInfo{\n\t\t\t\t\"a\": {\n\t\t\t\t\tIndex:     true,\n\t\t\t\t\tDocValues: true,\n\t\t\t\t},\n\t\t\t\t\"b\": {\n\t\t\t\t\tStore: true,\n\t\t\t\t},\n\t\t\t\t\"c\": {\n\t\t\t\t\tDocValues: true,\n\t\t\t\t},\n\t\t\t\t\"d\": {\n\t\t\t\t\tDeleted: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\terr: false,\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tinfo, err := DeletedFields(test.original, test.updated)\n\n\t\tif err == nil && test.err || err != nil && !test.err {\n\t\t\tt.Errorf(\"Unexpected error value for test %d, expecting %t, got %v\\n\", i, test.err, err)\n\t\t}\n\t\tif info == nil && test.fieldInfo != nil || info != nil && test.fieldInfo == nil || !reflect.DeepEqual(info, test.fieldInfo) {\n\t\t\tt.Errorf(\"Unexpected default info value for test %d, expecting %+v, got %+v, err %v\", i, test.fieldInfo, info, err)\n\t\t}\n\t}\n}\n\nfunc TestIndexUpdateText(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindexMappingBefore := mapping.NewIndexMapping()\n\tindexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{}\n\tindexMappingBefore.DefaultMapping = &mapping.DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: false,\n\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\"a\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"b\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"c\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"d\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t},\n\t\tFields:               []*mapping.FieldMapping{},\n\t\tDefaultAnalyzer:      \"standard\",\n\t\tDefaultSynonymSource: \"\",\n\t}\n\tindexMappingBefore.IndexDynamic = false\n\tindexMappingBefore.StoreDynamic = false\n\tindexMappingBefore.DocValuesDynamic = false\n\n\tindex, err := New(tmpIndexPath, indexMappingBefore)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc1 := map[string]interface{}{\"a\": \"xyz\", \"b\": \"abc\", \"c\": \"def\", \"d\": \"ghi\"}\n\tdoc2 := map[string]interface{}{\"a\": \"uvw\", \"b\": \"rst\", \"c\": \"klm\", \"d\": \"pqr\"}\n\tdoc3 := map[string]interface{}{\"a\": \"xyz\", \"b\": \"def\", \"c\": \"abc\", \"d\": \"mno\"}\n\tbatch := index.NewBatch()\n\terr = batch.Index(\"001\", doc1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = batch.Index(\"002\", doc2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = batch.Index(\"003\", doc3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexMappingAfter := mapping.NewIndexMapping()\n\tindexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{}\n\tindexMappingAfter.DefaultMapping = &mapping.DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: false,\n\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\"a\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"b\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: false,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"c\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t},\n\t\tFields:               []*mapping.FieldMapping{},\n\t\tDefaultAnalyzer:      \"standard\",\n\t\tDefaultSynonymSource: \"\",\n\t}\n\tindexMappingAfter.IndexDynamic = false\n\tindexMappingAfter.StoreDynamic = false\n\tindexMappingAfter.DocValuesDynamic = false\n\n\tmappingString, err := json.Marshal(indexMappingAfter)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tconfig := map[string]interface{}{\n\t\t\"updated_mapping\": string(mappingString),\n\t}\n\n\tindex, err = OpenUsing(tmpIndexPath, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tq1 := NewSearchRequest(NewQueryStringQuery(\"a:*\"))\n\tq1.Fields = append(q1.Fields, \"a\")\n\tres1, err := index.Search(q1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res1.Hits) != 3 {\n\t\tt.Errorf(\"Expected 3 hits, got %d\\n\", len(res1.Hits))\n\t}\n\tif len(res1.Hits[0].Fields) != 1 {\n\t\tt.Errorf(\"Expected 1 field, got %d\\n\", len(res1.Hits[0].Fields))\n\t}\n\tq2 := NewSearchRequest(NewQueryStringQuery(\"b:*\"))\n\tq2.Fields = append(q2.Fields, \"b\")\n\tres2, err := index.Search(q2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res2.Hits) != 0 {\n\t\tt.Errorf(\"Expected 0 hits, got %d\\n\", len(res2.Hits))\n\t}\n\tq3 := NewSearchRequest(NewQueryStringQuery(\"c:*\"))\n\tq3.Fields = append(q3.Fields, \"c\")\n\tres3, err := index.Search(q3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res3.Hits) != 3 {\n\t\tt.Errorf(\"Expected 3 hits, got %d\\n\", len(res3.Hits))\n\t}\n\tif len(res3.Hits[0].Fields) != 0 {\n\t\tt.Errorf(\"Expected 0 fields, got %d\\n\", len(res3.Hits[0].Fields))\n\t}\n\tq4 := NewSearchRequest(NewQueryStringQuery(\"d:*\"))\n\tq4.Fields = append(q4.Fields, \"d\")\n\tres4, err := index.Search(q4)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res4.Hits) != 0 {\n\t\tt.Errorf(\"Expected 0 hits, got %d\\n\", len(res4.Hits))\n\t}\n}\n\nfunc TestIndexUpdateSynonym(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tsynonymCollection := \"collection1\"\n\tsynonymSourceName := \"english\"\n\tanalyzer := en.AnalyzerName\n\tsynonymSourceConfig := map[string]interface{}{\n\t\t\"collection\": synonymCollection,\n\t\t\"analyzer\":   analyzer,\n\t}\n\n\ta := mapping.NewTextFieldMapping()\n\ta.Analyzer = analyzer\n\ta.SynonymSource = synonymSourceName\n\ta.IncludeInAll = false\n\n\tb := mapping.NewTextFieldMapping()\n\tb.Analyzer = analyzer\n\tb.SynonymSource = synonymSourceName\n\tb.IncludeInAll = false\n\n\tc := mapping.NewTextFieldMapping()\n\tc.Analyzer = analyzer\n\tc.SynonymSource = synonymSourceName\n\tc.IncludeInAll = false\n\n\tindexMappingBefore := mapping.NewIndexMapping()\n\tindexMappingBefore.DefaultMapping.AddFieldMappingsAt(\"a\", a)\n\tindexMappingBefore.DefaultMapping.AddFieldMappingsAt(\"b\", b)\n\tindexMappingBefore.DefaultMapping.AddFieldMappingsAt(\"c\", c)\n\terr := indexMappingBefore.AddSynonymSource(synonymSourceName, synonymSourceConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexMappingBefore.IndexDynamic = false\n\tindexMappingBefore.StoreDynamic = false\n\tindexMappingBefore.DocValuesDynamic = false\n\n\tindex, err := New(tmpIndexPath, indexMappingBefore)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc1 := map[string]interface{}{\n\t\t\"a\": `The hardworking employee consistently strives to exceed expectations.\n\t\t\t\tHis industrious nature makes him a valuable asset to any team.\n\t\t\t\tHis conscientious attention to detail ensures that projects are completed efficiently and accurately.\n\t\t\t\tHe remains persistent even in the face of challenges.`,\n\t\t\"b\": `The hardworking employee consistently strives to exceed expectations.\n\t\t\t\tHis industrious nature makes him a valuable asset to any team.\n\t\t\t\tHis conscientious attention to detail ensures that projects are completed efficiently and accurately.\n\t\t\t\tHe remains persistent even in the face of challenges.`,\n\t\t\"c\": `The hardworking employee consistently strives to exceed expectations.\n\t\t\t\tHis industrious nature makes him a valuable asset to any team.\n\t\t\t\tHis conscientious attention to detail ensures that projects are completed efficiently and accurately.\n\t\t\t\tHe remains persistent even in the face of challenges.`,\n\t}\n\tdoc2 := map[string]interface{}{\n\t\t\"a\": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. \n\t\t\t\tGuests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. \n\t\t\t\tThe calm environment offers the ideal place to meditate and connect with nature. \n\t\t\t\tEven the most stressed individuals find themselves feeling relaxed and at ease.`,\n\t\t\"b\": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. \n\t\t\t\tGuests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. \n\t\t\t\tThe calm environment offers the ideal place to meditate and connect with nature. \n\t\t\t\tEven the most stressed individuals find themselves feeling relaxed and at ease.`,\n\t\t\"c\": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. \n\t\t\t\tGuests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. \n\t\t\t\tThe calm environment offers the ideal place to meditate and connect with nature. \n\t\t\t\tEven the most stressed individuals find themselves feeling relaxed and at ease.`,\n\t}\n\tsynDoc1 := &SynonymDefinition{Synonyms: []string{\"hardworking\", \"industrious\", \"conscientious\", \"persistent\", \"focused\", \"devoted\"}}\n\tsynDoc2 := &SynonymDefinition{Synonyms: []string{\"tranquil\", \"peaceful\", \"calm\", \"relaxed\", \"unruffled\"}}\n\n\tbatch := index.NewBatch()\n\terr = batch.IndexSynonym(\"001\", synonymCollection, synDoc1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = batch.IndexSynonym(\"002\", synonymCollection, synDoc2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = batch.Index(\"003\", doc1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = batch.Index(\"004\", doc2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexMappingAfter := mapping.NewIndexMapping()\n\tindexMappingAfter.DefaultMapping.AddFieldMappingsAt(\"a\", a)\n\tb.Index = false\n\tindexMappingAfter.DefaultMapping.AddFieldMappingsAt(\"b\", b)\n\terr = indexMappingAfter.AddSynonymSource(synonymSourceName, synonymSourceConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexMappingAfter.IndexDynamic = false\n\tindexMappingAfter.StoreDynamic = false\n\tindexMappingAfter.DocValuesDynamic = false\n\n\tmappingString, err := json.Marshal(indexMappingAfter)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tconfig := map[string]interface{}{\n\t\t\"updated_mapping\": string(mappingString),\n\t}\n\n\tindex, err = OpenUsing(tmpIndexPath, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tq1 := NewSearchRequest(NewQueryStringQuery(\"a:devoted\"))\n\tres1, err := index.Search(q1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res1.Hits) != 1 {\n\t\tt.Errorf(\"Expected 1 hit, got %d\\n\", len(res1.Hits))\n\t}\n\n\tq2 := NewSearchRequest(NewQueryStringQuery(\"b:devoted\"))\n\tres2, err := index.Search(q2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res2.Hits) != 0 {\n\t\tt.Errorf(\"Expected 0 hits, got %d\\n\", len(res2.Hits))\n\t}\n\n\tq3 := NewSearchRequest(NewQueryStringQuery(\"c:unruffled\"))\n\tres3, err := index.Search(q3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res3.Hits) != 0 {\n\t\tt.Errorf(\"Expected 0 hits, got %d\\n\", len(res3.Hits))\n\t}\n}\n\nfunc TestIndexUpdateMerge(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindexMappingBefore := mapping.NewIndexMapping()\n\tindexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{}\n\tindexMappingBefore.DefaultMapping = &mapping.DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: false,\n\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\"a\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"b\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"c\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"d\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t},\n\t\tFields:               []*mapping.FieldMapping{},\n\t\tDefaultAnalyzer:      \"standard\",\n\t\tDefaultSynonymSource: \"\",\n\t}\n\tindexMappingBefore.IndexDynamic = false\n\tindexMappingBefore.StoreDynamic = false\n\tindexMappingBefore.DocValuesDynamic = false\n\n\tindex, err := New(tmpIndexPath, indexMappingBefore)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tnumDocsPerBatch := 1000\n\tnumBatches := 10\n\n\tvar batch *Batch\n\tdoc := make(map[string]interface{})\n\tconst letters = \"abcdefghijklmnopqrstuvwxyz\"\n\n\trandStr := func() string {\n\t\tresult := make([]byte, 3)\n\t\tfor i := 0; i < 3; i++ {\n\t\t\tresult[i] = letters[rand.Intn(len(letters))]\n\t\t}\n\t\treturn string(result)\n\t}\n\tfor i := 0; i < numBatches; i++ {\n\t\tbatch = index.NewBatch()\n\t\tfor j := 0; j < numDocsPerBatch; j++ {\n\t\t\tdoc[\"a\"] = randStr()\n\t\t\tdoc[\"b\"] = randStr()\n\t\t\tdoc[\"c\"] = randStr()\n\t\t\tdoc[\"d\"] = randStr()\n\t\t\terr = batch.Index(fmt.Sprintf(\"%d\", i*numDocsPerBatch+j), doc)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\t\terr = index.Batch(batch)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexMappingAfter := mapping.NewIndexMapping()\n\tindexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{}\n\tindexMappingAfter.DefaultMapping = &mapping.DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: false,\n\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\"a\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"b\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: false,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t\t\"c\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t},\n\t\tFields:               []*mapping.FieldMapping{},\n\t\tDefaultAnalyzer:      \"standard\",\n\t\tDefaultSynonymSource: \"\",\n\t}\n\tindexMappingAfter.IndexDynamic = false\n\tindexMappingAfter.StoreDynamic = false\n\tindexMappingAfter.DocValuesDynamic = false\n\n\tmappingString, err := json.Marshal(indexMappingAfter)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tconfig := map[string]interface{}{\n\t\t\"updated_mapping\": string(mappingString),\n\t}\n\n\tindex, err = OpenUsing(tmpIndexPath, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\timpl, ok := index.(*indexImpl)\n\tif !ok {\n\t\tt.Fatalf(\"Typecasting index to indexImpl failed\")\n\t}\n\tsindex, ok := impl.i.(*scorch.Scorch)\n\tif !ok {\n\t\tt.Fatalf(\"Typecasting index to scorch index failed\")\n\t}\n\n\terr = sindex.ForceMerge(context.Background(), &mergeplan.SingleSegmentMergePlanOptions)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tq1 := NewSearchRequest(NewQueryStringQuery(\"a:*\"))\n\tq1.Fields = append(q1.Fields, \"a\")\n\n\tres1, err := index.Search(q1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res1.Hits) != 10 {\n\t\tt.Errorf(\"Expected 10 hits, got %d\\n\", len(res1.Hits))\n\t}\n\tif len(res1.Hits[0].Fields) != 1 {\n\t\tt.Errorf(\"Expected 1 field, got %d\\n\", len(res1.Hits[0].Fields))\n\t}\n\tq2 := NewSearchRequest(NewQueryStringQuery(\"b:*\"))\n\tq2.Fields = append(q2.Fields, \"b\")\n\tres2, err := index.Search(q2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res2.Hits) != 0 {\n\t\tt.Errorf(\"Expected 0 hits, got %d\\n\", len(res2.Hits))\n\t}\n\tq3 := NewSearchRequest(NewQueryStringQuery(\"c:*\"))\n\tq3.Fields = append(q3.Fields, \"c\")\n\tres3, err := index.Search(q3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res3.Hits) != 10 {\n\t\tt.Errorf(\"Expected 10 hits, got %d\\n\", len(res3.Hits))\n\t}\n\tif len(res3.Hits[0].Fields) != 0 {\n\t\tt.Errorf(\"Expected 0 fields, got %d\\n\", len(res3.Hits[0].Fields))\n\t}\n\tq4 := NewSearchRequest(NewQueryStringQuery(\"d:*\"))\n\tq4.Fields = append(q4.Fields, \"d\")\n\tres4, err := index.Search(q4)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res4.Hits) != 0 {\n\t\tt.Errorf(\"Expected 0 hits, got %d\\n\", len(res4.Hits))\n\t}\n}\n\nfunc BenchmarkIndexUpdateText(b *testing.B) {\n\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tindexMappingBefore := mapping.NewIndexMapping()\n\tindexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{}\n\tindexMappingBefore.DefaultMapping = &mapping.DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: false,\n\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\"a\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t},\n\t\tFields:               []*mapping.FieldMapping{},\n\t\tDefaultAnalyzer:      \"standard\",\n\t\tDefaultSynonymSource: \"\",\n\t}\n\tindexMappingBefore.IndexDynamic = false\n\tindexMappingBefore.StoreDynamic = false\n\tindexMappingBefore.DocValuesDynamic = false\n\n\tindex, err := New(tmpIndexPath, indexMappingBefore)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tnumDocsPerBatch := 1000\n\tnumBatches := 5\n\n\tvar batch *Batch\n\tdoc := make(map[string]interface{})\n\tconst letters = \"abcdefghijklmnopqrstuvwxyz\"\n\n\trandStr := func() string {\n\t\tresult := make([]byte, 3)\n\t\tfor i := 0; i < 3; i++ {\n\t\t\tresult[i] = letters[rand.Intn(len(letters))]\n\t\t}\n\t\treturn string(result)\n\t}\n\tfor i := 0; i < numBatches; i++ {\n\t\tbatch = index.NewBatch()\n\t\tfor j := 0; j < numDocsPerBatch; j++ {\n\t\t\tdoc[\"a\"] = randStr()\n\t\t\terr = batch.Index(fmt.Sprintf(\"%d\", i*numDocsPerBatch+j), doc)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t}\n\t\terr = index.Batch(batch)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\terr = index.Close()\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tindexMappingAfter := mapping.NewIndexMapping()\n\tindexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{}\n\tindexMappingAfter.DefaultMapping = &mapping.DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: false,\n\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\"a\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:  \"text\",\n\t\t\t\t\t\tIndex: true,\n\t\t\t\t\t\tStore: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tDefaultAnalyzer:      \"standard\",\n\t\t\t\tDefaultSynonymSource: \"\",\n\t\t\t},\n\t\t},\n\t\tFields:               []*mapping.FieldMapping{},\n\t\tDefaultAnalyzer:      \"standard\",\n\t\tDefaultSynonymSource: \"\",\n\t}\n\tindexMappingAfter.IndexDynamic = false\n\tindexMappingAfter.StoreDynamic = false\n\tindexMappingAfter.DocValuesDynamic = false\n\n\tmappingString, err := json.Marshal(indexMappingAfter)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tconfig := map[string]interface{}{\n\t\t\"updated_mapping\": string(mappingString),\n\t}\n\n\tindex, err = OpenUsing(tmpIndexPath, config)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := NewQueryStringQuery(\"a:*\")\n\t\treq := NewSearchRequest(q)\n\t\tif _, err = index.Search(req); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc TestIndexUpdateNestedMapping(t *testing.T) {\n\t// Helper: create a mapping with optional nested structure\n\tcreateCompanyMapping := func(nestedEmployees, nestedDepartments, nestedProjects, nestedLocations bool) *mapping.IndexMappingImpl {\n\t\trv := mapping.NewIndexMapping()\n\t\tcompanyMapping := mapping.NewDocumentMapping()\n\n\t\t// Basic fields\n\t\tcompanyMapping.AddFieldMappingsAt(\"id\", mapping.NewTextFieldMapping())\n\t\tcompanyMapping.AddFieldMappingsAt(\"name\", mapping.NewTextFieldMapping())\n\n\t\tvar deptMapping *mapping.DocumentMapping\n\t\t// Departments nested conditionally\n\t\tif !nestedDepartments {\n\t\t\tdeptMapping = mapping.NewDocumentMapping()\n\t\t} else {\n\t\t\tdeptMapping = mapping.NewNestedDocumentMapping()\n\t\t}\n\t\tdeptMapping.AddFieldMappingsAt(\"name\", mapping.NewTextFieldMapping())\n\t\tdeptMapping.AddFieldMappingsAt(\"budget\", mapping.NewNumericFieldMapping())\n\n\t\t// Employees nested conditionally\n\t\tvar empMapping *mapping.DocumentMapping\n\t\tif !nestedEmployees {\n\t\t\tempMapping = mapping.NewNestedDocumentMapping()\n\t\t} else {\n\t\t\tempMapping = mapping.NewDocumentMapping()\n\t\t}\n\t\tempMapping.AddFieldMappingsAt(\"name\", mapping.NewTextFieldMapping())\n\t\tempMapping.AddFieldMappingsAt(\"role\", mapping.NewTextFieldMapping())\n\t\tdeptMapping.AddSubDocumentMapping(\"employees\", empMapping)\n\n\t\t// Projects nested conditionally\n\t\tvar projMapping *mapping.DocumentMapping\n\t\tif !nestedProjects {\n\t\t\tprojMapping = mapping.NewNestedDocumentMapping()\n\t\t} else {\n\t\t\tprojMapping = mapping.NewDocumentMapping()\n\t\t}\n\t\tprojMapping.AddFieldMappingsAt(\"title\", mapping.NewTextFieldMapping())\n\t\tprojMapping.AddFieldMappingsAt(\"status\", mapping.NewTextFieldMapping())\n\t\tdeptMapping.AddSubDocumentMapping(\"projects\", projMapping)\n\n\t\tcompanyMapping.AddSubDocumentMapping(\"departments\", deptMapping)\n\n\t\t// Locations nested conditionally\n\t\tvar locMapping *mapping.DocumentMapping\n\t\tif nestedLocations {\n\t\t\tlocMapping = mapping.NewNestedDocumentMapping()\n\t\t} else {\n\t\t\tlocMapping = mapping.NewDocumentMapping()\n\t\t}\n\t\tlocMapping.AddFieldMappingsAt(\"address\", mapping.NewTextFieldMapping())\n\t\tlocMapping.AddFieldMappingsAt(\"city\", mapping.NewTextFieldMapping())\n\n\t\tcompanyMapping.AddSubDocumentMapping(\"locations\", locMapping)\n\n\t\trv.DefaultMapping.AddSubDocumentMapping(\"company\", companyMapping)\n\t\treturn rv\n\t}\n\n\ttests := []struct {\n\t\tname      string\n\t\toriginal  *mapping.IndexMappingImpl\n\t\tupdated   *mapping.IndexMappingImpl\n\t\texpectErr bool\n\t}{\n\t\t{\n\t\t\tname:      \"No nested to all nested\",\n\t\t\toriginal:  createCompanyMapping(false, false, false, false),\n\t\t\tupdated:   createCompanyMapping(true, true, true, true),\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"No nested to mixed nested\",\n\t\t\toriginal:  createCompanyMapping(false, false, false, false),\n\t\t\tupdated:   createCompanyMapping(true, false, true, false),\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"No nested to mixed nested\",\n\t\t\toriginal:  createCompanyMapping(false, false, false, false),\n\t\t\tupdated:   createCompanyMapping(true, true, true, false),\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"Mixed nested to no nested\",\n\t\t\toriginal:  createCompanyMapping(false, true, false, true),\n\t\t\tupdated:   createCompanyMapping(false, false, true, true),\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"All nested to no nested\",\n\t\t\toriginal:  createCompanyMapping(true, true, true, true),\n\t\t\tupdated:   createCompanyMapping(false, false, false, false),\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"Mixed nested to all nested\",\n\t\t\toriginal:  createCompanyMapping(true, false, true, false),\n\t\t\tupdated:   createCompanyMapping(true, true, true, true),\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"All nested to mixed nested\",\n\t\t\toriginal:  createCompanyMapping(true, true, true, true),\n\t\t\tupdated:   createCompanyMapping(true, false, true, false),\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"No nested to no nested\",\n\t\t\toriginal:  createCompanyMapping(false, false, false, false),\n\t\t\tupdated:   createCompanyMapping(false, false, false, false),\n\t\t\texpectErr: false,\n\t\t},\n\t\t{\n\t\t\tname:      \"All nested to all nested\",\n\t\t\toriginal:  createCompanyMapping(true, true, true, true),\n\t\t\tupdated:   createCompanyMapping(true, true, true, true),\n\t\t\texpectErr: false,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\t_, err := DeletedFields(test.original, test.updated)\n\t\tif (err != nil) != test.expectErr {\n\t\t\tt.Errorf(\"Test '%s' unexpected error state: got %v, expectErr %t\", test.name, err, test.expectErr)\n\t\t}\n\t}\n}\n\nfunc TestTemp(t *testing.T) {\n\n\toriJSON := `\n\t\t{\n\t\t\"default_analyzer\": \"standard\",\n\t\t\"default_datetime_parser\": \"dateTimeOptional\",\n\t\t\"default_field\": \"_all\",\n\t\t\"default_mapping\": {\n\t\t\t\"dynamic\": true,\n\t\t\t\"enabled\": false\n\t\t},\n\t\t\"default_type\": \"_default\",\n\t\t\"docvalues_dynamic\": false,\n\t\t\"index_dynamic\": true,\n\t\t\"scoring_model\": \"tf-idf\",\n\t\t\"store_dynamic\": false,\n\t\t\"type_field\": \"_type\",\n\t\t\"types\": {\n\t\t\t\"inventory.hotel\": {\n\t\t\t\"dynamic\": false,\n\t\t\t\"enabled\": true,\n\t\t\t\"properties\": {\n\t\t\t\t\"city\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"keyword\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"city\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"country\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"keyword\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"country\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"description\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"en\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"description\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"free_breakfast\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"free_breakfast\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"boolean\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"geo\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"geo\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"geopoint\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"id\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"id\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"number\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"name\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"en\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"name\",\n\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\"store\": true\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"phone\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"keyword\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"phone\",\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"title\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"en\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"title\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"vacancy\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"vacancy\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"boolean\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t\t}\n\t\t}\n\t\t}`\n\n\tupdJSON := `\n\t\t{\n\t\t\"default_analyzer\": \"standard\",\n\t\t\"default_datetime_parser\": \"dateTimeOptional\",\n\t\t\"default_field\": \"_all\",\n\t\t\"default_mapping\": {\n\t\t\t\"dynamic\": true,\n\t\t\t\"enabled\": false\n\t\t},\n\t\t\"default_type\": \"_default\",\n\t\t\"docvalues_dynamic\": false,\n\t\t\"index_dynamic\": true,\n\t\t\"scoring_model\": \"tf-idf\",\n\t\t\"store_dynamic\": false,\n\t\t\"type_field\": \"_type\",\n\t\t\"types\": {\n\t\t\t\"inventory.hotel\": {\n\t\t\t\"dynamic\": false,\n\t\t\t\"enabled\": true,\n\t\t\t\"properties\": {\n\t\t\t\t\"city\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"keyword\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"city\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"country\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"keyword\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"country\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"description\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"en\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"description\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"free_breakfast\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"free_breakfast\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"boolean\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"geo\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"geo\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"geopoint\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"id\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"id\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"number\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"name\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"en\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"name\",\n\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\"store\": false\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"phone\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"keyword\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"phone\",\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"title\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"analyzer\": \"en\",\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"title\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"text\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t\"vacancy\": {\n\t\t\t\t\"enabled\": true,\n\t\t\t\t\"dynamic\": false,\n\t\t\t\t\"fields\": [\n\t\t\t\t\t{\n\t\t\t\t\t\"docvalues\": true,\n\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\"name\": \"vacancy\",\n\t\t\t\t\t\"store\": true,\n\t\t\t\t\t\"type\": \"boolean\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t\t}\n\t\t}\n\t\t}`\n\n\tvar originalMapping *mapping.IndexMappingImpl\n\terr := json.Unmarshal([]byte(oriJSON), &originalMapping)\n\tif err != nil {\n\t\tt.Fatalf(\"Error unmarshalling mapping JSON: %v\", err)\n\t}\n\n\tvar updatedMapping *mapping.IndexMappingImpl\n\terr = json.Unmarshal([]byte(updJSON), &updatedMapping)\n\tif err != nil {\n\t\tt.Fatalf(\"Error unmarshalling mapping JSON: %v\", err)\n\t}\n\n\tdeletedFields, err := DeletedFields(originalMapping, updatedMapping)\n\tif err != nil {\n\t\tt.Fatalf(\"Error comparing mappings: %v\", err)\n\t}\n\n\tfmt.Printf(\"Deleted fields: %v\\n\", deletedFields)\n\n}\n"
  },
  {
    "path": "mapping/analysis.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\ntype customAnalysis struct {\n\tCharFilters     map[string]map[string]interface{} `json:\"char_filters,omitempty\"`\n\tTokenizers      map[string]map[string]interface{} `json:\"tokenizers,omitempty\"`\n\tTokenMaps       map[string]map[string]interface{} `json:\"token_maps,omitempty\"`\n\tTokenFilters    map[string]map[string]interface{} `json:\"token_filters,omitempty\"`\n\tAnalyzers       map[string]map[string]interface{} `json:\"analyzers,omitempty\"`\n\tDateTimeParsers map[string]map[string]interface{} `json:\"date_time_parsers,omitempty\"`\n\tSynonymSources  map[string]map[string]interface{} `json:\"synonym_sources,omitempty\"`\n}\n\nfunc (c *customAnalysis) registerAll(i *IndexMappingImpl) error {\n\tfor name, config := range c.CharFilters {\n\t\t_, err := i.cache.DefineCharFilter(name, config)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif len(c.Tokenizers) > 0 {\n\t\t// put all the names in map tracking work to do\n\t\ttodo := map[string]struct{}{}\n\t\tfor name := range c.Tokenizers {\n\t\t\ttodo[name] = struct{}{}\n\t\t}\n\t\tregistered := 1\n\t\terrs := []error{}\n\t\t// as long as we keep making progress, keep going\n\t\tfor len(todo) > 0 && registered > 0 {\n\t\t\tregistered = 0\n\t\t\terrs = []error{}\n\t\t\tfor name := range todo {\n\t\t\t\tconfig := c.Tokenizers[name]\n\t\t\t\t_, err := i.cache.DefineTokenizer(name, config)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrs = append(errs, err)\n\t\t\t\t} else {\n\t\t\t\t\tdelete(todo, name)\n\t\t\t\t\tregistered++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif len(errs) > 0 {\n\t\t\treturn errs[0]\n\t\t}\n\t}\n\tfor name, config := range c.TokenMaps {\n\t\t_, err := i.cache.DefineTokenMap(name, config)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor name, config := range c.TokenFilters {\n\t\t_, err := i.cache.DefineTokenFilter(name, config)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor name, config := range c.Analyzers {\n\t\t_, err := i.cache.DefineAnalyzer(name, config)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor name, config := range c.DateTimeParsers {\n\t\t_, err := i.cache.DefineDateTimeParser(name, config)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor name, config := range c.SynonymSources {\n\t\t_, err := i.cache.DefineSynonymSource(name, config)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc newCustomAnalysis() *customAnalysis {\n\trv := customAnalysis{\n\t\tCharFilters:     make(map[string]map[string]interface{}),\n\t\tTokenizers:      make(map[string]map[string]interface{}),\n\t\tTokenMaps:       make(map[string]map[string]interface{}),\n\t\tTokenFilters:    make(map[string]map[string]interface{}),\n\t\tAnalyzers:       make(map[string]map[string]interface{}),\n\t\tDateTimeParsers: make(map[string]map[string]interface{}),\n\t\tSynonymSources:  make(map[string]map[string]interface{}),\n\t}\n\treturn &rv\n}\n"
  },
  {
    "path": "mapping/document.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\nimport (\n\t\"encoding\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"reflect\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n)\n\n// A DocumentMapping describes how a type of document\n// should be indexed.\n// As documents can be hierarchical, named sub-sections\n// of documents are mapped using the same structure in\n// the Properties field.\n// Each value inside a document can be indexed 0 or more\n// ways.  These index entries are called fields and\n// are stored in the Fields field.\n// Entire sections of a document can be ignored or\n// excluded by setting Enabled to false.\n// If not explicitly mapped, default mapping operations\n// are used.  To disable this automatic handling, set\n// Dynamic to false.\ntype DocumentMapping struct {\n\tEnabled              bool                        `json:\"enabled\"`\n\tDynamic              bool                        `json:\"dynamic\"`\n\tProperties           map[string]*DocumentMapping `json:\"properties,omitempty\"`\n\tFields               []*FieldMapping             `json:\"fields,omitempty\"`\n\tNested               bool                        `json:\"nested,omitempty\"`\n\tDefaultAnalyzer      string                      `json:\"default_analyzer,omitempty\"`\n\tDefaultSynonymSource string                      `json:\"default_synonym_source,omitempty\"`\n\n\t// StructTagKey overrides \"json\" when looking for field names in struct tags\n\tStructTagKey string `json:\"struct_tag_key,omitempty\"`\n}\n\nfunc (dm *DocumentMapping) Validate(cache *registry.Cache,\n\tpath []string, fieldAliasCtx map[string]*FieldMapping,\n) error {\n\tvar err error\n\tif dm.DefaultAnalyzer != \"\" {\n\t\t_, err := cache.AnalyzerNamed(dm.DefaultAnalyzer)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif dm.DefaultSynonymSource != \"\" {\n\t\t_, err := cache.SynonymSourceNamed(dm.DefaultSynonymSource)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor propertyName, property := range dm.Properties {\n\t\terr = property.Validate(cache, append(path, propertyName), fieldAliasCtx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tfor _, field := range dm.Fields {\n\t\tif field.Analyzer != \"\" {\n\t\t\t_, err = cache.AnalyzerNamed(field.Analyzer)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif field.DateFormat != \"\" {\n\t\t\t_, err = cache.DateTimeParserNamed(field.DateFormat)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif field.SynonymSource != \"\" {\n\t\t\t_, err = cache.SynonymSourceNamed(field.SynonymSource)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\terr := validateFieldMapping(field, path, fieldAliasCtx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc validateFieldType(field *FieldMapping) error {\n\tswitch field.Type {\n\tcase \"text\", \"datetime\", \"number\", \"boolean\", \"geopoint\", \"geoshape\", \"IP\":\n\t\treturn nil\n\tdefault:\n\t\treturn fmt.Errorf(\"field: '%s', unknown field type: '%s'\",\n\t\t\tfield.Name, field.Type)\n\t}\n}\n\n// analyzerNameForPath attempts to first find the field\n// described by this path, then returns the analyzer\n// configured for that field\nfunc (dm *DocumentMapping) analyzerNameForPath(path string) string {\n\tfield := dm.fieldDescribedByPath(path)\n\tif field != nil {\n\t\treturn field.Analyzer\n\t}\n\treturn \"\"\n}\n\n// synonymSourceForPath attempts to first find the field\n// described by this path, then returns the analyzer\n// configured for that field\nfunc (dm *DocumentMapping) synonymSourceForPath(path string) string {\n\tfield := dm.fieldDescribedByPath(path)\n\tif field != nil {\n\t\treturn field.SynonymSource\n\t}\n\treturn \"\"\n}\n\nfunc (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping {\n\tpathElements := decodePath(path)\n\tif len(pathElements) > 1 {\n\t\t// easy case, there is more than 1 path element remaining\n\t\t// the next path element must match a property name\n\t\t// at this level\n\t\tfor propName, subDocMapping := range dm.Properties {\n\t\t\tif propName == pathElements[0] {\n\t\t\t\treturn subDocMapping.fieldDescribedByPath(encodePath(pathElements[1:]))\n\t\t\t}\n\t\t}\n\t}\n\n\t// either the path just had one element\n\t// or it had multiple, but no match for the first element at this level\n\t// look for match with full path\n\n\t// first look for property name with empty field\n\tfor propName, subDocMapping := range dm.Properties {\n\t\tif propName == path {\n\t\t\t// found property name match, now look at its fields\n\t\t\tfor _, field := range subDocMapping.Fields {\n\t\t\t\tif field.Name == \"\" || field.Name == path {\n\t\t\t\t\t// match\n\t\t\t\t\treturn field\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t// next, walk the properties again, looking for field overriding the name\n\tfor propName, subDocMapping := range dm.Properties {\n\t\tif propName != path {\n\t\t\t// property name isn't a match, but field name could override it\n\t\t\tfor _, field := range subDocMapping.Fields {\n\t\t\t\tif field.Name == path {\n\t\t\t\t\treturn field\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// documentMappingForPathElements returns the EXACT and closest matches for a sub\n// document or for an explicitly mapped field; the closest most specific\n// document mapping could be one that matches part of the provided path.\nfunc (dm *DocumentMapping) documentMappingForPathElements(pathElements []string) (\n\t*DocumentMapping, *DocumentMapping,\n) {\n\tvar pathElementsCopy []string\n\tif len(pathElements) == 0 {\n\t\tpathElementsCopy = []string{\"\"}\n\t} else {\n\t\tpathElementsCopy = pathElements\n\t}\n\tcurrent := dm\nOUTER:\n\tfor i, pathElement := range pathElementsCopy {\n\t\tif subDocMapping, exists := current.Properties[pathElement]; exists {\n\t\t\tcurrent = subDocMapping\n\t\t\tcontinue OUTER\n\t\t}\n\n\t\t// no subDocMapping matches this pathElement\n\t\t// only if this is the last element check for field name\n\t\tif i == len(pathElementsCopy)-1 {\n\t\t\tfor _, field := range current.Fields {\n\t\t\t\tif field.Name == pathElement {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn nil, current\n\t}\n\treturn current, current\n}\n\n// documentMappingForPath returns the EXACT and closest matches for a sub\n// document or for an explicitly mapped field; the closest most specific\n// document mapping could be one that matches part of the provided path.\nfunc (dm *DocumentMapping) documentMappingForPath(path string) (\n\t*DocumentMapping, *DocumentMapping,\n) {\n\tpathElements := decodePath(path)\n\treturn dm.documentMappingForPathElements(pathElements)\n}\n\n// NewDocumentMapping returns a new document mapping\n// with all the default values.\nfunc NewDocumentMapping() *DocumentMapping {\n\treturn &DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: true,\n\t}\n}\n\n// NewNestedDocumentMapping returns a new document\n// mapping that treats sub-documents as nested\n// objects.\nfunc NewNestedDocumentMapping() *DocumentMapping {\n\treturn &DocumentMapping{\n\t\tNested:  true,\n\t\tEnabled: true,\n\t\tDynamic: true,\n\t}\n}\n\n// NewDocumentStaticMapping returns a new document\n// mapping that will not automatically index parts\n// of a document without an explicit mapping.\nfunc NewDocumentStaticMapping() *DocumentMapping {\n\treturn &DocumentMapping{\n\t\tEnabled: true,\n\t}\n}\n\n// NewNestedDocumentStaticMapping returns a new document\n// mapping that treats sub-documents as nested\n// objects and will not automatically index parts\n// of the nested document without an explicit mapping.\nfunc NewNestedDocumentStaticMapping() *DocumentMapping {\n\treturn &DocumentMapping{\n\t\tEnabled: true,\n\t\tNested:  true,\n\t}\n}\n\n// NewDocumentDisabledMapping returns a new document\n// mapping that will not perform any indexing.\nfunc NewDocumentDisabledMapping() *DocumentMapping {\n\treturn &DocumentMapping{}\n}\n\n// AddSubDocumentMapping adds the provided DocumentMapping as a sub-mapping\n// for the specified named subsection.\nfunc (dm *DocumentMapping) AddSubDocumentMapping(property string, sdm *DocumentMapping) {\n\tif dm.Properties == nil {\n\t\tdm.Properties = make(map[string]*DocumentMapping)\n\t}\n\tdm.Properties[property] = sdm\n}\n\n// AddFieldMappingsAt adds one or more FieldMappings\n// at the named sub-document.  If the named sub-document\n// doesn't yet exist it is created for you.\n// This is a convenience function to make most common\n// mappings more concise.\n// Otherwise, you would:\n//\n//\tsubMapping := NewDocumentMapping()\n//\tsubMapping.AddFieldMapping(fieldMapping)\n//\tparentMapping.AddSubDocumentMapping(property, subMapping)\nfunc (dm *DocumentMapping) AddFieldMappingsAt(property string, fms ...*FieldMapping) {\n\tif dm.Properties == nil {\n\t\tdm.Properties = make(map[string]*DocumentMapping)\n\t}\n\tsdm, ok := dm.Properties[property]\n\tif !ok {\n\t\tsdm = NewDocumentMapping()\n\t}\n\tfor _, fm := range fms {\n\t\tsdm.AddFieldMapping(fm)\n\t}\n\tdm.Properties[property] = sdm\n}\n\n// AddFieldMapping adds the provided FieldMapping for this section\n// of the document.\nfunc (dm *DocumentMapping) AddFieldMapping(fm *FieldMapping) {\n\tif dm.Fields == nil {\n\t\tdm.Fields = make([]*FieldMapping, 0)\n\t}\n\tdm.Fields = append(dm.Fields, fm)\n}\n\n// UnmarshalJSON offers custom unmarshaling with optional strict validation\nfunc (dm *DocumentMapping) UnmarshalJSON(data []byte) error {\n\tvar tmp map[string]json.RawMessage\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// set defaults for fields which might have been omitted\n\tdm.Enabled = true\n\tdm.Dynamic = true\n\n\tvar invalidKeys []string\n\tfor k, v := range tmp {\n\t\tswitch k {\n\t\tcase \"enabled\":\n\t\t\terr := util.UnmarshalJSON(v, &dm.Enabled)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"dynamic\":\n\t\t\terr := util.UnmarshalJSON(v, &dm.Dynamic)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"nested\":\n\t\t\terr := util.UnmarshalJSON(v, &dm.Nested)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"default_analyzer\":\n\t\t\terr := util.UnmarshalJSON(v, &dm.DefaultAnalyzer)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"default_synonym_source\":\n\t\t\terr := util.UnmarshalJSON(v, &dm.DefaultSynonymSource)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"properties\":\n\t\t\terr := util.UnmarshalJSON(v, &dm.Properties)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"fields\":\n\t\t\terr := util.UnmarshalJSON(v, &dm.Fields)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"struct_tag_key\":\n\t\t\terr := util.UnmarshalJSON(v, &dm.StructTagKey)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tdefault:\n\t\t\tinvalidKeys = append(invalidKeys, k)\n\t\t}\n\t}\n\n\tif MappingJSONStrict && len(invalidKeys) > 0 {\n\t\treturn fmt.Errorf(\"document mapping contains invalid keys: %v\", invalidKeys)\n\t}\n\n\treturn nil\n}\n\nfunc (dm *DocumentMapping) defaultAnalyzerName(path []string) string {\n\tcurrent := dm\n\trv := current.DefaultAnalyzer\n\tfor _, pathElement := range path {\n\t\tvar ok bool\n\t\tcurrent, ok = current.Properties[pathElement]\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tif current.DefaultAnalyzer != \"\" {\n\t\t\trv = current.DefaultAnalyzer\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (dm *DocumentMapping) defaultSynonymSource(path []string) string {\n\tcurrent := dm\n\trv := current.DefaultSynonymSource\n\tfor _, pathElement := range path {\n\t\tvar ok bool\n\t\tcurrent, ok = current.Properties[pathElement]\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tif current.DefaultSynonymSource != \"\" {\n\t\t\trv = current.DefaultSynonymSource\n\t\t}\n\t}\n\treturn rv\n}\n\n// baseType returns the base type of v by dereferencing pointers\nfunc baseType(v interface{}) reflect.Type {\n\tif v == nil {\n\t\treturn nil\n\t}\n\tt := reflect.TypeOf(v)\n\tfor t.Kind() == reflect.Pointer {\n\t\tt = t.Elem()\n\t}\n\treturn t\n}\n\nfunc (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes []uint64, context *walkContext) {\n\t// allow default \"json\" tag to be overridden\n\tstructTagKey := dm.StructTagKey\n\tif structTagKey == \"\" {\n\t\tstructTagKey = \"json\"\n\t}\n\n\tval := reflect.ValueOf(data)\n\tif !val.IsValid() {\n\t\treturn\n\t}\n\n\ttyp := val.Type()\n\tswitch typ.Kind() {\n\tcase reflect.Map:\n\t\t// FIXME can add support for other map keys in the future\n\t\tif typ.Key().Kind() == reflect.String {\n\t\t\tfor _, key := range val.MapKeys() {\n\t\t\t\tfieldName := key.String()\n\t\t\t\tfieldVal := val.MapIndex(key).Interface()\n\t\t\t\tdm.processProperty(fieldVal, append(path, fieldName), indexes, context)\n\t\t\t}\n\t\t}\n\tcase reflect.Struct:\n\t\tfor i := 0; i < val.NumField(); i++ {\n\t\t\tfield := typ.Field(i)\n\t\t\tfieldName := field.Name\n\t\t\t// anonymous fields of type struct can elide the type name\n\t\t\tif field.Anonymous && field.Type.Kind() == reflect.Struct {\n\t\t\t\tfieldName = \"\"\n\t\t\t}\n\n\t\t\t// if the field has a name under the specified tag, prefer that\n\t\t\ttag := field.Tag.Get(structTagKey)\n\t\t\ttagFieldName := parseTagName(tag)\n\t\t\tif tagFieldName == \"-\" {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// allow tag to set field name to empty, only if anonymous\n\t\t\tif field.Tag != \"\" && (tagFieldName != \"\" || field.Anonymous) {\n\t\t\t\tfieldName = tagFieldName\n\t\t\t}\n\n\t\t\tif val.Field(i).CanInterface() {\n\t\t\t\tfieldVal := val.Field(i).Interface()\n\t\t\t\tnewpath := path\n\t\t\t\tif fieldName != \"\" {\n\t\t\t\t\tnewpath = append(path, fieldName)\n\t\t\t\t}\n\t\t\t\tdm.processProperty(fieldVal, newpath, indexes, context)\n\t\t\t}\n\t\t}\n\tcase reflect.Slice, reflect.Array:\n\t\tsubDocMapping, _ := dm.documentMappingForPathElements(path)\n\t\tallowNested := subDocMapping != nil && subDocMapping.Nested\n\t\tfor i := 0; i < val.Len(); i++ {\n\t\t\t// for each array element, check if it can be represented as an interface\n\t\t\tidxVal := val.Index(i)\n\t\t\t// skip invalid values\n\t\t\tif !idxVal.CanInterface() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// get the actual value in interface form\n\t\t\tactual := idxVal.Interface()\n\t\t\t// if nested mapping, only create nested document for object elements\n\t\t\tif allowNested && actual != nil {\n\t\t\t\t// check the kind of the actual value, is it an object (struct or map)?\n\t\t\t\ttyp := baseType(actual)\n\t\t\t\tif typ == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tkind := typ.Kind()\n\t\t\t\t// only create nested docs for real JSON objects\n\t\t\t\tif kind == reflect.Struct || kind == reflect.Map {\n\t\t\t\t\t// Create nested document only for only object elements\n\t\t\t\t\tnestedDocument := document.NewDocument(\n\t\t\t\t\t\tfmt.Sprintf(\"%s_$%s_$%d\", context.doc.ID(), encodePath(path), i))\n\t\t\t\t\tnestedContext := context.im.newWalkContext(nestedDocument, dm)\n\t\t\t\t\tdm.processProperty(actual, path, append(indexes, uint64(i)), nestedContext)\n\t\t\t\t\tcontext.doc.AddNestedDocument(nestedDocument)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\t// non-nested mapping, or non-object element in nested mapping\n\t\t\t// process the element normally\n\t\t\tdm.processProperty(actual, path, append(indexes, uint64(i)), context)\n\t\t}\n\tcase reflect.Ptr:\n\t\tptrElem := val.Elem()\n\t\tif ptrElem.IsValid() && ptrElem.CanInterface() {\n\t\t\tdm.processProperty(ptrElem.Interface(), path, indexes, context)\n\t\t}\n\tcase reflect.String:\n\t\tdm.processProperty(val.String(), path, indexes, context)\n\tcase reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:\n\t\tdm.processProperty(float64(val.Int()), path, indexes, context)\n\tcase reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:\n\t\tdm.processProperty(float64(val.Uint()), path, indexes, context)\n\tcase reflect.Float32, reflect.Float64:\n\t\tdm.processProperty(float64(val.Float()), path, indexes, context)\n\tcase reflect.Bool:\n\t\tdm.processProperty(val.Bool(), path, indexes, context)\n\t}\n}\n\nfunc (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) {\n\t// look to see if there is a mapping for this field\n\tsubDocMapping, closestDocMapping := dm.documentMappingForPathElements(path)\n\n\t// check to see if we even need to do further processing\n\tif subDocMapping != nil && !subDocMapping.Enabled {\n\t\treturn\n\t}\n\n\tpropertyValue := reflect.ValueOf(property)\n\tif !propertyValue.IsValid() {\n\t\t// cannot do anything with the zero value\n\t\treturn\n\t}\n\n\tpathString := encodePath(path)\n\tpropertyType := propertyValue.Type()\n\tswitch propertyType.Kind() {\n\tcase reflect.String:\n\t\tpropertyValueString := propertyValue.String()\n\t\tif subDocMapping != nil {\n\t\t\t// index by explicit mapping\n\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\tswitch fieldMapping.Type {\n\t\t\t\tcase \"geoshape\":\n\t\t\t\t\tfieldMapping.processGeoShape(property, pathString, path, indexes, context)\n\t\t\t\tcase \"geopoint\":\n\t\t\t\t\tfieldMapping.processGeoPoint(property, pathString, path, indexes, context)\n\t\t\t\tcase \"vector_base64\":\n\t\t\t\t\tfieldMapping.processVectorBase64(property, pathString, path, indexes, context)\n\t\t\t\tdefault:\n\t\t\t\t\tfieldMapping.processString(propertyValueString, pathString, path, indexes, context)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if closestDocMapping.Dynamic {\n\t\t\t// automatic indexing behavior\n\n\t\t\t// first see if it can be parsed by the default date parser\n\t\t\tdateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser)\n\t\t\tif dateTimeParser != nil {\n\t\t\t\tparsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// index as text\n\t\t\t\t\tfieldMapping := newTextFieldMappingDynamic(context.im)\n\t\t\t\t\tfieldMapping.processString(propertyValueString, pathString, path, indexes, context)\n\t\t\t\t} else {\n\t\t\t\t\t// index as datetime\n\t\t\t\t\tfieldMapping := newDateTimeFieldMappingDynamic(context.im)\n\t\t\t\t\tfieldMapping.processTime(parsedDateTime, layout, pathString, path, indexes, context)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\tcase reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:\n\t\tdm.processProperty(float64(propertyValue.Int()), path, indexes, context)\n\t\treturn\n\tcase reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:\n\t\tdm.processProperty(float64(propertyValue.Uint()), path, indexes, context)\n\t\treturn\n\tcase reflect.Float64, reflect.Float32:\n\t\tpropertyValFloat := propertyValue.Float()\n\t\tif subDocMapping != nil {\n\t\t\t// index by explicit mapping\n\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\tfieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context)\n\t\t\t}\n\t\t} else if closestDocMapping.Dynamic {\n\t\t\t// automatic indexing behavior\n\t\t\tfieldMapping := newNumericFieldMappingDynamic(context.im)\n\t\t\tfieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context)\n\t\t}\n\tcase reflect.Bool:\n\t\tpropertyValBool := propertyValue.Bool()\n\t\tif subDocMapping != nil {\n\t\t\t// index by explicit mapping\n\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\tfieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context)\n\t\t\t}\n\t\t} else if closestDocMapping.Dynamic {\n\t\t\t// automatic indexing behavior\n\t\t\tfieldMapping := newBooleanFieldMappingDynamic(context.im)\n\t\t\tfieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context)\n\t\t}\n\tcase reflect.Struct:\n\t\tswitch property := property.(type) {\n\t\tcase time.Time:\n\t\t\t// don't descend into the time struct\n\t\t\tif subDocMapping != nil {\n\t\t\t\t// index by explicit mapping\n\t\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\t\tfieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context)\n\t\t\t\t}\n\t\t\t} else if closestDocMapping.Dynamic {\n\t\t\t\tfieldMapping := newDateTimeFieldMappingDynamic(context.im)\n\t\t\t\tfieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context)\n\t\t\t}\n\t\tcase encoding.TextMarshaler:\n\t\t\ttxt, err := property.MarshalText()\n\t\t\tif err == nil && subDocMapping != nil {\n\t\t\t\t// index by explicit mapping\n\t\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\t\tif fieldMapping.Type == \"text\" {\n\t\t\t\t\t\tfieldMapping.processString(string(txt), pathString, path, indexes, context)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tdm.walkDocument(property, path, indexes, context)\n\t\tdefault:\n\t\t\tif subDocMapping != nil {\n\t\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\t\tswitch fieldMapping.Type {\n\t\t\t\t\tcase \"geopoint\":\n\t\t\t\t\t\tfieldMapping.processGeoPoint(property, pathString, path, indexes, context)\n\t\t\t\t\tcase \"geoshape\":\n\t\t\t\t\t\tfieldMapping.processGeoShape(property, pathString, path, indexes, context)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tdm.walkDocument(property, path, indexes, context)\n\t\t}\n\tcase reflect.Map, reflect.Slice:\n\t\twalkDocument := false\n\t\tif subDocMapping != nil && len(subDocMapping.Fields) != 0 {\n\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\tswitch fieldMapping.Type {\n\t\t\t\tcase \"vector\":\n\t\t\t\t\tfieldMapping.processVector(property, pathString, path,\n\t\t\t\t\t\tindexes, context)\n\t\t\t\tcase \"geopoint\":\n\t\t\t\t\tfieldMapping.processGeoPoint(property, pathString, path, indexes, context)\n\t\t\t\t\twalkDocument = true\n\t\t\t\tcase \"IP\":\n\t\t\t\t\tip, ok := property.(net.IP)\n\t\t\t\t\tif ok {\n\t\t\t\t\t\tfieldMapping.processIP(ip, pathString, path, indexes, context)\n\t\t\t\t\t}\n\t\t\t\t\twalkDocument = true\n\t\t\t\tcase \"geoshape\":\n\t\t\t\t\tfieldMapping.processGeoShape(property, pathString, path, indexes, context)\n\t\t\t\t\twalkDocument = true\n\t\t\t\tdefault:\n\t\t\t\t\twalkDocument = true\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\twalkDocument = true\n\t\t}\n\t\tif walkDocument {\n\t\t\tdm.walkDocument(property, path, indexes, context)\n\t\t}\n\tcase reflect.Ptr:\n\t\tif !propertyValue.IsNil() {\n\t\t\tswitch property := property.(type) {\n\t\t\tcase encoding.TextMarshaler:\n\t\t\t\t// ONLY process TextMarshaler if there is an explicit mapping\n\t\t\t\t// AND all of the fields are of type text\n\t\t\t\t// OTHERWISE process field without TextMarshaler\n\t\t\t\tif subDocMapping != nil {\n\t\t\t\t\tallFieldsText := true\n\t\t\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\t\t\tif fieldMapping.Type != \"text\" {\n\t\t\t\t\t\t\tallFieldsText = false\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\ttxt, err := property.MarshalText()\n\t\t\t\t\tif err == nil && allFieldsText {\n\t\t\t\t\t\ttxtStr := string(txt)\n\t\t\t\t\t\tfor _, fieldMapping := range subDocMapping.Fields {\n\t\t\t\t\t\t\tfieldMapping.processString(txtStr, pathString, path, indexes, context)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tdm.walkDocument(property, path, indexes, context)\n\t\t\tdefault:\n\t\t\t\tdm.walkDocument(property, path, indexes, context)\n\t\t\t}\n\t\t}\n\tdefault:\n\t\tdm.walkDocument(property, path, indexes, context)\n\t}\n}\n"
  },
  {
    "path": "mapping/examples_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\nimport \"fmt\"\n\n// Examples for Mapping related functions\n\nfunc ExampleDocumentMapping_AddSubDocumentMapping() {\n\t// adds a document mapping for a property in a document\n\t// useful for mapping nested documents\n\tdocumentMapping := NewDocumentMapping()\n\tsubDocumentMapping := NewDocumentMapping()\n\tdocumentMapping.AddSubDocumentMapping(\"Property\", subDocumentMapping)\n\n\tfmt.Println(len(documentMapping.Properties))\n\t// Output:\n\t// 1\n}\n\nfunc ExampleDocumentMapping_AddFieldMapping() {\n\t// you can only add field mapping to those properties which already have a document mapping\n\tdocumentMapping := NewDocumentMapping()\n\tsubDocumentMapping := NewDocumentMapping()\n\tdocumentMapping.AddSubDocumentMapping(\"Property\", subDocumentMapping)\n\n\tfieldMapping := NewTextFieldMapping()\n\tfieldMapping.Analyzer = \"en\"\n\tsubDocumentMapping.AddFieldMapping(fieldMapping)\n\n\tfmt.Println(len(documentMapping.Properties[\"Property\"].Fields))\n\t// Output:\n\t// 1\n}\n\nfunc ExampleDocumentMapping_AddFieldMappingsAt() {\n\t// you can only add field mapping to those properties which already have a document mapping\n\tdocumentMapping := NewDocumentMapping()\n\tsubDocumentMapping := NewDocumentMapping()\n\tdocumentMapping.AddSubDocumentMapping(\"NestedProperty\", subDocumentMapping)\n\n\tfieldMapping := NewTextFieldMapping()\n\tfieldMapping.Analyzer = \"en\"\n\tdocumentMapping.AddFieldMappingsAt(\"NestedProperty\", fieldMapping)\n\n\tfmt.Println(len(documentMapping.Properties[\"NestedProperty\"].Fields))\n\t// Output:\n\t// 1\n}\n"
  },
  {
    "path": "mapping/field.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"github.com/blevesearch/geo/geojson\"\n)\n\n// control the default behavior for dynamic fields (those not explicitly mapped)\nvar (\n\tIndexDynamic     = true\n\tStoreDynamic     = true\n\tDocValuesDynamic = true // TODO revisit default?\n)\n\n// A FieldMapping describes how a specific item\n// should be put into the index.\ntype FieldMapping struct {\n\tName string `json:\"name,omitempty\"`\n\tType string `json:\"type,omitempty\"`\n\n\t// Analyzer specifies the name of the analyzer to use for this field.  If\n\t// Analyzer is empty, traverse the DocumentMapping tree toward the root and\n\t// pick the first non-empty DefaultAnalyzer found. If there is none, use\n\t// the IndexMapping.DefaultAnalyzer.\n\tAnalyzer string `json:\"analyzer,omitempty\"`\n\n\t// Store indicates whether to store field values in the index. Stored\n\t// values can be retrieved from search results using SearchRequest.Fields.\n\tStore bool `json:\"store,omitempty\"`\n\tIndex bool `json:\"index,omitempty\"`\n\n\t// IncludeTermVectors, if true, makes terms occurrences to be recorded for\n\t// this field. It includes the term position within the terms sequence and\n\t// the term offsets in the source document field. Term vectors are required\n\t// to perform phrase queries or terms highlighting in source documents.\n\tIncludeTermVectors bool   `json:\"include_term_vectors,omitempty\"`\n\tIncludeInAll       bool   `json:\"include_in_all,omitempty\"`\n\tDateFormat         string `json:\"date_format,omitempty\"`\n\n\t// DocValues, if true makes the index uninverting possible for this field\n\t// It is useful for faceting and sorting queries.\n\tDocValues bool `json:\"docvalues,omitempty\"`\n\n\t// SkipFreqNorm, if true, avoids the indexing of frequency and norm values\n\t// of the tokens for this field. This option would be useful for saving\n\t// the processing of freq/norm details when the default score based relevancy\n\t// isn't needed.\n\tSkipFreqNorm bool `json:\"skip_freq_norm,omitempty\"`\n\n\t// Dimensionality of the vector\n\tDims int `json:\"dims,omitempty\"`\n\n\t// Similarity is the similarity algorithm used for scoring\n\t// field's content while performing search on it.\n\t// See: index.SimilarityModels\n\tSimilarity string `json:\"similarity,omitempty\"`\n\n\t// Applicable to vector fields only - optimization string\n\tVectorIndexOptimizedFor string `json:\"vector_index_optimized_for,omitempty\"`\n\n\tSynonymSource string `json:\"synonym_source,omitempty\"`\n}\n\n// NewTextFieldMapping returns a default field mapping for text\nfunc NewTextFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:               \"text\",\n\t\tStore:              true,\n\t\tIndex:              true,\n\t\tIncludeTermVectors: true,\n\t\tIncludeInAll:       true,\n\t\tDocValues:          true,\n\t}\n}\n\nfunc newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {\n\trv := NewTextFieldMapping()\n\trv.Store = im.StoreDynamic\n\trv.Index = im.IndexDynamic\n\trv.DocValues = im.DocValuesDynamic\n\treturn rv\n}\n\n// NewKeywordFieldMapping returns a default field mapping for text with analyzer \"keyword\".\nfunc NewKeywordFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:               \"text\",\n\t\tAnalyzer:           keyword.Name,\n\t\tStore:              true,\n\t\tIndex:              true,\n\t\tIncludeTermVectors: true,\n\t\tIncludeInAll:       true,\n\t\tDocValues:          true,\n\t}\n}\n\n// NewNumericFieldMapping returns a default field mapping for numbers\nfunc NewNumericFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:         \"number\",\n\t\tStore:        true,\n\t\tIndex:        true,\n\t\tIncludeInAll: true,\n\t\tDocValues:    true,\n\t}\n}\n\nfunc newNumericFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {\n\trv := NewNumericFieldMapping()\n\trv.Store = im.StoreDynamic\n\trv.Index = im.IndexDynamic\n\trv.DocValues = im.DocValuesDynamic\n\treturn rv\n}\n\n// NewDateTimeFieldMapping returns a default field mapping for dates\nfunc NewDateTimeFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:         \"datetime\",\n\t\tStore:        true,\n\t\tIndex:        true,\n\t\tIncludeInAll: true,\n\t\tDocValues:    true,\n\t}\n}\n\nfunc newDateTimeFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {\n\trv := NewDateTimeFieldMapping()\n\trv.Store = im.StoreDynamic\n\trv.Index = im.IndexDynamic\n\trv.DocValues = im.DocValuesDynamic\n\treturn rv\n}\n\n// NewBooleanFieldMapping returns a default field mapping for booleans\nfunc NewBooleanFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:         \"boolean\",\n\t\tStore:        true,\n\t\tIndex:        true,\n\t\tIncludeInAll: true,\n\t\tDocValues:    true,\n\t}\n}\n\nfunc newBooleanFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {\n\trv := NewBooleanFieldMapping()\n\trv.Store = im.StoreDynamic\n\trv.Index = im.IndexDynamic\n\trv.DocValues = im.DocValuesDynamic\n\treturn rv\n}\n\n// NewGeoPointFieldMapping returns a default field mapping for geo points\nfunc NewGeoPointFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:         \"geopoint\",\n\t\tStore:        true,\n\t\tIndex:        true,\n\t\tIncludeInAll: true,\n\t\tDocValues:    true,\n\t}\n}\n\n// NewGeoShapeFieldMapping returns a default field mapping\n// for geoshapes\nfunc NewGeoShapeFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:         \"geoshape\",\n\t\tStore:        true,\n\t\tIndex:        true,\n\t\tIncludeInAll: true,\n\t\tDocValues:    true,\n\t}\n}\n\n// NewIPFieldMapping returns a default field mapping for IP points\nfunc NewIPFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:         \"IP\",\n\t\tStore:        true,\n\t\tIndex:        true,\n\t\tIncludeInAll: true,\n\t}\n}\n\n// Options returns the indexing options for this field.\nfunc (fm *FieldMapping) Options() index.FieldIndexingOptions {\n\tvar rv index.FieldIndexingOptions\n\tif fm.Store {\n\t\trv |= index.StoreField\n\t}\n\tif fm.Index {\n\t\trv |= index.IndexField\n\t}\n\tif fm.IncludeTermVectors {\n\t\trv |= index.IncludeTermVectors\n\t}\n\tif fm.DocValues {\n\t\trv |= index.DocValues\n\t}\n\tif fm.SkipFreqNorm {\n\t\trv |= index.SkipFreqNorm\n\t}\n\treturn rv\n}\n\nfunc (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) {\n\tfieldName := getFieldName(pathString, path, fm)\n\toptions := fm.Options()\n\n\tswitch fm.Type {\n\tcase \"text\":\n\t\tanalyzer := fm.analyzerForField(path, context)\n\t\tfield := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer)\n\t\tcontext.doc.AddField(field)\n\n\t\tif !fm.IncludeInAll {\n\t\t\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\t\t}\n\tcase \"datetime\":\n\t\tdateTimeFormat := context.im.DefaultDateTimeParser\n\t\tif fm.DateFormat != \"\" {\n\t\t\tdateTimeFormat = fm.DateFormat\n\t\t}\n\t\tdateTimeParser := context.im.DateTimeParserNamed(dateTimeFormat)\n\t\tif dateTimeParser != nil {\n\t\t\tparsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString)\n\t\t\tif err == nil {\n\t\t\t\tfm.processTime(parsedDateTime, layout, pathString, path, indexes, context)\n\t\t\t}\n\t\t}\n\tcase \"IP\":\n\t\tip := net.ParseIP(propertyValueString)\n\t\tif ip != nil {\n\t\t\tfm.processIP(ip, pathString, path, indexes, context)\n\t\t}\n\t}\n}\n\nfunc (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString string, path []string, indexes []uint64, context *walkContext) {\n\tfieldName := getFieldName(pathString, path, fm)\n\tif fm.Type == \"number\" {\n\t\toptions := fm.Options()\n\t\tfield := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options)\n\t\tcontext.doc.AddField(field)\n\n\t\tif !fm.IncludeInAll {\n\t\t\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\t\t}\n\t}\n}\n\nfunc (fm *FieldMapping) processTime(propertyValueTime time.Time, layout string, pathString string, path []string, indexes []uint64, context *walkContext) {\n\tfieldName := getFieldName(pathString, path, fm)\n\tif fm.Type == \"datetime\" {\n\t\toptions := fm.Options()\n\t\tfield, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, layout, options)\n\t\tif err == nil {\n\t\t\tcontext.doc.AddField(field)\n\t\t} else {\n\t\t\tlogger.Printf(\"could not build date %v\", err)\n\t\t}\n\n\t\tif !fm.IncludeInAll {\n\t\t\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\t\t}\n\t}\n}\n\nfunc (fm *FieldMapping) processBoolean(propertyValueBool bool, pathString string, path []string, indexes []uint64, context *walkContext) {\n\tfieldName := getFieldName(pathString, path, fm)\n\tif fm.Type == \"boolean\" {\n\t\toptions := fm.Options()\n\t\tfield := document.NewBooleanFieldWithIndexingOptions(fieldName, indexes, propertyValueBool, options)\n\t\tcontext.doc.AddField(field)\n\n\t\tif !fm.IncludeInAll {\n\t\t\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\t\t}\n\t}\n}\n\nfunc (fm *FieldMapping) processGeoPoint(propertyMightBeGeoPoint interface{}, pathString string, path []string, indexes []uint64, context *walkContext) {\n\tlon, lat, found := geo.ExtractGeoPoint(propertyMightBeGeoPoint)\n\tif found {\n\t\tfieldName := getFieldName(pathString, path, fm)\n\t\toptions := fm.Options()\n\t\tfield := document.NewGeoPointFieldWithIndexingOptions(fieldName, indexes, lon, lat, options)\n\t\tcontext.doc.AddField(field)\n\n\t\tif !fm.IncludeInAll {\n\t\t\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\t\t}\n\t}\n}\n\nfunc (fm *FieldMapping) processIP(ip net.IP, pathString string, path []string, indexes []uint64, context *walkContext) {\n\tfieldName := getFieldName(pathString, path, fm)\n\toptions := fm.Options()\n\tfield := document.NewIPFieldWithIndexingOptions(fieldName, indexes, ip, options)\n\tcontext.doc.AddField(field)\n\n\tif !fm.IncludeInAll {\n\t\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\t}\n}\n\nfunc (fm *FieldMapping) processGeoShape(propertyMightBeGeoShape interface{},\n\tpathString string, path []string, indexes []uint64, context *walkContext,\n) {\n\tcoordValue, shape, err := geo.ParseGeoShapeField(propertyMightBeGeoShape)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tif shape == geo.GeometryCollectionType {\n\t\tgeoShapes, found := geo.ExtractGeometryCollection(propertyMightBeGeoShape)\n\t\tif found {\n\t\t\tfieldName := getFieldName(pathString, path, fm)\n\t\t\toptions := fm.Options()\n\t\t\tfield := document.NewGeometryCollectionFieldFromShapesWithIndexingOptions(fieldName,\n\t\t\t\tindexes, geoShapes, options)\n\t\t\tcontext.doc.AddField(field)\n\n\t\t\tif !fm.IncludeInAll {\n\t\t\t\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tvar geoShape *geojson.GeoShape\n\t\tvar found bool\n\n\t\tif shape == geo.CircleType {\n\t\t\tgeoShape, found = geo.ExtractCircle(propertyMightBeGeoShape)\n\t\t} else {\n\t\t\tgeoShape, found = geo.ExtractGeoShapeCoordinates(coordValue, shape)\n\t\t}\n\n\t\tif found {\n\t\t\tfieldName := getFieldName(pathString, path, fm)\n\t\t\toptions := fm.Options()\n\t\t\tfield := document.NewGeoShapeFieldFromShapeWithIndexingOptions(fieldName,\n\t\t\t\tindexes, geoShape, options)\n\t\t\tcontext.doc.AddField(field)\n\n\t\t\tif !fm.IncludeInAll {\n\t\t\t\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (fm *FieldMapping) analyzerForField(path []string, context *walkContext) analysis.Analyzer {\n\tanalyzerName := fm.Analyzer\n\tif analyzerName == \"\" {\n\t\tanalyzerName = context.dm.defaultAnalyzerName(path)\n\t\tif analyzerName == \"\" {\n\t\t\tanalyzerName = context.im.DefaultAnalyzer\n\t\t}\n\t}\n\treturn context.im.AnalyzerNamed(analyzerName)\n}\n\nfunc getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string {\n\tfieldName := pathString\n\tif fieldMapping.Name != \"\" {\n\t\tparentName := \"\"\n\t\tif len(path) > 1 {\n\t\t\tparentName = encodePath(path[:len(path)-1]) + pathSeparator\n\t\t}\n\t\tfieldName = parentName + fieldMapping.Name\n\t}\n\treturn fieldName\n}\n\n// UnmarshalJSON offers custom unmarshaling with optional strict validation\nfunc (fm *FieldMapping) UnmarshalJSON(data []byte) error {\n\tvar tmp map[string]json.RawMessage\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar invalidKeys []string\n\tfor k, v := range tmp {\n\t\tswitch k {\n\t\tcase \"name\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.Name)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"type\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.Type)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"analyzer\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.Analyzer)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"store\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.Store)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"index\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.Index)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"include_term_vectors\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.IncludeTermVectors)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"include_in_all\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.IncludeInAll)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"date_format\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.DateFormat)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"docvalues\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.DocValues)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"skip_freq_norm\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.SkipFreqNorm)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"dims\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.Dims)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"similarity\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.Similarity)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"vector_index_optimized_for\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.VectorIndexOptimizedFor)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"synonym_source\":\n\t\t\terr := util.UnmarshalJSON(v, &fm.SynonymSource)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tdefault:\n\t\t\tinvalidKeys = append(invalidKeys, k)\n\t\t}\n\t}\n\n\tif MappingJSONStrict && len(invalidKeys) > 0 {\n\t\treturn fmt.Errorf(\"field mapping contains invalid keys: %v\", invalidKeys)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "mapping/index.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/optional\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar MappingJSONStrict = false\n\nconst defaultTypeField = \"_type\"\nconst defaultType = \"_default\"\nconst defaultField = \"_all\"\nconst defaultAnalyzer = standard.Name\nconst defaultDateTimeParser = optional.Name\n\n// An IndexMappingImpl controls how objects are placed\n// into an index.\n// First the type of the object is determined.\n// Once the type is know, the appropriate\n// DocumentMapping is selected by the type.\n// If no mapping was determined for that type,\n// a DefaultMapping will be used.\ntype IndexMappingImpl struct {\n\tTypeMapping           map[string]*DocumentMapping `json:\"types,omitempty\"`\n\tDefaultMapping        *DocumentMapping            `json:\"default_mapping\"`\n\tTypeField             string                      `json:\"type_field\"`\n\tDefaultType           string                      `json:\"default_type\"`\n\tDefaultAnalyzer       string                      `json:\"default_analyzer\"`\n\tDefaultDateTimeParser string                      `json:\"default_datetime_parser\"`\n\tDefaultSynonymSource  string                      `json:\"default_synonym_source,omitempty\"`\n\tScoringModel          string                      `json:\"scoring_model,omitempty\"`\n\tDefaultField          string                      `json:\"default_field\"`\n\tStoreDynamic          bool                        `json:\"store_dynamic\"`\n\tIndexDynamic          bool                        `json:\"index_dynamic\"`\n\tDocValuesDynamic      bool                        `json:\"docvalues_dynamic\"`\n\tCustomAnalysis        *customAnalysis             `json:\"analysis,omitempty\"`\n\tcache                 *registry.Cache\n}\n\n// AddCustomCharFilter defines a custom char filter for use in this mapping\nfunc (im *IndexMappingImpl) AddCustomCharFilter(name string, config map[string]interface{}) error {\n\t_, err := im.cache.DefineCharFilter(name, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tim.CustomAnalysis.CharFilters[name] = config\n\treturn nil\n}\n\n// AddCustomTokenizer defines a custom tokenizer for use in this mapping\nfunc (im *IndexMappingImpl) AddCustomTokenizer(name string, config map[string]interface{}) error {\n\t_, err := im.cache.DefineTokenizer(name, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tim.CustomAnalysis.Tokenizers[name] = config\n\treturn nil\n}\n\n// AddCustomTokenMap defines a custom token map for use in this mapping\nfunc (im *IndexMappingImpl) AddCustomTokenMap(name string, config map[string]interface{}) error {\n\t_, err := im.cache.DefineTokenMap(name, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tim.CustomAnalysis.TokenMaps[name] = config\n\treturn nil\n}\n\n// AddCustomTokenFilter defines a custom token filter for use in this mapping\nfunc (im *IndexMappingImpl) AddCustomTokenFilter(name string, config map[string]interface{}) error {\n\t_, err := im.cache.DefineTokenFilter(name, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tim.CustomAnalysis.TokenFilters[name] = config\n\treturn nil\n}\n\n// AddCustomAnalyzer defines a custom analyzer for use in this mapping. The\n// config map must have a \"type\" string entry to resolve the analyzer\n// constructor. The constructor is invoked with the remaining entries and\n// returned analyzer is registered in the IndexMapping.\n//\n// bleve comes with predefined analyzers, like\n// github.com/blevesearch/bleve/analysis/analyzer/custom. They are\n// available only if their package is imported by client code. To achieve this,\n// use their metadata to fill configuration entries:\n//\n//\timport (\n//\t    \"github.com/blevesearch/bleve/v2/analysis/analyzer/custom\"\n//\t    \"github.com/blevesearch/bleve/v2/analysis/char/html\"\n//\t    \"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n//\t    \"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode\"\n//\t)\n//\n//\tm := bleve.NewIndexMapping()\n//\terr := m.AddCustomAnalyzer(\"html\", map[string]interface{}{\n//\t    \"type\": custom.Name,\n//\t    \"char_filters\": []string{\n//\t        html.Name,\n//\t    },\n//\t    \"tokenizer\":     unicode.Name,\n//\t    \"token_filters\": []string{\n//\t        lowercase.Name,\n//\t        ...\n//\t    },\n//\t})\nfunc (im *IndexMappingImpl) AddCustomAnalyzer(name string, config map[string]interface{}) error {\n\t_, err := im.cache.DefineAnalyzer(name, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tim.CustomAnalysis.Analyzers[name] = config\n\treturn nil\n}\n\n// AddCustomDateTimeParser defines a custom date time parser for use in this mapping\nfunc (im *IndexMappingImpl) AddCustomDateTimeParser(name string, config map[string]interface{}) error {\n\t_, err := im.cache.DefineDateTimeParser(name, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tim.CustomAnalysis.DateTimeParsers[name] = config\n\treturn nil\n}\n\nfunc (im *IndexMappingImpl) AddSynonymSource(name string, config map[string]interface{}) error {\n\t_, err := im.cache.DefineSynonymSource(name, config)\n\tif err != nil {\n\t\treturn err\n\t}\n\tim.CustomAnalysis.SynonymSources[name] = config\n\treturn nil\n}\n\n// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules\nfunc NewIndexMapping() *IndexMappingImpl {\n\treturn &IndexMappingImpl{\n\t\tTypeMapping:           make(map[string]*DocumentMapping),\n\t\tDefaultMapping:        NewDocumentMapping(),\n\t\tTypeField:             defaultTypeField,\n\t\tDefaultType:           defaultType,\n\t\tDefaultAnalyzer:       defaultAnalyzer,\n\t\tDefaultDateTimeParser: defaultDateTimeParser,\n\t\tDefaultField:          defaultField,\n\t\tIndexDynamic:          IndexDynamic,\n\t\tStoreDynamic:          StoreDynamic,\n\t\tDocValuesDynamic:      DocValuesDynamic,\n\t\tCustomAnalysis:        newCustomAnalysis(),\n\t\tcache:                 registry.NewCache(),\n\t}\n}\n\n// Validate will walk the entire structure ensuring the following\n// explicitly named and default analyzers can be built\nfunc (im *IndexMappingImpl) Validate() error {\n\t_, err := im.cache.AnalyzerNamed(im.DefaultAnalyzer)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = im.cache.DateTimeParserNamed(im.DefaultDateTimeParser)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif im.DefaultSynonymSource != \"\" {\n\t\t_, err = im.cache.SynonymSourceNamed(im.DefaultSynonymSource)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// fieldAliasCtx is used to detect any field alias conflicts across the entire mapping\n\t// the map will hold the fully qualified field name to FieldMapping, so we can\n\t// check for conflicts as we validate each DocumentMapping.\n\tfieldAliasCtx := make(map[string]*FieldMapping)\n\t// ensure that the nested property is not set for top-level default mapping\n\tif im.DefaultMapping.Nested {\n\t\treturn fmt.Errorf(\"default mapping cannot be nested\")\n\t}\n\terr = im.DefaultMapping.Validate(im.cache, []string{}, fieldAliasCtx)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor name, docMapping := range im.TypeMapping {\n\t\t// ensure that the nested property is not set for top-level mappings\n\t\tif docMapping.Nested {\n\t\t\treturn fmt.Errorf(\"type mapping named: %s cannot be nested\", name)\n\t\t}\n\t\terr = docMapping.Validate(im.cache, []string{}, fieldAliasCtx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif _, ok := index.SupportedScoringModels[im.ScoringModel]; !ok && im.ScoringModel != \"\" {\n\t\treturn fmt.Errorf(\"unsupported scoring model: %s\", im.ScoringModel)\n\t}\n\n\treturn nil\n}\n\n// AddDocumentMapping sets a custom document mapping for the specified type\nfunc (im *IndexMappingImpl) AddDocumentMapping(doctype string, dm *DocumentMapping) {\n\tim.TypeMapping[doctype] = dm\n}\n\nfunc (im *IndexMappingImpl) mappingForType(docType string) *DocumentMapping {\n\tdocMapping := im.TypeMapping[docType]\n\tif docMapping == nil {\n\t\tdocMapping = im.DefaultMapping\n\t}\n\treturn docMapping\n}\n\n// UnmarshalJSON offers custom unmarshaling with optional strict validation\nfunc (im *IndexMappingImpl) UnmarshalJSON(data []byte) error {\n\n\tvar tmp map[string]json.RawMessage\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// set defaults for fields which might have been omitted\n\tim.cache = registry.NewCache()\n\tim.CustomAnalysis = newCustomAnalysis()\n\tim.TypeField = defaultTypeField\n\tim.DefaultType = defaultType\n\tim.DefaultAnalyzer = defaultAnalyzer\n\tim.DefaultDateTimeParser = defaultDateTimeParser\n\tim.DefaultField = defaultField\n\tim.DefaultMapping = NewDocumentMapping()\n\tim.TypeMapping = make(map[string]*DocumentMapping)\n\tim.StoreDynamic = StoreDynamic\n\tim.IndexDynamic = IndexDynamic\n\tim.DocValuesDynamic = DocValuesDynamic\n\n\tvar invalidKeys []string\n\tfor k, v := range tmp {\n\t\tswitch k {\n\t\tcase \"analysis\":\n\t\t\terr := util.UnmarshalJSON(v, &im.CustomAnalysis)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"type_field\":\n\t\t\terr := util.UnmarshalJSON(v, &im.TypeField)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"default_type\":\n\t\t\terr := util.UnmarshalJSON(v, &im.DefaultType)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"default_analyzer\":\n\t\t\terr := util.UnmarshalJSON(v, &im.DefaultAnalyzer)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"default_datetime_parser\":\n\t\t\terr := util.UnmarshalJSON(v, &im.DefaultDateTimeParser)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"default_synonym_source\":\n\t\t\terr := util.UnmarshalJSON(v, &im.DefaultSynonymSource)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"default_field\":\n\t\t\terr := util.UnmarshalJSON(v, &im.DefaultField)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"default_mapping\":\n\t\t\terr := util.UnmarshalJSON(v, &im.DefaultMapping)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"types\":\n\t\t\terr := util.UnmarshalJSON(v, &im.TypeMapping)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"store_dynamic\":\n\t\t\terr := util.UnmarshalJSON(v, &im.StoreDynamic)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"index_dynamic\":\n\t\t\terr := util.UnmarshalJSON(v, &im.IndexDynamic)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"docvalues_dynamic\":\n\t\t\terr := util.UnmarshalJSON(v, &im.DocValuesDynamic)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase \"scoring_model\":\n\t\t\terr := util.UnmarshalJSON(v, &im.ScoringModel)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\tdefault:\n\t\t\tinvalidKeys = append(invalidKeys, k)\n\t\t}\n\t}\n\n\tif MappingJSONStrict && len(invalidKeys) > 0 {\n\t\treturn fmt.Errorf(\"index mapping contains invalid keys: %v\", invalidKeys)\n\t}\n\n\terr = im.CustomAnalysis.registerAll(im)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (im *IndexMappingImpl) determineType(data interface{}) string {\n\t// first see if the object implements bleveClassifier\n\tbleveClassifier, ok := data.(bleveClassifier)\n\tif ok {\n\t\treturn bleveClassifier.BleveType()\n\t}\n\t// next see if the object implements Classifier\n\tclassifier, ok := data.(Classifier)\n\tif ok {\n\t\treturn classifier.Type()\n\t}\n\n\t// now see if we can find a type using the mapping\n\ttyp, ok := mustString(lookupPropertyPath(data, im.TypeField))\n\tif ok {\n\t\treturn typ\n\t}\n\n\treturn im.DefaultType\n}\n\nfunc (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}) error {\n\tdocType := im.determineType(data)\n\tdocMapping := im.mappingForType(docType)\n\tif docMapping.Enabled {\n\t\twalkContext := im.newWalkContext(doc, docMapping)\n\t\tdocMapping.walkDocument(data, []string{}, []uint64{}, walkContext)\n\n\t\t// see if the _all field was disabled\n\t\tallMapping, _ := docMapping.documentMappingForPath(\"_all\")\n\t\tif allMapping == nil || allMapping.Enabled {\n\t\t\tfield := document.NewCompositeFieldWithIndexingOptions(\"_all\", true, []string{}, walkContext.excludedFromAll, index.IndexField|index.IncludeTermVectors)\n\t\t\tdoc.AddField(field)\n\t\t}\n\t\tdoc.SetIndexed()\n\t}\n\n\treturn nil\n}\n\nfunc (im *IndexMappingImpl) MapSynonymDocument(doc *document.Document, collection string, input []string, synonyms []string) error {\n\t// determine all the synonym sources with the given collection\n\t// and create a synonym field for each\n\terr := im.SynonymSourceVisitor(func(name string, item analysis.SynonymSource) error {\n\t\tif item.Collection() == collection {\n\t\t\t// create a new field with the name of the synonym source\n\t\t\tanalyzer := im.AnalyzerNamed(item.Analyzer())\n\t\t\tif analyzer == nil {\n\t\t\t\treturn fmt.Errorf(\"unknown analyzer named: %s\", item.Analyzer())\n\t\t\t}\n\t\t\tfield := document.NewSynonymField(name, analyzer, input, synonyms)\n\t\t\tdoc.AddField(field)\n\t\t}\n\t\treturn nil\n\t})\n\treturn err\n}\n\ntype walkContext struct {\n\tdoc             *document.Document\n\tim              *IndexMappingImpl\n\tdm              *DocumentMapping\n\texcludedFromAll []string\n}\n\nfunc (im *IndexMappingImpl) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext {\n\treturn &walkContext{\n\t\tdoc:             doc,\n\t\tim:              im,\n\t\tdm:              dm,\n\t\texcludedFromAll: []string{\"_id\"},\n\t}\n}\n\n// AnalyzerNameForPath attempts to find the best analyzer to use with only a\n// field name will walk all the document types, look for field mappings at the\n// provided path, if one exists and it has an explicit analyzer that is\n// returned.\nfunc (im *IndexMappingImpl) AnalyzerNameForPath(path string) string {\n\t// first we look for explicit mapping on the field\n\tfor _, docMapping := range im.TypeMapping {\n\t\tanalyzerName := docMapping.analyzerNameForPath(path)\n\t\tif analyzerName != \"\" {\n\t\t\treturn analyzerName\n\t\t}\n\t}\n\n\t// now try the default mapping\n\tpathMapping, _ := im.DefaultMapping.documentMappingForPath(path)\n\tif pathMapping != nil {\n\t\tif len(pathMapping.Fields) > 0 {\n\t\t\tif pathMapping.Fields[0].Analyzer != \"\" {\n\t\t\t\treturn pathMapping.Fields[0].Analyzer\n\t\t\t}\n\t\t}\n\t}\n\n\t// next we will try default analyzers for the path\n\tpathDecoded := decodePath(path)\n\tfor _, docMapping := range im.TypeMapping {\n\t\tif docMapping.Enabled {\n\t\t\trv := docMapping.defaultAnalyzerName(pathDecoded)\n\t\t\tif rv != \"\" {\n\t\t\t\treturn rv\n\t\t\t}\n\t\t}\n\t}\n\t// now the default analyzer for the default mapping\n\tif im.DefaultMapping.Enabled {\n\t\trv := im.DefaultMapping.defaultAnalyzerName(pathDecoded)\n\t\tif rv != \"\" {\n\t\t\treturn rv\n\t\t}\n\t}\n\n\treturn im.DefaultAnalyzer\n}\n\nfunc (im *IndexMappingImpl) AnalyzerNamed(name string) analysis.Analyzer {\n\tanalyzer, err := im.cache.AnalyzerNamed(name)\n\tif err != nil {\n\t\tlogger.Printf(\"error using analyzer named: %s\", name)\n\t\treturn nil\n\t}\n\treturn analyzer\n}\n\nfunc (im *IndexMappingImpl) DateTimeParserNamed(name string) analysis.DateTimeParser {\n\tif name == \"\" {\n\t\tname = im.DefaultDateTimeParser\n\t}\n\tdateTimeParser, err := im.cache.DateTimeParserNamed(name)\n\tif err != nil {\n\t\tlogger.Printf(\"error using datetime parser named: %s\", name)\n\t\treturn nil\n\t}\n\treturn dateTimeParser\n}\n\nfunc (im *IndexMappingImpl) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) {\n\tanalyzer, err := im.cache.AnalyzerNamed(analyzerName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn analyzer.Analyze(text), nil\n}\n\n// FieldAnalyzer returns the name of the analyzer used on a field.\nfunc (im *IndexMappingImpl) FieldAnalyzer(field string) string {\n\treturn im.AnalyzerNameForPath(field)\n}\n\n// FieldMappingForPath returns the mapping for a specific field 'path'.\nfunc (im *IndexMappingImpl) FieldMappingForPath(path string) FieldMapping {\n\tif im.TypeMapping != nil {\n\t\tfor _, v := range im.TypeMapping {\n\t\t\tfm := v.fieldDescribedByPath(path)\n\t\t\tif fm != nil {\n\t\t\t\treturn *fm\n\t\t\t}\n\t\t}\n\t}\n\n\tfm := im.DefaultMapping.fieldDescribedByPath(path)\n\tif fm != nil {\n\t\treturn *fm\n\t}\n\n\treturn FieldMapping{}\n}\n\n// wrapper to satisfy new interface\n\nfunc (im *IndexMappingImpl) DefaultSearchField() string {\n\treturn im.DefaultField\n}\n\nfunc (im *IndexMappingImpl) SynonymSourceNamed(name string) analysis.SynonymSource {\n\tsyn, err := im.cache.SynonymSourceNamed(name)\n\tif err != nil {\n\t\tlogger.Printf(\"error using synonym source named: %s\", name)\n\t\treturn nil\n\t}\n\treturn syn\n}\n\nfunc (im *IndexMappingImpl) SynonymSourceForPath(path string) string {\n\t// first we look for explicit mapping on the field\n\tfor _, docMapping := range im.TypeMapping {\n\t\tsynonymSource := docMapping.synonymSourceForPath(path)\n\t\tif synonymSource != \"\" {\n\t\t\treturn synonymSource\n\t\t}\n\t}\n\n\t// now try the default mapping\n\tpathMapping, _ := im.DefaultMapping.documentMappingForPath(path)\n\tif pathMapping != nil {\n\t\tif len(pathMapping.Fields) > 0 {\n\t\t\tif pathMapping.Fields[0].SynonymSource != \"\" {\n\t\t\t\treturn pathMapping.Fields[0].SynonymSource\n\t\t\t}\n\t\t}\n\t}\n\n\t// next we will try default synonym sources for the path\n\tpathDecoded := decodePath(path)\n\tfor _, docMapping := range im.TypeMapping {\n\t\tif docMapping.Enabled {\n\t\t\trv := docMapping.defaultSynonymSource(pathDecoded)\n\t\t\tif rv != \"\" {\n\t\t\t\treturn rv\n\t\t\t}\n\t\t}\n\t}\n\t// now the default analyzer for the default mapping\n\tif im.DefaultMapping.Enabled {\n\t\trv := im.DefaultMapping.defaultSynonymSource(pathDecoded)\n\t\tif rv != \"\" {\n\t\t\treturn rv\n\t\t}\n\t}\n\n\treturn im.DefaultSynonymSource\n}\n\n// SynonymCount() returns the number of synonym sources defined in the mapping\nfunc (im *IndexMappingImpl) SynonymCount() int {\n\treturn len(im.CustomAnalysis.SynonymSources)\n}\n\n// SynonymSourceVisitor() allows a visitor to iterate over all synonym sources\nfunc (im *IndexMappingImpl) SynonymSourceVisitor(visitor analysis.SynonymSourceVisitor) error {\n\terr := im.cache.SynonymSources.VisitSynonymSources(visitor)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (im *IndexMappingImpl) buildNestedPrefixes() map[string]int {\n\tprefixDepth := make(map[string]int)\n\tvar collectNestedFields func(dm *DocumentMapping, pathComponents []string, currentDepth int)\n\tcollectNestedFields = func(dm *DocumentMapping, pathComponents []string, currentDepth int) {\n\t\tfor name, docMapping := range dm.Properties {\n\t\t\tnewPathComponents := append(pathComponents, name)\n\t\t\tif docMapping.Nested {\n\t\t\t\t// This is a nested field boundary\n\t\t\t\tnewDepth := currentDepth + 1\n\t\t\t\tprefixDepth[strings.Join(newPathComponents, pathSeparator)] = newDepth\n\t\t\t\t// Continue deeper with incremented depth\n\t\t\t\tcollectNestedFields(docMapping, newPathComponents, newDepth)\n\t\t\t} else {\n\t\t\t\t// Not nested, continue with same depth\n\t\t\t\tcollectNestedFields(docMapping, newPathComponents, currentDepth)\n\t\t\t}\n\t\t}\n\t}\n\t// Start from depth 0 (root)\n\tif im.DefaultMapping != nil && im.DefaultMapping.Enabled {\n\t\tcollectNestedFields(im.DefaultMapping, []string{}, 0)\n\t}\n\t// Now do this for each type mapping\n\tfor _, docMapping := range im.TypeMapping {\n\t\tif docMapping.Enabled {\n\t\t\tcollectNestedFields(docMapping, []string{}, 0)\n\t\t}\n\t}\n\treturn prefixDepth\n}\n\nfunc (im *IndexMappingImpl) NestedDepth(fs search.FieldSet) (int, int) {\n\tif im.cache == nil || im.cache.NestedPrefixes == nil {\n\t\treturn 0, 0\n\t}\n\n\tim.cache.NestedPrefixes.InitOnce(func() map[string]int {\n\t\treturn im.buildNestedPrefixes()\n\t})\n\n\treturn im.cache.NestedPrefixes.NestedDepth(fs)\n}\n\nfunc (im *IndexMappingImpl) CountNested() int {\n\tif im.cache == nil || im.cache.NestedPrefixes == nil {\n\t\treturn 0\n\t}\n\n\tim.cache.NestedPrefixes.InitOnce(func() map[string]int {\n\t\treturn im.buildNestedPrefixes()\n\t})\n\n\treturn im.cache.NestedPrefixes.CountNested()\n}\n\nfunc (im *IndexMappingImpl) IntersectsPrefix(fs search.FieldSet) bool {\n\tif im.cache == nil || im.cache.NestedPrefixes == nil {\n\t\treturn false\n\t}\n\n\tim.cache.NestedPrefixes.InitOnce(func() map[string]int {\n\t\treturn im.buildNestedPrefixes()\n\t})\n\n\treturn im.cache.NestedPrefixes.IntersectsPrefix(fs)\n}\n"
  },
  {
    "path": "mapping/mapping.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\nimport (\n\t\"io\"\n\t\"log\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\n// A Classifier is an interface describing any object which knows how to\n// identify its own type.  Alternatively, if a struct already has a Type\n// field or method in conflict, one can use BleveType instead.\ntype Classifier interface {\n\tType() string\n}\n\n// A bleveClassifier is an interface describing any object which knows how\n// to identify its own type.  This is introduced as an alternative to the\n// Classifier interface which often has naming conflicts with existing\n// structures.\ntype bleveClassifier interface {\n\tBleveType() string\n}\n\nvar logger = log.New(io.Discard, \"bleve mapping \", log.LstdFlags)\n\n// SetLog sets the logger used for logging\n// by default log messages are sent to io.Discard\nfunc SetLog(l *log.Logger) {\n\tlogger = l\n}\n\ntype IndexMapping interface {\n\tMapDocument(doc *document.Document, data interface{}) error\n\tValidate() error\n\n\tDateTimeParserNamed(name string) analysis.DateTimeParser\n\n\tDefaultSearchField() string\n\n\tAnalyzerNameForPath(path string) string\n\tAnalyzerNamed(name string) analysis.Analyzer\n\n\tFieldMappingForPath(path string) FieldMapping\n}\n\n// A SynonymMapping extends the IndexMapping interface to provide\n// additional methods for working with synonyms.\ntype SynonymMapping interface {\n\tIndexMapping\n\n\tMapSynonymDocument(doc *document.Document, collection string, input []string, synonyms []string) error\n\n\tSynonymSourceForPath(path string) string\n\n\tSynonymSourceNamed(name string) analysis.SynonymSource\n\n\tSynonymCount() int\n\n\tSynonymSourceVisitor(visitor analysis.SynonymSourceVisitor) error\n}\n\n// A NestedMapping extends the IndexMapping interface to provide\n// additional methods for working with nested object mappings.\ntype NestedMapping interface {\n\t// NestedDepth returns two values:\n\t//   - common: the highest nested level that is common to all given field paths,\n\t//     if 0 then there is no common nested level among the given field paths\n\t//   - max: the highest nested level that applies to at least one of the given field paths\n\t//     if 0 then none of the given field paths are nested\n\tNestedDepth(fieldPaths search.FieldSet) (int, int)\n\n\t// IntersectsPrefix returns true if any of the given\n\t// field paths intersect with a known nested prefix\n\tIntersectsPrefix(fieldPaths search.FieldSet) bool\n\n\t// CountNested returns the number of nested object mappings\n\tCountNested() int\n}\n"
  },
  {
    "path": "mapping/mapping_no_vectors.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !vectors\n// +build !vectors\n\npackage mapping\n\nfunc NewVectorFieldMapping() *FieldMapping {\n\treturn nil\n}\n\nfunc NewVectorBase64FieldMapping() *FieldMapping {\n\treturn nil\n}\n\nfunc (fm *FieldMapping) processVector(propertyMightBeVector interface{},\n\tpathString string, path []string, indexes []uint64, context *walkContext) bool {\n\treturn false\n}\n\nfunc (fm *FieldMapping) processVectorBase64(propertyMightBeVector interface{},\n\tpathString string, path []string, indexes []uint64, context *walkContext) {\n\n}\n\n// -----------------------------------------------------------------------------\n// document validation functions\n\nfunc validateFieldMapping(field *FieldMapping, path []string,\n\tfieldAliasCtx map[string]*FieldMapping) error {\n\treturn validateFieldType(field)\n}\n"
  },
  {
    "path": "mapping/mapping_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/exception\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n)\n\nvar mappingSource = []byte(`{\n    \"types\": {\n    \t\"beer\": {\n    \t\t\"properties\": {\n    \t\t\t\"name\": {\n    \t\t\t\t\"fields\": [\n    \t\t\t\t\t{\n    \t\t\t\t\t\t\"name\": \"name\",\n    \t\t\t\t\t\t\"type\": \"text\",\n    \t\t\t\t\t\t\"analyzer\": \"standard\",\n    \t\t\t\t\t\t\"store\": true,\n    \t\t\t\t\t\t\"index\": true,\n                            \"include_term_vectors\": true,\n                            \"include_in_all\": true,\n                            \"docvalues\": true\n    \t\t\t\t\t}\n    \t\t\t\t]\n    \t\t\t}\n    \t\t}\n    \t},\n    \t\"brewery\": {\n    \t}\n    },\n    \"type_field\": \"_type\",\n    \"default_type\": \"_default\"\n}`)\n\nfunc buildMapping() IndexMapping {\n\tnameFieldMapping := NewTextFieldMapping()\n\tnameFieldMapping.Name = \"name\"\n\tnameFieldMapping.Analyzer = \"standard\"\n\n\tbeerMapping := NewDocumentMapping()\n\tbeerMapping.AddFieldMappingsAt(\"name\", nameFieldMapping)\n\n\tbreweryMapping := NewDocumentMapping()\n\n\tmapping := NewIndexMapping()\n\tmapping.AddDocumentMapping(\"beer\", beerMapping)\n\tmapping.AddDocumentMapping(\"brewery\", breweryMapping)\n\n\treturn mapping\n}\n\nfunc TestUnmarshalMappingJSON(t *testing.T) {\n\tmapping := buildMapping()\n\n\tvar indexMapping IndexMappingImpl\n\terr := json.Unmarshal(mappingSource, &indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif !reflect.DeepEqual(&indexMapping, mapping) {\n\t\tt.Errorf(\"expected %#v,\\n got %#v\", mapping, &indexMapping)\n\t}\n}\n\nfunc TestMappingStructWithJSONTags(t *testing.T) {\n\n\tmapping := buildMapping()\n\n\tx := struct {\n\t\tNoJSONTag string\n\t\tName      string `json:\"name\"`\n\t}{\n\t\tName: \"marty\",\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\terr := mapping.MapDocument(doc, x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfoundJSONName := false\n\tfoundNoJSONName := false\n\tcount := 0\n\tfor _, f := range doc.Fields {\n\t\tif f.Name() == \"name\" {\n\t\t\tfoundJSONName = true\n\t\t}\n\t\tif f.Name() == \"NoJSONTag\" {\n\t\t\tfoundNoJSONName = true\n\t\t}\n\t\tcount++\n\t}\n\tif !foundJSONName {\n\t\tt.Errorf(\"expected to find field named 'name'\")\n\t}\n\tif !foundNoJSONName {\n\t\tt.Errorf(\"expected to find field named 'NoJSONTag'\")\n\t}\n\tif count != 2 {\n\t\tt.Errorf(\"expected to find 2 find, found %d\", count)\n\t}\n}\n\nfunc TestMappingStructWithJSONTagsOneDisabled(t *testing.T) {\n\n\tmapping := buildMapping()\n\n\tx := struct {\n\t\tName      string `json:\"name\"`\n\t\tTitle     string `json:\"-\"`\n\t\tNoJSONTag string\n\t}{\n\t\tName: \"marty\",\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\terr := mapping.MapDocument(doc, x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfoundJSONName := false\n\tfoundNoJSONName := false\n\tcount := 0\n\tfor _, f := range doc.Fields {\n\t\tif f.Name() == \"name\" {\n\t\t\tfoundJSONName = true\n\t\t}\n\t\tif f.Name() == \"NoJSONTag\" {\n\t\t\tfoundNoJSONName = true\n\t\t}\n\t\tcount++\n\t}\n\tif !foundJSONName {\n\t\tt.Errorf(\"expected to find field named 'name'\")\n\t}\n\tif !foundNoJSONName {\n\t\tt.Errorf(\"expected to find field named 'NoJSONTag'\")\n\t}\n\tif count != 2 {\n\t\tt.Errorf(\"expected to find 2 find, found %d\", count)\n\t}\n}\n\nfunc TestMappingStructWithAlternateTags(t *testing.T) {\n\n\tmapping := buildMapping()\n\tmapping.(*IndexMappingImpl).DefaultMapping.StructTagKey = \"bleve\"\n\n\tx := struct {\n\t\tNoBLEVETag string\n\t\tName       string `bleve:\"name\"`\n\t}{\n\t\tName: \"marty\",\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\terr := mapping.MapDocument(doc, x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfoundBLEVEName := false\n\tfoundNoBLEVEName := false\n\tcount := 0\n\tfor _, f := range doc.Fields {\n\t\tif f.Name() == \"name\" {\n\t\t\tfoundBLEVEName = true\n\t\t}\n\t\tif f.Name() == \"NoBLEVETag\" {\n\t\t\tfoundNoBLEVEName = true\n\t\t}\n\t\tcount++\n\t}\n\tif !foundBLEVEName {\n\t\tt.Errorf(\"expected to find field named 'name'\")\n\t}\n\tif !foundNoBLEVEName {\n\t\tt.Errorf(\"expected to find field named 'NoBLEVETag'\")\n\t}\n\tif count != 2 {\n\t\tt.Errorf(\"expected to find 2 find, found %d\", count)\n\t}\n}\n\nfunc TestMappingStructWithAlternateTagsTwoDisabled(t *testing.T) {\n\n\tmapping := buildMapping()\n\tmapping.(*IndexMappingImpl).DefaultMapping.StructTagKey = \"bleve\"\n\n\tx := struct {\n\t\tName       string `json:\"-\"     bleve:\"name\"`\n\t\tTitle      string `json:\"-\"     bleve:\"-\"`\n\t\tNoBLEVETag string `json:\"-\"`\n\t\tExtra      string `json:\"extra\" bleve:\"-\"`\n\t}{\n\t\tName: \"marty\",\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\terr := mapping.MapDocument(doc, x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfoundBLEVEName := false\n\tfoundNoBLEVEName := false\n\tcount := 0\n\tfor _, f := range doc.Fields {\n\t\tif f.Name() == \"name\" {\n\t\t\tfoundBLEVEName = true\n\t\t}\n\t\tif f.Name() == \"NoBLEVETag\" {\n\t\t\tfoundNoBLEVEName = true\n\t\t}\n\t\tcount++\n\t}\n\tif !foundBLEVEName {\n\t\tt.Errorf(\"expected to find field named 'name'\")\n\t}\n\tif !foundNoBLEVEName {\n\t\tt.Errorf(\"expected to find field named 'NoBLEVETag'\")\n\t}\n\tif count != 2 {\n\t\tt.Errorf(\"expected to find 2 find, found %d\", count)\n\t}\n}\n\nfunc TestMappingStructWithPointerToString(t *testing.T) {\n\n\tmapping := buildMapping()\n\n\tname := \"marty\"\n\n\tx := struct {\n\t\tName *string\n\t}{\n\t\tName: &name,\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\terr := mapping.MapDocument(doc, x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfound := false\n\tcount := 0\n\tfor _, f := range doc.Fields {\n\t\tif f.Name() == \"Name\" {\n\t\t\tfound = true\n\t\t}\n\t\tcount++\n\t}\n\tif !found {\n\t\tt.Errorf(\"expected to find field named 'Name'\")\n\t}\n\tif count != 1 {\n\t\tt.Errorf(\"expected to find 1 find, found %d\", count)\n\t}\n}\n\nfunc TestMappingJSONWithNull(t *testing.T) {\n\n\tmapping := NewIndexMapping()\n\n\tjsonbytes := []byte(`{\"name\":\"marty\", \"age\": null}`)\n\tvar jsondoc interface{}\n\terr := json.Unmarshal(jsonbytes, &jsondoc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\terr = mapping.MapDocument(doc, jsondoc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfound := false\n\tcount := 0\n\tfor _, f := range doc.Fields {\n\t\tif f.Name() == \"name\" {\n\t\t\tfound = true\n\t\t}\n\t\tcount++\n\t}\n\tif !found {\n\t\tt.Errorf(\"expected to find field named 'name'\")\n\t}\n\tif count != 1 {\n\t\tt.Errorf(\"expected to find 1 find, found %d\", count)\n\t}\n}\n\nfunc TestMappingForPath(t *testing.T) {\n\n\tenFieldMapping := NewTextFieldMapping()\n\tenFieldMapping.Analyzer = \"en\"\n\n\tdocMappingA := NewDocumentMapping()\n\tdocMappingA.AddFieldMappingsAt(\"name\", enFieldMapping)\n\n\tcustomMapping := NewTextFieldMapping()\n\tcustomMapping.Analyzer = \"xyz\"\n\tcustomMapping.Name = \"nameCustom\"\n\n\tsubDocMappingB := NewDocumentMapping()\n\tcustomFieldX := NewTextFieldMapping()\n\tcustomFieldX.Analyzer = \"analyzerx\"\n\tsubDocMappingB.AddFieldMappingsAt(\"desc\", customFieldX)\n\n\tdocMappingA.AddFieldMappingsAt(\"author\", enFieldMapping, customMapping)\n\tdocMappingA.AddSubDocumentMapping(\"child\", subDocMappingB)\n\n\tmapping := NewIndexMapping()\n\tmapping.AddDocumentMapping(\"a\", docMappingA)\n\n\tanalyzerName := mapping.AnalyzerNameForPath(\"name\")\n\tif analyzerName != enFieldMapping.Analyzer {\n\t\tt.Errorf(\"expected '%s' got '%s'\", enFieldMapping.Analyzer, analyzerName)\n\t}\n\n\tanalyzerName = mapping.AnalyzerNameForPath(\"nameCustom\")\n\tif analyzerName != customMapping.Analyzer {\n\t\tt.Errorf(\"expected '%s' got '%s'\", customMapping.Analyzer, analyzerName)\n\t}\n\n\tanalyzerName = mapping.AnalyzerNameForPath(\"child.desc\")\n\tif analyzerName != customFieldX.Analyzer {\n\t\tt.Errorf(\"expected '%s' got '%s'\", customFieldX.Analyzer, analyzerName)\n\t}\n\n}\n\nfunc TestMappingWithTokenizerDeps(t *testing.T) {\n\n\ttokNoDeps := map[string]interface{}{\n\t\t\"type\":   regexp.Name,\n\t\t\"regexp\": \"\",\n\t}\n\n\ttokDepsL1 := map[string]interface{}{\n\t\t\"type\":       exception.Name,\n\t\t\"tokenizer\":  \"a\",\n\t\t\"exceptions\": []string{\".*\"},\n\t}\n\n\t// this tests a 1-level dependency\n\t// it is run 100 times to increase the\n\t// likelihood that it fails along time way\n\t// (depends on key order iteration in map)\n\tfor i := 0; i < 100; i++ {\n\n\t\tm := NewIndexMapping()\n\t\tca := customAnalysis{\n\t\t\tTokenizers: map[string]map[string]interface{}{\n\t\t\t\t\"a\": tokNoDeps,\n\t\t\t\t\"b\": tokDepsL1,\n\t\t\t},\n\t\t}\n\t\terr := ca.registerAll(m)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\ttokDepsL2 := map[string]interface{}{\n\t\t\"type\":       \"exception\",\n\t\t\"tokenizer\":  \"b\",\n\t\t\"exceptions\": []string{\".*\"},\n\t}\n\n\t// now test a second-level dependency\n\tfor i := 0; i < 100; i++ {\n\n\t\tm := NewIndexMapping()\n\t\tca := customAnalysis{\n\t\t\tTokenizers: map[string]map[string]interface{}{\n\t\t\t\t\"a\": tokNoDeps,\n\t\t\t\t\"b\": tokDepsL1,\n\t\t\t\t\"c\": tokDepsL2,\n\t\t\t},\n\t\t}\n\t\terr := ca.registerAll(m)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\ttokUnsatisfied := map[string]interface{}{\n\t\t\"type\":      \"exception\",\n\t\t\"tokenizer\": \"e\",\n\t}\n\n\t// now make sure an unsatisfied dep still\n\t// results in an error\n\tm := NewIndexMapping()\n\tca := customAnalysis{\n\t\tTokenizers: map[string]map[string]interface{}{\n\t\t\t\"a\": tokNoDeps,\n\t\t\t\"b\": tokDepsL1,\n\t\t\t\"c\": tokDepsL2,\n\t\t\t\"d\": tokUnsatisfied,\n\t\t},\n\t}\n\terr := ca.registerAll(m)\n\tif err == nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestEnablingDisablingStoringDynamicFields(t *testing.T) {\n\n\t// first verify that with system defaults, dynamic field is stored\n\tdata := map[string]interface{}{\n\t\t\"name\": \"bleve\",\n\t}\n\tdoc := document.NewDocument(\"x\")\n\tmapping := NewIndexMapping()\n\terr := mapping.MapDocument(doc, data)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, field := range doc.Fields {\n\t\tif field.Name() == \"name\" && !field.Options().IsStored() {\n\t\t\tt.Errorf(\"expected field 'name' to be stored, isn't\")\n\t\t}\n\t}\n\n\t// now change system level defaults, verify dynamic field is not stored\n\tStoreDynamic = false\n\tdefer func() {\n\t\tStoreDynamic = true\n\t}()\n\n\tmapping = NewIndexMapping()\n\tdoc = document.NewDocument(\"y\")\n\terr = mapping.MapDocument(doc, data)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, field := range doc.Fields {\n\t\tif field.Name() == \"name\" && field.Options().IsStored() {\n\t\t\tt.Errorf(\"expected field 'name' to be not stored, is\")\n\t\t}\n\t}\n\n\t// now override the system level defaults inside the index mapping\n\tmapping = NewIndexMapping()\n\tmapping.StoreDynamic = true\n\tdoc = document.NewDocument(\"y\")\n\terr = mapping.MapDocument(doc, data)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, field := range doc.Fields {\n\t\tif field.Name() == \"name\" && !field.Options().IsStored() {\n\t\t\tt.Errorf(\"expected field 'name' to be stored, isn't\")\n\t\t}\n\t}\n}\n\nfunc TestMappingBool(t *testing.T) {\n\tboolMapping := NewBooleanFieldMapping()\n\tdocMapping := NewDocumentMapping()\n\tdocMapping.AddFieldMappingsAt(\"prop\", boolMapping)\n\tmapping := NewIndexMapping()\n\tmapping.AddDocumentMapping(\"doc\", docMapping)\n\n\tpprop := false\n\tx := struct {\n\t\tProp  bool  `json:\"prop\"`\n\t\tPProp *bool `json:\"pprop\"`\n\t}{\n\t\tProp:  true,\n\t\tPProp: &pprop,\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\terr := mapping.MapDocument(doc, x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfoundProp := false\n\tfoundPProp := false\n\tcount := 0\n\tfor _, f := range doc.Fields {\n\t\tif f.Name() == \"prop\" {\n\t\t\tfoundProp = true\n\t\t}\n\t\tif f.Name() == \"pprop\" {\n\t\t\tfoundPProp = true\n\t\t}\n\t\tcount++\n\t}\n\tif !foundProp {\n\t\tt.Errorf(\"expected to find bool field named 'prop'\")\n\t}\n\tif !foundPProp {\n\t\tt.Errorf(\"expected to find pointer to bool field named 'pprop'\")\n\t}\n\tif count != 2 {\n\t\tt.Errorf(\"expected to find 2 fields, found %d\", count)\n\t}\n}\n\nfunc TestDisableDefaultMapping(t *testing.T) {\n\tindexMapping := NewIndexMapping()\n\tindexMapping.DefaultMapping.Enabled = false\n\n\tdata := map[string]string{\n\t\t\"name\": \"bleve\",\n\t}\n\n\tdoc := document.NewDocument(\"x\")\n\terr := indexMapping.MapDocument(doc, data)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(doc.Fields) > 0 {\n\t\tt.Errorf(\"expected no fields, got %d\", len(doc.Fields))\n\t}\n}\n\nfunc TestInvalidFieldMappingStrict(t *testing.T) {\n\tmappingBytes := []byte(`{\"includeInAll\":true,\"name\":\"a parsed name\"}`)\n\n\t// first unmarhsal it without strict\n\tvar fm FieldMapping\n\terr := json.Unmarshal(mappingBytes, &fm)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif fm.Name != \"a parsed name\" {\n\t\tt.Fatalf(\"expect to find field mapping name 'a parsed name', got '%s'\", fm.Name)\n\t}\n\n\t// reset\n\tfm.Name = \"\"\n\n\t// now enable strict\n\tMappingJSONStrict = true\n\tdefer func() {\n\t\tMappingJSONStrict = false\n\t}()\n\n\texpectedInvalidKeys := []string{\"includeInAll\"}\n\texpectedErr := fmt.Errorf(\"field mapping contains invalid keys: %v\", expectedInvalidKeys)\n\terr = json.Unmarshal(mappingBytes, &fm)\n\tif err.Error() != expectedErr.Error() {\n\t\tt.Fatalf(\"expected err: %v, got err: %v\", expectedErr, err)\n\t}\n\n\tif fm.Name != \"a parsed name\" {\n\t\tt.Fatalf(\"expect to find field mapping name 'a parsed name', got '%s'\", fm.Name)\n\t}\n\n}\n\nfunc TestInvalidDocumentMappingStrict(t *testing.T) {\n\tmappingBytes := []byte(`{\"defaultAnalyzer\":true,\"enabled\":false}`)\n\n\t// first unmarhsal it without strict\n\tvar dm DocumentMapping\n\terr := json.Unmarshal(mappingBytes, &dm)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif dm.Enabled != false {\n\t\tt.Fatalf(\"expect to find document mapping enabled false, got '%t'\", dm.Enabled)\n\t}\n\n\t// reset\n\tdm.Enabled = true\n\n\t// now enable strict\n\tMappingJSONStrict = true\n\tdefer func() {\n\t\tMappingJSONStrict = false\n\t}()\n\n\texpectedInvalidKeys := []string{\"defaultAnalyzer\"}\n\texpectedErr := fmt.Errorf(\"document mapping contains invalid keys: %v\", expectedInvalidKeys)\n\terr = json.Unmarshal(mappingBytes, &dm)\n\tif err.Error() != expectedErr.Error() {\n\t\tt.Fatalf(\"expected err: %v, got err: %v\", expectedErr, err)\n\t}\n\n\tif dm.Enabled != false {\n\t\tt.Fatalf(\"expect to find document mapping enabled false, got '%t'\", dm.Enabled)\n\t}\n}\n\nfunc TestInvalidIndexMappingStrict(t *testing.T) {\n\tmappingBytes := []byte(`{\"typeField\":\"type\",\"default_field\":\"all\"}`)\n\n\t// first unmarhsal it without strict\n\tvar im IndexMappingImpl\n\terr := json.Unmarshal(mappingBytes, &im)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif im.DefaultField != \"all\" {\n\t\tt.Fatalf(\"expect to find index mapping default field 'all', got '%s'\", im.DefaultField)\n\t}\n\n\t// reset\n\tim.DefaultField = \"_all\"\n\n\t// now enable strict\n\tMappingJSONStrict = true\n\tdefer func() {\n\t\tMappingJSONStrict = false\n\t}()\n\n\texpectedInvalidKeys := []string{\"typeField\"}\n\texpectedErr := fmt.Errorf(\"index mapping contains invalid keys: %v\", expectedInvalidKeys)\n\terr = json.Unmarshal(mappingBytes, &im)\n\tif err.Error() != expectedErr.Error() {\n\t\tt.Fatalf(\"expected err: %v, got err: %v\", expectedErr, err)\n\t}\n\n\tif im.DefaultField != \"all\" {\n\t\tt.Fatalf(\"expect to find index mapping default field 'all', got '%s'\", im.DefaultField)\n\t}\n}\n\nfunc TestMappingBug353(t *testing.T) {\n\tdataBytes := `{\n  \"Reviews\": [\n    {\n      \"ReviewID\": \"RX16692001\",\n      \"Content\": \"Usually stay near the airport...\"\n    }\n\t],\n\t\"Other\": {\n\t  \"Inside\": \"text\"\n  },\n  \"Name\": \"The Inn at Baltimore White Marsh\"\n}`\n\n\tvar data map[string]interface{}\n\terr := json.Unmarshal([]byte(dataBytes), &data)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treviewContentFieldMapping := NewTextFieldMapping()\n\treviewContentFieldMapping.Analyzer = \"crazy\"\n\n\treviewsMapping := NewDocumentMapping()\n\treviewsMapping.Dynamic = false\n\treviewsMapping.AddFieldMappingsAt(\"Content\", reviewContentFieldMapping)\n\totherMapping := NewDocumentMapping()\n\totherMapping.Dynamic = false\n\tmapping := NewIndexMapping()\n\tmapping.DefaultMapping.AddSubDocumentMapping(\"Reviews\", reviewsMapping)\n\tmapping.DefaultMapping.AddSubDocumentMapping(\"Other\", otherMapping)\n\n\tdoc := document.NewDocument(\"x\")\n\terr = mapping.MapDocument(doc, data)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// expect doc has only 2 fields\n\tif len(doc.Fields) != 2 {\n\t\tt.Errorf(\"expected doc with 2 fields, got: %d\", len(doc.Fields))\n\t\tfor _, f := range doc.Fields {\n\t\t\tt.Logf(\"field named: %s\", f.Name())\n\t\t}\n\t}\n}\n\nfunc TestAnonymousStructFields(t *testing.T) {\n\n\ttype Contact0 string\n\n\ttype Contact1 struct {\n\t\tName string\n\t}\n\n\ttype Contact2 interface{}\n\n\ttype Contact3 interface{}\n\n\ttype Thing struct {\n\t\tContact0\n\t\tContact1\n\t\tContact2\n\t\tContact3\n\t}\n\n\tx := Thing{\n\t\tContact0: \"hello\",\n\t\tContact1: Contact1{\n\t\t\tName: \"marty\",\n\t\t},\n\t\tContact2: Contact1{\n\t\t\tName: \"will\",\n\t\t},\n\t\tContact3: \"steve\",\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tm := NewIndexMapping()\n\terr := m.MapDocument(doc, x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(doc.Fields) != 4 {\n\t\tt.Fatalf(\"expected 4 fields, got %d\", len(doc.Fields))\n\t}\n\tif doc.Fields[0].Name() != \"Contact0\" {\n\t\tt.Errorf(\"expected field named 'Contact0', got '%s'\", doc.Fields[0].Name())\n\t}\n\tif doc.Fields[1].Name() != \"Name\" {\n\t\tt.Errorf(\"expected field named 'Name', got '%s'\", doc.Fields[1].Name())\n\t}\n\tif doc.Fields[2].Name() != \"Contact2.Name\" {\n\t\tt.Errorf(\"expected field named 'Contact2.Name', got '%s'\", doc.Fields[2].Name())\n\t}\n\tif doc.Fields[3].Name() != \"Contact3\" {\n\t\tt.Errorf(\"expected field named 'Contact3', got '%s'\", doc.Fields[3].Name())\n\t}\n\n\ttype AnotherThing struct {\n\t\tContact0 `json:\"Alternate0\"`\n\t\tContact1 `json:\"Alternate1\"`\n\t\tContact2 `json:\"Alternate2\"`\n\t\tContact3 `json:\"Alternate3\"`\n\t}\n\n\ty := AnotherThing{\n\t\tContact0: \"hello\",\n\t\tContact1: Contact1{\n\t\t\tName: \"marty\",\n\t\t},\n\t\tContact2: Contact1{\n\t\t\tName: \"will\",\n\t\t},\n\t\tContact3: \"steve\",\n\t}\n\n\tdoc2 := document.NewDocument(\"2\")\n\terr = m.MapDocument(doc2, y)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(doc2.Fields) != 4 {\n\t\tt.Fatalf(\"expected 4 fields, got %d\", len(doc2.Fields))\n\t}\n\tif doc2.Fields[0].Name() != \"Alternate0\" {\n\t\tt.Errorf(\"expected field named 'Alternate0', got '%s'\", doc2.Fields[0].Name())\n\t}\n\tif doc2.Fields[1].Name() != \"Alternate1.Name\" {\n\t\tt.Errorf(\"expected field named 'Name', got '%s'\", doc2.Fields[1].Name())\n\t}\n\tif doc2.Fields[2].Name() != \"Alternate2.Name\" {\n\t\tt.Errorf(\"expected field named 'Alternate2.Name', got '%s'\", doc2.Fields[2].Name())\n\t}\n\tif doc2.Fields[3].Name() != \"Alternate3\" {\n\t\tt.Errorf(\"expected field named 'Alternate3', got '%s'\", doc2.Fields[3].Name())\n\t}\n}\n\nfunc TestAnonymousStructFieldWithJSONStructTagEmptString(t *testing.T) {\n\ttype InterfaceThing interface{}\n\ttype Thing struct {\n\t\tInterfaceThing `json:\"\"`\n\t}\n\n\tx := Thing{\n\t\tInterfaceThing: map[string]interface{}{\n\t\t\t\"key\": \"value\",\n\t\t},\n\t}\n\n\tdoc := document.NewDocument(\"1\")\n\tm := NewIndexMapping()\n\terr := m.MapDocument(doc, x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(doc.Fields) != 1 {\n\t\tt.Fatalf(\"expected 1 field, got %d\", len(doc.Fields))\n\t}\n\tif doc.Fields[0].Name() != \"key\" {\n\t\tt.Errorf(\"expected field named 'key', got '%s'\", doc.Fields[0].Name())\n\t}\n}\n\nfunc TestMappingPrimitives(t *testing.T) {\n\n\ttests := []struct {\n\t\tdata interface{}\n\t}{\n\t\t{data: \"marty\"},\n\t\t{data: int(1)},\n\t\t{data: int8(2)},\n\t\t{data: int16(3)},\n\t\t{data: int32(4)},\n\t\t{data: int64(5)},\n\t\t{data: uint(6)},\n\t\t{data: uint8(7)},\n\t\t{data: uint16(8)},\n\t\t{data: uint32(9)},\n\t\t{data: uint64(10)},\n\t\t{data: float32(11.0)},\n\t\t{data: float64(12.0)},\n\t\t{data: false},\n\t}\n\n\tm := NewIndexMapping()\n\tfor _, test := range tests {\n\t\tdoc := document.NewDocument(\"x\")\n\t\terr := m.MapDocument(doc, test.data)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif len(doc.Fields) != 1 {\n\t\t\tt.Errorf(\"expected 1 field, got %d for %v\", len(doc.Fields), test.data)\n\t\t}\n\t}\n}\n\nfunc TestMappingForGeo(t *testing.T) {\n\n\ttype Location struct {\n\t\tLat float64\n\t\tLon float64\n\t}\n\n\tnameFieldMapping := NewTextFieldMapping()\n\tnameFieldMapping.Name = \"name\"\n\tnameFieldMapping.Analyzer = \"standard\"\n\n\tlocFieldMapping := NewGeoPointFieldMapping()\n\n\tthingMapping := NewDocumentMapping()\n\tthingMapping.AddFieldMappingsAt(\"name\", nameFieldMapping)\n\tthingMapping.AddFieldMappingsAt(\"location\", locFieldMapping)\n\n\tmapping := NewIndexMapping()\n\tmapping.DefaultMapping = thingMapping\n\n\tgeopoints := []interface{}{}\n\texpect := [][]float64{} // to contain expected [lon,lat] for geopoints\n\n\t// geopoint as a struct\n\tgeopoints = append(geopoints, struct {\n\t\tName     string    `json:\"name\"`\n\t\tLocation *Location `json:\"location\"`\n\t}{\n\t\tName: \"struct\",\n\t\tLocation: &Location{\n\t\t\tLon: -180,\n\t\t\tLat: -90,\n\t\t},\n\t})\n\texpect = append(expect, []float64{-180, -90})\n\n\t// geopoint as a map\n\tgeopoints = append(geopoints, struct {\n\t\tName     string                 `json:\"name\"`\n\t\tLocation map[string]interface{} `json:\"location\"`\n\t}{\n\t\tName: \"map\",\n\t\tLocation: map[string]interface{}{\n\t\t\t\"lon\": -180,\n\t\t\t\"lat\": -90,\n\t\t},\n\t})\n\texpect = append(expect, []float64{-180, -90})\n\n\t// geopoint as a slice, format: {lon, lat}\n\tgeopoints = append(geopoints, struct {\n\t\tName     string        `json:\"name\"`\n\t\tLocation []interface{} `json:\"location\"`\n\t}{\n\t\tName: \"slice\",\n\t\tLocation: []interface{}{\n\t\t\t-180, -90,\n\t\t},\n\t})\n\texpect = append(expect, []float64{-180, -90})\n\n\t// geopoint as a string, format: \"lat,lon\"\n\tgeopoints = append(geopoints, struct {\n\t\tName     string        `json:\"name\"`\n\t\tLocation []interface{} `json:\"location\"`\n\t}{\n\t\tName: \"string\",\n\t\tLocation: []interface{}{\n\t\t\t\"-90,-180\",\n\t\t},\n\t})\n\texpect = append(expect, []float64{-180, -90})\n\n\t// geopoint as a string, format: \"lat , lon\" with leading/trailing whitespaces\n\tgeopoints = append(geopoints, struct {\n\t\tName     string        `json:\"name\"`\n\t\tLocation []interface{} `json:\"location\"`\n\t}{\n\t\tName: \"string\",\n\t\tLocation: []interface{}{\n\t\t\t\"-90    ,    -180\",\n\t\t},\n\t})\n\texpect = append(expect, []float64{-180, -90})\n\n\t// geopoint as a string - geohash\n\tgeopoints = append(geopoints, struct {\n\t\tName     string        `json:\"name\"`\n\t\tLocation []interface{} `json:\"location\"`\n\t}{\n\t\tName: \"string\",\n\t\tLocation: []interface{}{\n\t\t\t\"000000000000\",\n\t\t},\n\t})\n\texpect = append(expect, []float64{-180, -90})\n\n\t// geopoint as a string - geohash\n\tgeopoints = append(geopoints, struct {\n\t\tName     string        `json:\"name\"`\n\t\tLocation []interface{} `json:\"location\"`\n\t}{\n\t\tName: \"string\",\n\t\tLocation: []interface{}{\n\t\t\t\"drm3btev3e86\",\n\t\t},\n\t})\n\texpect = append(expect, []float64{-71.34, 41.12})\n\n\tfor i, geopoint := range geopoints {\n\t\tdoc := document.NewDocument(fmt.Sprint(i))\n\t\terr := mapping.MapDocument(doc, geopoint)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tvar foundGeo bool\n\t\tfor _, f := range doc.Fields {\n\t\t\tif f.Name() == \"location\" {\n\t\t\t\tfoundGeo = true\n\t\t\t\tgeoF, ok := f.(index.GeoPointField)\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Errorf(\"expected a geopoint field!\")\n\t\t\t\t}\n\t\t\t\tlon, err := geoF.Lon()\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"error in fetching lon, err: %v\", err)\n\t\t\t\t}\n\t\t\t\tlat, err := geoF.Lat()\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"error in fetching lat, err: %v\", err)\n\t\t\t\t}\n\t\t\t\t// round obtained lon, lat to 2 decimal places\n\t\t\t\troundLon, _ := strconv.ParseFloat(fmt.Sprintf(\"%.2f\", lon), 64)\n\t\t\t\troundLat, _ := strconv.ParseFloat(fmt.Sprintf(\"%.2f\", lat), 64)\n\t\t\t\tif roundLon != expect[i][0] || roundLat != expect[i][1] {\n\t\t\t\t\tt.Errorf(\"expected geo point: {%v, %v}, got {%v, %v}\",\n\t\t\t\t\t\texpect[i][0], expect[i][1], lon, lat)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif !foundGeo {\n\t\t\tt.Errorf(\"expected to find geo point, did not\")\n\t\t}\n\t}\n}\n\ntype textMarshalable struct {\n\tbody  string\n\tExtra string\n}\n\nfunc (t *textMarshalable) MarshalText() ([]byte, error) {\n\treturn []byte(t.body), nil\n}\n\nfunc TestMappingForTextMarshaler(t *testing.T) {\n\ttm := struct {\n\t\tMarshalable *textMarshalable\n\t}{\n\t\tMarshalable: &textMarshalable{\n\t\t\tbody:  \"text\",\n\t\t\tExtra: \"stuff\",\n\t\t},\n\t}\n\n\t// first verify that when using a mapping that doesn't explicitly\n\t// map the struct field as text, then we traverse inside the struct\n\t// and do our best\n\tm := NewIndexMapping()\n\tdoc := document.NewDocument(\"x\")\n\terr := m.MapDocument(doc, tm)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(doc.Fields) != 1 {\n\t\tt.Fatalf(\"expected 1 field, got: %d\", len(doc.Fields))\n\t}\n\tif doc.Fields[0].Name() != \"Marshalable.Extra\" {\n\t\tt.Errorf(\"expected field to be named 'Marshalable.Extra', got: '%s'\", doc.Fields[0].Name())\n\t}\n\tif string(doc.Fields[0].Value()) != tm.Marshalable.Extra {\n\t\tt.Errorf(\"expected field value to be '%s', got: '%s'\", tm.Marshalable.Extra, string(doc.Fields[0].Value()))\n\t}\n\n\t// now verify that when a mapping explicitly\n\tm = NewIndexMapping()\n\ttxt := NewTextFieldMapping()\n\tm.DefaultMapping.AddFieldMappingsAt(\"Marshalable\", txt)\n\tdoc = document.NewDocument(\"x\")\n\terr = m.MapDocument(doc, tm)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(doc.Fields) != 1 {\n\t\tt.Fatalf(\"expected 1 field, got: %d\", len(doc.Fields))\n\n\t}\n\tif doc.Fields[0].Name() != \"Marshalable\" {\n\t\tt.Errorf(\"expected field to be named 'Marshalable', got: '%s'\", doc.Fields[0].Name())\n\t}\n\twant, err := tm.Marshalable.MarshalText()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(doc.Fields[0].Value()) != string(want) {\n\t\tt.Errorf(\"expected field value to be  '%s', got: '%s'\", string(want), string(doc.Fields[0].Value()))\n\t}\n\n}\n\nfunc TestMappingForNilTextMarshaler(t *testing.T) {\n\ttm := struct {\n\t\tMarshalable *time.Time\n\t}{\n\t\tMarshalable: nil,\n\t}\n\n\t// now verify that when a mapping explicitly\n\tm := NewIndexMapping()\n\ttxt := NewTextFieldMapping()\n\tm.DefaultMapping.AddFieldMappingsAt(\"Marshalable\", txt)\n\tdoc := document.NewDocument(\"x\")\n\terr := m.MapDocument(doc, tm)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(doc.Fields) != 0 {\n\t\tt.Fatalf(\"expected 1 field, got: %d\", len(doc.Fields))\n\n\t}\n\n}\n\nfunc TestClosestDocDynamicMapping(t *testing.T) {\n\tmapping := NewIndexMapping()\n\tmapping.IndexDynamic = false\n\tmapping.DefaultMapping = NewDocumentStaticMapping()\n\tmapping.DefaultMapping.AddFieldMappingsAt(\"foo\", NewTextFieldMapping())\n\n\tdoc := document.NewDocument(\"x\")\n\terr := mapping.MapDocument(doc, map[string]interface{}{\n\t\t\"foo\": \"value\",\n\t\t\"bar\": map[string]string{\n\t\t\t\"foo\": \"value2\",\n\t\t\t\"baz\": \"value3\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(doc.Fields) != 1 {\n\t\tt.Fatalf(\"expected 1 field, got: %d\", len(doc.Fields))\n\t}\n}\n\nfunc TestMappingPointerToTimeBug1152(t *testing.T) {\n\twhen, err := time.Parse(time.RFC3339, \"2019-03-06T15:04:05Z\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tthing := struct {\n\t\tWhen *time.Time\n\t}{\n\t\tWhen: &when,\n\t}\n\n\t// this case tests when there WAS an explicit mapping, but it was NOT type text\n\t// as this was the specific case that was problematic\n\tm := NewIndexMapping()\n\tdtf := NewDateTimeFieldMapping()\n\tm.DefaultMapping.AddFieldMappingsAt(\"When\", dtf)\n\tdoc := document.NewDocument(\"x\")\n\terr = m.MapDocument(doc, thing)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(doc.Fields) != 1 {\n\t\tt.Fatalf(\"expected 1 field, got: %d\", len(doc.Fields))\n\t}\n\tif _, ok := doc.Fields[0].(index.DateTimeField); !ok {\n\t\tt.Fatalf(\"expected field to be type *document.DateTimeField, got %T\", doc.Fields[0])\n\t}\n}\n\nfunc TestDefaultAnalyzerInheritance(t *testing.T) {\n\tdocMapping := NewDocumentMapping()\n\tdocMapping.DefaultAnalyzer = \"xyz\"\n\tchildMapping := NewTextFieldMapping()\n\tdocMapping.AddFieldMappingsAt(\"field\", childMapping)\n\n\tif analyzer := docMapping.defaultAnalyzerName([]string{\"field\"}); analyzer != \"xyz\" {\n\t\tt.Fatalf(\"Expected analyzer: xyz to be inherited by field, but got: '%v'\", analyzer)\n\t}\n}\n\nfunc TestWrongAnalyzerSearchableAs(t *testing.T) {\n\tfieldMapping := NewTextFieldMapping()\n\tfieldMapping.Name = \"geo.accuracy\"\n\tfieldMapping.Analyzer = \"xyz\"\n\n\tnestedMapping := NewDocumentMapping()\n\tnestedMapping.AddFieldMappingsAt(\"accuracy\", fieldMapping)\n\n\tdocMapping := NewDocumentMapping()\n\tdocMapping.AddSubDocumentMapping(\"geo\", nestedMapping)\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.AddDocumentMapping(\"brewery\", docMapping)\n\n\tanalyzerName := indexMapping.AnalyzerNameForPath(\"geo.geo.accuracy\")\n\tif analyzerName != \"xyz\" {\n\t\tt.Errorf(\"expected analyzer name `xyz`, got `%s`\", analyzerName)\n\t}\n}\n\nfunc TestMappingArrayOfStringGeoPoints(t *testing.T) {\n\tnameFieldMapping := NewTextFieldMapping()\n\tnameFieldMapping.Name = \"name\"\n\tnameFieldMapping.Analyzer = \"standard\"\n\n\tlocFieldMapping := NewGeoPointFieldMapping()\n\n\tthingMapping := NewDocumentMapping()\n\tthingMapping.AddFieldMappingsAt(\"points\", locFieldMapping)\n\n\tmapping := NewIndexMapping()\n\tmapping.DefaultMapping = thingMapping\n\n\tdocs := []map[string]interface{}{\n\t\t{\n\t\t\t// string: \"lat,lon\"\n\t\t\t\"points\": []string{\n\t\t\t\t\"1.0, 2.0\",\n\t\t\t\t\"3.0, 4.0\",\n\t\t\t\t\"5.0, 6.0\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t// slice: {lon, lat}\n\t\t\t\"points\": [][]float64{\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{4.0, 3.0},\n\t\t\t\t{6.0, 5.0},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t// struct: {\"lon/lng\": .., \"lat\": ..}\n\t\t\t\"points\": []map[string]interface{}{\n\t\t\t\t{\"lon\": 2.0, \"lat\": 1.0},\n\t\t\t\t{\"lng\": 4.0, \"lat\": 3.0},\n\t\t\t\t{\"lng\": 6.0, \"lat\": 5.0},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, docSrc := range docs {\n\t\tdoc := document.NewDocument(\"x\")\n\t\terr := mapping.MapDocument(doc, docSrc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\t// points here in lon, lat order\n\t\texpectPoints := map[string][]float64{\n\t\t\t\"first\":  {2.0, 1.0},\n\t\t\t\"second\": {4.0, 3.0},\n\t\t\t\"third\":  {6.0, 5.0},\n\t\t}\n\n\t\tfor _, f := range doc.Fields {\n\t\t\tif f.Name() == \"points\" {\n\t\t\t\tgeoF, ok := f.(*document.GeoPointField)\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Errorf(\"expected a geopoint field!\")\n\t\t\t\t}\n\t\t\t\tlon, err := geoF.Lon()\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"error in fetching lon, err: %v\", err)\n\t\t\t\t}\n\t\t\t\tlat, err := geoF.Lat()\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"error in fetching lat, err: %v\", err)\n\t\t\t\t}\n\t\t\t\t// round obtained lon, lat to 2 decimal places\n\t\t\t\troundLon, _ := strconv.ParseFloat(fmt.Sprintf(\"%.2f\", lon), 64)\n\t\t\t\troundLat, _ := strconv.ParseFloat(fmt.Sprintf(\"%.2f\", lat), 64)\n\n\t\t\t\tfor key, point := range expectPoints {\n\t\t\t\t\tif roundLon == point[0] && roundLat == point[1] {\n\t\t\t\t\t\tdelete(expectPoints, key)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif len(expectPoints) > 0 {\n\t\t\tt.Errorf(\"some points not found: %v\", expectPoints)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "mapping/mapping_vectors.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage mapping\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"slices\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\tfaiss \"github.com/blevesearch/go-faiss\"\n)\n\n// Min and Max allowed dimensions for a vector field;\n// p.s must be set/updated at process init() _only_\nvar (\n\tMinVectorDims = 1\n\tMaxVectorDims = 4096\n)\n\nfunc NewVectorFieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:         \"vector\",\n\t\tStore:        false,\n\t\tIndex:        true,\n\t\tIncludeInAll: false,\n\t\tDocValues:    false,\n\t\tSkipFreqNorm: true,\n\t}\n}\n\nfunc NewVectorBase64FieldMapping() *FieldMapping {\n\treturn &FieldMapping{\n\t\tType:         \"vector_base64\",\n\t\tStore:        false,\n\t\tIndex:        true,\n\t\tIncludeInAll: false,\n\t\tDocValues:    false,\n\t\tSkipFreqNorm: true,\n\t}\n}\n\n// validate and process a flat vector\nfunc processFlatVector(vecV reflect.Value, dims int) ([]float32, bool) {\n\tif vecV.Len() != dims {\n\t\treturn nil, false\n\t}\n\n\trv := make([]float32, dims)\n\tfor i := 0; i < vecV.Len(); i++ {\n\t\titem := vecV.Index(i)\n\t\tif !item.CanInterface() {\n\t\t\treturn nil, false\n\t\t}\n\t\titemI := item.Interface()\n\t\titemFloat, ok := util.ExtractNumericValFloat32(itemI)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\trv[i] = itemFloat\n\t}\n\n\treturn rv, true\n}\n\n// validate and process a vector\n// max supported depth of nesting is 2 ([][]float32)\nfunc processVector(vecI interface{}, dims int) ([]float32, bool) {\n\tvecV := reflect.ValueOf(vecI)\n\tif !vecV.IsValid() || vecV.Kind() != reflect.Slice || vecV.Len() == 0 {\n\t\treturn nil, false\n\t}\n\n\t// Let's examine the first element (head) of the vector.\n\t// If head is a slice, then vector is nested, otherwise flat.\n\thead := vecV.Index(0)\n\tif !head.CanInterface() {\n\t\treturn nil, false\n\t}\n\theadI := head.Interface()\n\theadV := reflect.ValueOf(headI)\n\tif !headV.IsValid() {\n\t\treturn nil, false\n\t}\n\tif headV.Kind() != reflect.Slice { // vector is flat\n\t\treturn processFlatVector(vecV, dims)\n\t}\n\n\t// # process nested vector\n\n\t// pre-allocate memory for the flattened vector\n\t// so that we can use copy() later\n\trv := make([]float32, dims*vecV.Len())\n\n\tfor i := 0; i < vecV.Len(); i++ {\n\t\tsubVec := vecV.Index(i)\n\t\tif !subVec.CanInterface() {\n\t\t\treturn nil, false\n\t\t}\n\t\tsubVecI := subVec.Interface()\n\t\tsubVecV := reflect.ValueOf(subVecI)\n\t\tif !subVecV.IsValid() {\n\t\t\treturn nil, false\n\t\t}\n\n\t\tif subVecV.Kind() != reflect.Slice {\n\t\t\treturn nil, false\n\t\t}\n\n\t\tflatVector, ok := processFlatVector(subVecV, dims)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\n\t\tcopy(rv[i*dims:(i+1)*dims], flatVector)\n\t}\n\n\treturn rv, true\n}\n\nfunc (fm *FieldMapping) processVector(propertyMightBeVector interface{},\n\tpathString string, path []string, indexes []uint64, context *walkContext) bool {\n\tvector, ok := processVector(propertyMightBeVector, fm.Dims)\n\t// Don't add field to document if vector is invalid\n\tif !ok {\n\t\treturn false\n\t}\n\t// Apply defaults for similarity and optimization if not set\n\tsimilarity := fm.Similarity\n\tif similarity == \"\" {\n\t\tsimilarity = index.DefaultVectorSimilarityMetric\n\t}\n\tvectorIndexOptimizedFor := fm.VectorIndexOptimizedFor\n\tif vectorIndexOptimizedFor == \"\" {\n\t\tvectorIndexOptimizedFor = index.DefaultIndexOptimization\n\t}\n\t// bivf indexes only supports hamming distance for the primary\n\t// binary index. Similarity here is used for the backing flat index,\n\t// which is set to cosine similarity for recall reasons\n\tif index.OptimizationRequiresBinaryIndex(vectorIndexOptimizedFor) {\n\t\tsimilarity = index.CosineSimilarity\n\t}\n\t// normalize raw vector if similarity is cosine\n\t// Since the vector can be multi-vector (flattened array of multiple vectors),\n\t// we use NormalizeMultiVector to normalize each sub-vector independently.\n\tif similarity == index.CosineSimilarity {\n\t\tvector = NormalizeMultiVector(vector, fm.Dims)\n\t}\n\n\tfieldName := getFieldName(pathString, path, fm)\n\n\toptions := fm.Options()\n\tfield := document.NewVectorFieldWithIndexingOptions(fieldName, indexes, vector,\n\t\tfm.Dims, similarity, vectorIndexOptimizedFor, options)\n\tcontext.doc.AddField(field)\n\n\t// \"_all\" composite field is not applicable for vector field\n\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n\treturn true\n}\n\nfunc (fm *FieldMapping) processVectorBase64(propertyMightBeVectorBase64 interface{},\n\tpathString string, path []string, indexes []uint64, context *walkContext) {\n\tencodedString, ok := propertyMightBeVectorBase64.(string)\n\tif !ok {\n\t\treturn\n\t}\n\t// Apply defaults for similarity and optimization if not set\n\tsimilarity := fm.Similarity\n\tif similarity == \"\" {\n\t\tsimilarity = index.DefaultVectorSimilarityMetric\n\t}\n\tvectorIndexOptimizedFor := fm.VectorIndexOptimizedFor\n\tif vectorIndexOptimizedFor == \"\" {\n\t\tvectorIndexOptimizedFor = index.DefaultIndexOptimization\n\t}\n\t// bivf indexes only supports hamming distance for the primary\n\t// binary index. Similarity here is used for the backing flat index,\n\t// which is set to cosine similarity for recall reasons\n\tif index.OptimizationRequiresBinaryIndex(vectorIndexOptimizedFor) {\n\t\tsimilarity = index.CosineSimilarity\n\t}\n\tdecodedVector, err := document.DecodeVector(encodedString)\n\tif err != nil || len(decodedVector) != fm.Dims {\n\t\treturn\n\t}\n\t// normalize raw vector if similarity is cosine, multi-vector is not supported\n\t// for base64 encoded vectors, so we use NormalizeVector directly.\n\tif similarity == index.CosineSimilarity {\n\t\tdecodedVector = NormalizeVector(decodedVector)\n\t}\n\n\tfieldName := getFieldName(pathString, path, fm)\n\toptions := fm.Options()\n\n\tfield := document.NewVectorFieldWithIndexingOptions(fieldName, indexes, decodedVector,\n\t\tfm.Dims, similarity, vectorIndexOptimizedFor, options)\n\tcontext.doc.AddField(field)\n\n\t// \"_all\" composite field is not applicable for vector_base64 field\n\tcontext.excludedFromAll = append(context.excludedFromAll, fieldName)\n}\n\n// -----------------------------------------------------------------------------\n// document validation functions\n\nfunc validateFieldMapping(field *FieldMapping, path []string,\n\tfieldAliasCtx map[string]*FieldMapping) error {\n\tswitch field.Type {\n\tcase \"vector\", \"vector_base64\":\n\t\treturn validateVectorFieldAlias(field, path, fieldAliasCtx)\n\tdefault: // non-vector field\n\t\treturn validateFieldType(field)\n\t}\n}\n\nfunc validateVectorFieldAlias(field *FieldMapping, path []string,\n\tfieldAliasCtx map[string]*FieldMapping) error {\n\t// fully qualified field name\n\tpathString := encodePath(path)\n\t// check if field has a name set, else use path to compute effective name\n\teffectiveFieldName := getFieldName(pathString, path, field)\n\t// Compute effective values for validation\n\teffectiveSimilarity := field.Similarity\n\tif effectiveSimilarity == \"\" {\n\t\teffectiveSimilarity = index.DefaultVectorSimilarityMetric\n\t}\n\teffectiveOptimizedFor := field.VectorIndexOptimizedFor\n\tif effectiveOptimizedFor == \"\" {\n\t\teffectiveOptimizedFor = index.DefaultIndexOptimization\n\t}\n\n\t// # If alias is present, validate the field options as per the alias.\n\t// note: reading from a nil map is safe\n\tif fieldAlias, ok := fieldAliasCtx[effectiveFieldName]; ok {\n\t\tif field.Dims != fieldAlias.Dims {\n\t\t\treturn fmt.Errorf(\"field: '%s', invalid alias \"+\n\t\t\t\t\"(different dimensions %d and %d)\", effectiveFieldName, field.Dims,\n\t\t\t\tfieldAlias.Dims)\n\t\t}\n\n\t\t// Compare effective similarity values\n\t\taliasSimilarity := fieldAlias.Similarity\n\t\tif aliasSimilarity == \"\" {\n\t\t\taliasSimilarity = index.DefaultVectorSimilarityMetric\n\t\t}\n\t\tif effectiveSimilarity != aliasSimilarity {\n\t\t\treturn fmt.Errorf(\"field: '%s', invalid alias \"+\n\t\t\t\t\"(different similarity values %s and %s)\", effectiveFieldName,\n\t\t\t\teffectiveSimilarity, aliasSimilarity)\n\t\t}\n\n\t\t// Compare effective vector index optimization values\n\t\taliasOptimizedFor := fieldAlias.VectorIndexOptimizedFor\n\t\tif aliasOptimizedFor == \"\" {\n\t\t\taliasOptimizedFor = index.DefaultIndexOptimization\n\t\t}\n\t\tif effectiveOptimizedFor != aliasOptimizedFor {\n\t\t\treturn fmt.Errorf(\"field: '%s', invalid alias \"+\n\t\t\t\t\"(different vector index optimization values %s and %s)\", effectiveFieldName,\n\t\t\t\teffectiveOptimizedFor, aliasOptimizedFor)\n\t\t}\n\n\t\treturn nil\n\t}\n\n\t// # Validate field options\n\t// Vector dimensions must be within allowed range\n\tif field.Dims < MinVectorDims || field.Dims > MaxVectorDims {\n\t\treturn fmt.Errorf(\"field: '%s', invalid vector dimension: %d,\"+\n\t\t\t\" value should be in range [%d, %d]\", effectiveFieldName, field.Dims,\n\t\t\tMinVectorDims, MaxVectorDims)\n\t}\n\t// Similarity metric must be supported\n\tif _, ok := index.SupportedVectorSimilarityMetrics[effectiveSimilarity]; !ok {\n\t\treturn fmt.Errorf(\"field: '%s', invalid similarity \"+\n\t\t\t\"metric: '%s', valid metrics are: %+v\", effectiveFieldName, effectiveSimilarity,\n\t\t\treflect.ValueOf(index.SupportedVectorSimilarityMetrics).MapKeys())\n\t}\n\t// Vector index optimization must be supported\n\tif _, ok := index.SupportedVectorIndexOptimizations[effectiveOptimizedFor]; !ok {\n\t\treturn fmt.Errorf(\"field: '%s', invalid vector index \"+\n\t\t\t\"optimization: '%s', valid optimizations are: %+v\", effectiveFieldName,\n\t\t\teffectiveOptimizedFor,\n\t\t\treflect.ValueOf(index.SupportedVectorIndexOptimizations).MapKeys())\n\t}\n\t// bivf indexes requires vector dimensionality to be a multiple of 8\n\tif index.OptimizationRequiresBinaryIndex(effectiveOptimizedFor) && field.Dims%8 != 0 {\n\t\treturn fmt.Errorf(\"field: '%s', incompatible vector dimensionality for BIVF: %d,\"+\n\t\t\t\" dimension should be a multiple of 8\", effectiveFieldName, field.Dims)\n\t}\n\n\tif fieldAliasCtx != nil { // writing to a nil map is unsafe\n\t\tfieldAliasCtx[effectiveFieldName] = field\n\t}\n\n\treturn nil\n}\n\n// NormalizeVector normalizes a single vector to unit length.\n// It makes a copy of the input vector to avoid modifying it in-place.\nfunc NormalizeVector(vec []float32) []float32 {\n\t// make a copy of the vector to avoid modifying the original\n\t// vector in-place\n\tvecCopy := slices.Clone(vec)\n\t// normalize the vector copy using in-place normalization provided by faiss\n\treturn faiss.NormalizeVector(vecCopy)\n}\n\n// NormalizeMultiVector normalizes each sub-vector of size `dims` independently.\n// For a flattened array containing multiple vectors, each sub-vector is\n// normalized separately to unit length.\n// It makes a copy of the input vector to avoid modifying it in-place.\nfunc NormalizeMultiVector(vec []float32, dims int) []float32 {\n\tif len(vec) == 0 || dims <= 0 || len(vec)%dims != 0 {\n\t\treturn vec\n\t}\n\t// Single vector - delegate to NormalizeVector\n\tif len(vec) == dims {\n\t\treturn NormalizeVector(vec)\n\t}\n\t// Multi-vector - make a copy to avoid modifying the original\n\tresult := slices.Clone(vec)\n\t// Normalize each sub-vector in-place\n\tfor i := 0; i < len(result); i += dims {\n\t\tfaiss.NormalizeVector(result[i : i+dims])\n\t}\n\treturn result\n}\n"
  },
  {
    "path": "mapping/mapping_vectors_test.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage mapping\n\nimport (\n\t\"math\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestVectorFieldAliasValidation(t *testing.T) {\n\ttests := []struct {\n\t\t// input\n\t\tname       string // name of the test\n\t\tmappingStr string // index mapping json string\n\n\t\t// expected output\n\t\texpValidity bool     // validity of the mapping\n\t\terrMsgs     []string // error message, given expValidity is false\n\t}{\n\t\t{\n\t\t\tname: \"test1\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"cityVec\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 4\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs:     []string{`field: 'cityVec', invalid alias (different dimensions 4 and 3)`},\n\t\t},\n\t\t{\n\t\t\tname: \"test2\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"l2_norm\"\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"cityVec\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"dot_product\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs:     []string{`field: 'cityVec', invalid alias (different similarity values dot_product and l2_norm)`},\n\t\t},\n\t\t{\n\t\t\tname: \"test3\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"cityVec\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t{\n\t\t\tname: \"test4\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 4\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"countryVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs:     []string{`field: 'vecData', invalid alias (different dimensions 3 and 4)`, `field: 'vecData', invalid alias (different dimensions 4 and 3)`},\n\t\t},\n\t\t{\n\t\t\tname: \"test5\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t},\n\t\t\t\t\t\"types\": {\n\t\t\t\t\t\t\"type1\": {\n\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\"dims\": 4\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs:     []string{`field: 'vecData', invalid alias (different dimensions 4 and 3)`},\n\t\t},\n\t\t// Test 6: Different vector index optimization values (alias case)\n\t\t{\n\t\t\tname: \"different_optimization_alias\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"recall\"\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"cityVec\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"latency\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs:     []string{`field: 'cityVec', invalid alias (different vector index optimization values latency and recall)`},\n\t\t},\n\t\t// Test 7: Invalid dimensions - below minimum\n\t\t{\n\t\t\tname: \"dims_below_minimum\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 0\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs:     []string{`field: 'cityVec', invalid vector dimension: 0, value should be in range [1, 4096]`},\n\t\t},\n\t\t// Test 8: Invalid dimensions - above maximum\n\t\t{\n\t\t\tname: \"dims_above_maximum\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 5000\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs:     []string{`field: 'cityVec', invalid vector dimension: 5000, value should be in range [1, 4096]`},\n\t\t},\n\t\t// Test 9: Invalid similarity metric\n\t\t{\n\t\t\tname: \"invalid_similarity_metric\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"invalid_metric\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\t// Note: error message contains map keys which have non-deterministic order\n\t\t\terrMsgs: []string{`invalid similarity metric: 'invalid_metric'`},\n\t\t},\n\t\t// Test 10: Invalid vector index optimization\n\t\t{\n\t\t\tname: \"invalid_optimization\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"invalid_opt\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\t// Note: error message contains map keys which have non-deterministic order\n\t\t\terrMsgs: []string{`invalid vector index optimization: 'invalid_opt'`},\n\t\t},\n\t\t// Test 11: vector_base64 type with valid dimensions\n\t\t{\n\t\t\tname: \"vector_base64_valid\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector_base64\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 128\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 12: vector_base64 alias with different dimensions\n\t\t{\n\t\t\tname: \"vector_base64_different_dims_alias\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector_base64\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 128\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"cityVec\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector_base64\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 256\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs:     []string{`field: 'cityVec', invalid alias (different dimensions 256 and 128)`},\n\t\t},\n\t\t// Test 13: Default similarity matching explicit similarity in alias\n\t\t{\n\t\t\tname: \"default_similarity_matches_explicit\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"cityVec\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"l2_norm\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 14: Default optimization matching explicit optimization in alias\n\t\t{\n\t\t\tname: \"default_optimization_matches_explicit\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"cityVec\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"recall\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 15: Valid alias with all explicit matching values\n\t\t{\n\t\t\tname: \"valid_alias_all_explicit_matching\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 64,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"dot_product\",\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"latency\"\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"cityVec\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 64,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"dot_product\",\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"latency\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 16: Cross-property alias with different similarity\n\t\t{\n\t\t\tname: \"cross_property_different_similarity\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"cosine\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"countryVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"l2_norm\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs: []string{\n\t\t\t\t`field: 'vecData', invalid alias (different similarity values l2_norm and cosine)`,\n\t\t\t\t`field: 'vecData', invalid alias (different similarity values cosine and l2_norm)`,\n\t\t\t},\n\t\t},\n\t\t// Test 17: Cross-property alias with different optimization\n\t\t{\n\t\t\tname: \"cross_property_different_optimization\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"recall\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"countryVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"memory-efficient\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs: []string{\n\t\t\t\t`field: 'vecData', invalid alias (different vector index optimization values memory-efficient and recall)`,\n\t\t\t\t`field: 'vecData', invalid alias (different vector index optimization values recall and memory-efficient)`,\n\t\t\t},\n\t\t},\n\t\t// Test 18: Valid cross-property alias with matching values\n\t\t{\n\t\t\tname: \"valid_cross_property_alias\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"cityVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 64,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"dot_product\",\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"latency\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"countryVec\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vecData\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 64,\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"dot_product\",\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"latency\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 20: Different fully qualified paths - a.b.c.f vs f (different effective names, no conflict)\n\t\t{\n\t\t\tname: \"different_fq_paths_no_conflict\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"f\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 64\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"x\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"f\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 128\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 21: Same leaf property name at different paths (a.b.vec vs x.y.vec) - no conflict\n\t\t{\n\t\t\tname: \"same_leaf_different_paths_no_conflict\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\t\t\"vec\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 64\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"x\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"y\": {\n\t\t\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\t\t\"vec\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 128\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 22: Field name override creates same effective name - alias conflict\n\t\t// a.b with name \"data\" → effective \"a.data\"\n\t\t// a with name \"data\" → effective \"data\"\n\t\t// These are different, so no conflict\n\t\t{\n\t\t\tname: \"field_name_override_different_parents_no_conflict\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"data\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 64\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"a2\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"name\": \"data\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 128\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 23: Same effective field name via name override - should conflict\n\t\t// a.b with name \"sharedVec\" → effective \"a.sharedVec\"\n\t\t// a.c with name \"sharedVec\" → effective \"a.sharedVec\"\n\t\t// Both resolve to same effective name with different dims → conflict\n\t\t{\n\t\t\tname: \"same_effective_name_via_override_conflict\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"sharedVec\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 64\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"sharedVec\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 128\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs: []string{\n\t\t\t\t`field: 'a.sharedVec', invalid alias (different dimensions 128 and 64)`,\n\t\t\t\t`field: 'a.sharedVec', invalid alias (different dimensions 64 and 128)`,\n\t\t\t},\n\t\t},\n\t\t// Test 24: Deep nesting with same effective name via name override - should conflict\n\t\t// level1.level2.propA with name \"vec\" → effective \"level1.level2.vec\"\n\t\t// level1.level2.propB with name \"vec\" → effective \"level1.level2.vec\"\n\t\t{\n\t\t\tname: \"deep_nesting_same_effective_name_conflict\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"level1\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"level2\": {\n\t\t\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\t\t\"propA\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"vec\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 64\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\"propB\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"vec\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 128\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs: []string{\n\t\t\t\t`field: 'level1.level2.vec', invalid alias (different dimensions 128 and 64)`,\n\t\t\t\t`field: 'level1.level2.vec', invalid alias (different dimensions 64 and 128)`,\n\t\t\t},\n\t\t},\n\t\t// Test 25: Root level field vs nested field with same name - no conflict\n\t\t// Root: \"embedding\" → effective \"embedding\"\n\t\t// Nested: a.b.embedding → effective \"a.b.embedding\"\n\t\t{\n\t\t\tname: \"root_vs_nested_same_name_no_conflict\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"embedding\": {\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"dims\": 64\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"nested\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"deep\": {\n\t\t\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\t\t\"embedding\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 256\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 26: Multiple levels with name override targeting same effective path\n\t\t// a.b.x with name \"target\" → effective \"a.b.target\"\n\t\t// a.b.target (no override) → effective \"a.b.target\"\n\t\t// Same effective name, different dims → conflict\n\t\t{\n\t\t\tname: \"name_override_matches_sibling_path_conflict\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\t\t\"x\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"target\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 64\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\"target\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 128\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: false,\n\t\t\terrMsgs: []string{\n\t\t\t\t`field: 'a.b.target', invalid alias (different dimensions 128 and 64)`,\n\t\t\t\t`field: 'a.b.target', invalid alias (different dimensions 64 and 128)`,\n\t\t\t},\n\t\t},\n\t\t// Test 27: Valid alias at deep nesting level\n\t\t{\n\t\t\tname: \"valid_alias_deep_nesting\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"a\": {\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"b\": {\n\t\t\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\t\t\"c\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\t\"vec\": {\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 128,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"similarity\": \"dot_product\"\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"vec\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 128,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"similarity\": \"dot_product\"\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t\t// Test 28: Valid alias with different paths but same effective field name\n\t\t// vectors.vec with name \"vec\" → effective \"vectors.vec\"\n\t\t// vec with name \"vec\" → effective \"vec\"\n\t\t// Different effective names, so no conflict\n\t\t{\n\t\t\tname: \"valid_alias_different_paths_same_field_name\",\n\t\t\tmappingStr: `\n\t\t\t\t{\n\t\t\t\t\t\"default_mapping\": {\n\t\t\t\t\t\t\"dynamic\": false,\n\t\t\t\t\t\t\"enabled\": true,\n\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\"vectors\": {\n\t\t\t\t\t\t\t\t\"dynamic\": true,\n\t\t\t\t\t\t\t\t\"enabled\": true,\n\t\t\t\t\t\t\t\t\"properties\": {\n\t\t\t\t\t\t\t\t\t\"vec\": {\n\t\t\t\t\t\t\t\t\t\t\"enabled\": true,\n\t\t\t\t\t\t\t\t\t\t\"dynamic\": false,\n\t\t\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\"name\": \"vec\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\"\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\"vec\": {\n\t\t\t\t\t\t\t\t\"enabled\": true,\n\t\t\t\t\t\t\t\t\"dynamic\": false,\n\t\t\t\t\t\t\t\t\"fields\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"dims\": 3,\n\t\t\t\t\t\t\t\t\t\t\"index\": true,\n\t\t\t\t\t\t\t\t\t\t\"name\": \"vec\",\n\t\t\t\t\t\t\t\t\t\t\"similarity\": \"l2_norm\",\n\t\t\t\t\t\t\t\t\t\t\"type\": \"vector\",\n\t\t\t\t\t\t\t\t\t\t\"vector_index_optimized_for\": \"recall\"\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}`,\n\t\t\texpValidity: true,\n\t\t\terrMsgs:     []string{},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tim := NewIndexMapping()\n\t\t\terr := im.UnmarshalJSON([]byte(test.mappingStr))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"failed to unmarshal index mapping: %v\", err)\n\t\t\t}\n\n\t\t\terr = im.Validate()\n\t\t\tisValid := err == nil\n\t\t\tif test.expValidity != isValid {\n\t\t\t\tt.Fatalf(\"validity mismatch, expected: %v, got: %v\",\n\t\t\t\t\ttest.expValidity, isValid)\n\t\t\t}\n\n\t\t\tif !isValid {\n\t\t\t\terrStringMatched := false\n\t\t\t\tfor _, possibleErrMsg := range test.errMsgs {\n\t\t\t\t\t// Use Contains for matching since some error messages include\n\t\t\t\t\t// map keys which have non-deterministic ordering\n\t\t\t\t\tif err.Error() == possibleErrMsg || strings.Contains(err.Error(), possibleErrMsg) {\n\t\t\t\t\t\terrStringMatched = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !errStringMatched {\n\t\t\t\t\tt.Fatalf(\"invalid error message, expected one of: %v, got: %v\",\n\t\t\t\t\t\ttest.errMsgs, err.Error())\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// A test case for processVector function\ntype vectorTest struct {\n\t// Input\n\n\tipVec interface{} // input vector\n\tdims  int         // dimensionality of input vector\n\n\t// Expected Output\n\n\texpValidity bool      // expected validity of the input\n\texpOpVec    []float32 // expected output vector, given the input is valid\n}\n\nfunc TestProcessVector(t *testing.T) {\n\t// Note: while creating vectors, we are using []any instead of []float32,\n\t// this is done to enhance our test coverage.\n\t// When we unmarshal a vector from a JSON, we get []any, not []float32.\n\ttests := []vectorTest{\n\t\t// # Flat vectors\n\n\t\t// ## numeric cases\n\t\t// (all numeric elements)\n\t\t{[]any{1, 2.2, 3}, 3, true, []float32{1, 2.2, 3}}, // len==dims\n\t\t{[]any{1, 2.2, 3}, 2, false, nil},                 // len>dims\n\t\t{[]any{1, 2.2, 3}, 4, false, nil},                 // len<dims\n\n\t\t// ## imposter cases\n\t\t// (len==dims, some elements are non-numeric)\n\t\t{[]any{1, 2, \"three\"}, 3, false, nil},    // string\n\t\t{[]any{1, nil, 3}, 3, false, nil},        // nil\n\t\t{[]any{nil, 1}, 2, false, nil},           // nil head\n\t\t{[]any{1, 2, struct{}{}}, 3, false, nil}, // struct\n\n\t\t// non-slice cases\n\t\t// (vector is of types other than slice)\n\t\t{nil, 1, false, nil},\n\t\t{struct{}{}, 1, false, nil},\n\t\t{1, 1, false, nil},\n\n\t\t// # Nested vectors\n\n\t\t// ## numeric cases\n\t\t// (all numeric elements)\n\t\t{[]any{[]any{1, 2, 3}, []any{4, 5, 6}}, 3, true,\n\t\t\t[]float32{1, 2, 3, 4, 5, 6}}, // len==dims\n\t\t{[]any{[]any{1, 2, 3}}, 3, true, []float32{1, 2, 3}}, // len==dims\n\t\t{[]any{[]any{1, 2, 3}}, 4, false, nil},               // len>dims\n\t\t{[]any{[]any{1, 2, 3}}, 2, false, nil},               // len<dims\n\n\t\t// ## imposter cases\n\t\t// some inner vectors are short\n\t\t{[]any{[]any{1, 2, 3}, []any{4, 5}}, 3, false, nil},\n\t\t// some inner vectors are long\n\t\t{[]any{[]any{1, 2, 3}, []any{4, 5, 6, 7}}, 3, false, nil},\n\t\t// contains string\n\t\t{[]any{[]any{1, 2, \"three\"}, []any{4, 5, 6}}, 3, false, nil},\n\t\t// contains nil\n\t\t{[]any{[]any{1, 2, nil}, []any{4, 5, 6}}, 3, false, nil},\n\n\t\t// non-slice cases (inner vectors)\n\t\t{[]any{[]any{1, 2, 3}, nil}, 3, false, nil},        // nil\n\t\t{[]any{nil, []any{1, 2, 3}}, 3, false, nil},        // nil head\n\t\t{[]any{[]any{1, 2, 3}, struct{}{}}, 3, false, nil}, // struct\n\t\t{[]any{[]any{1, 2, 3}, 4}, 3, false, nil},          // int\n\t}\n\n\tfor _, test := range tests {\n\t\topVec, valid := processVector(test.ipVec, test.dims)\n\n\t\t// check the validity of the input, as returned by processVector,\n\t\t// against the expected validity.\n\t\tif valid != test.expValidity {\n\t\t\tt.Errorf(\"validity mismatch, ipVec:%v, dims:%v, expected:%v, got:%v\",\n\t\t\t\ttest.ipVec, test.dims, test.expValidity, valid)\n\t\t\tt.Fail()\n\t\t}\n\n\t\t// If input vector is valid, check the correctness of the output vector\n\t\t// against the expected output vector.\n\t\tif valid {\n\t\t\tif len(opVec) != len(test.expOpVec) {\n\t\t\t\tt.Errorf(\"output vector mismatch, ipVec:%v, dims:%v, \"+\n\t\t\t\t\t\"expected:%v, got:%v\", test.ipVec, test.dims, test.expOpVec,\n\t\t\t\t\topVec)\n\t\t\t\tt.Fail()\n\t\t\t}\n\n\t\t\tfor i := 0; i < len(opVec); i++ {\n\t\t\t\tif opVec[i] != test.expOpVec[i] {\n\t\t\t\t\tt.Errorf(\"output vector mismatch, ipVec:%v, dims:%v, \"+\n\t\t\t\t\t\t\"expected:%v, got:%v\", test.ipVec, test.dims, test.expOpVec,\n\t\t\t\t\t\topVec)\n\t\t\t\t\tt.Fail()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestNormalizeVector(t *testing.T) {\n\tvectors := [][]float32{\n\t\t{1, 2, 3, 4, 5},\n\t\t{1, 0, 0, 0, 0},\n\t\t{0.182574183, 0.365148365, 0.547722578, 0.730296731},\n\t\t{1, 1, 1, 1, 1, 1, 1, 1},\n\t\t{0},\n\t}\n\n\texpectedNormalizedVectors := [][]float32{\n\t\t{0.13483998, 0.26967996, 0.40451995, 0.5393599, 0.67419994},\n\t\t{1, 0, 0, 0, 0},\n\t\t{0.18257418, 0.36514837, 0.5477226, 0.73029673},\n\t\t{0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338, 0.35355338},\n\t\t{0},\n\t}\n\n\tfor i := 0; i < len(vectors); i++ {\n\t\tnormalizedVector := NormalizeVector(vectors[i])\n\t\tif !reflect.DeepEqual(normalizedVector, expectedNormalizedVectors[i]) {\n\t\t\tt.Errorf(\"[vector-%d] Expected: %v, Got: %v\", i+1, expectedNormalizedVectors[i], normalizedVector)\n\t\t}\n\t}\n}\n\nfunc TestNormalizeMultiVectors(t *testing.T) {\n\ttests := []struct {\n\t\tname     string\n\t\tinput    []float32\n\t\tdims     int\n\t\texpected []float32\n\t}{\n\t\t{\n\t\t\tname:     \"single vector - already normalized\",\n\t\t\tinput:    []float32{1, 0, 0},\n\t\t\tdims:     3,\n\t\t\texpected: []float32{1, 0, 0},\n\t\t},\n\t\t{\n\t\t\tname:     \"single vector - needs normalization\",\n\t\t\tinput:    []float32{3, 0, 0},\n\t\t\tdims:     3,\n\t\t\texpected: []float32{1, 0, 0},\n\t\t},\n\t\t{\n\t\t\tname:     \"two vectors - X and Y directions\",\n\t\t\tinput:    []float32{3, 0, 0, 0, 4, 0},\n\t\t\tdims:     3,\n\t\t\texpected: []float32{1, 0, 0, 0, 1, 0},\n\t\t},\n\t\t{\n\t\t\tname:     \"three vectors\",\n\t\t\tinput:    []float32{3, 0, 0, 0, 4, 0, 0, 0, 5},\n\t\t\tdims:     3,\n\t\t\texpected: []float32{1, 0, 0, 0, 1, 0, 0, 0, 1},\n\t\t},\n\t\t{\n\t\t\tname:     \"two 2D vectors\",\n\t\t\tinput:    []float32{3, 4, 5, 12},\n\t\t\tdims:     2,\n\t\t\texpected: []float32{0.6, 0.8, 0.38461538, 0.92307693},\n\t\t},\n\t\t{\n\t\t\tname:     \"empty vector\",\n\t\t\tinput:    []float32{},\n\t\t\tdims:     3,\n\t\t\texpected: []float32{},\n\t\t},\n\t\t{\n\t\t\tname:     \"zero dims\",\n\t\t\tinput:    []float32{1, 2, 3},\n\t\t\tdims:     0,\n\t\t\texpected: []float32{1, 2, 3},\n\t\t},\n\t\t{\n\t\t\tname:     \"negative dims\",\n\t\t\tinput:    []float32{1, 2, 3},\n\t\t\tdims:     -1,\n\t\t\texpected: []float32{1, 2, 3},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Make a copy of input to verify original is not modified\n\t\t\tinputCopy := make([]float32, len(tt.input))\n\t\t\tcopy(inputCopy, tt.input)\n\n\t\t\tresult := NormalizeMultiVector(tt.input, tt.dims)\n\n\t\t\t// Check result matches expected\n\t\t\tif len(result) != len(tt.expected) {\n\t\t\t\tt.Errorf(\"length mismatch: expected %d, got %d\", len(tt.expected), len(result))\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfor i := range result {\n\t\t\t\tif !floatApproxEqual(result[i], tt.expected[i], 1e-5) {\n\t\t\t\t\tt.Errorf(\"value mismatch at index %d: expected %v, got %v\",\n\t\t\t\t\t\ti, tt.expected[i], result[i])\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Verify original input was not modified\n\t\t\tif !reflect.DeepEqual(tt.input, inputCopy) {\n\t\t\t\tt.Errorf(\"original input was modified: was %v, now %v\", inputCopy, tt.input)\n\t\t\t}\n\n\t\t\t// For valid multi-vectors, verify each sub-vector has unit magnitude\n\t\t\tif tt.dims > 0 && len(tt.input) > 0 && len(tt.input)%tt.dims == 0 {\n\t\t\t\tnumVecs := len(result) / tt.dims\n\t\t\t\tfor i := 0; i < numVecs; i++ {\n\t\t\t\t\tsubVec := result[i*tt.dims : (i+1)*tt.dims]\n\t\t\t\t\tmag := magnitude(subVec)\n\t\t\t\t\t// Allow for zero vectors (magnitude 0) or unit vectors (magnitude 1)\n\t\t\t\t\tif mag > 1e-6 && !floatApproxEqual(mag, 1.0, 1e-5) {\n\t\t\t\t\t\tt.Errorf(\"sub-vector %d has magnitude %v, expected 1.0\", i, mag)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Helper to compute magnitude of a vector\nfunc magnitude(v []float32) float32 {\n\tvar sum float32\n\tfor _, x := range v {\n\t\tsum += x * x\n\t}\n\treturn float32(math.Sqrt(float64(sum)))\n}\n\n// Helper for approximate float comparison\nfunc floatApproxEqual(a, b, epsilon float32) bool {\n\tdiff := a - b\n\tif diff < 0 {\n\t\tdiff = -diff\n\t}\n\treturn diff < epsilon\n}\n"
  },
  {
    "path": "mapping/reflect.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\nimport (\n\t\"reflect\"\n\t\"strings\"\n)\n\nfunc lookupPropertyPath(data interface{}, path string) interface{} {\n\tpathParts := decodePath(path)\n\n\tcurrent := data\n\tfor _, part := range pathParts {\n\t\tcurrent = lookupPropertyPathPart(current, part)\n\t\tif current == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn current\n}\n\nfunc lookupPropertyPathPart(data interface{}, part string) interface{} {\n\tval := reflect.ValueOf(data)\n\tif !val.IsValid() {\n\t\treturn nil\n\t}\n\ttyp := val.Type()\n\tswitch typ.Kind() {\n\tcase reflect.Map:\n\t\t// FIXME can add support for other map keys in the future\n\t\tif typ.Key().Kind() == reflect.String {\n\t\t\tkey := reflect.ValueOf(part)\n\t\t\tentry := val.MapIndex(key)\n\t\t\tif entry.IsValid() {\n\t\t\t\treturn entry.Interface()\n\t\t\t}\n\t\t}\n\tcase reflect.Struct:\n\t\tfield := val.FieldByName(part)\n\t\tif field.IsValid() && field.CanInterface() {\n\t\t\treturn field.Interface()\n\t\t}\n\tcase reflect.Ptr:\n\t\tptrElem := val.Elem()\n\t\tif ptrElem.IsValid() && ptrElem.CanInterface() {\n\t\t\treturn lookupPropertyPathPart(ptrElem.Interface(), part)\n\t\t}\n\t}\n\treturn nil\n}\n\nconst pathSeparator = \".\"\n\nfunc decodePath(path string) []string {\n\treturn strings.Split(path, pathSeparator)\n}\n\nfunc encodePath(pathElements []string) string {\n\treturn strings.Join(pathElements, pathSeparator)\n}\n\nfunc mustString(data interface{}) (string, bool) {\n\tif data != nil {\n\t\tstr, ok := data.(string)\n\t\tif ok {\n\t\t\treturn str, true\n\t\t}\n\t}\n\treturn \"\", false\n}\n\n// parseTagName extracts the field name from a struct tag\nfunc parseTagName(tag string) string {\n\tif idx := strings.Index(tag, \",\"); idx != -1 {\n\t\treturn tag[:idx]\n\t}\n\treturn tag\n}\n"
  },
  {
    "path": "mapping/reflect_test.go",
    "content": "package mapping\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestLookupPropertyPath(t *testing.T) {\n\ttests := []struct {\n\t\tinput  interface{}\n\t\tpath   string\n\t\toutput interface{}\n\t}{\n\t\t{\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"Type\": \"a\",\n\t\t\t},\n\t\t\tpath:   \"Type\",\n\t\t\toutput: \"a\",\n\t\t},\n\t\t{\n\t\t\tinput: struct {\n\t\t\t\tType string\n\t\t\t}{\n\t\t\t\tType: \"b\",\n\t\t\t},\n\t\t\tpath:   \"Type\",\n\t\t\toutput: \"b\",\n\t\t},\n\t\t{\n\t\t\tinput: &struct {\n\t\t\t\tType string\n\t\t\t}{\n\t\t\t\tType: \"b\",\n\t\t\t},\n\t\t\tpath:   \"Type\",\n\t\t\toutput: \"b\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := lookupPropertyPath(test.input, test.path)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Fatalf(\"expected '%v', got '%v', for path '%s' in  %+v\", test.output, actual, test.path, test.input)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "mapping/synonym.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mapping\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n)\n\ntype SynonymSource struct {\n\tCollectionName string `json:\"collection\"`\n\tAnalyzerName   string `json:\"analyzer\"`\n}\n\nfunc NewSynonymSource(collection, analyzer string) *SynonymSource {\n\treturn &SynonymSource{\n\t\tCollectionName: collection,\n\t\tAnalyzerName:   analyzer,\n\t}\n}\n\nfunc (s *SynonymSource) Collection() string {\n\treturn s.CollectionName\n}\n\nfunc (s *SynonymSource) Analyzer() string {\n\treturn s.AnalyzerName\n}\n\nfunc (s *SynonymSource) SetCollection(c string) {\n\ts.CollectionName = c\n}\n\nfunc (s *SynonymSource) SetAnalyzer(a string) {\n\ts.AnalyzerName = a\n}\nfunc SynonymSourceConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.SynonymSource, error) {\n\tcollection, ok := config[\"collection\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify collection\")\n\t}\n\tanalyzer, ok := config[\"analyzer\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify analyzer\")\n\t}\n\tif _, err := cache.AnalyzerNamed(analyzer); err != nil {\n\t\treturn nil, fmt.Errorf(\"analyzer named '%s' not found\", analyzer)\n\t}\n\treturn NewSynonymSource(collection, analyzer), nil\n}\n\nfunc init() {\n\terr := registry.RegisterSynonymSource(analysis.SynonymSourceType, SynonymSourceConstructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "mapping.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport \"github.com/blevesearch/bleve/v2/mapping\"\n\n// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules\nfunc NewIndexMapping() *mapping.IndexMappingImpl {\n\treturn mapping.NewIndexMapping()\n}\n\n// NewDocumentMapping returns a new document mapping\n// with all the default values.\nfunc NewDocumentMapping() *mapping.DocumentMapping {\n\treturn mapping.NewDocumentMapping()\n}\n\n// NewDocumentStaticMapping returns a new document\n// mapping that will not automatically index parts\n// of a document without an explicit mapping.\nfunc NewDocumentStaticMapping() *mapping.DocumentMapping {\n\treturn mapping.NewDocumentStaticMapping()\n}\n\n// NewNestedDocumentMapping returns a new document mapping\n// that will treat all objects as nested documents.\nfunc NewNestedDocumentMapping() *mapping.DocumentMapping {\n\treturn mapping.NewNestedDocumentMapping()\n}\n\n// NewNestedDocumentStaticMapping returns a new document mapping\n// that will treat all objects as nested documents and\n// will not automatically index parts of a nested document\n// without an explicit mapping.\nfunc NewNestedDocumentStaticMapping() *mapping.DocumentMapping {\n\treturn mapping.NewNestedDocumentStaticMapping()\n}\n\n// NewDocumentDisabledMapping returns a new document\n// mapping that will not perform any indexing.\nfunc NewDocumentDisabledMapping() *mapping.DocumentMapping {\n\treturn mapping.NewDocumentDisabledMapping()\n}\n\n// NewTextFieldMapping returns a default field mapping for text\nfunc NewTextFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewTextFieldMapping()\n}\n\n// NewKeywordFieldMapping returns a field mapping for text using the keyword\n// analyzer, which essentially doesn't apply any specific text analysis.\nfunc NewKeywordFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewKeywordFieldMapping()\n}\n\n// NewNumericFieldMapping returns a default field mapping for numbers\nfunc NewNumericFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewNumericFieldMapping()\n}\n\n// NewDateTimeFieldMapping returns a default field mapping for dates\nfunc NewDateTimeFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewDateTimeFieldMapping()\n}\n\n// NewBooleanFieldMapping returns a default field mapping for booleans\nfunc NewBooleanFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewBooleanFieldMapping()\n}\n\nfunc NewGeoPointFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewGeoPointFieldMapping()\n}\n\nfunc NewGeoShapeFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewGeoShapeFieldMapping()\n}\n\nfunc NewIPFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewIPFieldMapping()\n}\n"
  },
  {
    "path": "mapping_vector.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage bleve\n\nimport \"github.com/blevesearch/bleve/v2/mapping\"\n\nfunc NewVectorFieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewVectorFieldMapping()\n}\n\nfunc NewVectorBase64FieldMapping() *mapping.FieldMapping {\n\treturn mapping.NewVectorBase64FieldMapping()\n}\n"
  },
  {
    "path": "numeric/bin.go",
    "content": "package numeric\n\nvar interleaveMagic = []uint64{\n\t0x5555555555555555,\n\t0x3333333333333333,\n\t0x0F0F0F0F0F0F0F0F,\n\t0x00FF00FF00FF00FF,\n\t0x0000FFFF0000FFFF,\n\t0x00000000FFFFFFFF,\n\t0xAAAAAAAAAAAAAAAA,\n}\n\nvar interleaveShift = []uint{1, 2, 4, 8, 16}\n\n// Interleave the first 32 bits of each uint64\n// adapted from org.apache.lucene.util.BitUtil\n// which was adapted from:\n// http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN\nfunc Interleave(v1, v2 uint64) uint64 {\n\tv1 = (v1 | (v1 << interleaveShift[4])) & interleaveMagic[4]\n\tv1 = (v1 | (v1 << interleaveShift[3])) & interleaveMagic[3]\n\tv1 = (v1 | (v1 << interleaveShift[2])) & interleaveMagic[2]\n\tv1 = (v1 | (v1 << interleaveShift[1])) & interleaveMagic[1]\n\tv1 = (v1 | (v1 << interleaveShift[0])) & interleaveMagic[0]\n\tv2 = (v2 | (v2 << interleaveShift[4])) & interleaveMagic[4]\n\tv2 = (v2 | (v2 << interleaveShift[3])) & interleaveMagic[3]\n\tv2 = (v2 | (v2 << interleaveShift[2])) & interleaveMagic[2]\n\tv2 = (v2 | (v2 << interleaveShift[1])) & interleaveMagic[1]\n\tv2 = (v2 | (v2 << interleaveShift[0])) & interleaveMagic[0]\n\treturn (v2 << 1) | v1\n}\n\n// Deinterleave the 32-bit value starting at position 0\n// to get the other 32-bit value, shift it by 1 first\nfunc Deinterleave(b uint64) uint64 {\n\tb &= interleaveMagic[0]\n\tb = (b ^ (b >> interleaveShift[0])) & interleaveMagic[1]\n\tb = (b ^ (b >> interleaveShift[1])) & interleaveMagic[2]\n\tb = (b ^ (b >> interleaveShift[2])) & interleaveMagic[3]\n\tb = (b ^ (b >> interleaveShift[3])) & interleaveMagic[4]\n\tb = (b ^ (b >> interleaveShift[4])) & interleaveMagic[5]\n\treturn b\n}\n"
  },
  {
    "path": "numeric/bin_test.go",
    "content": "package numeric\n\nimport \"testing\"\n\nfunc TestInterleaveDeinterleave(t *testing.T) {\n\ttests := []struct {\n\t\tv1 uint64\n\t\tv2 uint64\n\t}{\n\t\t{0, 0},\n\t\t{1, 1},\n\t\t{27, 39},\n\t\t{1<<32 - 1, 1<<32 - 1}, // largest that should still work\n\t}\n\n\tfor _, test := range tests {\n\t\ti := Interleave(test.v1, test.v2)\n\t\tgotv1 := Deinterleave(i)\n\t\tgotv2 := Deinterleave(i >> 1)\n\t\tif gotv1 != test.v1 {\n\t\t\tt.Errorf(\"expected v1: %d, got %d, interleaved was %x\", test.v1, gotv1, i)\n\t\t}\n\t\tif gotv2 != test.v2 {\n\t\t\tt.Errorf(\"expected v2: %d, got %d, interleaved was %x\", test.v2, gotv2, i)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "numeric/float.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage numeric\n\nimport (\n\t\"math\"\n)\n\nfunc Float64ToInt64(f float64) int64 {\n\tfasint := int64(math.Float64bits(f))\n\tif fasint < 0 {\n\t\tfasint = fasint ^ 0x7fffffffffffffff\n\t}\n\treturn fasint\n}\n\nfunc Int64ToFloat64(i int64) float64 {\n\tif i < 0 {\n\t\ti ^= 0x7fffffffffffffff\n\t}\n\treturn math.Float64frombits(uint64(i))\n}\n"
  },
  {
    "path": "numeric/float_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage numeric\n\nimport (\n\t\"testing\"\n)\n\n// test that the float/sortable int operations work both ways\n// and that the corresponding integers sort the same as\n// the original floats would have\nfunc TestSortabledFloat64ToInt64(t *testing.T) {\n\ttests := []struct {\n\t\tinput float64\n\t}{\n\t\t{\n\t\t\tinput: -4640094584139352638,\n\t\t},\n\t\t{\n\t\t\tinput: -167.42,\n\t\t},\n\t\t{\n\t\t\tinput: -1.11,\n\t\t},\n\t\t{\n\t\t\tinput: 0,\n\t\t},\n\t\t{\n\t\t\tinput: 3.14,\n\t\t},\n\t\t{\n\t\t\tinput: 167.42,\n\t\t},\n\t}\n\n\tvar lastInt64 *int64\n\tfor _, test := range tests {\n\t\tactual := Float64ToInt64(test.input)\n\t\tif lastInt64 != nil {\n\t\t\t// check that this float is greater than the last one\n\t\t\tif actual <= *lastInt64 {\n\t\t\t\tt.Errorf(\"expected greater than prev, this: %d, last %d\", actual, *lastInt64)\n\t\t\t}\n\t\t}\n\t\tlastInt64 = &actual\n\t\tconvertedBack := Int64ToFloat64(actual)\n\t\t// assert that we got back what we started with\n\t\tif convertedBack != test.input {\n\t\t\tt.Errorf(\"expected %f, got %f\", test.input, convertedBack)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "numeric/prefix_coded.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage numeric\n\nimport \"fmt\"\n\nconst ShiftStartInt64 byte = 0x20\n\n// PrefixCoded is a byte array encoding of\n// 64-bit numeric values shifted by 0-63 bits\ntype PrefixCoded []byte\n\nfunc NewPrefixCodedInt64(in int64, shift uint) (PrefixCoded, error) {\n\trv, _, err := NewPrefixCodedInt64Prealloc(in, shift, nil)\n\treturn rv, err\n}\n\nfunc NewPrefixCodedInt64Prealloc(in int64, shift uint, prealloc []byte) (\n\trv PrefixCoded, preallocRest []byte, err error) {\n\tif shift > 63 {\n\t\treturn nil, prealloc, fmt.Errorf(\"cannot shift %d, must be between 0 and 63\", shift)\n\t}\n\n\tnChars := ((63 - shift) / 7) + 1\n\n\tsize := int(nChars + 1)\n\tif len(prealloc) >= size {\n\t\trv = PrefixCoded(prealloc[0:size])\n\t\tpreallocRest = prealloc[size:]\n\t} else {\n\t\trv = make(PrefixCoded, size)\n\t}\n\n\trv[0] = ShiftStartInt64 + byte(shift)\n\n\tsortableBits := int64(uint64(in) ^ 0x8000000000000000)\n\tsortableBits = int64(uint64(sortableBits) >> shift)\n\tfor nChars > 0 {\n\t\t// Store 7 bits per byte for compatibility\n\t\t// with UTF-8 encoding of terms\n\t\trv[nChars] = byte(sortableBits & 0x7f)\n\t\tnChars--\n\t\tsortableBits = int64(uint64(sortableBits) >> 7)\n\t}\n\n\treturn rv, preallocRest, nil\n}\n\nfunc MustNewPrefixCodedInt64(in int64, shift uint) PrefixCoded {\n\trv, err := NewPrefixCodedInt64(in, shift)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn rv\n}\n\nfunc MustNewPrefixCodedInt64Prealloc(in int64, shift uint, prealloc []byte) PrefixCoded {\n\trv, _, err := NewPrefixCodedInt64Prealloc(in, shift, prealloc)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn rv\n}\n\n// Shift returns the number of bits shifted\n// returns 0 if in uninitialized state\nfunc (p PrefixCoded) Shift() (uint, error) {\n\tif len(p) > 0 {\n\t\tshift := p[0] - ShiftStartInt64\n\t\tif shift < 0 || shift < 63 {\n\t\t\treturn uint(shift), nil\n\t\t}\n\t}\n\treturn 0, fmt.Errorf(\"invalid prefix coded value\")\n}\n\nfunc (p PrefixCoded) Int64() (int64, error) {\n\tshift, err := p.Shift()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tvar sortableBits int64\n\tfor _, inbyte := range p[1:] {\n\t\tsortableBits <<= 7\n\t\tsortableBits |= int64(inbyte)\n\t}\n\treturn int64(uint64((sortableBits << shift)) ^ 0x8000000000000000), nil\n}\n\nfunc ValidPrefixCodedTerm(p string) (bool, int) {\n\treturn ValidPrefixCodedTermBytes([]byte(p))\n}\n\nfunc ValidPrefixCodedTermBytes(p []byte) (bool, int) {\n\tif len(p) > 0 {\n\t\tif p[0] < ShiftStartInt64 || p[0] > ShiftStartInt64+63 {\n\t\t\treturn false, 0\n\t\t}\n\t\tshift := p[0] - ShiftStartInt64\n\t\tnChars := ((63 - int(shift)) / 7) + 1\n\t\tif len(p) != nChars+1 {\n\t\t\treturn false, 0\n\t\t}\n\t\treturn true, int(shift)\n\t}\n\treturn false, 0\n}\n"
  },
  {
    "path": "numeric/prefix_coded_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage numeric\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nvar tests = []struct {\n\tinput  int64\n\tshift  uint\n\toutput PrefixCoded\n}{\n\t{\n\t\tinput:  1,\n\t\tshift:  0,\n\t\toutput: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},\n\t},\n\t{\n\t\tinput:  -1,\n\t\tshift:  0,\n\t\toutput: PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f},\n\t},\n\t{\n\t\tinput:  -94582,\n\t\tshift:  0,\n\t\toutput: PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7a, 0x1d, 0xa},\n\t},\n\t{\n\t\tinput:  314729851,\n\t\tshift:  0,\n\t\toutput: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x16, 0x9, 0x4a, 0x7b},\n\t},\n\t{\n\t\tinput:  314729851,\n\t\tshift:  4,\n\t\toutput: PrefixCoded{0x24, 0x8, 0x0, 0x0, 0x0, 0x0, 0x9, 0x30, 0x4c, 0x57},\n\t},\n\t{\n\t\tinput:  314729851,\n\t\tshift:  8,\n\t\toutput: PrefixCoded{0x28, 0x40, 0x0, 0x0, 0x0, 0x0, 0x4b, 0x4, 0x65},\n\t},\n\t{\n\t\tinput:  314729851,\n\t\tshift:  16,\n\t\toutput: PrefixCoded{0x30, 0x20, 0x0, 0x0, 0x0, 0x0, 0x25, 0x42},\n\t},\n\t{\n\t\tinput:  314729851,\n\t\tshift:  32,\n\t\toutput: PrefixCoded{0x40, 0x8, 0x0, 0x0, 0x0, 0x0},\n\t},\n\t{\n\t\tinput:  1234729851,\n\t\tshift:  32,\n\t\toutput: PrefixCoded{0x40, 0x8, 0x0, 0x0, 0x0, 0x0},\n\t},\n}\n\n// these array encoding values have been verified manually\n// against the lucene implementation\nfunc TestPrefixCoded(t *testing.T) {\n\n\tfor _, test := range tests {\n\t\tactual, err := NewPrefixCodedInt64(test.input, test.shift)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t}\n\t\tcheckedShift, err := actual.Shift()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif checkedShift != test.shift {\n\t\t\tt.Errorf(\"expected %d, got %d\", test.shift, checkedShift)\n\t\t}\n\t\t// if the shift was 0, make sure we can go back to the original\n\t\tif test.shift == 0 {\n\t\t\tbackToLong, err := actual.Int64()\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tif backToLong != test.input {\n\t\t\t\tt.Errorf(\"expected %v, got %v\", test.input, backToLong)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestPrefixCodedValid(t *testing.T) {\n\t// all of the shared tests should be valid\n\tfor _, test := range tests {\n\t\tvalid, _ := ValidPrefixCodedTerm(string(test.output))\n\t\tif !valid {\n\t\t\tt.Errorf(\"expected %s to be valid prefix coded, is not\", string(test.output))\n\t\t}\n\t}\n\n\tinvalidTests := []struct {\n\t\tdata PrefixCoded\n\t}{\n\t\t// first byte invalid skip (too low)\n\t\t{\n\t\t\tdata: PrefixCoded{0x19, 'c', 'a', 't'},\n\t\t},\n\t\t// first byte invalid skip (too high)\n\t\t{\n\t\t\tdata: PrefixCoded{0x20 + 64, 'c'},\n\t\t},\n\t\t// length of trailing bytes wrong (too long)\n\t\t{\n\t\t\tdata: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},\n\t\t},\n\t\t// length of trailing bytes wrong (too short)\n\t\t{\n\t\t\tdata: PrefixCoded{0x20 + 63},\n\t\t},\n\t}\n\n\t// all of the shared tests should be valid\n\tfor _, test := range invalidTests {\n\t\tvalid, _ := ValidPrefixCodedTerm(string(test.data))\n\t\tif valid {\n\t\t\tt.Errorf(\"expected %s to be invalid prefix coded, it is\", string(test.data))\n\t\t}\n\t}\n}\n\nfunc BenchmarkTestPrefixCoded(b *testing.B) {\n\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, test := range tests {\n\t\t\tactual, err := NewPrefixCodedInt64(test.input, test.shift)\n\t\t\tif err != nil {\n\t\t\t\tb.Error(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\t\tb.Errorf(\"expected %#v, got %#v\", test.output, actual)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pre_search.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\n// A preSearchResultProcessor processes the data in\n// the preSearch result from multiple\n// indexes in an alias and merges them together to\n// create the final preSearch result\ntype preSearchResultProcessor interface {\n\t// adds the preSearch result to the processor\n\tadd(*SearchResult, string)\n\t// updates the final search result with the finalized\n\t// data from the processor\n\tfinalize(*SearchResult)\n}\n\n// -----------------------------------------------------------------------------\n// KNN preSearchResultProcessor for handling KNN presearch results\ntype knnPreSearchResultProcessor struct {\n\taddFn      func(sr *SearchResult, indexName string)\n\tfinalizeFn func(sr *SearchResult)\n}\n\nfunc (k *knnPreSearchResultProcessor) add(sr *SearchResult, indexName string) {\n\tif k.addFn != nil {\n\t\tk.addFn(sr, indexName)\n\t}\n}\n\nfunc (k *knnPreSearchResultProcessor) finalize(sr *SearchResult) {\n\tif k.finalizeFn != nil {\n\t\tk.finalizeFn(sr)\n\t}\n}\n\n// -----------------------------------------------------------------------------\n// Synonym preSearchResultProcessor for handling Synonym presearch results\ntype synonymPreSearchResultProcessor struct {\n\tfinalizedFts search.FieldTermSynonymMap\n}\n\nfunc newSynonymPreSearchResultProcessor() *synonymPreSearchResultProcessor {\n\treturn &synonymPreSearchResultProcessor{}\n}\n\nfunc (s *synonymPreSearchResultProcessor) add(sr *SearchResult, indexName string) {\n\t// Check if SynonymResult or the synonym data key is nil\n\tif sr.SynonymResult == nil {\n\t\treturn\n\t}\n\n\t// Attempt to cast PreSearchResults to FieldTermSynonymMap\n\n\t// Merge with finalizedFts or initialize it if nil\n\tif s.finalizedFts == nil {\n\t\ts.finalizedFts = sr.SynonymResult\n\t} else {\n\t\ts.finalizedFts.MergeWith(sr.SynonymResult)\n\t}\n}\n\nfunc (s *synonymPreSearchResultProcessor) finalize(sr *SearchResult) {\n\t// Set the finalized synonym data to the PreSearchResults\n\tif s.finalizedFts != nil {\n\t\tsr.SynonymResult = s.finalizedFts\n\t}\n}\n\ntype bm25PreSearchResultProcessor struct {\n\tdocCount         float64 // bm25 specific stats\n\tfieldCardinality map[string]int\n}\n\nfunc newBM25PreSearchResultProcessor() *bm25PreSearchResultProcessor {\n\treturn &bm25PreSearchResultProcessor{\n\t\tfieldCardinality: make(map[string]int),\n\t}\n}\n\n// TODO How will this work for queries other than term queries?\nfunc (b *bm25PreSearchResultProcessor) add(sr *SearchResult, indexName string) {\n\tif sr.BM25Stats != nil {\n\t\tb.docCount += sr.BM25Stats.DocCount\n\t\tfor field, cardinality := range sr.BM25Stats.FieldCardinality {\n\t\t\tb.fieldCardinality[field] += cardinality\n\t\t}\n\t}\n}\n\nfunc (b *bm25PreSearchResultProcessor) finalize(sr *SearchResult) {\n\tsr.BM25Stats = &search.BM25Stats{\n\t\tDocCount:         b.docCount,\n\t\tFieldCardinality: b.fieldCardinality,\n\t}\n}\n\n// -----------------------------------------------------------------------------\n// Master struct that can hold any number of presearch result processors\ntype compositePreSearchResultProcessor struct {\n\tpresearchResultProcessors []preSearchResultProcessor\n}\n\n// Implements the add method, which forwards to all the internal processors\nfunc (m *compositePreSearchResultProcessor) add(sr *SearchResult, indexName string) {\n\tfor _, p := range m.presearchResultProcessors {\n\t\tp.add(sr, indexName)\n\t}\n}\n\n// Implements the finalize method, which forwards to all the internal processors\nfunc (m *compositePreSearchResultProcessor) finalize(sr *SearchResult) {\n\tfor _, p := range m.presearchResultProcessors {\n\t\tp.finalize(sr)\n\t}\n}\n\n// -----------------------------------------------------------------------------\n// Function to create the appropriate preSearchResultProcessor(s)\nfunc createPreSearchResultProcessor(req *SearchRequest, flags *preSearchFlags) preSearchResultProcessor {\n\t// return nil for invalid input\n\tif flags == nil || req == nil {\n\t\treturn nil\n\t}\n\tvar processors []preSearchResultProcessor\n\t// Add KNN processor if the request has KNN\n\tif flags.knn {\n\t\tif knnProcessor := newKnnPreSearchResultProcessor(req); knnProcessor != nil {\n\t\t\tprocessors = append(processors, knnProcessor)\n\t\t}\n\t}\n\t// Add Synonym processor if the request has Synonym\n\tif flags.synonyms {\n\t\tif synonymProcessor := newSynonymPreSearchResultProcessor(); synonymProcessor != nil {\n\t\t\tprocessors = append(processors, synonymProcessor)\n\t\t}\n\t}\n\tif flags.bm25 {\n\t\tif bm25Processtor := newBM25PreSearchResultProcessor(); bm25Processtor != nil {\n\t\t\tprocessors = append(processors, bm25Processtor)\n\t\t}\n\t}\n\t// Return based on the number of processors, optimizing for the common case of 1 processor\n\t// If there are no processors, return nil\n\tswitch len(processors) {\n\tcase 0:\n\t\treturn nil\n\tcase 1:\n\t\treturn processors[0]\n\tdefault:\n\t\treturn &compositePreSearchResultProcessor{\n\t\t\tpresearchResultProcessors: processors,\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "query.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n)\n\n// NewBoolFieldQuery creates a new Query for boolean fields\nfunc NewBoolFieldQuery(val bool) *query.BoolFieldQuery {\n\treturn query.NewBoolFieldQuery(val)\n}\n\n// NewBooleanQuery creates a compound Query composed\n// of several other Query objects.\n// These other query objects are added using the\n// AddMust() AddShould() and AddMustNot() methods.\n// Result documents must satisfy ALL of the\n// must Queries.\n// Result documents must satisfy NONE of the must not\n// Queries.\n// Result documents that ALSO satisfy any of the should\n// Queries will score higher.\nfunc NewBooleanQuery() *query.BooleanQuery {\n\treturn query.NewBooleanQuery(nil, nil, nil)\n}\n\n// NewConjunctionQuery creates a new compound Query.\n// Result documents must satisfy all of the queries.\nfunc NewConjunctionQuery(conjuncts ...query.Query) *query.ConjunctionQuery {\n\treturn query.NewConjunctionQuery(conjuncts)\n}\n\n// NewDateRangeQuery creates a new Query for ranges\n// of date values.\n// Date strings are parsed using the DateTimeParser configured in the\n//\n//\ttop-level config.QueryDateTimeParser\n//\n// Either, but not both endpoints can be nil.\nfunc NewDateRangeQuery(start, end time.Time) *query.DateRangeQuery {\n\treturn query.NewDateRangeQuery(start, end)\n}\n\n// NewDateRangeInclusiveQuery creates a new Query for ranges\n// of date values.\n// Date strings are parsed using the DateTimeParser configured in the\n//\n//\ttop-level config.QueryDateTimeParser\n//\n// Either, but not both endpoints can be nil.\n// startInclusive and endInclusive control inclusion of the endpoints.\nfunc NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *query.DateRangeQuery {\n\treturn query.NewDateRangeInclusiveQuery(start, end, startInclusive, endInclusive)\n}\n\n// NewDateRangeStringQuery creates a new Query for ranges\n// of date values.\n// Date strings are parsed using the DateTimeParser set using\n//\n//\tthe DateRangeStringQuery.SetDateTimeParser() method.\n//\n// If no DateTimeParser is set, then the\n//\n//\ttop-level config.QueryDateTimeParser\n//\n// is used.\nfunc NewDateRangeStringQuery(start, end string) *query.DateRangeStringQuery {\n\treturn query.NewDateRangeStringQuery(start, end)\n}\n\n// NewDateRangeInclusiveStringQuery creates a new Query for ranges\n// of date values.\n// Date strings are parsed using the DateTimeParser set using\n//\n//\tthe DateRangeStringQuery.SetDateTimeParser() method.\n//\n// this DateTimeParser is a custom date time parser defined in the index mapping,\n// using AddCustomDateTimeParser() method.\n// If no DateTimeParser is set, then the\n//\n//\ttop-level config.QueryDateTimeParser\n//\n// is used.\n// Either, but not both endpoints can be nil.\n// startInclusive and endInclusive control inclusion of the endpoints.\nfunc NewDateRangeInclusiveStringQuery(start, end string, startInclusive, endInclusive *bool) *query.DateRangeStringQuery {\n\treturn query.NewDateRangeStringInclusiveQuery(start, end, startInclusive, endInclusive)\n}\n\n// NewDisjunctionQuery creates a new compound Query.\n// Result documents satisfy at least one Query.\nfunc NewDisjunctionQuery(disjuncts ...query.Query) *query.DisjunctionQuery {\n\treturn query.NewDisjunctionQuery(disjuncts)\n}\n\n// NewDocIDQuery creates a new Query object returning indexed documents among\n// the specified set. Combine it with ConjunctionQuery to restrict the scope of\n// other queries output.\nfunc NewDocIDQuery(ids []string) *query.DocIDQuery {\n\treturn query.NewDocIDQuery(ids)\n}\n\n// NewFuzzyQuery creates a new Query which finds\n// documents containing terms within a specific\n// fuzziness of the specified term.\n// The default fuzziness is 1.\n//\n// The current implementation uses Levenshtein edit\n// distance as the fuzziness metric.\nfunc NewFuzzyQuery(term string) *query.FuzzyQuery {\n\treturn query.NewFuzzyQuery(term)\n}\n\n// NewMatchAllQuery creates a Query which will\n// match all documents in the index.\nfunc NewMatchAllQuery() *query.MatchAllQuery {\n\treturn query.NewMatchAllQuery()\n}\n\n// NewMatchNoneQuery creates a Query which will not\n// match any documents in the index.\nfunc NewMatchNoneQuery() *query.MatchNoneQuery {\n\treturn query.NewMatchNoneQuery()\n}\n\n// NewMatchPhraseQuery creates a new Query object\n// for matching phrases in the index.\n// An Analyzer is chosen based on the field.\n// Input text is analyzed using this analyzer.\n// Token terms resulting from this analysis are\n// used to build a search phrase.  Result documents\n// must match this phrase. Queried field must have been indexed with\n// IncludeTermVectors set to true.\nfunc NewMatchPhraseQuery(matchPhrase string) *query.MatchPhraseQuery {\n\treturn query.NewMatchPhraseQuery(matchPhrase)\n}\n\n// NewMatchQuery creates a Query for matching text.\n// An Analyzer is chosen based on the field.\n// Input text is analyzed using this analyzer.\n// Token terms resulting from this analysis are\n// used to perform term searches.  Result documents\n// must satisfy at least one of these term searches.\nfunc NewMatchQuery(match string) *query.MatchQuery {\n\treturn query.NewMatchQuery(match)\n}\n\n// NewNumericRangeQuery creates a new Query for ranges\n// of numeric values.\n// Either, but not both endpoints can be nil.\n// The minimum value is inclusive.\n// The maximum value is exclusive.\nfunc NewNumericRangeQuery(min, max *float64) *query.NumericRangeQuery {\n\treturn query.NewNumericRangeQuery(min, max)\n}\n\n// NewNumericRangeInclusiveQuery creates a new Query for ranges\n// of numeric values.\n// Either, but not both endpoints can be nil.\n// Control endpoint inclusion with inclusiveMin, inclusiveMax.\nfunc NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *query.NumericRangeQuery {\n\treturn query.NewNumericRangeInclusiveQuery(min, max, minInclusive, maxInclusive)\n}\n\n// NewTermRangeQuery creates a new Query for ranges\n// of text terms.\n// Either, but not both endpoints can be \"\".\n// The minimum value is inclusive.\n// The maximum value is exclusive.\nfunc NewTermRangeQuery(min, max string) *query.TermRangeQuery {\n\treturn query.NewTermRangeQuery(min, max)\n}\n\n// NewTermRangeInclusiveQuery creates a new Query for ranges\n// of text terms.\n// Either, but not both endpoints can be \"\".\n// Control endpoint inclusion with inclusiveMin, inclusiveMax.\nfunc NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *query.TermRangeQuery {\n\treturn query.NewTermRangeInclusiveQuery(min, max, minInclusive, maxInclusive)\n}\n\n// NewPhraseQuery creates a new Query for finding\n// exact term phrases in the index.\n// The provided terms must exist in the correct\n// order, at the correct index offsets, in the\n// specified field. Queried field must have been indexed with\n// IncludeTermVectors set to true.\nfunc NewPhraseQuery(terms []string, field string) *query.PhraseQuery {\n\treturn query.NewPhraseQuery(terms, field)\n}\n\n// NewPrefixQuery creates a new Query which finds\n// documents containing terms that start with the\n// specified prefix.\nfunc NewPrefixQuery(prefix string) *query.PrefixQuery {\n\treturn query.NewPrefixQuery(prefix)\n}\n\n// NewRegexpQuery creates a new Query which finds\n// documents containing terms that match the\n// specified regular expression.\nfunc NewRegexpQuery(regexp string) *query.RegexpQuery {\n\treturn query.NewRegexpQuery(regexp)\n}\n\n// NewQueryStringQuery creates a new Query used for\n// finding documents that satisfy a query string.  The\n// query string is a small query language for humans.\nfunc NewQueryStringQuery(q string) *query.QueryStringQuery {\n\treturn query.NewQueryStringQuery(q)\n}\n\n// NewTermQuery creates a new Query for finding an\n// exact term match in the index.\nfunc NewTermQuery(term string) *query.TermQuery {\n\treturn query.NewTermQuery(term)\n}\n\n// NewWildcardQuery creates a new Query which finds\n// documents containing terms that match the\n// specified wildcard.  In the wildcard pattern '*'\n// will match any sequence of 0 or more characters,\n// and '?' will match any single character.\nfunc NewWildcardQuery(wildcard string) *query.WildcardQuery {\n\treturn query.NewWildcardQuery(wildcard)\n}\n\n// NewGeoBoundingBoxQuery creates a new Query for performing geo bounding\n// box searches. The arguments describe the position of the box and documents\n// which have an indexed geo point inside the box will be returned.\nfunc NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *query.GeoBoundingBoxQuery {\n\treturn query.NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat)\n}\n\n// NewGeoDistanceQuery creates a new Query for performing geo distance\n// searches. The arguments describe a position and a distance. Documents\n// which have an indexed geo point which is less than or equal to the provided\n// distance from the given position will be returned.\nfunc NewGeoDistanceQuery(lon, lat float64, distance string) *query.GeoDistanceQuery {\n\treturn query.NewGeoDistanceQuery(lon, lat, distance)\n}\n\n// NewIPRangeQuery creates a new Query for matching IP addresses.\n// If the argument is in CIDR format, then the query will match all\n// IP addresses in the network specified. If the argument is an IP address,\n// then the query will return documents which contain that IP.\n// Both ipv4 and ipv6 are supported.\nfunc NewIPRangeQuery(cidr string) *query.IPRangeQuery {\n\treturn query.NewIPRangeQuery(cidr)\n}\n\n// NewGeoShapeQuery creates a new Query for matching the given geo shape.\n// This method can be used for creating geoshape queries for shape types\n// like: point, linestring, polygon, multipoint, multilinestring,\n// multipolygon and envelope.\nfunc NewGeoShapeQuery(coordinates [][][][]float64, typ, relation string) (*query.GeoShapeQuery, error) {\n\treturn query.NewGeoShapeQuery(coordinates, typ, relation)\n}\n\n// NewGeoShapeCircleQuery creates a new query for a geoshape that is a\n// circle given center point and the radius. Radius formats supported:\n// \"5in\" \"5inch\" \"7yd\" \"7yards\" \"9ft\" \"9feet\" \"11km\" \"11kilometers\"\n// \"3nm\" \"3nauticalmiles\" \"13mm\" \"13millimeters\" \"15cm\" \"15centimeters\"\n// \"17mi\" \"17miles\" \"19m\" \"19meters\" If the unit cannot be determined,\n// the entire string is parsed and the unit of meters is assumed.\nfunc NewGeoShapeCircleQuery(coordinates []float64, radius, relation string) (*query.GeoShapeQuery, error) {\n\treturn query.NewGeoShapeCircleQuery(coordinates, radius, relation)\n}\n\n// NewGeometryCollectionQuery creates a new query for the provided\n// geometrycollection coordinates and types, which could contain\n// multiple geo shapes.\nfunc NewGeometryCollectionQuery(coordinates [][][][][]float64, types []string, relation string) (*query.GeoShapeQuery, error) {\n\treturn query.NewGeometryCollectionQuery(coordinates, types, relation)\n}\n"
  },
  {
    "path": "query_bench_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n)\n\nfunc BenchmarkQueryTerm(b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tfm := mapping.NewTextFieldMapping()\n\tfm.Analyzer = keyword.Name\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"text\", fm)\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping = dmap\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\tmembers := []string{\"abc\", \"abcdef\", \"ghi\", \"jkl\", \"jklmno\"}\n\tfor i := 0; i < 100; i++ {\n\t\tif err = idx.Index(strconv.Itoa(i),\n\t\t\tmap[string]interface{}{\"text\": members[i%len(members)]}); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := NewTermQuery(members[i%len(members)])\n\t\tq.SetField(\"text\")\n\t\treq := NewSearchRequest(q)\n\t\tif _, err = idx.Search(req); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkQueryTermRange(b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tfm := mapping.NewTextFieldMapping()\n\tfm.Analyzer = keyword.Name\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"text\", fm)\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping = dmap\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\tmembers := []string{\"abc\", \"abcdef\", \"ghi\", \"jkl\", \"jklmno\"}\n\tfor i := 0; i < 100; i++ {\n\t\tif err = idx.Index(strconv.Itoa(i),\n\t\t\tmap[string]interface{}{\"text\": members[i%len(members)]}); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tinclusive := true\n\tfor i := 0; i < b.N; i++ {\n\t\tq := NewTermRangeInclusiveQuery(\n\t\t\tmembers[i%(len(members)-2)],\n\t\t\tmembers[(i+2)%(len(members)-2)],\n\t\t\t&inclusive,\n\t\t\t&inclusive,\n\t\t)\n\t\tq.SetField(\"text\")\n\t\treq := NewSearchRequest(q)\n\t\tif _, err = idx.Search(req); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkQueryWildcard(b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tfm := mapping.NewTextFieldMapping()\n\tfm.Analyzer = keyword.Name\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"text\", fm)\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping = dmap\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\tmembers := []string{\"abc\", \"abcdef\", \"ghi\", \"jkl\", \"jklmno\"}\n\tfor i := 0; i < 100; i++ {\n\t\tif err = idx.Index(strconv.Itoa(i),\n\t\t\tmap[string]interface{}{\"text\": members[i%len(members)]}); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\twildcards := []string{\"ab*\", \"jk*\"}\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := NewWildcardQuery(wildcards[i%len(wildcards)])\n\t\tq.SetField(\"text\")\n\t\treq := NewSearchRequest(q)\n\t\tif _, err = idx.Search(req); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkQueryNumericRange(b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tfm := mapping.NewNumericFieldMapping()\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"number\", fm)\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping = dmap\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\tfor i := 0; i < 100; i++ {\n\t\tif err = idx.Index(strconv.Itoa(i),\n\t\t\tmap[string]interface{}{\"number\": i}); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tinclusive := true\n\tfor i := 0; i < b.N; i++ {\n\t\tstart := float64(i % 90)\n\t\tend := float64((i + 10) % 90)\n\t\tq := NewNumericRangeInclusiveQuery(&start, &end, &inclusive, &inclusive)\n\t\tq.SetField(\"number\")\n\t\treq := NewSearchRequest(q)\n\t\tif _, err = idx.Search(req); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkQueryDateRange(b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tfm := mapping.NewDateTimeFieldMapping()\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"date\", fm)\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping = dmap\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\tmembers := []string{\n\t\t\"2022-11-16T18:45:45Z\",\n\t\t\"2022-11-17T18:45:45Z\",\n\t\t\"2022-11-18T18:45:45Z\",\n\t\t\"2022-11-19T18:45:45Z\",\n\t\t\"2022-11-20T18:45:45Z\",\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\tif err = idx.Index(strconv.Itoa(i),\n\t\t\tmap[string]interface{}{\"date\": members[i%len(members)]}); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tinclusive := true\n\tfor i := 0; i < b.N; i++ {\n\t\tstart, _ := time.Parse(\"2006-01-02T15:04:05Z\", members[i%(len(members)-2)])\n\t\tend, _ := time.Parse(\"2006-01-02T15:04:05Z\", members[(i+2)%(len(members)-2)])\n\t\tq := NewDateRangeInclusiveQuery(start, end, &inclusive, &inclusive)\n\t\tq.SetField(\"date\")\n\t\treq := NewSearchRequest(q)\n\t\tif _, err = idx.Search(req); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkQueryGeoDistance(b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tfm := mapping.NewGeoPointFieldMapping()\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"geo\", fm)\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping = dmap\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\tmembers := [][]float64{\n\t\t{-121.96713072883645, 37.380331474621045},\n\t\t{-97.75518866579938, 30.38974491308761},\n\t\t{-0.08653451918110022, 51.51063984942306},\n\t\t{-2.230759791360498, 53.481514330841236},\n\t\t{77.59542326042589, 12.97215865921956},\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\tif err = idx.Index(strconv.Itoa(i),\n\t\t\tmap[string]interface{}{\"geo\": members[i%len(members)]}); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tcoordinates := members[i%len(members)]\n\t\tq := NewGeoDistanceQuery(coordinates[0], coordinates[1], \"1mi\")\n\t\tq.SetField(\"geo\")\n\t\treq := NewSearchRequest(q)\n\t\tif _, err = idx.Search(req); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkQueryGeoBoundingBox(b *testing.B) {\n\ttmpIndexPath := createTmpIndexPath(b)\n\tdefer cleanupTmpIndexPath(b, tmpIndexPath)\n\n\tfm := mapping.NewGeoPointFieldMapping()\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"geo\", fm)\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping = dmap\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}()\n\n\tmembers := [][]float64{\n\t\t{-121.96713072883645, 37.380331474621045},\n\t\t{-97.75518866579938, 30.38974491308761},\n\t\t{-0.08653451918110022, 51.51063984942306},\n\t\t{-2.230759791360498, 53.481514330841236},\n\t\t{77.59542326042589, 12.97215865921956},\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\tif err = idx.Index(strconv.Itoa(i),\n\t\t\tmap[string]interface{}{\"geo\": members[i%len(members)]}); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tboundingBoxes := []struct {\n\t\ttopLeft     []float64\n\t\tbottomRight []float64\n\t}{\n\t\t{\n\t\t\ttopLeft:     []float64{-122.14424992609722, 37.49751487670511},\n\t\t\tbottomRight: []float64{-121.78076546622579, 37.26963069737202},\n\t\t},\n\t\t{\n\t\t\ttopLeft:     []float64{-97.85362236226437, 30.473743975245725},\n\t\t\tbottomRight: []float64{-97.58691085968482, 30.285211697102895},\n\t\t},\n\t\t{\n\t\t\ttopLeft:     []float64{-0.28538822102223094, 51.61106497119687},\n\t\t\tbottomRight: []float64{0.16776748108466677, 51.395702237541286},\n\t\t},\n\t\t{\n\t\t\ttopLeft:     []float64{-2.373683904907921, 53.54371945714075},\n\t\t\tbottomRight: []float64{-2.134365533113197, 53.41788831720595},\n\t\t},\n\t\t{\n\t\t\ttopLeft:     []float64{77.52617635172015, 13.037587208986437},\n\t\t\tbottomRight: []float64{77.66508989028102, 12.924426170584738},\n\t\t},\n\t}\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttopLeftCoordinates := boundingBoxes[i%len(boundingBoxes)].topLeft\n\t\tbottomRightCoordinates := boundingBoxes[i%len(boundingBoxes)].bottomRight\n\t\tq := NewGeoBoundingBoxQuery(\n\t\t\ttopLeftCoordinates[0],\n\t\t\ttopLeftCoordinates[1],\n\t\t\tbottomRightCoordinates[0],\n\t\t\tbottomRightCoordinates[1],\n\t\t)\n\t\tq.SetField(\"geo\")\n\t\treq := NewSearchRequest(q)\n\t\tif _, err = idx.Search(req); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "registry/analyzer.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc RegisterAnalyzer(name string, constructor AnalyzerConstructor) error {\n\t_, exists := analyzers[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate analyzer named '%s'\", name)\n\t}\n\tanalyzers[name] = constructor\n\treturn nil\n}\n\ntype AnalyzerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Analyzer, error)\ntype AnalyzerRegistry map[string]AnalyzerConstructor\n\ntype AnalyzerCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewAnalyzerCache() *AnalyzerCache {\n\treturn &AnalyzerCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc AnalyzerBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := analyzers[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no analyzer with name or type '%s' registered\", name)\n\t}\n\tanalyzer, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building analyzer: %v\", err)\n\t}\n\treturn analyzer, nil\n}\n\nfunc (c *AnalyzerCache) AnalyzerNamed(name string, cache *Cache) (analysis.Analyzer, error) {\n\titem, err := c.ItemNamed(name, cache, AnalyzerBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.Analyzer), nil\n}\n\nfunc (c *AnalyzerCache) DefineAnalyzer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Analyzer, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, AnalyzerBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"analyzer named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.Analyzer), nil\n}\n\nfunc AnalyzerTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range analyzers {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/cache.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n)\n\nvar ErrAlreadyDefined = fmt.Errorf(\"item already defined\")\n\ntype CacheBuild func(name string, config map[string]interface{}, cache *Cache) (interface{}, error)\n\ntype ConcurrentCache struct {\n\tmutex sync.RWMutex\n\tdata  map[string]interface{}\n}\n\nfunc NewConcurrentCache() *ConcurrentCache {\n\treturn &ConcurrentCache{\n\t\tdata: make(map[string]interface{}),\n\t}\n}\n\nfunc (c *ConcurrentCache) ItemNamed(name string, cache *Cache, build CacheBuild) (interface{}, error) {\n\tc.mutex.RLock()\n\titem, cached := c.data[name]\n\tif cached {\n\t\tc.mutex.RUnlock()\n\t\treturn item, nil\n\t}\n\t// give up read lock\n\tc.mutex.RUnlock()\n\t// try to build it\n\tnewItem, err := build(name, nil, cache)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// acquire write lock\n\tc.mutex.Lock()\n\tdefer c.mutex.Unlock()\n\t// check again because it could have been created while trading locks\n\titem, cached = c.data[name]\n\tif cached {\n\t\treturn item, nil\n\t}\n\tc.data[name] = newItem\n\treturn newItem, nil\n}\n\nfunc (c *ConcurrentCache) DefineItem(name string, typ string, config map[string]interface{}, cache *Cache, build CacheBuild) (interface{}, error) {\n\tc.mutex.RLock()\n\t_, cached := c.data[name]\n\tif cached {\n\t\tc.mutex.RUnlock()\n\t\treturn nil, ErrAlreadyDefined\n\t}\n\t// give up read lock so others lookups can proceed\n\tc.mutex.RUnlock()\n\t// really not there, try to build it\n\tnewItem, err := build(typ, config, cache)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// now we've built it, acquire lock\n\tc.mutex.Lock()\n\tdefer c.mutex.Unlock()\n\t// check again because it could have been created while trading locks\n\t_, cached = c.data[name]\n\tif cached {\n\t\treturn nil, ErrAlreadyDefined\n\t}\n\tc.data[name] = newItem\n\treturn newItem, nil\n}\n"
  },
  {
    "path": "registry/char_filter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc RegisterCharFilter(name string, constructor CharFilterConstructor) error {\n\t_, exists := charFilters[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate char filter named '%s'\", name)\n\t}\n\tcharFilters[name] = constructor\n\treturn nil\n}\n\ntype CharFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.CharFilter, error)\ntype CharFilterRegistry map[string]CharFilterConstructor\n\ntype CharFilterCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewCharFilterCache() *CharFilterCache {\n\treturn &CharFilterCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc CharFilterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := charFilters[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no char filter with name or type '%s' registered\", name)\n\t}\n\tcharFilter, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building char filter: %v\", err)\n\t}\n\treturn charFilter, nil\n}\n\nfunc (c *CharFilterCache) CharFilterNamed(name string, cache *Cache) (analysis.CharFilter, error) {\n\titem, err := c.ItemNamed(name, cache, CharFilterBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.CharFilter), nil\n}\n\nfunc (c *CharFilterCache) DefineCharFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.CharFilter, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, CharFilterBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"char filter named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.CharFilter), nil\n}\n\nfunc CharFilterTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range charFilters {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/datetime_parser.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc RegisterDateTimeParser(name string, constructor DateTimeParserConstructor) error {\n\t_, exists := dateTimeParsers[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate date time parser named '%s'\", name)\n\t}\n\tdateTimeParsers[name] = constructor\n\treturn nil\n}\n\ntype DateTimeParserConstructor func(config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error)\ntype DateTimeParserRegistry map[string]DateTimeParserConstructor\n\ntype DateTimeParserCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewDateTimeParserCache() *DateTimeParserCache {\n\treturn &DateTimeParserCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc DateTimeParserBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := dateTimeParsers[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no date time parser with name or type '%s' registered\", name)\n\t}\n\tdateTimeParser, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building date time parser: %v\", err)\n\t}\n\treturn dateTimeParser, nil\n}\n\nfunc (c *DateTimeParserCache) DateTimeParserNamed(name string, cache *Cache) (analysis.DateTimeParser, error) {\n\titem, err := c.ItemNamed(name, cache, DateTimeParserBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.DateTimeParser), nil\n}\n\nfunc (c *DateTimeParserCache) DefineDateTimeParser(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, DateTimeParserBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"date time parser named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.DateTimeParser), nil\n}\n\nfunc DateTimeParserTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range dateTimeParsers {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/fragment_formatter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nfunc RegisterFragmentFormatter(name string, constructor FragmentFormatterConstructor) error {\n\t_, exists := fragmentFormatters[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate fragment formatter named '%s'\", name)\n\t}\n\tfragmentFormatters[name] = constructor\n\treturn nil\n}\n\ntype FragmentFormatterConstructor func(config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error)\ntype FragmentFormatterRegistry map[string]FragmentFormatterConstructor\n\ntype FragmentFormatterCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewFragmentFormatterCache() *FragmentFormatterCache {\n\treturn &FragmentFormatterCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc FragmentFormatterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := fragmentFormatters[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no fragment formatter with name or type '%s' registered\", name)\n\t}\n\tfragmentFormatter, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building fragment formatter: %v\", err)\n\t}\n\treturn fragmentFormatter, nil\n}\n\nfunc (c *FragmentFormatterCache) FragmentFormatterNamed(name string, cache *Cache) (highlight.FragmentFormatter, error) {\n\titem, err := c.ItemNamed(name, cache, FragmentFormatterBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(highlight.FragmentFormatter), nil\n}\n\nfunc (c *FragmentFormatterCache) DefineFragmentFormatter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, FragmentFormatterBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"fragment formatter named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(highlight.FragmentFormatter), nil\n}\n\nfunc FragmentFormatterTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range fragmentFormatters {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/fragmenter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nfunc RegisterFragmenter(name string, constructor FragmenterConstructor) error {\n\t_, exists := fragmenters[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate fragmenter named '%s'\", name)\n\t}\n\tfragmenters[name] = constructor\n\treturn nil\n}\n\ntype FragmenterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error)\ntype FragmenterRegistry map[string]FragmenterConstructor\n\ntype FragmenterCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewFragmenterCache() *FragmenterCache {\n\treturn &FragmenterCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc FragmenterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := fragmenters[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no fragmenter with name or type '%s' registered\", name)\n\t}\n\tfragmenter, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building fragmenter: %v\", err)\n\t}\n\treturn fragmenter, nil\n}\n\nfunc (c *FragmenterCache) FragmenterNamed(name string, cache *Cache) (highlight.Fragmenter, error) {\n\titem, err := c.ItemNamed(name, cache, FragmenterBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(highlight.Fragmenter), nil\n}\n\nfunc (c *FragmenterCache) DefineFragmenter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, FragmenterBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"fragmenter named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(highlight.Fragmenter), nil\n}\n\nfunc FragmenterTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range fragmenters {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/highlighter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nfunc RegisterHighlighter(name string, constructor HighlighterConstructor) error {\n\t_, exists := highlighters[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate highlighter named '%s'\", name)\n\t}\n\thighlighters[name] = constructor\n\treturn nil\n}\n\ntype HighlighterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Highlighter, error)\ntype HighlighterRegistry map[string]HighlighterConstructor\n\ntype HighlighterCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewHighlighterCache() *HighlighterCache {\n\treturn &HighlighterCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc HighlighterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := highlighters[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no highlighter with name or type '%s' registered\", name)\n\t}\n\thighlighter, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building highlighter: %v\", err)\n\t}\n\treturn highlighter, nil\n}\n\nfunc (c *HighlighterCache) HighlighterNamed(name string, cache *Cache) (highlight.Highlighter, error) {\n\titem, err := c.ItemNamed(name, cache, HighlighterBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(highlight.Highlighter), nil\n}\n\nfunc (c *HighlighterCache) DefineHighlighter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, HighlighterBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"highlighter named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(highlight.Highlighter), nil\n}\n\nfunc HighlighterTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range highlighters {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/index_type.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc RegisterIndexType(name string, constructor IndexTypeConstructor) error {\n\t_, exists := indexTypes[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate index encoding named '%s'\", name)\n\t}\n\tindexTypes[name] = constructor\n\treturn nil\n}\n\ntype IndexTypeConstructor func(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error)\ntype IndexTypeRegistry map[string]IndexTypeConstructor\n\nfunc IndexTypeConstructorByName(name string) IndexTypeConstructor {\n\treturn indexTypes[name]\n}\n\nfunc IndexTypesAndInstances() ([]string, []string) {\n\tvar types []string\n\tvar instances []string\n\tfor name := range indexTypes {\n\t\ttypes = append(types, name)\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/nested.go",
    "content": "//  Copyright (c) 2026 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\n// NestedFieldCache caches nested field prefixes and their corresponding nesting levels.\n// A nested field prefix is a field path prefix that indicates the start of a nested document.\n// The nesting level indicates how deep the nested document is in the overall document structure.\ntype NestedFieldCache struct {\n\t// nested prefix -> nested level\n\tprefixDepth map[string]int\n\tonce        sync.Once\n\tm           sync.RWMutex\n}\n\nfunc NewNestedFieldCache() *NestedFieldCache {\n\treturn &NestedFieldCache{}\n}\n\nfunc (nfc *NestedFieldCache) InitOnce(buildFunc func() map[string]int) {\n\tnfc.once.Do(func() {\n\t\tnfc.m.Lock()\n\t\tdefer nfc.m.Unlock()\n\t\tnfc.prefixDepth = buildFunc()\n\t})\n}\n\n// NestedDepth returns two values:\n//   - common: The nesting level of the longest prefix that applies to every field path\n//     in the provided FieldSet. A value of 0 means no nested prefix is shared\n//     across all field paths.\n//   - max: The nesting level of the longest prefix that applies to at least one\n//     field path in the provided FieldSet. A value of 0 means none of the\n//     field paths match any nested prefix.\nfunc (nfc *NestedFieldCache) NestedDepth(fieldPaths search.FieldSet) (common int, max int) {\n\t// if no field paths, no nested depth\n\tif len(fieldPaths) == 0 {\n\t\treturn\n\t}\n\tnfc.m.RLock()\n\tdefer nfc.m.RUnlock()\n\t// if no cached prefixes, no nested depth\n\tif len(nfc.prefixDepth) == 0 {\n\t\treturn\n\t}\n\t// for each prefix, check if its a common prefix or matches any path\n\t// update common and max accordingly with the highest nesting level\n\t// possible for each respective case\n\tfor prefix, level := range nfc.prefixDepth {\n\t\t// only check prefixes that could increase one of the results\n\t\tif level <= common && level <= max {\n\t\t\tcontinue\n\t\t}\n\t\t// check prefix against field paths, getting whether it matches all paths (common)\n\t\t// and whether it matches at least one path (any)\n\t\tmatchAll, matchAny := nfc.prefixMatch(prefix, fieldPaths)\n\t\t// if it matches all paths, update common\n\t\tif matchAll && level > common {\n\t\t\tcommon = level\n\t\t}\n\t\t// if it matches any path, update max\n\t\tif matchAny && level > max {\n\t\t\tmax = level\n\t\t}\n\t}\n\treturn common, max\n}\n\n// CountNested returns the number of nested prefixes\nfunc (nfc *NestedFieldCache) CountNested() int {\n\tnfc.m.RLock()\n\tdefer nfc.m.RUnlock()\n\n\treturn len(nfc.prefixDepth)\n}\n\n// IntersectsPrefix returns true if any of the given\n// field paths have a nested prefix\nfunc (nfc *NestedFieldCache) IntersectsPrefix(fieldPaths search.FieldSet) bool {\n\t// if no field paths, no intersection\n\tif len(fieldPaths) == 0 {\n\t\treturn false\n\t}\n\tnfc.m.RLock()\n\tdefer nfc.m.RUnlock()\n\t// if no cached prefixes, no intersection\n\tif len(nfc.prefixDepth) == 0 {\n\t\treturn false\n\t}\n\t// Check each cached nested prefix to see if it intersects with any path\n\tfor prefix := range nfc.prefixDepth {\n\t\t_, matchAny := nfc.prefixMatch(prefix, fieldPaths)\n\t\tif matchAny {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// prefixMatch checks whether the prefix matches all paths (common) and whether it matches at least one path (any)\n// Caller must hold the read lock.\nfunc (nfc *NestedFieldCache) prefixMatch(prefix string, fieldPaths search.FieldSet) (common bool, any bool) {\n\tcommon = true\n\tany = false\n\tfor path := range fieldPaths {\n\t\thas := strings.HasPrefix(path, prefix)\n\t\tif has {\n\t\t\tany = true\n\t\t} else {\n\t\t\tcommon = false\n\t\t}\n\t\t// early exit if we have determined both values\n\t\tif any && !common {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn common, any\n}\n"
  },
  {
    "path": "registry/registry.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nvar stores = make(KVStoreRegistry, 0)\nvar indexTypes = make(IndexTypeRegistry, 0)\n\n// highlight\nvar fragmentFormatters = make(FragmentFormatterRegistry, 0)\nvar fragmenters = make(FragmenterRegistry, 0)\nvar highlighters = make(HighlighterRegistry, 0)\n\n// analysis\nvar charFilters = make(CharFilterRegistry, 0)\nvar tokenizers = make(TokenizerRegistry, 0)\nvar tokenMaps = make(TokenMapRegistry, 0)\nvar tokenFilters = make(TokenFilterRegistry, 0)\nvar analyzers = make(AnalyzerRegistry, 0)\nvar dateTimeParsers = make(DateTimeParserRegistry, 0)\nvar synonymSources = make(SynonymSourceRegistry, 0)\n\ntype Cache struct {\n\tCharFilters        *CharFilterCache\n\tTokenizers         *TokenizerCache\n\tTokenMaps          *TokenMapCache\n\tTokenFilters       *TokenFilterCache\n\tAnalyzers          *AnalyzerCache\n\tDateTimeParsers    *DateTimeParserCache\n\tFragmentFormatters *FragmentFormatterCache\n\tFragmenters        *FragmenterCache\n\tHighlighters       *HighlighterCache\n\tSynonymSources     *SynonymSourceCache\n\tNestedPrefixes     *NestedFieldCache\n}\n\nfunc NewCache() *Cache {\n\treturn &Cache{\n\t\tCharFilters:        NewCharFilterCache(),\n\t\tTokenizers:         NewTokenizerCache(),\n\t\tTokenMaps:          NewTokenMapCache(),\n\t\tTokenFilters:       NewTokenFilterCache(),\n\t\tAnalyzers:          NewAnalyzerCache(),\n\t\tDateTimeParsers:    NewDateTimeParserCache(),\n\t\tFragmentFormatters: NewFragmentFormatterCache(),\n\t\tFragmenters:        NewFragmenterCache(),\n\t\tHighlighters:       NewHighlighterCache(),\n\t\tSynonymSources:     NewSynonymSourceCache(),\n\t\tNestedPrefixes:     NewNestedFieldCache(),\n\t}\n}\n\nfunc typeFromConfig(config map[string]interface{}) (string, error) {\n\tprop, ok := config[\"type\"]\n\tif !ok {\n\t\treturn \"\", fmt.Errorf(\"'type' property is not defined\")\n\t}\n\ttyp, ok := prop.(string)\n\tif !ok {\n\t\treturn \"\", fmt.Errorf(\"'type' property must be a string, not %T\", prop)\n\t}\n\treturn typ, nil\n}\n\nfunc (c *Cache) CharFilterNamed(name string) (analysis.CharFilter, error) {\n\treturn c.CharFilters.CharFilterNamed(name, c)\n}\n\nfunc (c *Cache) DefineCharFilter(name string, config map[string]interface{}) (analysis.CharFilter, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.CharFilters.DefineCharFilter(name, typ, config, c)\n}\n\nfunc (c *Cache) TokenizerNamed(name string) (analysis.Tokenizer, error) {\n\treturn c.Tokenizers.TokenizerNamed(name, c)\n}\n\nfunc (c *Cache) DefineTokenizer(name string, config map[string]interface{}) (analysis.Tokenizer, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"cannot resolve '%s' tokenizer type: %s\", name, err)\n\t}\n\treturn c.Tokenizers.DefineTokenizer(name, typ, config, c)\n}\n\nfunc (c *Cache) TokenMapNamed(name string) (analysis.TokenMap, error) {\n\treturn c.TokenMaps.TokenMapNamed(name, c)\n}\n\nfunc (c *Cache) DefineTokenMap(name string, config map[string]interface{}) (analysis.TokenMap, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.TokenMaps.DefineTokenMap(name, typ, config, c)\n}\n\nfunc (c *Cache) TokenFilterNamed(name string) (analysis.TokenFilter, error) {\n\treturn c.TokenFilters.TokenFilterNamed(name, c)\n}\n\nfunc (c *Cache) DefineTokenFilter(name string, config map[string]interface{}) (analysis.TokenFilter, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.TokenFilters.DefineTokenFilter(name, typ, config, c)\n}\n\nfunc (c *Cache) AnalyzerNamed(name string) (analysis.Analyzer, error) {\n\treturn c.Analyzers.AnalyzerNamed(name, c)\n}\n\nfunc (c *Cache) DefineAnalyzer(name string, config map[string]interface{}) (analysis.Analyzer, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.Analyzers.DefineAnalyzer(name, typ, config, c)\n}\n\nfunc (c *Cache) DateTimeParserNamed(name string) (analysis.DateTimeParser, error) {\n\treturn c.DateTimeParsers.DateTimeParserNamed(name, c)\n}\n\nfunc (c *Cache) DefineDateTimeParser(name string, config map[string]interface{}) (analysis.DateTimeParser, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.DateTimeParsers.DefineDateTimeParser(name, typ, config, c)\n}\n\nfunc (c *Cache) SynonymSourceNamed(name string) (analysis.SynonymSource, error) {\n\treturn c.SynonymSources.SynonymSourceNamed(name, c)\n}\n\nfunc (c *Cache) DefineSynonymSource(name string, config map[string]interface{}) (analysis.SynonymSource, error) {\n\treturn c.SynonymSources.DefineSynonymSource(name, analysis.SynonymSourceType, config, c)\n}\n\nfunc (c *Cache) FragmentFormatterNamed(name string) (highlight.FragmentFormatter, error) {\n\treturn c.FragmentFormatters.FragmentFormatterNamed(name, c)\n}\n\nfunc (c *Cache) DefineFragmentFormatter(name string, config map[string]interface{}) (highlight.FragmentFormatter, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.FragmentFormatters.DefineFragmentFormatter(name, typ, config, c)\n}\n\nfunc (c *Cache) FragmenterNamed(name string) (highlight.Fragmenter, error) {\n\treturn c.Fragmenters.FragmenterNamed(name, c)\n}\n\nfunc (c *Cache) DefineFragmenter(name string, config map[string]interface{}) (highlight.Fragmenter, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.Fragmenters.DefineFragmenter(name, typ, config, c)\n}\n\nfunc (c *Cache) HighlighterNamed(name string) (highlight.Highlighter, error) {\n\treturn c.Highlighters.HighlighterNamed(name, c)\n}\n\nfunc (c *Cache) DefineHighlighter(name string, config map[string]interface{}) (highlight.Highlighter, error) {\n\ttyp, err := typeFromConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.Highlighters.DefineHighlighter(name, typ, config, c)\n}\n"
  },
  {
    "path": "registry/store.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\tstore \"github.com/blevesearch/upsidedown_store_api\"\n)\n\nfunc RegisterKVStore(name string, constructor KVStoreConstructor) error {\n\t_, exists := stores[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate store named '%s'\", name)\n\t}\n\tstores[name] = constructor\n\treturn nil\n}\n\n// KVStoreConstructor is used to build a KVStore of a specific type when\n// specified by the index configuration. In addition to meeting the\n// store.KVStore interface, KVStores must also support this constructor.\n// Note that currently the values of config must\n// be able to be marshaled and unmarshaled using the encoding/json library (used\n// when reading/writing the index metadata file).\ntype KVStoreConstructor func(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error)\ntype KVStoreRegistry map[string]KVStoreConstructor\n\nfunc KVStoreConstructorByName(name string) KVStoreConstructor {\n\treturn stores[name]\n}\n\nfunc KVStoreTypesAndInstances() ([]string, []string) {\n\tvar types []string\n\tvar instances []string\n\tfor name := range stores {\n\t\ttypes = append(types, name)\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/synonym_source.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc RegisterSynonymSource(typ string, constructor SynonymSourceConstructor) error {\n\t_, exists := synonymSources[typ]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate synonym source with type '%s'\", typ)\n\t}\n\tsynonymSources[typ] = constructor\n\treturn nil\n}\n\ntype SynonymSourceCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewSynonymSourceCache() *SynonymSourceCache {\n\treturn &SynonymSourceCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\ntype SynonymSourceConstructor func(config map[string]interface{}, cache *Cache) (analysis.SynonymSource, error)\ntype SynonymSourceRegistry map[string]SynonymSourceConstructor\n\nfunc SynonymSourceBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := synonymSources[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no synonym source with name '%s' registered\", name)\n\t}\n\tsynonymSource, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building synonym source: %v\", err)\n\t}\n\treturn synonymSource, nil\n}\n\nfunc (c *SynonymSourceCache) SynonymSourceNamed(name string, cache *Cache) (analysis.SynonymSource, error) {\n\titem, err := c.ItemNamed(name, cache, SynonymSourceBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.SynonymSource), nil\n}\n\nfunc (c *SynonymSourceCache) DefineSynonymSource(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.SynonymSource, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, SynonymSourceBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"synonym source named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.SynonymSource), nil\n}\n\nfunc (c *SynonymSourceCache) VisitSynonymSources(visitor analysis.SynonymSourceVisitor) error {\n\tc.mutex.RLock()\n\tdefer c.mutex.RUnlock()\n\tfor k, v := range c.data {\n\t\terr := visitor(k, v.(analysis.SynonymSource))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "registry/token_filter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc RegisterTokenFilter(name string, constructor TokenFilterConstructor) error {\n\t_, exists := tokenFilters[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate token filter named '%s'\", name)\n\t}\n\ttokenFilters[name] = constructor\n\treturn nil\n}\n\ntype TokenFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error)\ntype TokenFilterRegistry map[string]TokenFilterConstructor\n\ntype TokenFilterCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewTokenFilterCache() *TokenFilterCache {\n\treturn &TokenFilterCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc TokenFilterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := tokenFilters[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no token filter with name or type '%s' registered\", name)\n\t}\n\ttokenFilter, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building token filter: %v\", err)\n\t}\n\treturn tokenFilter, nil\n}\n\nfunc (c *TokenFilterCache) TokenFilterNamed(name string, cache *Cache) (analysis.TokenFilter, error) {\n\titem, err := c.ItemNamed(name, cache, TokenFilterBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.TokenFilter), nil\n}\n\nfunc (c *TokenFilterCache) DefineTokenFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, TokenFilterBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"token filter named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.TokenFilter), nil\n}\n\nfunc TokenFilterTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range tokenFilters {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/token_maps.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc RegisterTokenMap(name string, constructor TokenMapConstructor) error {\n\t_, exists := tokenMaps[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate token map named '%s'\", name)\n\t}\n\ttokenMaps[name] = constructor\n\treturn nil\n}\n\ntype TokenMapConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenMap, error)\ntype TokenMapRegistry map[string]TokenMapConstructor\n\ntype TokenMapCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewTokenMapCache() *TokenMapCache {\n\treturn &TokenMapCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc TokenMapBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := tokenMaps[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no token map with name or type '%s' registered\", name)\n\t}\n\ttokenMap, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building token map: %v\", err)\n\t}\n\treturn tokenMap, nil\n}\n\nfunc (c *TokenMapCache) TokenMapNamed(name string, cache *Cache) (analysis.TokenMap, error) {\n\titem, err := c.ItemNamed(name, cache, TokenMapBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.TokenMap), nil\n}\n\nfunc (c *TokenMapCache) DefineTokenMap(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenMap, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, TokenMapBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"token map named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.TokenMap), nil\n}\n\nfunc TokenMapTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range tokenMaps {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "registry/tokenizer.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage registry\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc RegisterTokenizer(name string, constructor TokenizerConstructor) error {\n\t_, exists := tokenizers[name]\n\tif exists {\n\t\treturn fmt.Errorf(\"attempted to register duplicate tokenizer named '%s'\", name)\n\t}\n\ttokenizers[name] = constructor\n\treturn nil\n}\n\ntype TokenizerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error)\ntype TokenizerRegistry map[string]TokenizerConstructor\n\ntype TokenizerCache struct {\n\t*ConcurrentCache\n}\n\nfunc NewTokenizerCache() *TokenizerCache {\n\treturn &TokenizerCache{\n\t\tNewConcurrentCache(),\n\t}\n}\n\nfunc TokenizerBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {\n\tcons, registered := tokenizers[name]\n\tif !registered {\n\t\treturn nil, fmt.Errorf(\"no tokenizer with name or type '%s' registered\", name)\n\t}\n\ttokenizer, err := cons(config, cache)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building tokenizer: %v\", err)\n\t}\n\treturn tokenizer, nil\n}\n\nfunc (c *TokenizerCache) TokenizerNamed(name string, cache *Cache) (analysis.Tokenizer, error) {\n\titem, err := c.ItemNamed(name, cache, TokenizerBuild)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.Tokenizer), nil\n}\n\nfunc (c *TokenizerCache) DefineTokenizer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error) {\n\titem, err := c.DefineItem(name, typ, config, cache, TokenizerBuild)\n\tif err != nil {\n\t\tif err == ErrAlreadyDefined {\n\t\t\treturn nil, fmt.Errorf(\"tokenizer named '%s' already defined\", name)\n\t\t}\n\t\treturn nil, err\n\t}\n\treturn item.(analysis.Tokenizer), nil\n}\n\nfunc TokenizerTypesAndInstances() ([]string, []string) {\n\temptyConfig := map[string]interface{}{}\n\temptyCache := NewCache()\n\tvar types []string\n\tvar instances []string\n\tfor name, cons := range tokenizers {\n\t\t_, err := cons(emptyConfig, emptyCache)\n\t\tif err == nil {\n\t\t\tinstances = append(instances, name)\n\t\t} else {\n\t\t\ttypes = append(types, name)\n\t\t}\n\t}\n\treturn types, instances\n}\n"
  },
  {
    "path": "rescorer.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/fusion\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n)\n\nconst (\n\tDefaultScoreRankConstant = 60\n)\n\n// Rescorer is applied after all the query and knn results are obtained.\n// The main use of Rescorer is in hybrid search; all the individual scores\n// for query and knn are combined using Rescorer. Makes use of algorithms\n// defined in `fusion`\ntype rescorer struct {\n\treq *SearchRequest\n\n\t// Stores the original From, Size and Boost parameters from the request\n\torigFrom   int\n\torigSize   int\n\torigBoosts []float64\n\n\t// Flag variable to make sure that restoreSearchRequest is only run once\n\t// when it is deferred\n\trestored bool\n}\n\n// Stores information about the hybrid search into FusionRescorer.\n// Also mutates the SearchRequest by:\n// - Setting boosts to 1: top level boosts only used for rescoring\n// - Setting From and Size to 0 and ScoreWindowSize\nfunc (r *rescorer) prepareSearchRequest() error {\n\tif r.req.Params == nil {\n\t\tr.req.Params = NewDefaultParams(r.req.From, r.req.Size)\n\t}\n\n\tr.origFrom = r.req.From\n\tr.origSize = r.req.Size\n\n\tr.req.From = 0\n\tr.req.Size = r.req.Params.ScoreWindowSize\n\n\t// req.Query's top level boost comes first, followed by the KNN queries\n\tnumQueries := numKNNQueries(r.req) + 1\n\tr.origBoosts = make([]float64, numQueries)\n\n\t// only modify queries if it is boostable. If not, ignore\n\tif bQuery, ok := r.req.Query.(query.BoostableQuery); ok {\n\t\tr.origBoosts[0] = bQuery.Boost()\n\t\tbQuery.SetBoost(1.0)\n\t} else {\n\t\tr.origBoosts[0] = 1.0\n\t}\n\n\t// for all the knn queries, replace boost values\n\tr.prepareKnnRequest()\n\n\treturn nil\n}\n\nfunc (r *rescorer) restoreSearchRequest() {\n\t// Skip if already restored\n\tif r.restored {\n\t\treturn\n\t}\n\tr.restored = true\n\n\tr.req.From = r.origFrom\n\tr.req.Size = r.origSize\n\n\tif bQuery, ok := r.req.Query.(query.BoostableQuery); ok {\n\t\tbQuery.SetBoost(r.origBoosts[0])\n\t}\n\n\t// for all the knn queries, restore boost values\n\tr.restoreKnnRequest()\n}\n\nfunc (r *rescorer) rescore(ftsHits, knnHits search.DocumentMatchCollection) (search.DocumentMatchCollection, uint64, float64) {\n\tmergedHits := r.mergeDocs(ftsHits, knnHits)\n\n\tvar fusionResult *fusion.FusionResult\n\n\tswitch r.req.Score {\n\tcase ScoreRRF:\n\t\tfusionResult = fusion.ReciprocalRankFusion(\n\t\t\tmergedHits,\n\t\t\tr.origBoosts,\n\t\t\tr.req.Params.ScoreRankConstant,\n\t\t\tr.req.Params.ScoreWindowSize,\n\t\t\tnumKNNQueries(r.req),\n\t\t\tr.req.Explain,\n\t\t)\n\tcase ScoreRSF:\n\t\tfusionResult = fusion.RelativeScoreFusion(\n\t\t\tmergedHits,\n\t\t\tr.origBoosts,\n\t\t\tr.req.Params.ScoreWindowSize,\n\t\t\tnumKNNQueries(r.req),\n\t\t\tr.req.Explain,\n\t\t)\n\t}\n\n\treturn fusionResult.Hits, fusionResult.Total, fusionResult.MaxScore\n}\n\n// Merge all the FTS and KNN docs along with explanations\nfunc (r *rescorer) mergeDocs(ftsHits, knnHits search.DocumentMatchCollection) search.DocumentMatchCollection {\n\tif len(knnHits) == 0 {\n\t\treturn ftsHits\n\t}\n\n\tknnHitMap := make(map[string]*search.DocumentMatch, len(knnHits))\n\n\tfor _, hit := range knnHits {\n\t\tknnHitMap[hit.ID] = hit\n\t}\n\n\tfor _, hit := range ftsHits {\n\t\tif knnHit, ok := knnHitMap[hit.ID]; ok {\n\t\t\thit.ScoreBreakdown = knnHit.ScoreBreakdown\n\t\t\tif r.req.Explain {\n\t\t\t\thit.Expl = &search.Explanation{Value: 0.0, Message: \"\", Children: append([]*search.Explanation{hit.Expl}, knnHit.Expl.Children...)}\n\t\t\t}\n\t\t\tdelete(knnHitMap, hit.ID)\n\t\t}\n\t}\n\n\tfor _, hit := range knnHitMap {\n\t\thit.Score = 0\n\t\tftsHits = append(ftsHits, hit)\n\t\tif r.req.Explain {\n\t\t\thit.Expl = &search.Explanation{Value: 0.0, Message: \"\", Children: append([]*search.Explanation{nil}, hit.Expl.Children...)}\n\t\t}\n\t}\n\n\treturn ftsHits\n}\n\nfunc newRescorer(req *SearchRequest) *rescorer {\n\treturn &rescorer{\n\t\treq: req,\n\t}\n}\n"
  },
  {
    "path": "rescorer_knn_test.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"math\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc createHybridSearchIndex(path string) (Index, error) {\n\t// Index mapping\n\tindexMapping := NewIndexMapping()\n\n\t// Disable default mapping to match expected configuration\n\tindexMapping.DefaultMapping.Enabled = false\n\tindexMapping.DefaultMapping.Dynamic = false\n\n\t// Create a specific document mapping type\n\tdocMapping := NewDocumentMapping()\n\tdocMapping.Enabled = true\n\tdocMapping.Dynamic = false\n\n\t// Text field for color with specific properties\n\tcolorFieldMapping := NewTextFieldMapping()\n\tcolorFieldMapping.Analyzer = \"en\" // Use \"en\" analyzer as specified\n\tcolorFieldMapping.DocValues = true\n\tcolorFieldMapping.IncludeInAll = true\n\tcolorFieldMapping.Store = true\n\tcolorFieldMapping.Index = true\n\tdocMapping.AddFieldMappingsAt(\"color\", colorFieldMapping)\n\n\t// Vector field for color vector with L2 similarity\n\tvecFieldMapping := mapping.NewVectorFieldMapping()\n\tvecFieldMapping.Dims = 3\n\tvecFieldMapping.Similarity = index.EuclideanDistance // l2_norm equivalent\n\tvecFieldMapping.VectorIndexOptimizedFor = \"recall\"\n\tdocMapping.AddFieldMappingsAt(\"colorvect_l2\", vecFieldMapping)\n\n\t// Add the document mapping to the index\n\tindexMapping.AddDocumentMapping(\"_default\", docMapping)\n\n\t// Create index\n\treturn New(path, indexMapping)\n}\n\nfunc getHybridSearchDocuments() []map[string]interface{} {\n\tdocuments := []map[string]interface{}{\n\t\t{\n\t\t\t\"color\":        \"dark slate blue\",\n\t\t\t\"colorvect_l2\": []float32{72, 61, 139},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"blue\",\n\t\t\t\"colorvect_l2\": []float32{0, 0, 255},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"navy\",\n\t\t\t\"colorvect_l2\": []float32{0, 0, 128},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"steel blue\",\n\t\t\t\"colorvect_l2\": []float32{70, 130, 180},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"light blue\",\n\t\t\t\"colorvect_l2\": []float32{173, 216, 230},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"deep sky blue\",\n\t\t\t\"colorvect_l2\": []float32{0, 191, 255},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"royal blue\",\n\t\t\t\"colorvect_l2\": []float32{65, 105, 225},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"powder blue\",\n\t\t\t\"colorvect_l2\": []float32{176, 224, 230},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"corn flower blue\",\n\t\t\t\"colorvect_l2\": []float32{100, 149, 237},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"alice blue\",\n\t\t\t\"colorvect_l2\": []float32{240, 248, 255},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"blue violet\",\n\t\t\t\"colorvect_l2\": []float32{138, 43, 226},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"sky blue\",\n\t\t\t\"colorvect_l2\": []float32{135, 206, 235},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"indigo\",\n\t\t\t\"colorvect_l2\": []float32{75, 0, 130},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"midnight blue\",\n\t\t\t\"colorvect_l2\": []float32{25, 25, 112},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"dark blue\",\n\t\t\t\"colorvect_l2\": []float32{0, 0, 139},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"medium slate blue\",\n\t\t\t\"colorvect_l2\": []float32{123, 104, 238},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"cadet blue\",\n\t\t\t\"colorvect_l2\": []float32{95, 158, 160},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"light steel blue\",\n\t\t\t\"colorvect_l2\": []float32{176, 196, 222},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"dodger blue\",\n\t\t\t\"colorvect_l2\": []float32{30, 144, 255},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"medium blue\",\n\t\t\t\"colorvect_l2\": []float32{0, 0, 205},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"slate blue\",\n\t\t\t\"colorvect_l2\": []float32{106, 90, 205},\n\t\t},\n\t\t{\n\t\t\t\"color\":        \"light sky blue\",\n\t\t\t\"colorvect_l2\": []float32{135, 206, 250},\n\t\t},\n\t}\n\n\treturn documents\n}\n\nfunc createScoreFusionRequest(scoreMethod string, knn bool) *SearchRequest {\n\t// Create hybrid search request (FTS + KNN)\n\ttextQuery := query.NewMatchPhraseQuery(\"dark\")\n\tsearchRequest := NewSearchRequest(textQuery)\n\n\tif knn {\n\t\tqueryVector_1 := []float32{0, 0, 129} // Similar to blue colors\n\t\tsearchRequest.AddKNN(\"colorvect_l2\", queryVector_1, 5, 1.0)\n\n\t\tqueryVector_2 := []float32{0, 0, 250} // lighter blue\n\t\tsearchRequest.AddKNN(\"colorvect_l2\", queryVector_2, 5, 1.0)\n\t}\n\n\tparams := RequestParams{1, 10}\n\tsearchRequest.AddParams(params)\n\n\tsearchRequest.Size = 10\n\n\tsearchRequest.Score = scoreMethod\n\tsearchRequest.Explain = false\n\treturn searchRequest\n}\n\n// verifyRRFResults verifies that the search hits match the expected RRF ranking and scores\nfunc verifyRRFResults(t *testing.T, hits search.DocumentMatchCollection) {\n\t// Manual RRF calculation for verification\n\t// With k=1 (ScoreRankConstant), RRF formula: 1/(1+rank)\n\t//\n\t// FTS \"dark\" ranks:\n\t// 1. dark blue, 2. dark slate blue\n\t//\n\t// kNN1 [0,0,129] ranks:\n\t// 1. navy, 2. dark blue, 3. midnight blue, 4. indigo, 5. medium blue\n\t//\n\t// kNN2 [0,0,250] ranks:\n\t// 1. blue, 2. medium blue, 3. dark blue, 4. navy, 5. royal blue\n\n\texpectedRRFScores := map[string]float64{\n\t\t\"dark blue\":       1.083333, // FTS(1): 1/2 + kNN1(2): 1/3 + kNN2(3): 1/4 = 1.083333\n\t\t\"navy\":            0.7,      // kNN1(1): 1/2 + kNN2(4): 1/5 = 0.7\n\t\t\"blue\":            0.5,      // kNN2(1): 1/2 = 0.5\n\t\t\"medium blue\":     0.5,      // kNN1(5): 1/6 + kNN2(2): 1/3 = 0.5\n\t\t\"dark slate blue\": 0.333333, // FTS(2): 1/3 = 0.333333\n\t\t\"midnight blue\":   0.25,     // kNN1(3): 1/4 = 0.25\n\t\t\"indigo\":          0.2,      // kNN1(4): 1/5 = 0.2\n\t\t\"royal blue\":      0.166667, // kNN2(5): 1/6 = 0.166667\n\t}\n\n\t// Verify top results match expected RRF ranking\n\texpectedOrder := []string{\"dark blue\", \"navy\", \"blue\", \"medium blue\", \"dark slate blue\", \"midnight blue\", \"indigo\", \"royal blue\"}\n\n\tif len(hits) < len(expectedOrder) {\n\t\tt.Fatalf(\"Expected at least %d results, got %d\", len(expectedOrder), len(hits))\n\t}\n\n\tfor i, expectedID := range expectedOrder {\n\t\tif hits[i].ID != expectedID {\n\t\t\tid := hits[i].ID\n\t\t\tif !(id == \"blue\" || id == \"medium blue\") { // Don't throw an error, since these scores are the same\n\t\t\t\tt.Errorf(\"Position %d: expected %s, got %s\", i+1, expectedID, hits[i].ID)\n\t\t\t}\n\t\t}\n\n\t\texpectedScore := expectedRRFScores[expectedID]\n\t\tactualScore := hits[i].Score\n\t\ttolerance := 0.001\n\n\t\tif math.Abs(actualScore-expectedScore) > tolerance {\n\t\t\tt.Errorf(\"Score for %s: expected %.6f, got %.6f (diff: %.6f)\",\n\t\t\t\texpectedID, expectedScore, actualScore, math.Abs(actualScore-expectedScore))\n\t\t}\n\t}\n}\n\n// setupSingleIndex creates a single index with all documents\nfunc setupSingleIndex(t *testing.T) (Index, func()) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\n\tindex, err := createHybridSearchIndex(tmpIndexPath)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdocuments := getHybridSearchDocuments()\n\n\t// Index documents\n\tbatch := index.NewBatch()\n\tfor _, doc := range documents {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcleanup := func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, tmpIndexPath)\n\t}\n\n\treturn index, cleanup\n}\n\n// setupAliasWithSingleIndex creates an alias containing one index with all documents\nfunc setupAliasWithSingleIndex(t *testing.T) (Index, func()) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\n\tindex, err := createHybridSearchIndex(tmpIndexPath)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdocuments := getHybridSearchDocuments()\n\n\t// Create alias and add the single index\n\talias := NewIndexAlias()\n\talias.Add(index)\n\n\t// Index all documents\n\tbatch := alias.NewBatch()\n\tfor _, doc := range documents {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = alias.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcleanup := func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, tmpIndexPath)\n\t}\n\n\treturn alias, cleanup\n}\n\n// setupAliasWithTwoIndexes creates an alias containing two indexes with documents split between them\nfunc setupAliasWithTwoIndexes(t *testing.T) (Index, func()) {\n\tdocuments := getHybridSearchDocuments()\n\n\t// Split documents into two groups\n\tmidpoint := len(documents) / 2\n\tdocs1 := documents[:midpoint]\n\tdocs2 := documents[midpoint:]\n\n\t// Create first index\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tindex1, err := createHybridSearchIndex(tmpIndexPath1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index first half of documents\n\tbatch1 := index1.NewBatch()\n\tfor _, doc := range docs1 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch1.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index1.Batch(batch1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create second index\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tindex2, err := createHybridSearchIndex(tmpIndexPath2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index second half of documents\n\tbatch2 := index2.NewBatch()\n\tfor _, doc := range docs2 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch2.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index2.Batch(batch2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create alias and add both indexes\n\talias := NewIndexAlias()\n\talias.Add(index1, index2)\n\n\tcleanup := func() {\n\t\terr := index1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = index2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, tmpIndexPath1)\n\t\tcleanupTmpIndexPath(t, tmpIndexPath2)\n\t}\n\n\treturn alias, cleanup\n}\n\n// setupNestedAliases creates nested aliases with three indexes spread across sub-aliases\nfunc setupNestedAliases(t *testing.T) (Index, func()) {\n\tdocuments := getHybridSearchDocuments()\n\n\t// Split documents into three groups\n\tthirdPoint1 := len(documents) / 3\n\tthirdPoint2 := 2 * len(documents) / 3\n\tdocs1 := documents[:thirdPoint1]\n\tdocs2 := documents[thirdPoint1:thirdPoint2]\n\tdocs3 := documents[thirdPoint2:]\n\n\t// Create first index\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tindex1, err := createHybridSearchIndex(tmpIndexPath1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index first third of documents\n\tbatch1 := index1.NewBatch()\n\tfor _, doc := range docs1 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch1.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index1.Batch(batch1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create second index\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tindex2, err := createHybridSearchIndex(tmpIndexPath2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index second third of documents\n\tbatch2 := index2.NewBatch()\n\tfor _, doc := range docs2 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch2.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index2.Batch(batch2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create third index\n\ttmpIndexPath3 := createTmpIndexPath(t)\n\tindex3, err := createHybridSearchIndex(tmpIndexPath3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index third third of documents\n\tbatch3 := index3.NewBatch()\n\tfor _, doc := range docs3 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch3.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index3.Batch(batch3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create first sub-alias (contains 1 index)\n\tsubAlias1 := NewIndexAlias()\n\tsubAlias1.SetName(\"subAlias1\")\n\tsubAlias1.Add(index1)\n\n\t// Create second sub-alias (contains 2 indexes)\n\tsubAlias2 := NewIndexAlias()\n\tsubAlias2.SetName(\"subAlias2\")\n\tsubAlias2.Add(index2, index3)\n\n\t// Create master alias containing the two sub-aliases\n\tmasterAlias := NewIndexAlias()\n\tmasterAlias.SetName(\"masterAlias\")\n\tmasterAlias.Add(subAlias1, subAlias2)\n\n\tcleanup := func() {\n\t\terr := index1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = index2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = index3.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, tmpIndexPath1)\n\t\tcleanupTmpIndexPath(t, tmpIndexPath2)\n\t\tcleanupTmpIndexPath(t, tmpIndexPath3)\n\t}\n\n\treturn masterAlias, cleanup\n}\n\nfunc TestRRFEndToEnd(t *testing.T) {\n\t// Setup the index configuration\n\tindex, cleanup := setupSingleIndex(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createScoreFusionRequest(ScoreRRF, true)\n\n\t// Execute search\n\tresult, err := index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify RRF results\n\tverifyRRFResults(t, result.Hits)\n}\n\n// TestRRFAliasWithSingleIndex tests RRF with an alias containing one index\nfunc TestRRFAliasWithSingleIndex(t *testing.T) {\n\t// Setup the alias configuration\n\talias, cleanup := setupAliasWithSingleIndex(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createScoreFusionRequest(ScoreRRF, true)\n\n\t// Execute search through alias\n\tresult, err := alias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify RRF results - should be identical to direct index search\n\tverifyRRFResults(t, result.Hits)\n}\n\n// TestRRFAliasWithTwoIndexes tests RRF with an alias containing two indexes\nfunc TestRRFAliasWithTwoIndexes(t *testing.T) {\n\t// Setup the alias configuration\n\talias, cleanup := setupAliasWithTwoIndexes(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createScoreFusionRequest(ScoreRRF, true)\n\n\t// Execute search through alias\n\tresult, err := alias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify RRF results - should be identical to single index results\n\tverifyRRFResults(t, result.Hits)\n}\n\n// TestRRFNestedAliases tests RRF with an alias containing two index aliases\nfunc TestRRFNestedAliases(t *testing.T) {\n\t// Setup the nested aliases configuration\n\tmasterAlias, cleanup := setupNestedAliases(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createScoreFusionRequest(ScoreRRF, true)\n\n\t// Execute search through master alias\n\tresult, err := masterAlias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify RRF results - should be identical to single index results\n\tverifyRRFResults(t, result.Hits)\n}\n\n// TestRRFPagination tests RRF with pagination across different index/alias configurations\nfunc TestRRFPagination(t *testing.T) {\n\tscenarios := []struct {\n\t\tname  string\n\t\tsetup func(t *testing.T) (Index, func())\n\t}{\n\t\t{\n\t\t\tname:  \"SingleIndex\",\n\t\t\tsetup: setupSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithSingleIndex\",\n\t\t\tsetup: setupAliasWithSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithTwoIndexes\",\n\t\t\tsetup: setupAliasWithTwoIndexes,\n\t\t},\n\t\t{\n\t\t\tname:  \"NestedAliases\",\n\t\t\tsetup: setupNestedAliases,\n\t\t},\n\t}\n\n\tfor _, scenario := range scenarios {\n\t\tt.Run(scenario.name, func(t *testing.T) {\n\t\t\t// Setup the index/alias configuration\n\t\t\tindex, cleanup := scenario.setup(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create first page request (first 5 results)\n\t\t\tfirstPageRequest := createScoreFusionRequest(ScoreRRF, true)\n\t\t\tfirstPageRequest.From = 0\n\t\t\tfirstPageRequest.Size = 5\n\n\t\t\t// Execute first page search\n\t\t\tfirstPageResult, err := index.Search(firstPageRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Create second page request (next 5 results, starting from index 5)\n\t\t\tsecondPageRequest := createScoreFusionRequest(ScoreRRF, true)\n\t\t\tsecondPageRequest.From = 5\n\t\t\tsecondPageRequest.Size = 5\n\n\t\t\t// Execute second page search\n\t\t\tsecondPageResult, err := index.Search(secondPageRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Combine results from both pages\n\t\t\tcombinedHits := make(search.DocumentMatchCollection, 0, len(firstPageResult.Hits)+len(secondPageResult.Hits))\n\t\t\tcombinedHits = append(combinedHits, firstPageResult.Hits...)\n\t\t\tcombinedHits = append(combinedHits, secondPageResult.Hits...)\n\n\t\t\t// Verify we have the expected number of results\n\t\t\tif len(firstPageResult.Hits) != 5 {\n\t\t\t\tt.Errorf(\"Expected 5 results in first page, got %d\", len(firstPageResult.Hits))\n\t\t\t}\n\t\t\tif len(secondPageResult.Hits) != 3 {\n\t\t\t\tt.Errorf(\"Expected 3 results in second page, got %d\", len(secondPageResult.Hits))\n\t\t\t}\n\n\t\t\t// Verify combined RRF results match expected ranking\n\t\t\tverifyRRFResults(t, combinedHits)\n\t\t})\n\t}\n}\n\n// TestHybridRRFFaceting tests that facet results are identical whether using RRF or default scoring in hybrid search\nfunc TestRRFFaceting(t *testing.T) {\n\tscenarios := []struct {\n\t\tname  string\n\t\tsetup func(t *testing.T) (Index, func())\n\t}{\n\t\t{\n\t\t\tname:  \"SingleIndex\",\n\t\t\tsetup: setupSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithSingleIndex\",\n\t\t\tsetup: setupAliasWithSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithTwoIndexes\",\n\t\t\tsetup: setupAliasWithTwoIndexes,\n\t\t},\n\t\t{\n\t\t\tname:  \"NestedAliases\",\n\t\t\tsetup: setupNestedAliases,\n\t\t},\n\t}\n\n\tfor _, scenario := range scenarios {\n\t\tt.Run(scenario.name, func(t *testing.T) {\n\t\t\t// Setup the index/alias configuration\n\t\t\tindex, cleanup := scenario.setup(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create search request with default scoring and facets\n\t\t\tdefaultRequest := createScoreFusionRequest(ScoreDefault, false)\n\t\t\tdefaultRequest.Score = ScoreDefault // Use default scoring\n\t\t\tdefaultRequest.Size = 10\n\t\t\t// Add facet for color field with size 10\n\t\t\tcolorFacet := NewFacetRequest(\"color\", 10)\n\t\t\tdefaultRequest.AddFacet(\"color\", colorFacet)\n\n\t\t\t// Create search request with RRF scoring and identical facets\n\t\t\trrfRequest := createScoreFusionRequest(ScoreRRF, true)\n\t\t\trrfRequest.Score = ScoreRRF // Use RRF scoring\n\t\t\trrfRequest.Size = 10\n\t\t\t// Add identical facet for color field with size 10\n\t\t\tcolorFacetRRF := NewFacetRequest(\"color\", 10)\n\t\t\trrfRequest.AddFacet(\"color\", colorFacetRRF)\n\n\t\t\t// Execute both searches\n\t\t\tdefaultResult, err := index.Search(defaultRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Default scoring search failed: %v\", err)\n\t\t\t}\n\n\t\t\trrfResult, err := index.Search(rrfRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"RRF scoring search failed: %v\", err)\n\t\t\t}\n\n\t\t\t// Verify both searches returned results\n\t\t\tif len(defaultResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected search results with default scoring, got none\")\n\t\t\t}\n\t\t\tif len(rrfResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected search results with RRF scoring, got none\")\n\t\t\t}\n\n\t\t\t// Verify both searches returned facets\n\t\t\tif defaultResult.Facets == nil {\n\t\t\t\tt.Fatal(\"Expected facets with default scoring, got nil\")\n\t\t\t}\n\t\t\tif rrfResult.Facets == nil {\n\t\t\t\tt.Fatal(\"Expected facets with RRF scoring, got nil\")\n\t\t\t}\n\n\t\t\t// Check that color facet exists in both results\n\t\t\tdefaultColorFacet, defaultExists := defaultResult.Facets[\"color\"]\n\t\t\trrfColorFacet, rrfExists := rrfResult.Facets[\"color\"]\n\n\t\t\tif !defaultExists {\n\t\t\t\tt.Fatal(\"Expected color facet in default scoring results\")\n\t\t\t}\n\t\t\tif !rrfExists {\n\t\t\t\tt.Fatal(\"Expected color facet in RRF scoring results\")\n\t\t\t}\n\n\t\t\t// Compare the facet results - they should be identical\n\t\t\t// Since facets are based on the document corpus and not scoring,\n\t\t\t// they should not be affected by the scoring method (even with KNN)\n\t\t\tif defaultColorFacet.Total != rrfColorFacet.Total {\n\t\t\t\tt.Errorf(\"Facet totals differ: default=%d, RRF=%d\",\n\t\t\t\t\tdefaultColorFacet.Total, rrfColorFacet.Total)\n\t\t\t}\n\n\t\t\tif defaultColorFacet.Missing != rrfColorFacet.Missing {\n\t\t\t\tt.Errorf(\"Facet missing counts differ: default=%d, RRF=%d\",\n\t\t\t\t\tdefaultColorFacet.Missing, rrfColorFacet.Missing)\n\t\t\t}\n\n\t\t\tif defaultColorFacet.Other != rrfColorFacet.Other {\n\t\t\t\tt.Errorf(\"Facet other counts differ: default=%d, RRF=%d\",\n\t\t\t\t\tdefaultColorFacet.Other, rrfColorFacet.Other)\n\t\t\t}\n\n\t\t\t// Compare the facet terms\n\t\t\tdefaultTerms := defaultColorFacet.Terms.Terms()\n\t\t\trrfTerms := rrfColorFacet.Terms.Terms()\n\n\t\t\tif len(defaultTerms) != len(rrfTerms) {\n\t\t\t\tt.Errorf(\"Facet terms count differs: default=%d, RRF=%d\",\n\t\t\t\t\tlen(defaultTerms), len(rrfTerms))\n\t\t\t} else {\n\t\t\t\t// Compare each term\n\t\t\t\tfor i, defaultTerm := range defaultTerms {\n\t\t\t\t\trrfTerm := rrfTerms[i]\n\t\t\t\t\tif defaultTerm.Term != rrfTerm.Term {\n\t\t\t\t\t\tt.Errorf(\"Facet term differs at position %d: default=%s, RRF=%s\",\n\t\t\t\t\t\t\ti, defaultTerm.Term, rrfTerm.Term)\n\t\t\t\t\t}\n\t\t\t\t\tif defaultTerm.Count != rrfTerm.Count {\n\t\t\t\t\t\tt.Errorf(\"Facet term count differs for %s: default=%d, RRF=%d\",\n\t\t\t\t\t\t\tdefaultTerm.Term, defaultTerm.Count, rrfTerm.Count)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// verifyRSFResults verifies that the search hits match expected RSF ranking and scores\nfunc verifyRSFResults(t *testing.T, hits search.DocumentMatchCollection) {\n\t// For RSF, we expect similar high-level results to RRF but with different scoring methodology\n\t// RSF uses min-max normalization of scores within the window\n\t// Expected top documents should include those matching \"dark\" query and similar vectors\n\n\t// Verify we have reasonable number of results\n\tif len(hits) == 0 {\n\t\tt.Fatal(\"Expected non-empty search results for RSF\")\n\t}\n\n\t// Verify we have at least 8 results\n\tif len(hits) < 8 {\n\t\tt.Errorf(\"Expected at least 6 results for RSF, got %d\", len(hits))\n\t}\n\n\t// Documents that should definitely appear in top results (high relevance)\n\t// These all get both text relevance (for \"dark blue\") or strong vector similarity\n\ttopExpectedDocs := []string{\"dark blue\", \"navy\", \"blue\", \"medium blue\"}\n\n\t// Create map of all hits for easier lookup\n\tdocMap := make(map[string]int) // doc -> position (0-based)\n\tfor i, hit := range hits {\n\t\tdocMap[hit.ID] = i\n\t}\n\n\t// Verify that \"dark blue\" appears in top 5 positions (high text + vector relevance)\n\tif pos, found := docMap[\"dark blue\"]; !found {\n\t\tt.Error(\"Expected 'dark blue' to appear in results but not found\")\n\t} else if pos >= 5 {\n\t\tt.Errorf(\"Expected 'dark blue' in top 3 positions, found at position %d\", pos+1)\n\t}\n\n\t// Verify that at least 3 of the top expected documents appear in top 5 results\n\ttopFoundCount := 0\n\tfor _, expectedDoc := range topExpectedDocs {\n\t\tif pos, found := docMap[expectedDoc]; found && pos < 5 {\n\t\t\ttopFoundCount++\n\t\t}\n\t}\n\tif topFoundCount < 3 {\n\t\tt.Errorf(\"Expected at least 3 of top expected documents in top 5 results, found %d\", topFoundCount)\n\t}\n\n\t// Verify scores are reasonable and within expected range\n\t// RSF scores should be between 0 and sum of weights (3.0 with default weights)\n\t// but typically should be more constrained than the full range\n\tfor i, hit := range hits {\n\t\tif hit.Score < 0 || hit.Score > 3.0 {\n\t\t\tt.Errorf(\"Hit %d (%s) has unreasonable score: %.6f\", i, hit.ID, hit.Score)\n\t\t}\n\t\t// First hit should have a substantial score (at least 0.1)\n\t\tif i == 0 && hit.Score < 0.1 {\n\t\t\tt.Errorf(\"Top hit (%s) has unexpectedly low score: %.6f\", hit.ID, hit.Score)\n\t\t}\n\t}\n\n\t// Verify hits are sorted by score descending with strict ordering\n\tfor i := 1; i < len(hits); i++ {\n\t\tif hits[i-1].Score < hits[i].Score {\n\t\t\tt.Errorf(\"Hits not sorted properly: hit %d (%s, score %.6f) < hit %d (%s, score %.6f)\",\n\t\t\t\ti, hits[i-1].ID, hits[i-1].Score, i+1, hits[i].ID, hits[i].Score)\n\t\t}\n\t}\n\n\t// Verify score range is reasonable - top score should be significantly higher than bottom\n\ttopScore := hits[0].Score\n\tfifthScore := hits[4].Score\n\tif topScore-fifthScore < 0.001 {\n\t\tt.Errorf(\"Insufficient score differentiation: top score %.6f, 5rd score %.6f (diff: %.6f)\",\n\t\t\ttopScore, fifthScore, topScore-fifthScore)\n\t}\n}\n\n// TestRSFEndToEnd tests RSF scoring with a single index\nfunc TestRSFEndToEnd(t *testing.T) {\n\t// Setup the index configuration\n\tindex, cleanup := setupSingleIndex(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createScoreFusionRequest(ScoreRSF, true)\n\n\tctx := context.Background()\n\tctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring)\n\n\t// Execute search\n\tresult, err := index.SearchInContext(ctx, searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify RSF results\n\tverifyRSFResults(t, result.Hits)\n}\n\n// TestRSFAliasWithSingleIndex tests RSF with an alias containing one index\nfunc TestRSFAliasWithSingleIndex(t *testing.T) {\n\t// Setup the alias configuration\n\talias, cleanup := setupAliasWithSingleIndex(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createScoreFusionRequest(ScoreRSF, true)\n\n\tctx := context.Background()\n\tctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring)\n\n\t// Execute search\n\tresult, err := alias.SearchInContext(ctx, searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify RSF results - should be identical to direct index search\n\tverifyRSFResults(t, result.Hits)\n}\n\n// TestRSFAliasWithTwoIndexes tests RSF with an alias containing two indexes\nfunc TestRSFAliasWithTwoIndexes(t *testing.T) {\n\t// Setup the alias configuration\n\talias, cleanup := setupAliasWithTwoIndexes(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createScoreFusionRequest(ScoreRSF, true)\n\n\tctx := context.Background()\n\tctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring)\n\n\t// Execute search\n\tresult, err := alias.SearchInContext(ctx, searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify RSF results - should be identical to single index results\n\tverifyRSFResults(t, result.Hits)\n}\n\n// TestRSFNestedAliases tests RSF with an alias containing two index aliases\nfunc TestRSFNestedAliases(t *testing.T) {\n\t// Setup the nested aliases configuration\n\tmasterAlias, cleanup := setupNestedAliases(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createScoreFusionRequest(ScoreRSF, true)\n\n\tctx := context.Background()\n\tctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring)\n\n\t// Execute search\n\tresult, err := masterAlias.SearchInContext(ctx, searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify RSF results - should be identical to single index results\n\tverifyRSFResults(t, result.Hits)\n}\n\n// TestRSFPagination tests RSF with pagination across different index/alias configurations\nfunc TestRSFPagination(t *testing.T) {\n\tscenarios := []struct {\n\t\tname  string\n\t\tsetup func(t *testing.T) (Index, func())\n\t}{\n\t\t{\n\t\t\tname:  \"SingleIndex\",\n\t\t\tsetup: setupSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithSingleIndex\",\n\t\t\tsetup: setupAliasWithSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithTwoIndexes\",\n\t\t\tsetup: setupAliasWithTwoIndexes,\n\t\t},\n\t\t{\n\t\t\tname:  \"NestedAliases\",\n\t\t\tsetup: setupNestedAliases,\n\t\t},\n\t}\n\n\tfor _, scenario := range scenarios {\n\t\tt.Run(scenario.name, func(t *testing.T) {\n\t\t\t// Setup the index/alias configuration\n\t\t\tindex, cleanup := scenario.setup(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create first page request (first 5 results)\n\t\t\tfirstPageRequest := createScoreFusionRequest(ScoreDefault, true)\n\t\t\tfirstPageRequest.From = 0\n\t\t\tfirstPageRequest.Size = 5\n\n\t\t\t// Execute first page search\n\t\t\tfirstPageResult, err := index.Search(firstPageRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Create second page request (next 5 results, starting from index 5)\n\t\t\tsecondPageRequest := createScoreFusionRequest(ScoreDefault, true)\n\t\t\tsecondPageRequest.From = 5\n\t\t\tsecondPageRequest.Size = 5\n\n\t\t\t// Execute second page search\n\t\t\tsecondPageResult, err := index.Search(secondPageRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Combine results from both pages\n\t\t\tcombinedHits := make(search.DocumentMatchCollection, 0, len(firstPageResult.Hits)+len(secondPageResult.Hits))\n\t\t\tcombinedHits = append(combinedHits, firstPageResult.Hits...)\n\t\t\tcombinedHits = append(combinedHits, secondPageResult.Hits...)\n\n\t\t\t// Verify we have reasonable number of results\n\t\t\tif len(firstPageResult.Hits) == 0 {\n\t\t\t\tt.Error(\"Expected results in first page, got none\")\n\t\t\t}\n\t\t\tif len(combinedHits) == 0 {\n\t\t\t\tt.Error(\"Expected combined results, got none\")\n\t\t\t}\n\n\t\t\t// Verify combined RSF results\n\t\t\tverifyRSFResults(t, combinedHits)\n\t\t})\n\t}\n}\n\n// TestRSFFaceting tests that facet results are identical whether using RSF or default scoring in hybrid search\nfunc TestRSFFaceting(t *testing.T) {\n\tscenarios := []struct {\n\t\tname  string\n\t\tsetup func(t *testing.T) (Index, func())\n\t}{\n\t\t{\n\t\t\tname:  \"SingleIndex\",\n\t\t\tsetup: setupSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithSingleIndex\",\n\t\t\tsetup: setupAliasWithSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithTwoIndexes\",\n\t\t\tsetup: setupAliasWithTwoIndexes,\n\t\t},\n\t\t{\n\t\t\tname:  \"NestedAliases\",\n\t\t\tsetup: setupNestedAliases,\n\t\t},\n\t}\n\n\tfor _, scenario := range scenarios {\n\t\tt.Run(scenario.name, func(t *testing.T) {\n\t\t\t// Setup the index/alias configuration\n\t\t\tindex, cleanup := scenario.setup(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create search request with default scoring and facets\n\t\t\tdefaultRequest := createScoreFusionRequest(ScoreDefault, false)\n\t\t\tdefaultRequest.Score = ScoreDefault // Use default scoring\n\t\t\tdefaultRequest.Size = 10\n\t\t\t// Add facet for color field with size 10\n\t\t\tcolorFacet := NewFacetRequest(\"color\", 10)\n\t\t\tdefaultRequest.AddFacet(\"color\", colorFacet)\n\n\t\t\t// Create search request with RSF scoring and identical facets\n\t\t\trsfRequest := createScoreFusionRequest(ScoreRSF, true)\n\t\t\trsfRequest.Size = 10\n\t\t\t// Add identical facet for color field with size 10\n\t\t\tcolorFacetRSF := NewFacetRequest(\"color\", 10)\n\t\t\trsfRequest.AddFacet(\"color\", colorFacetRSF)\n\n\t\t\t// Execute both searches\n\t\t\tdefaultResult, err := index.Search(defaultRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Default scoring search failed: %v\", err)\n\t\t\t}\n\n\t\t\trsfResult, err := index.Search(rsfRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"RSF scoring search failed: %v\", err)\n\t\t\t}\n\n\t\t\t// Verify both searches returned results\n\t\t\tif len(defaultResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected search results with default scoring, got none\")\n\t\t\t}\n\t\t\tif len(rsfResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected search results with RSF scoring, got none\")\n\t\t\t}\n\n\t\t\t// Verify both searches returned facets\n\t\t\tif defaultResult.Facets == nil {\n\t\t\t\tt.Fatal(\"Expected facets with default scoring, got nil\")\n\t\t\t}\n\t\t\tif rsfResult.Facets == nil {\n\t\t\t\tt.Fatal(\"Expected facets with RSF scoring, got nil\")\n\t\t\t}\n\n\t\t\t// Check that color facet exists in both results\n\t\t\tdefaultColorFacet, defaultExists := defaultResult.Facets[\"color\"]\n\t\t\trsfColorFacet, rsfExists := rsfResult.Facets[\"color\"]\n\n\t\t\tif !defaultExists {\n\t\t\t\tt.Fatal(\"Expected color facet in default scoring results\")\n\t\t\t}\n\t\t\tif !rsfExists {\n\t\t\t\tt.Fatal(\"Expected color facet in RSF scoring results\")\n\t\t\t}\n\n\t\t\t// Compare the facet results - they should be identical\n\t\t\t// Since facets are based on the document corpus and not scoring,\n\t\t\t// they should not be affected by the scoring method (even with KNN)\n\t\t\tif defaultColorFacet.Total != rsfColorFacet.Total {\n\t\t\t\tt.Errorf(\"Facet totals differ: default=%d, RSF=%d\",\n\t\t\t\t\tdefaultColorFacet.Total, rsfColorFacet.Total)\n\t\t\t}\n\n\t\t\tif defaultColorFacet.Missing != rsfColorFacet.Missing {\n\t\t\t\tt.Errorf(\"Facet missing counts differ: default=%d, RSF=%d\",\n\t\t\t\t\tdefaultColorFacet.Missing, rsfColorFacet.Missing)\n\t\t\t}\n\n\t\t\tif defaultColorFacet.Other != rsfColorFacet.Other {\n\t\t\t\tt.Errorf(\"Facet other counts differ: default=%d, RSF=%d\",\n\t\t\t\t\tdefaultColorFacet.Other, rsfColorFacet.Other)\n\t\t\t}\n\n\t\t\t// Compare the facet terms\n\t\t\tdefaultTerms := defaultColorFacet.Terms.Terms()\n\t\t\trsfTerms := rsfColorFacet.Terms.Terms()\n\n\t\t\tif len(defaultTerms) != len(rsfTerms) {\n\t\t\t\tt.Errorf(\"Facet terms count differs: default=%d, RSF=%d\",\n\t\t\t\t\tlen(defaultTerms), len(rsfTerms))\n\t\t\t} else {\n\t\t\t\t// Compare each term\n\t\t\t\tfor i, defaultTerm := range defaultTerms {\n\t\t\t\t\trsfTerm := rsfTerms[i]\n\t\t\t\t\tif defaultTerm.Term != rsfTerm.Term {\n\t\t\t\t\t\tt.Errorf(\"Facet term differs at position %d: default=%s, RSF=%s\",\n\t\t\t\t\t\t\ti, defaultTerm.Term, rsfTerm.Term)\n\t\t\t\t\t}\n\t\t\t\t\tif defaultTerm.Count != rsfTerm.Count {\n\t\t\t\t\t\tt.Errorf(\"Facet term count differs for %s: default=%d, RSF=%d\",\n\t\t\t\t\t\t\tdefaultTerm.Term, defaultTerm.Count, rsfTerm.Count)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "rescorer_test.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n)\n\nfunc createFTSIndex(path string) (Index, error) {\n\t// Index mapping for FTS-only testing\n\tindexMapping := NewIndexMapping()\n\n\t// Disable default mapping to match expected configuration\n\tindexMapping.DefaultMapping.Enabled = false\n\tindexMapping.DefaultMapping.Dynamic = false\n\n\t// Create a specific document mapping type\n\tdocMapping := NewDocumentMapping()\n\tdocMapping.Enabled = true\n\tdocMapping.Dynamic = false\n\n\t// Text field for color with specific properties\n\tcolorFieldMapping := NewTextFieldMapping()\n\tcolorFieldMapping.Analyzer = \"en\" // Use \"en\" analyzer as specified\n\tcolorFieldMapping.DocValues = true\n\tcolorFieldMapping.IncludeInAll = true\n\tcolorFieldMapping.Store = true\n\tcolorFieldMapping.Index = true\n\tdocMapping.AddFieldMappingsAt(\"color\", colorFieldMapping)\n\n\t// Text field for description with specific properties\n\tdescriptionFieldMapping := NewTextFieldMapping()\n\tdescriptionFieldMapping.Analyzer = \"en\"\n\tdescriptionFieldMapping.DocValues = true\n\tdescriptionFieldMapping.IncludeInAll = true\n\tdescriptionFieldMapping.Store = true\n\tdescriptionFieldMapping.Index = true\n\tdocMapping.AddFieldMappingsAt(\"description\", descriptionFieldMapping)\n\n\t// Text field for category with specific properties\n\tcategoryFieldMapping := NewTextFieldMapping()\n\tcategoryFieldMapping.Analyzer = \"en\"\n\tcategoryFieldMapping.DocValues = true\n\tcategoryFieldMapping.IncludeInAll = true\n\tcategoryFieldMapping.Store = true\n\tcategoryFieldMapping.Index = true\n\tdocMapping.AddFieldMappingsAt(\"category\", categoryFieldMapping)\n\n\t// Add the document mapping to the index\n\tindexMapping.AddDocumentMapping(\"_default\", docMapping)\n\n\t// Create index\n\treturn New(path, indexMapping)\n}\n\nvar benchmarkResult search.DocumentMatchCollection\n\ntype benchmarkConfig struct {\n\tname       string\n\tftsHits    int\n\tknnHits    int\n\tknnQueries int\n}\n\nfunc BenchmarkRescorerRRF(b *testing.B) {\n\trunRescorerBenchmarks(b, ScoreRRF)\n}\n\nfunc BenchmarkRescorerRSF(b *testing.B) {\n\trunRescorerBenchmarks(b, ScoreRSF)\n}\n\nfunc runRescorerBenchmarks(b *testing.B, scoreMode string) {\n\tconfigs := []benchmarkConfig{\n\t\t{name: \"small\", ftsHits: 256, knnHits: 192, knnQueries: 1},\n\t\t{name: \"medium\", ftsHits: 1024, knnHits: 896, knnQueries: 2},\n\t\t{name: \"large\", ftsHits: 4096, knnHits: 3584, knnQueries: 3},\n\t}\n\n\tfor _, cfg := range configs {\n\t\tb.Run(fmt.Sprintf(\"%s/%s\", scoreMode, cfg.name), func(b *testing.B) {\n\t\t\tb.ReportAllocs()\n\n\t\t\trescorer, baseFTSHits, baseKNNHits := buildBenchmarkInputs(cfg, scoreMode)\n\n\t\t\tvar last search.DocumentMatchCollection\n\t\t\tb.StopTimer()\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\tftsHits := cloneDocumentMatches(baseFTSHits)\n\t\t\t\tknnHits := cloneDocumentMatches(baseKNNHits)\n\n\t\t\t\tb.StartTimer()\n\t\t\t\thits, _, _ := rescorer.rescore(ftsHits, knnHits)\n\t\t\t\tb.StopTimer()\n\n\t\t\t\tlast = hits\n\t\t\t}\n\n\t\t\tif len(last) == 0 {\n\t\t\t\tb.Fatalf(\"rescorer returned no hits for config %q\", cfg.name)\n\t\t\t}\n\n\t\t\tbenchmarkResult = last\n\t\t})\n\t}\n}\n\nfunc buildBenchmarkInputs(cfg benchmarkConfig, scoreMode string) (*rescorer, search.DocumentMatchCollection, search.DocumentMatchCollection) {\n\twindowSize := cfg.ftsHits\n\tif cfg.knnHits > windowSize {\n\t\twindowSize = cfg.knnHits\n\t}\n\n\tmatchQuery := query.NewMatchQuery(\"rescorer benchmark payload\")\n\tmatchQuery.SetBoost(1.0)\n\n\treq := &SearchRequest{\n\t\tQuery:  matchQuery,\n\t\tSize:   cfg.ftsHits,\n\t\tFrom:   0,\n\t\tScore:  scoreMode,\n\t\tParams: &RequestParams{ScoreRankConstant: DefaultScoreRankConstant, ScoreWindowSize: windowSize},\n\t}\n\n\tactiveKNNQueries := cfg.knnQueries\n\tif knnAdder, ok := interface{}(req).(interface {\n\t\tAddKNN(field string, vector []float32, k int64, boost float64)\n\t}); ok {\n\t\tfor i := 0; i < cfg.knnQueries; i++ {\n\t\t\tknnAdder.AddKNN(fmt.Sprintf(\"vector_%d\", i), []float32{1.0, 0.5, 0.25}, int64(cfg.knnHits), 1.0)\n\t\t}\n\t} else {\n\t\tactiveKNNQueries = 0\n\t}\n\n\tr := newRescorer(req)\n\tr.origBoosts = make([]float64, activeKNNQueries+1)\n\tfor i := range r.origBoosts {\n\t\tr.origBoosts[i] = 1.0\n\t}\n\n\tftsHits, knnHits := buildBenchmarkHits(cfg, activeKNNQueries)\n\treturn r, ftsHits, knnHits\n}\n\nfunc buildBenchmarkHits(cfg benchmarkConfig, activeKNNQueries int) (search.DocumentMatchCollection, search.DocumentMatchCollection) {\n\tftsHits := make(search.DocumentMatchCollection, cfg.ftsHits)\n\tfor i := 0; i < cfg.ftsHits; i++ {\n\t\tftsHits[i] = &search.DocumentMatch{\n\t\t\tID:        fmt.Sprintf(\"doc-%06d\", i),\n\t\t\tScore:     float64(cfg.ftsHits - i),\n\t\t\tHitNumber: uint64(i + 1),\n\t\t}\n\t}\n\n\tknnHits := make(search.DocumentMatchCollection, cfg.knnHits)\n\tfor i := 0; i < cfg.knnHits; i++ {\n\t\tid := fmt.Sprintf(\"doc-%06d\", i)\n\t\tif cfg.ftsHits > 0 {\n\t\t\tid = fmt.Sprintf(\"doc-%06d\", i%cfg.ftsHits)\n\t\t}\n\t\tif cfg.ftsHits == 0 || i%4 == 0 {\n\t\t\tid = fmt.Sprintf(\"knn-only-%06d\", i/4)\n\t\t}\n\n\t\tscoreBreakdown := make(map[int]float64, activeKNNQueries)\n\t\tfor q := 0; q < activeKNNQueries; q++ {\n\t\t\tscoreBreakdown[q] = float64(cfg.knnHits - i + q + 1)\n\t\t}\n\n\t\tknnHits[i] = &search.DocumentMatch{\n\t\t\tID:             id,\n\t\t\tScore:          float64(cfg.knnHits - i),\n\t\t\tScoreBreakdown: scoreBreakdown,\n\t\t\tHitNumber:      uint64(i + 1),\n\t\t}\n\t}\n\n\treturn ftsHits, knnHits\n}\n\nfunc cloneDocumentMatches(src search.DocumentMatchCollection) search.DocumentMatchCollection {\n\tdst := make(search.DocumentMatchCollection, len(src))\n\tfor i, hit := range src {\n\t\tif hit == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tcloned := *hit\n\n\t\tif hit.ScoreBreakdown != nil {\n\t\t\tcloned.ScoreBreakdown = make(map[int]float64, len(hit.ScoreBreakdown))\n\t\t\tfor k, v := range hit.ScoreBreakdown {\n\t\t\t\tcloned.ScoreBreakdown[k] = v\n\t\t\t}\n\t\t}\n\n\t\tif len(hit.Sort) > 0 {\n\t\t\tcloned.Sort = append([]string(nil), hit.Sort...)\n\t\t}\n\t\tif len(hit.DecodedSort) > 0 {\n\t\t\tcloned.DecodedSort = append([]string(nil), hit.DecodedSort...)\n\t\t}\n\t\tif len(hit.IndexNames) > 0 {\n\t\t\tcloned.IndexNames = append([]string(nil), hit.IndexNames...)\n\t\t}\n\n\t\tdst[i] = &cloned\n\t}\n\treturn dst\n}\n\nfunc getFTSDocuments() []map[string]interface{} {\n\tdocuments := []map[string]interface{}{\n\t\t{\n\t\t\t\"color\":       \"dark slate blue\",\n\t\t\t\"description\": \"deep and rich color with dark undertones\",\n\t\t\t\"category\":    \"blue shades\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"blue\",\n\t\t\t\"description\": \"primary color that is bright and vibrant\",\n\t\t\t\"category\":    \"primary colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"navy\",\n\t\t\t\"description\": \"dark blue color often used in uniforms\",\n\t\t\t\"category\":    \"dark colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"steel blue\",\n\t\t\t\"description\": \"metallic blue with gray undertones\",\n\t\t\t\"category\":    \"metallic shades\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"light blue\",\n\t\t\t\"description\": \"pale and soft blue color with light appearance\",\n\t\t\t\"category\":    \"light colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"deep sky blue\",\n\t\t\t\"description\": \"bright blue reminiscent of clear skies\",\n\t\t\t\"category\":    \"sky colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"royal blue\",\n\t\t\t\"description\": \"rich and regal blue color fit for royalty\",\n\t\t\t\"category\":    \"rich colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"powder blue\",\n\t\t\t\"description\": \"very light blue with powder-like softness\",\n\t\t\t\"category\":    \"light colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"corn flower blue\",\n\t\t\t\"description\": \"medium blue color named after the flower\",\n\t\t\t\"category\":    \"floral colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"alice blue\",\n\t\t\t\"description\": \"very pale blue with light and airy quality\",\n\t\t\t\"category\":    \"light colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"blue violet\",\n\t\t\t\"description\": \"purple-blue color with violet undertones\",\n\t\t\t\"category\":    \"purple shades\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"sky blue\",\n\t\t\t\"description\": \"bright blue color of a clear day sky\",\n\t\t\t\"category\":    \"sky colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"indigo\",\n\t\t\t\"description\": \"deep purple-blue color with dark intensity\",\n\t\t\t\"category\":    \"dark colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"midnight blue\",\n\t\t\t\"description\": \"very dark blue like the night sky\",\n\t\t\t\"category\":    \"dark colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"dark blue\",\n\t\t\t\"description\": \"deep blue color with dark characteristics\",\n\t\t\t\"category\":    \"dark colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"medium slate blue\",\n\t\t\t\"description\": \"medium intensity blue with slate properties\",\n\t\t\t\"category\":    \"blue shades\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"cadet blue\",\n\t\t\t\"description\": \"grayish blue color often used in uniforms\",\n\t\t\t\"category\":    \"metallic shades\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"light steel blue\",\n\t\t\t\"description\": \"light metallic blue with steel-like appearance\",\n\t\t\t\"category\":    \"light colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"dodger blue\",\n\t\t\t\"description\": \"bright medium blue with vibrant intensity\",\n\t\t\t\"category\":    \"bright colors\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"medium blue\",\n\t\t\t\"description\": \"standard blue with medium intensity and saturation\",\n\t\t\t\"category\":    \"blue shades\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"slate blue\",\n\t\t\t\"description\": \"blue-gray color with slate-like properties\",\n\t\t\t\"category\":    \"blue shades\",\n\t\t},\n\t\t{\n\t\t\t\"color\":       \"light sky blue\",\n\t\t\t\"description\": \"light version of sky blue with airy quality\",\n\t\t\t\"category\":    \"light colors\",\n\t\t},\n\t}\n\n\treturn documents\n}\n\nfunc createFTSSearchRequest(scoreMethod string) *SearchRequest {\n\t// Create multi-FTS search request (multiple FTS queries for fusion scoring)\n\t// Query 1: Search for \"dark\" in color field\n\tquery1 := query.NewMatchPhraseQuery(\"dark\")\n\tquery1.SetField(\"color\")\n\n\t// Query 2: Search for \"light\" in description field\n\tquery2 := query.NewMatchPhraseQuery(\"light\")\n\tquery2.SetField(\"description\")\n\n\t// // Query 3: Search for \"blue\" in category field\n\tquery3 := query.NewMatchPhraseQuery(\"blue\")\n\tquery3.SetField(\"category\")\n\n\t// Use the first query as the main query for the search request\n\tsearchRequest := NewSearchRequest(query1)\n\n\t// Add additional queries for fusion scoring (this simulates multiple query sources)\n\t// Since SearchRequest doesn't have a direct way to add multiple FTS queries,\n\t// we'll use a disjunction query to combine them for fusion scoring simulation\n\tqueries := []query.Query{query1, query2, query3}\n\tdisjunctionQuery := query.NewDisjunctionQuery(queries)\n\tsearchRequest.Query = disjunctionQuery\n\n\tparams := RequestParams{1, 10}\n\tsearchRequest.AddParams(params)\n\n\tsearchRequest.Size = 10\n\tsearchRequest.Score = scoreMethod\n\n\tsearchRequest.Explain = false\n\treturn searchRequest\n}\n\n// verifyFTSRRFResults verifies that the search hits match the expected RRF ranking and scores\nfunc verifyFTSRRFResults(t *testing.T, hits search.DocumentMatchCollection) {\n\t// Manual RRF calculation for verification\n\t// With k=1 (ScoreRankConstant), RRF formula: 1/(1+rank)\n\t//\n\t// For FTS-only with disjunction query, we need to consider how each document\n\t// matches each of the three query components:\n\t// 1. \"dark\" in color field\n\t// 2. \"light\" in description field\n\t// 3. \"blue\" in category field\n\t//\n\t// Documents that match multiple query components will rank higher\n\n\t// Expected matches:\n\t// Query 1 (\"dark\" in color): dark slate blue, dark blue, midnight blue (has \"dark\")\n\t// Query 2 (\"light\" in description): light blue, powder blue, alice blue, light steel blue, light sky blue\n\t// Query 3 (\"blue\" in category): dark slate blue, medium slate blue, medium blue, slate blue\n\n\texpectedTopDocuments := []string{\n\t\t\"dark slate blue\",   // matches query 1 and 3\n\t\t\"light blue\",        // matches query 2\n\t\t\"dark blue\",         // matches query 1\n\t\t\"light steel blue\",  // matches query 2\n\t\t\"medium slate blue\", // matches query 3\n\t}\n\n\tif len(hits) == 0 {\n\t\tt.Fatal(\"Expected search results, got none\")\n\t}\n\n\t// Verify we have results and they're ranked by score\n\tfor i := 0; i < len(hits)-1; i++ {\n\t\tif hits[i].Score < hits[i+1].Score {\n\t\t\tt.Errorf(\"Results not properly ranked by score: position %d (%.6f) < position %d (%.6f)\",\n\t\t\t\ti, hits[i].Score, i+1, hits[i+1].Score)\n\t\t}\n\t}\n\n\t// Check that some expected top documents are present in results\n\tfoundExpected := 0\n\tfor _, hit := range hits {\n\t\tfor _, expected := range expectedTopDocuments {\n\t\t\tif hit.ID == expected {\n\t\t\t\tfoundExpected++\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tif foundExpected < 3 {\n\t\tt.Errorf(\"Expected to find at least 3 of the top expected documents, found %d\", foundExpected)\n\t\tt.Logf(\"Actual results:\")\n\t\tfor i, hit := range hits {\n\t\t\tt.Logf(\"  %d: %s (score: %.6f)\", i+1, hit.ID, hit.Score)\n\t\t}\n\t}\n}\n\n// setupFTSSingleIndex creates a single index with all FTS documents\nfunc setupFTSSingleIndex(t *testing.T) (Index, func()) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\n\tindex, err := createFTSIndex(tmpIndexPath)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdocuments := getFTSDocuments()\n\n\t// Index documents\n\tbatch := index.NewBatch()\n\tfor _, doc := range documents {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcleanup := func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, tmpIndexPath)\n\t}\n\n\treturn index, cleanup\n}\n\n// setupFTSAliasWithSingleIndex creates an alias containing one index with all FTS documents\nfunc setupFTSAliasWithSingleIndex(t *testing.T) (Index, func()) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\n\tindex, err := createFTSIndex(tmpIndexPath)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdocuments := getFTSDocuments()\n\n\t// Create alias and add the single index\n\talias := NewIndexAlias()\n\talias.Add(index)\n\n\t// Index all documents\n\tbatch := alias.NewBatch()\n\tfor _, doc := range documents {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = alias.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcleanup := func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, tmpIndexPath)\n\t}\n\n\treturn alias, cleanup\n}\n\n// setupFTSAliasWithTwoIndexes creates an alias containing two indexes with FTS documents split between them\nfunc setupFTSAliasWithTwoIndexes(t *testing.T) (Index, func()) {\n\tdocuments := getFTSDocuments()\n\n\t// Split documents into two groups\n\tmidpoint := len(documents) / 2\n\tdocs1 := documents[:midpoint]\n\tdocs2 := documents[midpoint:]\n\n\t// Create first index\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tindex1, err := createFTSIndex(tmpIndexPath1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index first half of documents\n\tbatch1 := index1.NewBatch()\n\tfor _, doc := range docs1 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch1.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index1.Batch(batch1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create second index\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tindex2, err := createFTSIndex(tmpIndexPath2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index second half of documents\n\tbatch2 := index2.NewBatch()\n\tfor _, doc := range docs2 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch2.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index2.Batch(batch2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create alias and add both indexes\n\talias := NewIndexAlias()\n\talias.Add(index1, index2)\n\n\tcleanup := func() {\n\t\terr := index1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = index2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, tmpIndexPath1)\n\t\tcleanupTmpIndexPath(t, tmpIndexPath2)\n\t}\n\n\treturn alias, cleanup\n}\n\n// setupFTSNestedAliases creates nested aliases with three indexes spread across sub-aliases\nfunc setupFTSNestedAliases(t *testing.T) (Index, func()) {\n\tdocuments := getFTSDocuments()\n\n\t// Split documents into three groups\n\tthirdPoint1 := len(documents) / 3\n\tthirdPoint2 := 2 * len(documents) / 3\n\tdocs1 := documents[:thirdPoint1]\n\tdocs2 := documents[thirdPoint1:thirdPoint2]\n\tdocs3 := documents[thirdPoint2:]\n\n\t// Create first index\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tindex1, err := createFTSIndex(tmpIndexPath1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index first third of documents\n\tbatch1 := index1.NewBatch()\n\tfor _, doc := range docs1 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch1.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index1.Batch(batch1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create second index\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tindex2, err := createFTSIndex(tmpIndexPath2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index second third of documents\n\tbatch2 := index2.NewBatch()\n\tfor _, doc := range docs2 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch2.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index2.Batch(batch2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create third index\n\ttmpIndexPath3 := createTmpIndexPath(t)\n\tindex3, err := createFTSIndex(tmpIndexPath3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Index third third of documents\n\tbatch3 := index3.NewBatch()\n\tfor _, doc := range docs3 {\n\t\tcolorName := doc[\"color\"].(string)\n\t\terr = batch3.Index(colorName, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index3.Batch(batch3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Create first sub-alias (contains 1 index)\n\tsubAlias1 := NewIndexAlias()\n\tsubAlias1.SetName(\"subAlias1\")\n\tsubAlias1.Add(index1)\n\n\t// Create second sub-alias (contains 2 indexes)\n\tsubAlias2 := NewIndexAlias()\n\tsubAlias2.SetName(\"subAlias2\")\n\tsubAlias2.Add(index2, index3)\n\n\t// Create master alias containing the two sub-aliases\n\tmasterAlias := NewIndexAlias()\n\tmasterAlias.SetName(\"masterAlias\")\n\tmasterAlias.Add(subAlias1, subAlias2)\n\n\tcleanup := func() {\n\t\terr := index1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = index2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = index3.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, tmpIndexPath1)\n\t\tcleanupTmpIndexPath(t, tmpIndexPath2)\n\t\tcleanupTmpIndexPath(t, tmpIndexPath3)\n\t}\n\n\treturn masterAlias, cleanup\n}\n\nfunc TestFTSRRFEndToEnd(t *testing.T) {\n\t// Setup the index configuration\n\tindex, cleanup := setupFTSSingleIndex(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createFTSSearchRequest(ScoreRRF)\n\n\t// Execute search\n\tresult, err := index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify FTS RRF results\n\tverifyFTSRRFResults(t, result.Hits)\n}\n\n// TestFTSRRFAliasWithSingleIndex tests RRF with an alias containing one index\nfunc TestFTSRRFAliasWithSingleIndex(t *testing.T) {\n\t// Setup the alias configuration\n\talias, cleanup := setupFTSAliasWithSingleIndex(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createFTSSearchRequest(ScoreRRF)\n\n\t// Execute search through alias\n\tresult, err := alias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify FTS RRF results - should be identical to direct index search\n\tverifyFTSRRFResults(t, result.Hits)\n}\n\n// TestFTSRRFAliasWithTwoIndexes tests RRF with an alias containing two indexes\nfunc TestFTSRRFAliasWithTwoIndexes(t *testing.T) {\n\t// Setup the alias configuration\n\talias, cleanup := setupFTSAliasWithTwoIndexes(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createFTSSearchRequest(ScoreRRF)\n\n\t// Execute search through alias\n\tresult, err := alias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify FTS RRF results - should be consistent across distributed indexes\n\tverifyFTSRRFResults(t, result.Hits)\n}\n\n// TestFTSRRFNestedAliases tests RRF with an alias containing two index aliases\nfunc TestFTSRRFNestedAliases(t *testing.T) {\n\t// Setup the nested aliases configuration\n\tmasterAlias, cleanup := setupFTSNestedAliases(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createFTSSearchRequest(ScoreRRF)\n\n\t// Execute search through master alias\n\tresult, err := masterAlias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify FTS RRF results - should be consistent across nested aliases\n\tverifyFTSRRFResults(t, result.Hits)\n}\n\n// TestFTSRRFPagination tests FTS RRF with pagination across different index/alias configurations\nfunc TestFTSRRFPagination(t *testing.T) {\n\tscenarios := []struct {\n\t\tname  string\n\t\tsetup func(t *testing.T) (Index, func())\n\t}{\n\t\t{\n\t\t\tname:  \"SingleIndex\",\n\t\t\tsetup: setupFTSSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithSingleIndex\",\n\t\t\tsetup: setupFTSAliasWithSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithTwoIndexes\",\n\t\t\tsetup: setupFTSAliasWithTwoIndexes,\n\t\t},\n\t\t{\n\t\t\tname:  \"NestedAliases\",\n\t\t\tsetup: setupFTSNestedAliases,\n\t\t},\n\t}\n\n\tfor _, scenario := range scenarios {\n\t\tt.Run(scenario.name, func(t *testing.T) {\n\t\t\t// Setup the index/alias configuration\n\t\t\tindex, cleanup := scenario.setup(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create first page request (first 5 results)\n\t\t\tfirstPageRequest := createFTSSearchRequest(ScoreRRF)\n\t\t\tfirstPageRequest.From = 0\n\t\t\tfirstPageRequest.Size = 5\n\n\t\t\t// Execute first page search\n\t\t\tfirstPageResult, err := index.Search(firstPageRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Create second page request (next 5 results, starting from index 5)\n\t\t\tsecondPageRequest := createFTSSearchRequest(ScoreRRF)\n\t\t\tsecondPageRequest.From = 5\n\t\t\tsecondPageRequest.Size = 5\n\n\t\t\t// Execute second page search\n\t\t\tsecondPageResult, err := index.Search(secondPageRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Combine results from both pages\n\t\t\tcombinedHits := make(search.DocumentMatchCollection, 0, len(firstPageResult.Hits)+len(secondPageResult.Hits))\n\t\t\tcombinedHits = append(combinedHits, firstPageResult.Hits...)\n\t\t\tcombinedHits = append(combinedHits, secondPageResult.Hits...)\n\n\t\t\t// Verify we have results (FTS may have variable results based on matches)\n\t\t\tif len(firstPageResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected at least some results in first page, got 0\")\n\t\t\t}\n\t\t\tif len(firstPageResult.Hits) > 5 {\n\t\t\t\tt.Errorf(\"Expected at most 5 results in first page, got %d\", len(firstPageResult.Hits))\n\t\t\t}\n\n\t\t\t// Total hits should not exceed the number of documents that match our queries\n\t\t\ttotalHits := len(combinedHits)\n\t\t\tif totalHits == 0 {\n\t\t\t\tt.Fatal(\"Expected at least some combined results, got 0\")\n\t\t\t}\n\n\t\t\t// Verify combined FTS RRF results\n\t\t\tverifyFTSRRFResults(t, combinedHits)\n\t\t})\n\t}\n}\n\n// TestFTSRRFFaceting tests that facet results are identical whether using RRF or default scoring\nfunc TestFTSRRFFaceting(t *testing.T) {\n\tscenarios := []struct {\n\t\tname  string\n\t\tsetup func(t *testing.T) (Index, func())\n\t}{\n\t\t{\n\t\t\tname:  \"SingleIndex\",\n\t\t\tsetup: setupFTSSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithSingleIndex\",\n\t\t\tsetup: setupFTSAliasWithSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithTwoIndexes\",\n\t\t\tsetup: setupFTSAliasWithTwoIndexes,\n\t\t},\n\t\t{\n\t\t\tname:  \"NestedAliases\",\n\t\t\tsetup: setupFTSNestedAliases,\n\t\t},\n\t}\n\n\tfor _, scenario := range scenarios {\n\t\tt.Run(scenario.name, func(t *testing.T) {\n\t\t\t// Setup the index/alias configuration\n\t\t\tindex, cleanup := scenario.setup(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create search request with default scoring and facets\n\t\t\tdefaultRequest := createFTSSearchRequest(ScoreRRF)\n\t\t\tdefaultRequest.Score = ScoreDefault // Use default scoring\n\t\t\tdefaultRequest.Size = 10\n\t\t\t// Add facet for color field with size 10\n\t\t\tcolorFacet := NewFacetRequest(\"color\", 10)\n\t\t\tdefaultRequest.AddFacet(\"color\", colorFacet)\n\n\t\t\t// Create search request with RRF scoring and identical facets\n\t\t\trrfRequest := createFTSSearchRequest(ScoreRRF)\n\t\t\trrfRequest.Size = 10\n\t\t\trrfRequest.Score = ScoreRRF\n\t\t\t// Add identical facet for color field with size 10\n\t\t\tcolorFacetRRF := NewFacetRequest(\"color\", 10)\n\t\t\trrfRequest.AddFacet(\"color\", colorFacetRRF)\n\n\t\t\t// Execute both searches\n\t\t\tdefaultResult, err := index.Search(defaultRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Default scoring search failed: %v\", err)\n\t\t\t}\n\n\t\t\trrfResult, err := index.Search(rrfRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"RRF scoring search failed: %v\", err)\n\t\t\t}\n\n\t\t\t// Verify both searches returned results\n\t\t\tif len(defaultResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected search results with default scoring, got none\")\n\t\t\t}\n\t\t\tif len(rrfResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected search results with RRF scoring, got none\")\n\t\t\t}\n\n\t\t\t// Verify both searches returned facets\n\t\t\tif defaultResult.Facets == nil {\n\t\t\t\tt.Fatal(\"Expected facets with default scoring, got nil\")\n\t\t\t}\n\t\t\tif rrfResult.Facets == nil {\n\t\t\t\tt.Fatal(\"Expected facets with RRF scoring, got nil\")\n\t\t\t}\n\n\t\t\t// Check that color facet exists in both results\n\t\t\tdefaultColorFacet, defaultExists := defaultResult.Facets[\"color\"]\n\t\t\trrfColorFacet, rrfExists := rrfResult.Facets[\"color\"]\n\n\t\t\tif !defaultExists {\n\t\t\t\tt.Fatal(\"Expected color facet in default scoring results\")\n\t\t\t}\n\t\t\tif !rrfExists {\n\t\t\t\tt.Fatal(\"Expected color facet in RRF scoring results\")\n\t\t\t}\n\n\t\t\t// Compare the facet results - they should be identical\n\t\t\t// Since facets are based on the document corpus and not scoring,\n\t\t\t// they should not be affected by the scoring method\n\t\t\tif defaultColorFacet.Total != rrfColorFacet.Total {\n\t\t\t\tt.Errorf(\"Facet totals differ: default=%d, RRF=%d\",\n\t\t\t\t\tdefaultColorFacet.Total, rrfColorFacet.Total)\n\t\t\t}\n\n\t\t\tif defaultColorFacet.Missing != rrfColorFacet.Missing {\n\t\t\t\tt.Errorf(\"Facet missing counts differ: default=%d, RRF=%d\",\n\t\t\t\t\tdefaultColorFacet.Missing, rrfColorFacet.Missing)\n\t\t\t}\n\n\t\t\tif defaultColorFacet.Other != rrfColorFacet.Other {\n\t\t\t\tt.Errorf(\"Facet other counts differ: default=%d, RRF=%d\",\n\t\t\t\t\tdefaultColorFacet.Other, rrfColorFacet.Other)\n\t\t\t}\n\n\t\t\t// Compare the facet terms\n\t\t\tdefaultTerms := defaultColorFacet.Terms.Terms()\n\t\t\trrfTerms := rrfColorFacet.Terms.Terms()\n\n\t\t\tif len(defaultTerms) != len(rrfTerms) {\n\t\t\t\tt.Errorf(\"Facet terms count differs: default=%d, RRF=%d\",\n\t\t\t\t\tlen(defaultTerms), len(rrfTerms))\n\t\t\t} else {\n\t\t\t\t// Compare each term\n\t\t\t\tfor i, defaultTerm := range defaultTerms {\n\t\t\t\t\trrfTerm := rrfTerms[i]\n\t\t\t\t\tif defaultTerm.Term != rrfTerm.Term {\n\t\t\t\t\t\tt.Errorf(\"Facet term differs at position %d: default=%s, RRF=%s\",\n\t\t\t\t\t\t\ti, defaultTerm.Term, rrfTerm.Term)\n\t\t\t\t\t}\n\t\t\t\t\tif defaultTerm.Count != rrfTerm.Count {\n\t\t\t\t\t\tt.Errorf(\"Facet term count differs for %s: default=%d, RRF=%d\",\n\t\t\t\t\t\t\tdefaultTerm.Term, defaultTerm.Count, rrfTerm.Count)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// verifyFTSRSFResults verifies that the search hits match expected RSF ranking and scores for FTS-only search\nfunc verifyFTSRSFResults(t *testing.T, hits search.DocumentMatchCollection) {\n\t// For FTS RSF, we expect documents that match multiple query components to rank higher\n\t// Query components: \"dark\" in color, \"light\" in description, \"blue\" in category\n\t// RSF uses min-max normalization of scores within the window\n\n\t// Verify we have reasonable number of results\n\tif len(hits) == 0 {\n\t\tt.Fatal(\"Expected non-empty search results for FTS RSF\")\n\t}\n\n\t// Verify we have at least 5 results for meaningful comparison\n\tif len(hits) < 5 {\n\t\tt.Errorf(\"Expected at least 5 results for FTS RSF, got %d\", len(hits))\n\t}\n\n\t// Documents that should appear in top results based on multi-query matching:\n\t// - \"dark slate blue\": matches \"dark\" in color AND \"blue\" in category (2 matches)\n\t// - \"light blue\": matches \"light\" in description (1 strong match)\n\t// - \"dark blue\": matches \"dark\" in color (1 match)\n\t// - Documents with \"light\" in description should rank well\n\ttopExpectedDocs := []string{\"dark slate blue\", \"light blue\", \"dark blue\", \"medium slate blue\", \"light sky blue\"}\n\n\t// Create map of all hits for easier lookup\n\tdocMap := make(map[string]int) // doc -> position (0-based)\n\tfor i, hit := range hits {\n\t\tdocMap[hit.ID] = i\n\t}\n\n\t// Verify that \"dark slate blue\" appears in top 3 positions (matches 2 query components)\n\tif pos, found := docMap[\"dark slate blue\"]; !found {\n\t\tt.Error(\"Expected 'dark slate blue' to appear in results but not found\")\n\t} else if pos >= 3 {\n\t\tt.Errorf(\"Expected 'dark slate blue' in top 3 positions, found at position %d\", pos+1)\n\t}\n\n\t// Verify that at least 3 of the top expected documents appear in top 5 results\n\ttopFoundCount := 0\n\tfor _, expectedDoc := range topExpectedDocs {\n\t\tif pos, found := docMap[expectedDoc]; found && pos < 5 {\n\t\t\ttopFoundCount++\n\t\t}\n\t}\n\tif topFoundCount < 3 {\n\t\tt.Errorf(\"Expected at least 3 of top expected documents in top 5 results, found %d\", topFoundCount)\n\t}\n\n\t// Verify scores are reasonable and within expected range\n\t// RSF scores should be between 0 and sum of weights (3.0 with default weights)\n\tfor i, hit := range hits {\n\t\tif hit.Score < 0 || hit.Score > 3.0 {\n\t\t\tt.Errorf(\"Hit %d (%s) has unreasonable score: %.6f\", i, hit.ID, hit.Score)\n\t\t}\n\t\t// First hit should have a substantial score (at least 0.1)\n\t\tif i == 0 && hit.Score < 0.1 {\n\t\t\tt.Errorf(\"Top hit (%s) has unexpectedly low score: %.6f\", hit.ID, hit.Score)\n\t\t}\n\t}\n\n\t// Verify hits are sorted by score descending with strict ordering\n\tfor i := 1; i < len(hits); i++ {\n\t\tif hits[i-1].Score < hits[i].Score {\n\t\t\tt.Errorf(\"Hits not sorted properly: hit %d (%s, score %.6f) < hit %d (%s, score %.6f)\",\n\t\t\t\ti, hits[i-1].ID, hits[i-1].Score, i+1, hits[i].ID, hits[i].Score)\n\t\t}\n\t}\n\n\t// Verify score range is reasonable - top score should be significantly higher than 5th\n\tif len(hits) >= 5 {\n\t\ttopScore := hits[0].Score\n\t\tfifthScore := hits[4].Score\n\t\tif topScore-fifthScore < 0.001 {\n\t\t\tt.Errorf(\"Insufficient score differentiation: top score %.6f, 5th score %.6f (diff: %.6f)\",\n\t\t\t\ttopScore, fifthScore, topScore-fifthScore)\n\t\t}\n\t}\n}\n\n// TestFTSRSFEndToEnd tests RSF scoring with a single FTS index\nfunc TestFTSRSFEndToEnd(t *testing.T) {\n\t// Setup the index configuration\n\tindex, cleanup := setupFTSSingleIndex(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createFTSSearchRequest(ScoreRSF)\n\n\t// Execute search\n\tresult, err := index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify FTS RSF results\n\tverifyFTSRSFResults(t, result.Hits)\n}\n\n// TestFTSRSFAliasWithSingleIndex tests RSF with an alias containing one FTS index\nfunc TestFTSRSFAliasWithSingleIndex(t *testing.T) {\n\t// Setup the alias configuration\n\talias, cleanup := setupFTSAliasWithSingleIndex(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createFTSSearchRequest(ScoreRSF)\n\n\t// Execute search through alias\n\tresult, err := alias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify FTS RSF results - should be identical to direct index search\n\tverifyFTSRSFResults(t, result.Hits)\n}\n\n// TestFTSRSFAliasWithTwoIndexes tests RSF with an alias containing two FTS indexes\nfunc TestFTSRSFAliasWithTwoIndexes(t *testing.T) {\n\t// Setup the alias configuration\n\talias, cleanup := setupFTSAliasWithTwoIndexes(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createFTSSearchRequest(ScoreRSF)\n\n\t// Execute search through alias\n\tresult, err := alias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify FTS RSF results - should be consistent across distributed indexes\n\tverifyFTSRSFResults(t, result.Hits)\n}\n\n// TestFTSRSFNestedAliases tests RSF with an alias containing two index aliases\nfunc TestFTSRSFNestedAliases(t *testing.T) {\n\t// Setup the nested aliases configuration\n\tmasterAlias, cleanup := setupFTSNestedAliases(t)\n\tdefer cleanup()\n\n\t// Create the search request\n\tsearchRequest := createFTSSearchRequest(ScoreRSF)\n\n\t// Execute search through master alias\n\tresult, err := masterAlias.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Verify FTS RSF results - should be consistent across nested aliases\n\tverifyFTSRSFResults(t, result.Hits)\n}\n\n// TestFTSRSFPagination tests FTS RSF with pagination across different index/alias configurations\nfunc TestFTSRSFPagination(t *testing.T) {\n\tscenarios := []struct {\n\t\tname  string\n\t\tsetup func(t *testing.T) (Index, func())\n\t}{\n\t\t{\n\t\t\tname:  \"SingleIndex\",\n\t\t\tsetup: setupFTSSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithSingleIndex\",\n\t\t\tsetup: setupFTSAliasWithSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithTwoIndexes\",\n\t\t\tsetup: setupFTSAliasWithTwoIndexes,\n\t\t},\n\t\t{\n\t\t\tname:  \"NestedAliases\",\n\t\t\tsetup: setupFTSNestedAliases,\n\t\t},\n\t}\n\n\tfor _, scenario := range scenarios {\n\t\tt.Run(scenario.name, func(t *testing.T) {\n\t\t\t// Setup the index/alias configuration\n\t\t\tindex, cleanup := scenario.setup(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create first page request (first 5 results)\n\t\t\tfirstPageRequest := createFTSSearchRequest(ScoreRSF)\n\t\t\tfirstPageRequest.From = 0\n\t\t\tfirstPageRequest.Size = 5\n\n\t\t\t// Execute first page search\n\t\t\tfirstPageResult, err := index.Search(firstPageRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Create second page request (next 5 results, starting from index 5)\n\t\t\tsecondPageRequest := createFTSSearchRequest(ScoreRSF)\n\t\t\tsecondPageRequest.From = 5\n\t\t\tsecondPageRequest.Size = 5\n\n\t\t\t// Execute second page search\n\t\t\tsecondPageResult, err := index.Search(secondPageRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Combine results from both pages\n\t\t\tcombinedHits := make(search.DocumentMatchCollection, 0, len(firstPageResult.Hits)+len(secondPageResult.Hits))\n\t\t\tcombinedHits = append(combinedHits, firstPageResult.Hits...)\n\t\t\tcombinedHits = append(combinedHits, secondPageResult.Hits...)\n\n\t\t\t// Verify we have results (FTS may have variable results based on matches)\n\t\t\tif len(firstPageResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected at least some results in first page, got 0\")\n\t\t\t}\n\t\t\tif len(firstPageResult.Hits) > 5 {\n\t\t\t\tt.Errorf(\"Expected at most 5 results in first page, got %d\", len(firstPageResult.Hits))\n\t\t\t}\n\n\t\t\t// Total hits should not exceed the number of documents that match our queries\n\t\t\ttotalHits := len(combinedHits)\n\t\t\tif totalHits == 0 {\n\t\t\t\tt.Fatal(\"Expected at least some combined results, got 0\")\n\t\t\t}\n\n\t\t\t// Verify combined FTS RSF results\n\t\t\tverifyFTSRSFResults(t, combinedHits)\n\t\t})\n\t}\n}\n\n// TestFTSRSFFaceting tests that facet results are identical whether using RSF or default scoring\nfunc TestFTSRSFFaceting(t *testing.T) {\n\tscenarios := []struct {\n\t\tname  string\n\t\tsetup func(t *testing.T) (Index, func())\n\t}{\n\t\t{\n\t\t\tname:  \"SingleIndex\",\n\t\t\tsetup: setupFTSSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithSingleIndex\",\n\t\t\tsetup: setupFTSAliasWithSingleIndex,\n\t\t},\n\t\t{\n\t\t\tname:  \"AliasWithTwoIndexes\",\n\t\t\tsetup: setupFTSAliasWithTwoIndexes,\n\t\t},\n\t\t{\n\t\t\tname:  \"NestedAliases\",\n\t\t\tsetup: setupFTSNestedAliases,\n\t\t},\n\t}\n\n\tfor _, scenario := range scenarios {\n\t\tt.Run(scenario.name, func(t *testing.T) {\n\t\t\t// Setup the index/alias configuration\n\t\t\tindex, cleanup := scenario.setup(t)\n\t\t\tdefer cleanup()\n\n\t\t\t// Create search request with default scoring and facets\n\t\t\tdefaultRequest := createFTSSearchRequest(ScoreRRF)\n\t\t\tdefaultRequest.Score = ScoreDefault // Use default scoring\n\t\t\tdefaultRequest.Size = 10\n\t\t\t// Add facet for category field with size 10\n\t\t\tcategoryFacet := NewFacetRequest(\"category\", 10)\n\t\t\tdefaultRequest.AddFacet(\"category\", categoryFacet)\n\n\t\t\t// Create search request with RSF scoring and identical facets\n\t\t\trsfRequest := createFTSSearchRequest(ScoreRSF)\n\t\t\trsfRequest.Size = 10\n\t\t\t// Add identical facet for category field with size 10\n\t\t\tcategoryFacetRSF := NewFacetRequest(\"category\", 10)\n\t\t\trsfRequest.AddFacet(\"category\", categoryFacetRSF)\n\n\t\t\t// Execute both searches\n\t\t\tdefaultResult, err := index.Search(defaultRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Default scoring search failed: %v\", err)\n\t\t\t}\n\n\t\t\trsfResult, err := index.Search(rsfRequest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"RSF scoring search failed: %v\", err)\n\t\t\t}\n\n\t\t\t// Verify both searches returned results\n\t\t\tif len(defaultResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected search results with default scoring, got none\")\n\t\t\t}\n\t\t\tif len(rsfResult.Hits) == 0 {\n\t\t\t\tt.Fatal(\"Expected search results with RSF scoring, got none\")\n\t\t\t}\n\n\t\t\t// Verify both searches returned facets\n\t\t\tif defaultResult.Facets == nil {\n\t\t\t\tt.Fatal(\"Expected facets with default scoring, got nil\")\n\t\t\t}\n\t\t\tif rsfResult.Facets == nil {\n\t\t\t\tt.Fatal(\"Expected facets with RSF scoring, got nil\")\n\t\t\t}\n\n\t\t\t// Check that category facet exists in both results\n\t\t\tdefaultCategoryFacet, defaultExists := defaultResult.Facets[\"category\"]\n\t\t\trsfCategoryFacet, rsfExists := rsfResult.Facets[\"category\"]\n\n\t\t\tif !defaultExists {\n\t\t\t\tt.Fatal(\"Expected category facet in default scoring results\")\n\t\t\t}\n\t\t\tif !rsfExists {\n\t\t\t\tt.Fatal(\"Expected category facet in RSF scoring results\")\n\t\t\t}\n\n\t\t\t// Compare the facet results - they should be identical\n\t\t\t// Since facets are based on the document corpus and not scoring,\n\t\t\t// they should not be affected by the scoring method\n\t\t\tif defaultCategoryFacet.Total != rsfCategoryFacet.Total {\n\t\t\t\tt.Errorf(\"Facet totals differ: default=%d, RSF=%d\",\n\t\t\t\t\tdefaultCategoryFacet.Total, rsfCategoryFacet.Total)\n\t\t\t}\n\n\t\t\tif defaultCategoryFacet.Missing != rsfCategoryFacet.Missing {\n\t\t\t\tt.Errorf(\"Facet missing counts differ: default=%d, RSF=%d\",\n\t\t\t\t\tdefaultCategoryFacet.Missing, rsfCategoryFacet.Missing)\n\t\t\t}\n\n\t\t\tif defaultCategoryFacet.Other != rsfCategoryFacet.Other {\n\t\t\t\tt.Errorf(\"Facet other counts differ: default=%d, RSF=%d\",\n\t\t\t\t\tdefaultCategoryFacet.Other, rsfCategoryFacet.Other)\n\t\t\t}\n\n\t\t\t// Compare the facet terms\n\t\t\tdefaultTerms := defaultCategoryFacet.Terms.Terms()\n\t\t\trsfTerms := rsfCategoryFacet.Terms.Terms()\n\n\t\t\tif len(defaultTerms) != len(rsfTerms) {\n\t\t\t\tt.Errorf(\"Facet terms count differs: default=%d, RSF=%d\",\n\t\t\t\t\tlen(defaultTerms), len(rsfTerms))\n\t\t\t} else {\n\t\t\t\t// Compare each term\n\t\t\t\tfor i, defaultTerm := range defaultTerms {\n\t\t\t\t\trsfTerm := rsfTerms[i]\n\t\t\t\t\tif defaultTerm.Term != rsfTerm.Term {\n\t\t\t\t\t\tt.Errorf(\"Facet term differs at position %d: default=%s, RSF=%s\",\n\t\t\t\t\t\t\ti, defaultTerm.Term, rsfTerm.Term)\n\t\t\t\t\t}\n\t\t\t\t\tif defaultTerm.Count != rsfTerm.Count {\n\t\t\t\t\t\tt.Errorf(\"Facet term count differs for %s: default=%d, RSF=%d\",\n\t\t\t\t\t\t\tdefaultTerm.Term, defaultTerm.Count, rsfTerm.Count)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "search/collector/bench_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage collector\n\nimport (\n\t\"context\"\n\t\"math/rand\"\n\t\"strconv\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype createCollector func() search.Collector\n\nfunc benchHelper(numOfMatches int, cc createCollector, b *testing.B) {\n\tmatches := make([]*search.DocumentMatch, 0, numOfMatches)\n\tfor i := 0; i < numOfMatches; i++ {\n\t\tmatches = append(matches, &search.DocumentMatch{\n\t\t\tIndexInternalID: index.IndexInternalID(strconv.Itoa(i)),\n\t\t\tScore:           rand.Float64(),\n\t\t})\n\t}\n\n\tb.ResetTimer()\n\n\tfor run := 0; run < b.N; run++ {\n\t\tsearcher := &stubSearcher{\n\t\t\tmatches: matches,\n\t\t}\n\t\tcollector := cc()\n\t\terr := collector.Collect(context.Background(), searcher, &stubReader{})\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/collector/eligible.go",
    "content": "//  Copyright (c) 2024 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage collector\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype EligibleCollector struct {\n\tsize             int\n\ttotal            uint64\n\ttook             time.Duration\n\teligibleSelector index.EligibleDocumentSelector\n}\n\nfunc NewEligibleCollector(size int) *EligibleCollector {\n\treturn newEligibleCollector(size)\n}\n\nfunc newEligibleCollector(size int) *EligibleCollector {\n\t// No sort order & skip always 0 since this is only to filter eligible docs.\n\tec := &EligibleCollector{\n\t\tsize: size,\n\t}\n\treturn ec\n}\n\nfunc makeEligibleDocumentMatchHandler(ctx *search.SearchContext, reader index.IndexReader) (search.DocumentMatchHandler, error) {\n\tif ec, ok := ctx.Collector.(*EligibleCollector); ok {\n\t\tif vr, ok := reader.(index.VectorIndexReader); ok {\n\t\t\t// create a new eligible document selector to add eligible document matches\n\t\t\tec.eligibleSelector = vr.NewEligibleDocumentSelector()\n\t\t\t// return a document match handler that adds eligible document matches\n\t\t\t// to the eligible document selector\n\t\t\treturn func(d *search.DocumentMatch) error {\n\t\t\t\tif d == nil {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\terr := ec.eligibleSelector.AddEligibleDocumentMatch(d.IndexInternalID)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\t// recycle the DocumentMatch\n\t\t\t\tctx.DocumentMatchPool.Put(d)\n\t\t\t\treturn nil\n\t\t\t}, nil\n\t\t}\n\t\treturn nil, fmt.Errorf(\"reader is not a VectorIndexReader\")\n\t}\n\n\treturn nil, fmt.Errorf(\"eligiblity collector not available\")\n}\n\nfunc (ec *EligibleCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error {\n\tstartTime := time.Now()\n\tvar err error\n\tvar next *search.DocumentMatch\n\n\tbackingSize := ec.size\n\tif backingSize > PreAllocSizeSkipCap {\n\t\tbackingSize = PreAllocSizeSkipCap + 1\n\t}\n\tsearchContext := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), 0),\n\t\tCollector:         ec,\n\t\tIndexReader:       reader,\n\t}\n\n\tdmHandler, err := makeEligibleDocumentMatchHandler(searchContext, reader)\n\tif err != nil {\n\t\treturn err\n\t}\n\tselect {\n\tcase <-ctx.Done():\n\t\tsearch.RecordSearchCost(ctx, search.AbortM, 0)\n\t\treturn ctx.Err()\n\tdefault:\n\t\tnext, err = searcher.Next(searchContext)\n\t}\n\tfor err == nil && next != nil {\n\t\tif ec.total%CheckDoneEvery == 0 {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\tsearch.RecordSearchCost(ctx, search.AbortM, 0)\n\t\t\t\treturn ctx.Err()\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t\tec.total++\n\n\t\terr = dmHandler(next)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\n\t\tnext, err = searcher.Next(searchContext)\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// help finalize/flush the results in case\n\t// of custom document match handlers.\n\terr = dmHandler(nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// compute search duration\n\tec.took = time.Since(startTime)\n\n\treturn nil\n}\n\n// The eligible collector does not return any document matches and hence\n// this method is a dummy method returning nil, to conform to the\n// search.Collector interface.\nfunc (ec *EligibleCollector) Results() search.DocumentMatchCollection {\n\treturn nil\n}\n\n// EligibleSelector returns the eligible document selector, which can be used\n// to retrieve the list of eligible documents from this collector.\n// If the collector has no results, it returns nil.\nfunc (ec *EligibleCollector) EligibleSelector() index.EligibleDocumentSelector {\n\tif ec.total == 0 {\n\t\treturn nil\n\t}\n\treturn ec.eligibleSelector\n}\n\nfunc (ec *EligibleCollector) Total() uint64 {\n\treturn ec.total\n}\n\n// No concept of scoring in the eligible collector.\nfunc (ec *EligibleCollector) MaxScore() float64 {\n\treturn 0\n}\n\nfunc (ec *EligibleCollector) Took() time.Duration {\n\treturn ec.took\n}\n\nfunc (ec *EligibleCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) {\n\t// facet unsupported for pre-filtering in KNN search\n}\n\nfunc (ec *EligibleCollector) FacetResults() search.FacetResults {\n\t// facet unsupported for pre-filtering in KNN search\n\treturn nil\n}\n"
  },
  {
    "path": "search/collector/heap.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage collector\n\nimport (\n\t\"container/heap\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\ntype collectStoreHeap struct {\n\theap    search.DocumentMatchCollection\n\tcompare collectorCompare\n}\n\nfunc newStoreHeap(capacity int, compare collectorCompare) *collectStoreHeap {\n\trv := &collectStoreHeap{\n\t\theap:    make(search.DocumentMatchCollection, 0, capacity),\n\t\tcompare: compare,\n\t}\n\theap.Init(rv)\n\treturn rv\n}\n\nfunc (c *collectStoreHeap) AddNotExceedingSize(doc *search.DocumentMatch,\n\tsize int) *search.DocumentMatch {\n\tc.add(doc)\n\tif c.Len() > size {\n\t\treturn c.removeLast()\n\t}\n\treturn nil\n}\n\nfunc (c *collectStoreHeap) add(doc *search.DocumentMatch) {\n\theap.Push(c, doc)\n}\n\nfunc (c *collectStoreHeap) removeLast() *search.DocumentMatch {\n\treturn heap.Pop(c).(*search.DocumentMatch)\n}\n\nfunc (c *collectStoreHeap) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) {\n\tcount := c.Len()\n\tsize := count - skip\n\tif size <= 0 {\n\t\treturn make(search.DocumentMatchCollection, 0), nil\n\t}\n\trv := make(search.DocumentMatchCollection, size)\n\tfor i := size - 1; i >= 0; i-- {\n\t\tdoc := heap.Pop(c).(*search.DocumentMatch)\n\t\trv[i] = doc\n\t\terr := fixup(doc)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn rv, nil\n}\n\nfunc (c *collectStoreHeap) Internal() search.DocumentMatchCollection {\n\treturn c.heap\n}\n\n// heap interface implementation\n\nfunc (c *collectStoreHeap) Len() int {\n\treturn len(c.heap)\n}\n\nfunc (c *collectStoreHeap) Less(i, j int) bool {\n\tso := c.compare(c.heap[i], c.heap[j])\n\treturn -so < 0\n}\n\nfunc (c *collectStoreHeap) Swap(i, j int) {\n\tc.heap[i], c.heap[j] = c.heap[j], c.heap[i]\n}\n\nfunc (c *collectStoreHeap) Push(x interface{}) {\n\tc.heap = append(c.heap, x.(*search.DocumentMatch))\n}\n\nfunc (c *collectStoreHeap) Pop() interface{} {\n\tvar rv *search.DocumentMatch\n\trv, c.heap = c.heap[len(c.heap)-1], c.heap[:len(c.heap)-1]\n\treturn rv\n}\n"
  },
  {
    "path": "search/collector/knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage collector\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype collectStoreKNN struct {\n\tinternalHeaps []collectorStore\n\tkValues       []int64\n\tallHits       map[*search.DocumentMatch]struct{}\n\tejectedDocs   map[*search.DocumentMatch]struct{}\n}\n\nfunc newStoreKNN(internalHeaps []collectorStore, kValues []int64) *collectStoreKNN {\n\treturn &collectStoreKNN{\n\t\tinternalHeaps: internalHeaps,\n\t\tkValues:       kValues,\n\t\tejectedDocs:   make(map[*search.DocumentMatch]struct{}),\n\t\tallHits:       make(map[*search.DocumentMatch]struct{}),\n\t}\n}\n\n// Adds a document to the collector store and returns the documents that were ejected\n// from the store. The documents that were ejected from the store are the ones that\n// were not in the top K documents for any of the heaps.\n// These document are put back into the pool document match pool in the KNN Collector.\nfunc (c *collectStoreKNN) AddDocument(doc *search.DocumentMatch) []*search.DocumentMatch {\n\tfor heapIdx := 0; heapIdx < len(c.internalHeaps); heapIdx++ {\n\t\tif _, ok := doc.ScoreBreakdown[heapIdx]; !ok {\n\t\t\tcontinue\n\t\t}\n\t\tejectedDoc := c.internalHeaps[heapIdx].AddNotExceedingSize(doc, int(c.kValues[heapIdx]))\n\t\tif ejectedDoc != nil {\n\t\t\tdelete(ejectedDoc.ScoreBreakdown, heapIdx)\n\t\t\tc.ejectedDocs[ejectedDoc] = struct{}{}\n\t\t}\n\t}\n\tvar rv []*search.DocumentMatch\n\tfor doc := range c.ejectedDocs {\n\t\tif len(doc.ScoreBreakdown) == 0 {\n\t\t\trv = append(rv, doc)\n\t\t}\n\t\t// clear out the ejectedDocs map to reuse it in the next AddDocument call\n\t\tdelete(c.ejectedDocs, doc)\n\t}\n\treturn rv\n}\n\nfunc (c *collectStoreKNN) Final(fixup collectorFixup) (search.DocumentMatchCollection, error) {\n\tfor _, heap := range c.internalHeaps {\n\t\tfor _, doc := range heap.Internal() {\n\t\t\t// duplicates may be present across the internal heaps\n\t\t\t// meaning the same document match may be in the top K\n\t\t\t// for multiple KNN queries.\n\t\t\tc.allHits[doc] = struct{}{}\n\t\t}\n\t}\n\tsize := len(c.allHits)\n\tif size <= 0 {\n\t\treturn make(search.DocumentMatchCollection, 0), nil\n\t}\n\trv := make(search.DocumentMatchCollection, size)\n\ti := 0\n\tfor doc := range c.allHits {\n\t\tif fixup != nil {\n\t\t\terr := fixup(doc)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\trv[i] = doc\n\t\ti++\n\t}\n\treturn rv, nil\n}\n\nfunc MakeKNNDocMatchHandler(ctx *search.SearchContext) (search.DocumentMatchHandler, error) {\n\tvar hc *KNNCollector\n\tvar ok bool\n\tif hc, ok = ctx.Collector.(*KNNCollector); ok {\n\t\treturn func(d *search.DocumentMatch) error {\n\t\t\tif d == nil {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\ttoRelease := hc.knnStore.AddDocument(d)\n\t\t\tfor _, doc := range toRelease {\n\t\t\t\tctx.DocumentMatchPool.Put(doc)\n\t\t\t}\n\t\t\treturn nil\n\t\t}, nil\n\t}\n\treturn nil, nil\n}\n\nfunc GetNewKNNCollectorStore(kArray []int64) *collectStoreKNN {\n\tinternalHeaps := make([]collectorStore, len(kArray))\n\tfor knnIdx, k := range kArray {\n\t\t// TODO - Check if the datatype of k can be made into an int instead of int64\n\t\tidx := knnIdx\n\t\tinternalHeaps[idx] = getOptimalCollectorStore(int(k), 0, func(i, j *search.DocumentMatch) int {\n\t\t\tif i.ScoreBreakdown[idx] < j.ScoreBreakdown[idx] {\n\t\t\t\treturn 1\n\t\t\t}\n\t\t\treturn -1\n\t\t})\n\t}\n\treturn newStoreKNN(internalHeaps, kArray)\n}\n\n// implements Collector interface\ntype KNNCollector struct {\n\tknnStore *collectStoreKNN\n\tsize     int\n\ttotal    uint64\n\ttook     time.Duration\n\tresults  search.DocumentMatchCollection\n\tmaxScore float64\n}\n\nfunc NewKNNCollector(kArray []int64, size int64) *KNNCollector {\n\treturn &KNNCollector{\n\t\tknnStore: GetNewKNNCollectorStore(kArray),\n\t\tsize:     int(size),\n\t}\n}\n\nfunc (hc *KNNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error {\n\tstartTime := time.Now()\n\tvar err error\n\tvar next *search.DocumentMatch\n\n\t// pre-allocate enough space in the DocumentMatchPool\n\t// unless the sum of K is too large, then cap it\n\t// everything should still work, just allocates DocumentMatches on demand\n\tbackingSize := hc.size\n\tif backingSize > PreAllocSizeSkipCap {\n\t\tbackingSize = PreAllocSizeSkipCap + 1\n\t}\n\tsearchContext := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), 0),\n\t\tCollector:         hc,\n\t\tIndexReader:       reader,\n\t}\n\n\tdmHandlerMakerKNN := MakeKNNDocMatchHandler\n\tif cv := ctx.Value(search.MakeKNNDocumentMatchHandlerKey); cv != nil {\n\t\tdmHandlerMakerKNN = cv.(search.MakeKNNDocumentMatchHandler)\n\t}\n\t// use the application given builder for making the custom document match\n\t// handler and perform callbacks/invocations on the newly made handler.\n\tdmHandler, err := dmHandlerMakerKNN(searchContext)\n\tif err != nil {\n\t\treturn err\n\t}\n\tselect {\n\tcase <-ctx.Done():\n\t\tsearch.RecordSearchCost(ctx, search.AbortM, 0)\n\t\treturn ctx.Err()\n\tdefault:\n\t\tnext, err = searcher.Next(searchContext)\n\t}\n\tfor err == nil && next != nil {\n\t\tif hc.total%CheckDoneEvery == 0 {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\tsearch.RecordSearchCost(ctx, search.AbortM, 0)\n\t\t\t\treturn ctx.Err()\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t\thc.total++\n\n\t\terr = dmHandler(next)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\n\t\tnext, err = searcher.Next(searchContext)\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// help finalize/flush the results in case\n\t// of custom document match handlers.\n\terr = dmHandler(nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// compute search duration\n\thc.took = time.Since(startTime)\n\n\t// finalize actual results\n\terr = hc.finalizeResults(reader)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (hc *KNNCollector) finalizeResults(r index.IndexReader) error {\n\tvar err error\n\thc.results, err = hc.knnStore.Final(func(doc *search.DocumentMatch) error {\n\t\tif doc.ID == \"\" {\n\t\t\t// look up the id since we need it for lookup\n\t\t\tvar err error\n\t\t\tdoc.ID, err = r.ExternalID(doc.IndexInternalID)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n\treturn err\n}\n\nfunc (hc *KNNCollector) Results() search.DocumentMatchCollection {\n\treturn hc.results\n}\n\nfunc (hc *KNNCollector) Total() uint64 {\n\treturn hc.total\n}\n\nfunc (hc *KNNCollector) MaxScore() float64 {\n\treturn hc.maxScore\n}\n\nfunc (hc *KNNCollector) Took() time.Duration {\n\treturn hc.took\n}\n\nfunc (hc *KNNCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) {\n\t// facet unsupported for vector search\n}\n\nfunc (hc *KNNCollector) FacetResults() search.FacetResults {\n\t// facet unsupported for vector search\n\treturn nil\n}\n"
  },
  {
    "path": "search/collector/list.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage collector\n\nimport (\n\t\"container/list\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\ntype collectStoreList struct {\n\tresults *list.List\n\tcompare collectorCompare\n}\n\nfunc newStoreList(capacity int, compare collectorCompare) *collectStoreList {\n\trv := &collectStoreList{\n\t\tresults: list.New(),\n\t\tcompare: compare,\n\t}\n\n\treturn rv\n}\n\nfunc (c *collectStoreList) AddNotExceedingSize(doc *search.DocumentMatch, size int) *search.DocumentMatch {\n\tc.add(doc)\n\tif c.len() > size {\n\t\treturn c.removeLast()\n\t}\n\treturn nil\n}\n\nfunc (c *collectStoreList) add(doc *search.DocumentMatch) {\n\tfor e := c.results.Front(); e != nil; e = e.Next() {\n\t\tcurr := e.Value.(*search.DocumentMatch)\n\t\tif c.compare(doc, curr) >= 0 {\n\t\t\tc.results.InsertBefore(doc, e)\n\t\t\treturn\n\t\t}\n\t}\n\t// if we got to the end, we still have to add it\n\tc.results.PushBack(doc)\n}\n\nfunc (c *collectStoreList) removeLast() *search.DocumentMatch {\n\treturn c.results.Remove(c.results.Front()).(*search.DocumentMatch)\n}\n\nfunc (c *collectStoreList) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) {\n\tif c.results.Len()-skip > 0 {\n\t\trv := make(search.DocumentMatchCollection, c.results.Len()-skip)\n\t\ti := 0\n\t\tskipped := 0\n\t\tfor e := c.results.Back(); e != nil; e = e.Prev() {\n\t\t\tif skipped < skip {\n\t\t\t\tskipped++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\trv[i] = e.Value.(*search.DocumentMatch)\n\t\t\terr := fixup(rv[i])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\ti++\n\t\t}\n\t\treturn rv, nil\n\t}\n\treturn search.DocumentMatchCollection{}, nil\n}\n\nfunc (c *collectStoreList) Internal() search.DocumentMatchCollection {\n\trv := make(search.DocumentMatchCollection, c.results.Len())\n\ti := 0\n\tfor e := c.results.Front(); e != nil; e = e.Next() {\n\t\trv[i] = e.Value.(*search.DocumentMatch)\n\t\ti++\n\t}\n\treturn rv\n}\n\nfunc (c *collectStoreList) len() int {\n\treturn c.results.Len()\n}\n"
  },
  {
    "path": "search/collector/nested.go",
    "content": "//  Copyright (c) 2026 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage collector\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype collectStoreNested struct {\n\t// descAdder is used to customize how descendants are merged into their parent\n\tdescAdder search.DescendantAdderCallbackFn\n\t// nested reader to retrieve ancestor information\n\tnr index.NestedReader\n\t// the current root document match being built\n\tcurrRoot *search.DocumentMatch\n\t// the ancestor ID of the current root document being built\n\tcurrRootAncestorID index.AncestorID\n\t// prealloc slice for ancestor IDs\n\tancestors []index.AncestorID\n}\n\nfunc newStoreNested(nr index.NestedReader, descAdder search.DescendantAdderCallbackFn) *collectStoreNested {\n\trv := &collectStoreNested{\n\t\tdescAdder: descAdder,\n\t\tnr:        nr,\n\t}\n\treturn rv\n}\n\n// ProcessNestedDocument adds a document to the nested store, merging it into its root document\n// as needed. If the returned DocumentMatch is nil, the incoming doc has been merged\n// into its parent and should not be processed further. If the returned DocumentMatch\n// is non-nil, it represents a complete root document that should be processed further.\n// NOTE: This implementation assumes that documents are added in increasing order of their internal IDs\n// which is guaranteed by all searchers in bleve.\nfunc (c *collectStoreNested) ProcessNestedDocument(ctx *search.SearchContext, doc *search.DocumentMatch) (*search.DocumentMatch, error) {\n\t// find ancestors for the doc\n\tvar err error\n\tc.ancestors, err = c.nr.Ancestors(doc.IndexInternalID, c.ancestors[:0])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(c.ancestors) == 0 {\n\t\t// should not happen, every doc should have at least itself as ancestor\n\t\treturn nil, nil\n\t}\n\t// root docID is the last ancestor\n\trootID := c.ancestors[len(c.ancestors)-1]\n\t// check if there is an interim root already and if the incoming doc belongs to it\n\tif c.currRoot != nil && c.currRootAncestorID.Equals(rootID) {\n\t\t// there is an interim root already, and the incoming doc belongs to it\n\t\tif err := c.descAdder(c.currRoot, doc); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// recycle the child document now that it's merged into the interim root\n\t\tctx.DocumentMatchPool.Put(doc)\n\t\treturn nil, nil\n\t}\n\t// completedRoot is the root document match to return, if any\n\tvar completedRoot *search.DocumentMatch\n\tif c.currRoot != nil {\n\t\t// we have an existing interim root, return it for processing\n\t\tcompletedRoot = c.currRoot\n\t}\n\t// no interim root for now so either we have a root document incoming\n\t// or we have a child doc and need to create an interim root\n\tif len(c.ancestors) == 1 {\n\t\t// incoming doc is the root itself\n\t\tc.currRoot = doc\n\t\tc.currRootAncestorID = rootID\n\t\treturn completedRoot, nil\n\t}\n\t// this is a child doc, create interim root\n\tnewDM := ctx.DocumentMatchPool.Get()\n\tnewDM.IndexInternalID = rootID.ToIndexInternalID(newDM.IndexInternalID)\n\t// merge the incoming doc into the new interim root\n\tc.currRoot = newDM\n\tc.currRootAncestorID = rootID\n\tif err := c.descAdder(c.currRoot, doc); err != nil {\n\t\treturn nil, err\n\t}\n\t// recycle the child document now that it's merged into the interim root\n\tctx.DocumentMatchPool.Put(doc)\n\treturn completedRoot, nil\n}\n\n// Current returns the current interim root document match being built, if any\nfunc (c *collectStoreNested) Current() *search.DocumentMatch {\n\treturn c.currRoot\n}\n"
  },
  {
    "path": "search/collector/search_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage collector\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype stubSearcher struct {\n\tindex   int\n\tmatches []*search.DocumentMatch\n}\n\nfunc (ss *stubSearcher) SetBytesRead(val uint64) {\n\n}\n\nfunc (ss *stubSearcher) BytesRead() uint64 {\n\treturn 0\n}\n\nfunc (ss *stubSearcher) Size() int {\n\tsizeInBytes := int(reflect.TypeOf(*ss).Size())\n\n\tfor _, entry := range ss.matches {\n\t\tif entry != nil {\n\t\t\tsizeInBytes += entry.Size()\n\t\t}\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (ss *stubSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\tif ss.index < len(ss.matches) {\n\t\trv := ctx.DocumentMatchPool.Get()\n\t\trv.IndexInternalID = ss.matches[ss.index].IndexInternalID\n\t\trv.Score = ss.matches[ss.index].Score\n\t\tss.index++\n\t\treturn rv, nil\n\t}\n\treturn nil, nil\n}\n\nfunc (ss *stubSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\n\tfor ss.index < len(ss.matches) && ss.matches[ss.index].IndexInternalID.Compare(ID) < 0 {\n\t\tss.index++\n\t}\n\tif ss.index < len(ss.matches) {\n\t\trv := ctx.DocumentMatchPool.Get()\n\t\trv.IndexInternalID = ss.matches[ss.index].IndexInternalID\n\t\trv.Score = ss.matches[ss.index].Score\n\t\tss.index++\n\t\treturn rv, nil\n\t}\n\treturn nil, nil\n}\n\nfunc (ss *stubSearcher) Close() error {\n\treturn nil\n}\n\nfunc (ss *stubSearcher) Weight() float64 {\n\treturn 0.0\n}\n\nfunc (ss *stubSearcher) SetQueryNorm(float64) {\n}\n\nfunc (ss *stubSearcher) Count() uint64 {\n\treturn uint64(len(ss.matches))\n}\n\nfunc (ss *stubSearcher) Min() int {\n\treturn 0\n}\n\nfunc (ss *stubSearcher) DocumentMatchPoolSize() int {\n\treturn 0\n}\n\ntype stubReader struct{}\n\nfunc (sr *stubReader) Size() int {\n\treturn 0\n}\n\nfunc (sr *stubReader) TermFieldReader(ctx context.Context, term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) DocIDReaderAll() (index.DocIDReader, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) FieldDict(field string) (index.FieldDict, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) Document(id string) (index.Document, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) DocumentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocValueVisitor) error {\n\treturn nil\n}\n\nfunc (sr *stubReader) Fields() ([]string, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) GetInternal(key []byte) ([]byte, error) {\n\treturn nil, nil\n}\n\nfunc (sr *stubReader) DocCount() (uint64, error) {\n\treturn 0, nil\n}\n\nfunc (sr *stubReader) ExternalID(id index.IndexInternalID) (string, error) {\n\treturn string(id), nil\n}\n\nfunc (sr *stubReader) InternalID(id string) (index.IndexInternalID, error) {\n\treturn []byte(id), nil\n}\n\nfunc (sr *stubReader) DumpAll() chan interface{} {\n\treturn nil\n}\n\nfunc (sr *stubReader) DumpDoc(id string) chan interface{} {\n\treturn nil\n}\n\nfunc (sr *stubReader) DumpFields() chan interface{} {\n\treturn nil\n}\n\nfunc (sr *stubReader) Close() error {\n\treturn nil\n}\n\nfunc (sr *stubReader) DocValueReader(fields []string) (index.DocValueReader, error) {\n\treturn &DocValueReader{i: sr, fields: fields}, nil\n}\n\ntype DocValueReader struct {\n\ti      *stubReader\n\tfields []string\n}\n\nfunc (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, visitor index.DocValueVisitor) error {\n\treturn dvr.i.DocumentVisitFieldTerms(id, dvr.fields, visitor)\n}\nfunc (dvr *DocValueReader) BytesRead() uint64 {\n\treturn 0\n}\n"
  },
  {
    "path": "search/collector/slice.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage collector\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\ntype collectStoreSlice struct {\n\tslice   search.DocumentMatchCollection\n\tcompare collectorCompare\n}\n\nfunc newStoreSlice(capacity int, compare collectorCompare) *collectStoreSlice {\n\trv := &collectStoreSlice{\n\t\tslice:   make(search.DocumentMatchCollection, 0, capacity),\n\t\tcompare: compare,\n\t}\n\treturn rv\n}\n\nfunc (c *collectStoreSlice) AddNotExceedingSize(doc *search.DocumentMatch,\n\tsize int) *search.DocumentMatch {\n\tc.add(doc)\n\tif c.len() > size {\n\t\treturn c.removeLast()\n\t}\n\treturn nil\n}\n\nfunc (c *collectStoreSlice) add(doc *search.DocumentMatch) {\n\t// find where to insert, starting at end (lowest)\n\ti := len(c.slice)\n\tfor ; i > 0; i-- {\n\t\tcmp := c.compare(doc, c.slice[i-1])\n\t\tif cmp >= 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\t// insert at i\n\tc.slice = append(c.slice, nil)\n\tcopy(c.slice[i+1:], c.slice[i:])\n\tc.slice[i] = doc\n}\n\nfunc (c *collectStoreSlice) removeLast() *search.DocumentMatch {\n\tvar rv *search.DocumentMatch\n\trv, c.slice = c.slice[len(c.slice)-1], c.slice[:len(c.slice)-1]\n\treturn rv\n}\n\nfunc (c *collectStoreSlice) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) {\n\tfor i := skip; i < len(c.slice); i++ {\n\t\terr := fixup(c.slice[i])\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tif skip <= len(c.slice) {\n\t\treturn c.slice[skip:], nil\n\t}\n\treturn search.DocumentMatchCollection{}, nil\n}\n\nfunc (c *collectStoreSlice) Internal() search.DocumentMatchCollection {\n\treturn c.slice\n}\n\nfunc (c *collectStoreSlice) len() int {\n\treturn len(c.slice)\n}\n"
  },
  {
    "path": "search/collector/topn.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage collector\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeTopNCollector int\n\nfunc init() {\n\tvar coll TopNCollector\n\treflectStaticSizeTopNCollector = int(reflect.TypeOf(coll).Size())\n}\n\ntype collectorStore interface {\n\t// Add the document, and if the new store size exceeds the provided size\n\t// the last element is removed and returned.  If the size has not been\n\t// exceeded, nil is returned.\n\tAddNotExceedingSize(doc *search.DocumentMatch, size int) *search.DocumentMatch\n\n\tFinal(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error)\n\n\t// Provide access the internal heap implementation\n\tInternal() search.DocumentMatchCollection\n}\n\n// PreAllocSizeSkipCap will cap preallocation to this amount when\n// size+skip exceeds this value\nvar PreAllocSizeSkipCap = 1000\n\ntype collectorCompare func(i, j *search.DocumentMatch) int\n\ntype collectorFixup func(d *search.DocumentMatch) error\n\n// TopNCollector collects the top N hits, optionally skipping some results\ntype TopNCollector struct {\n\tsize          int\n\tskip          int\n\ttotal         uint64\n\tbytesRead     uint64\n\tmaxScore      float64\n\ttook          time.Duration\n\tsort          search.SortOrder\n\tresults       search.DocumentMatchCollection\n\tfacetsBuilder *search.FacetsBuilder\n\n\tstore collectorStore\n\n\tneedDocIds    bool\n\tneededFields  []string\n\tcachedScoring []bool\n\tcachedDesc    []bool\n\n\tlowestMatchOutsideResults *search.DocumentMatch\n\tupdateFieldVisitor        index.DocValueVisitor\n\tdvReader                  index.DocValueReader\n\tsearchAfter               *search.DocumentMatch\n\n\tknnHits             map[string]*search.DocumentMatch\n\thybridMergeCallback search.HybridMergeCallbackFn\n\n\tnestedStore *collectStoreNested\n}\n\n// CheckDoneEvery controls how frequently we check the context deadline\nconst CheckDoneEvery = uint64(1024)\n\n// NewTopNCollector builds a collector to find the top 'size' hits\n// skipping over the first 'skip' hits\n// ordering hits by the provided sort order\nfunc NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector {\n\treturn newTopNCollector(size, skip, sort, nil)\n}\n\n// NewTopNCollectorAfter builds a collector to find the top 'size' hits\n// skipping over the first 'skip' hits\n// ordering hits by the provided sort order\n// starting after the provided 'after' sort values\nfunc NewTopNCollectorAfter(size int, sort search.SortOrder, after []string) *TopNCollector {\n\trv := newTopNCollector(size, 0, sort, nil)\n\trv.searchAfter = createSearchAfterDocument(sort, after)\n\treturn rv\n}\n\n// NewNestedTopNCollector builds a collector to find the top 'size' hits\n// skipping over the first 'skip' hits\n// ordering hits by the provided sort order\n// while ensuring the nested documents are handled correctly\n// (i.e. parent document is returned instead of nested document)\nfunc NewNestedTopNCollector(size int, skip int, sort search.SortOrder, nr index.NestedReader) *TopNCollector {\n\treturn newTopNCollector(size, skip, sort, nr)\n}\n\n// NewNestedTopNCollectorAfter builds a collector to find the top 'size' hits\n// skipping over the first 'skip' hits\n// ordering hits by the provided sort order\n// starting after the provided 'after' sort values\n// while ensuring the nested documents are handled correctly\n// (i.e. parent document is returned instead of nested document)\nfunc NewNestedTopNCollectorAfter(size int, sort search.SortOrder, after []string, nr index.NestedReader) *TopNCollector {\n\trv := newTopNCollector(size, 0, sort, nr)\n\trv.searchAfter = createSearchAfterDocument(sort, after)\n\treturn rv\n}\n\nfunc newTopNCollector(size int, skip int, sort search.SortOrder, nr index.NestedReader) *TopNCollector {\n\thc := &TopNCollector{size: size, skip: skip, sort: sort}\n\n\thc.store = getOptimalCollectorStore(size, skip, func(i, j *search.DocumentMatch) int {\n\t\treturn hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, i, j)\n\t})\n\n\tif nr != nil {\n\t\tdescAdder := func(parent, child *search.DocumentMatch) error {\n\t\t\t// add descendant score to parent score\n\t\t\tparent.Score += child.Score\n\t\t\t// merge explanations\n\t\t\tparent.Expl = parent.Expl.MergeWith(child.Expl)\n\t\t\t// merge field term locations\n\t\t\tparent.FieldTermLocations = search.MergeFieldTermLocationsFromMatch(parent.FieldTermLocations, child)\n\t\t\t// add child's ID to parent's Descendants\n\t\t\t// add other as descendant only if it is not the same document\n\t\t\tif !parent.IndexInternalID.Equals(child.IndexInternalID) {\n\t\t\t\t// Add a copy of child.IndexInternalID to descendants, because\n\t\t\t\t// child.IndexInternalID will be reset when 'child' is recycled.\n\t\t\t\tvar descendantID index.IndexInternalID\n\t\t\t\t// first check if parent's descendants slice has capacity to reuse\n\t\t\t\tif len(parent.Descendants) < cap(parent.Descendants) {\n\t\t\t\t\t// reuse the buffer element at len(parent.Descendants)\n\t\t\t\t\tdescendantID = parent.Descendants[:len(parent.Descendants)+1][len(parent.Descendants)]\n\t\t\t\t}\n\t\t\t\t// copy the contents of id into descendantID, allocating if needed\n\t\t\t\tparent.Descendants = append(parent.Descendants, index.NewIndexInternalIDFrom(descendantID, child.IndexInternalID))\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\thc.nestedStore = newStoreNested(nr, search.DescendantAdderCallbackFn(descAdder))\n\t}\n\n\t// these lookups traverse an interface, so do once up-front\n\tif sort.RequiresDocID() {\n\t\thc.needDocIds = true\n\t}\n\thc.neededFields = sort.RequiredFields()\n\thc.cachedScoring = sort.CacheIsScore()\n\thc.cachedDesc = sort.CacheDescending()\n\n\treturn hc\n}\n\n// Creates a dummy document to compare with for pagination.\nfunc createSearchAfterDocument(sort search.SortOrder, after []string) *search.DocumentMatch {\n\tencodedAfter := make([]string, len(after))\n\tfor i, ss := range sort {\n\t\tencodedAfter[i] = encodeSearchAfter(ss, after[i])\n\t}\n\n\trv := &search.DocumentMatch{\n\t\tSort: encodedAfter,\n\t}\n\tfor pos, ss := range sort {\n\t\tif ss.RequiresDocID() {\n\t\t\trv.ID = after[pos]\n\t\t}\n\t\tif ss.RequiresScoring() {\n\t\t\tif score, err := strconv.ParseFloat(after[pos], 64); err == nil {\n\t\t\t\trv.Score = score\n\t\t\t}\n\t\t}\n\t}\n\treturn rv\n}\n\n// encodeSearchAfter applies prefix-coding to SearchAfter\n// if required to enable pagination on numeric, datetime,\n// and geo fields\nfunc encodeSearchAfter(ss search.SearchSort, after string) string {\n\tencodeFloat := func() string {\n\t\tf64, _ := strconv.ParseFloat(after, 64) // error checking in SearchRequest.Validate\n\t\ti64 := numeric.Float64ToInt64(f64)\n\t\treturn string(numeric.MustNewPrefixCodedInt64(i64, 0))\n\t}\n\n\tencodeDate := func() string {\n\t\tt, _ := time.Parse(time.RFC3339Nano, after) // error checking in SearchRequest.Validate\n\t\ti64 := t.UnixNano()\n\t\treturn string(numeric.MustNewPrefixCodedInt64(i64, 0))\n\t}\n\n\tswitch ss := ss.(type) {\n\tcase *search.SortGeoDistance:\n\t\treturn encodeFloat()\n\tcase *search.SortField:\n\t\tswitch ss.Type {\n\t\tcase search.SortFieldAsNumber:\n\t\t\treturn encodeFloat()\n\t\tcase search.SortFieldAsDate:\n\t\t\treturn encodeDate()\n\t\tdefault:\n\t\t\t// For SortFieldAsString and SortFieldAuto\n\t\t\t// NOTE: SortFieldAuto is used if you set Sort with a string\n\t\t\t// or if the type of the field is not set in the object\n\t\t\t// in the Sort slice. We cannot perform type inference in\n\t\t\t// this case, so we return the original string, even if\n\t\t\t// its actually numeric or date.\n\t\t\treturn after\n\t\t}\n\tdefault:\n\t\t// For SortDocID and SortScore\n\t\treturn after\n\t}\n}\n\n// Filter document matches based on the SearchAfter field in the SearchRequest.\nfunc FilterHitsBySearchAfter(hits []*search.DocumentMatch, sort search.SortOrder, after []string) []*search.DocumentMatch {\n\tif len(hits) == 0 {\n\t\treturn hits\n\t}\n\t// create a search after document\n\tsearchAfter := createSearchAfterDocument(sort, after)\n\t// filter the hits\n\tidx := 0\n\tcachedScoring := sort.CacheIsScore()\n\tcachedDesc := sort.CacheDescending()\n\tfor _, hit := range hits {\n\t\tif sort.Compare(cachedScoring, cachedDesc, hit, searchAfter) > 0 {\n\t\t\thits[idx] = hit\n\t\t\tidx++\n\t\t}\n\t}\n\treturn hits[:idx]\n}\n\nfunc getOptimalCollectorStore(size, skip int, comparator collectorCompare) collectorStore {\n\t// pre-allocate space on the store to avoid reslicing\n\t// unless the size + skip is too large, then cap it\n\t// everything should still work, just reslices as necessary\n\tbackingSize := size + skip + 1\n\tif size+skip > PreAllocSizeSkipCap {\n\t\tbackingSize = PreAllocSizeSkipCap + 1\n\t}\n\n\tif size+skip > 10 {\n\t\treturn newStoreHeap(backingSize, comparator)\n\t} else {\n\t\treturn newStoreSlice(backingSize, comparator)\n\t}\n}\n\nfunc (hc *TopNCollector) Size() int {\n\tsizeInBytes := reflectStaticSizeTopNCollector + size.SizeOfPtr\n\n\tif hc.facetsBuilder != nil {\n\t\tsizeInBytes += hc.facetsBuilder.Size()\n\t}\n\n\tfor _, entry := range hc.neededFields {\n\t\tsizeInBytes += len(entry) + size.SizeOfString\n\t}\n\n\tsizeInBytes += len(hc.cachedScoring) + len(hc.cachedDesc)\n\n\treturn sizeInBytes\n}\n\n// Collect goes to the index to find the matching documents\nfunc (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error {\n\tstartTime := time.Now()\n\tvar err error\n\tvar next *search.DocumentMatch\n\n\t// pre-allocate enough space in the DocumentMatchPool\n\t// unless the size + skip is too large, then cap it\n\t// everything should still work, just allocates DocumentMatches on demand\n\tbackingSize := hc.size + hc.skip + 1\n\tif hc.size+hc.skip > PreAllocSizeSkipCap {\n\t\tbackingSize = PreAllocSizeSkipCap + 1\n\t}\n\tsearchContext := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), len(hc.sort)),\n\t\tCollector:         hc,\n\t\tIndexReader:       reader,\n\t}\n\n\thc.dvReader, err = reader.DocValueReader(hc.neededFields)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\thc.updateFieldVisitor = func(field string, term []byte) {\n\t\tif hc.facetsBuilder != nil {\n\t\t\thc.facetsBuilder.UpdateVisitor(field, term)\n\t\t}\n\t\thc.sort.UpdateVisitor(field, term)\n\t}\n\n\tdmHandlerMaker := MakeTopNDocumentMatchHandler\n\tif cv := ctx.Value(search.MakeDocumentMatchHandlerKey); cv != nil {\n\t\tdmHandlerMaker = cv.(search.MakeDocumentMatchHandler)\n\t}\n\t// use the application given builder for making the custom document match\n\t// handler and perform callbacks/invocations on the newly made handler.\n\tdmHandler, loadID, err := dmHandlerMaker(searchContext)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\thc.needDocIds = hc.needDocIds || loadID\n\tselect {\n\tcase <-ctx.Done():\n\t\tsearch.RecordSearchCost(ctx, search.AbortM, 0)\n\t\treturn ctx.Err()\n\tdefault:\n\t\tnext, err = searcher.Next(searchContext)\n\t}\n\t// use a local totalDocs for counting total docs seen\n\t// for context deadline checking, as hc.total is only\n\t// incremented for actual(root) collected documents, and\n\t// we need to check deadline for every document seen (root or nested)\n\tvar totalDocs uint64\n\tfor err == nil && next != nil {\n\t\tif totalDocs%CheckDoneEvery == 0 {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\tsearch.RecordSearchCost(ctx, search.AbortM, 0)\n\t\t\t\treturn ctx.Err()\n\t\t\tdefault:\n\t\t\t}\n\t\t}\n\t\ttotalDocs++\n\t\tif hc.nestedStore != nil {\n\t\t\t// This may be a nested document — add it to the nested store first.\n\t\t\t// If the nested store returns nil, the document was merged into its parent\n\t\t\t// and should not be processed further.\n\t\t\t// If it returns a non-nil document, it represents a complete root document\n\t\t\t// and should be processed further.\n\t\t\tnext, err = hc.nestedStore.ProcessNestedDocument(searchContext, next)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif next != nil {\n\t\t\terr = hc.adjustDocumentMatch(searchContext, reader, next)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\terr = hc.prepareDocumentMatch(searchContext, reader, next, false)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\terr = dmHandler(next)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tnext, err = searcher.Next(searchContext)\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// if we have a nested store, we may have an interim root\n\t// that needs to be returned for processing\n\tif hc.nestedStore != nil {\n\t\tcurrRoot := hc.nestedStore.Current()\n\t\tif currRoot != nil {\n\t\t\terr = hc.adjustDocumentMatch(searchContext, reader, currRoot)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t// no descendants at this point\n\t\t\terr = hc.prepareDocumentMatch(searchContext, reader, currRoot, false)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\terr = dmHandler(currRoot)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\tif hc.knnHits != nil {\n\t\t// we may have some knn hits left that did not match any of the top N tf-idf hits\n\t\t// we need to add them to the collector store to consider them as well.\n\t\tfor _, knnDoc := range hc.knnHits {\n\t\t\terr = hc.prepareDocumentMatch(searchContext, reader, knnDoc, true)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terr = dmHandler(knnDoc)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\tstatsCallbackFn := ctx.Value(search.SearchIOStatsCallbackKey)\n\tif statsCallbackFn != nil {\n\t\t// hc.bytesRead corresponds to the\n\t\t// total bytes read as part of docValues being read every hit\n\t\t// which must be accounted by invoking the callback.\n\t\tstatsCallbackFn.(search.SearchIOStatsCallbackFunc)(hc.bytesRead)\n\n\t\tsearch.RecordSearchCost(ctx, search.AddM, hc.bytesRead)\n\t}\n\n\t// help finalize/flush the results in case\n\t// of custom document match handlers.\n\terr = dmHandler(nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// compute search duration\n\thc.took = time.Since(startTime)\n\n\t// finalize actual results\n\terr = hc.finalizeResults(reader)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nvar sortByScoreOpt = []string{\"_score\"}\n\nfunc (hc *TopNCollector) adjustDocumentMatch(ctx *search.SearchContext,\n\treader index.IndexReader, d *search.DocumentMatch) (err error) {\n\tif hc.knnHits != nil {\n\t\td.ID, err = reader.ExternalID(d.IndexInternalID)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif knnHit, ok := hc.knnHits[d.ID]; ok {\n\t\t\t// we have a knn hit corresponding to this document\n\t\t\thc.hybridMergeCallback(d, knnHit)\n\t\t\t// remove this knn hit from the map as it's already\n\t\t\t// been merged\n\t\t\tdelete(hc.knnHits, d.ID)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (hc *TopNCollector) prepareDocumentMatch(ctx *search.SearchContext,\n\treader index.IndexReader, d *search.DocumentMatch, isKnnDoc bool) (err error) {\n\n\t// visit field terms for features that require it (sort, facets)\n\tif !isKnnDoc && len(hc.neededFields) > 0 {\n\t\terr = hc.visitFieldTerms(reader, d, hc.updateFieldVisitor)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if isKnnDoc && hc.facetsBuilder != nil {\n\t\t// we need to visit the field terms for the knn document\n\t\t// only for those fields that are required for faceting\n\t\t// and not for sorting. This is because the knn document's\n\t\t// sort value is already computed in the knn collector.\n\t\terr = hc.visitFieldTerms(reader, d, func(field string, term []byte) {\n\t\t\tif hc.facetsBuilder != nil {\n\t\t\t\thc.facetsBuilder.UpdateVisitor(field, term)\n\t\t\t}\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// increment total hits\n\thc.total++\n\td.HitNumber = hc.total\n\n\t// update max score\n\tif d.Score > hc.maxScore {\n\t\thc.maxScore = d.Score\n\t}\n\t// early exit as the document match had its sort value calculated in the knn\n\t// collector itself\n\tif isKnnDoc {\n\t\treturn nil\n\t}\n\n\t// see if we need to load ID (at this early stage, for example to sort on it)\n\tif hc.needDocIds && d.ID == \"\" {\n\t\td.ID, err = reader.ExternalID(d.IndexInternalID)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// compute this hits sort value\n\tif len(hc.sort) == 1 && hc.cachedScoring[0] {\n\t\td.Sort = sortByScoreOpt\n\t} else {\n\t\thc.sort.Value(d)\n\t}\n\n\treturn nil\n}\n\nfunc MakeTopNDocumentMatchHandler(\n\tctx *search.SearchContext) (search.DocumentMatchHandler, bool, error) {\n\tvar hc *TopNCollector\n\tvar ok bool\n\tif hc, ok = ctx.Collector.(*TopNCollector); ok {\n\t\treturn func(d *search.DocumentMatch) error {\n\t\t\tif d == nil {\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\t// support search after based pagination,\n\t\t\t// if this hit is <= the search after sort key\n\t\t\t// we should skip it\n\t\t\tif hc.searchAfter != nil {\n\t\t\t\t// exact sort order matches use hit number to break tie\n\t\t\t\t// but we want to allow for exact match, so we pretend\n\t\t\t\thc.searchAfter.HitNumber = d.HitNumber\n\t\t\t\tif hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, d, hc.searchAfter) <= 0 {\n\t\t\t\t\tctx.DocumentMatchPool.Put(d)\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// optimization, we track lowest sorting hit already removed from heap\n\t\t\t// with this one comparison, we can avoid all heap operations if\n\t\t\t// this hit would have been added and then immediately removed\n\t\t\tif hc.lowestMatchOutsideResults != nil {\n\t\t\t\tcmp := hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, d,\n\t\t\t\t\thc.lowestMatchOutsideResults)\n\t\t\t\tif cmp >= 0 {\n\t\t\t\t\t// this hit can't possibly be in the result set, so avoid heap ops\n\t\t\t\t\tctx.DocumentMatchPool.Put(d)\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tremoved := hc.store.AddNotExceedingSize(d, hc.size+hc.skip)\n\t\t\tif removed != nil {\n\t\t\t\tif hc.lowestMatchOutsideResults == nil {\n\t\t\t\t\thc.lowestMatchOutsideResults = removed\n\t\t\t\t} else {\n\t\t\t\t\tcmp := hc.sort.Compare(hc.cachedScoring, hc.cachedDesc,\n\t\t\t\t\t\tremoved, hc.lowestMatchOutsideResults)\n\t\t\t\t\tif cmp < 0 {\n\t\t\t\t\t\ttmp := hc.lowestMatchOutsideResults\n\t\t\t\t\t\thc.lowestMatchOutsideResults = removed\n\t\t\t\t\t\tctx.DocumentMatchPool.Put(tmp)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}, false, nil\n\t}\n\treturn nil, false, nil\n}\n\n// visitFieldTerms is responsible for visiting the field terms of the\n// search hit, and passing visited terms to the sort and facet builder\nfunc (hc *TopNCollector) visitFieldTerms(reader index.IndexReader, d *search.DocumentMatch, v index.DocValueVisitor) error {\n\tif hc.facetsBuilder != nil {\n\t\thc.facetsBuilder.StartDoc()\n\t}\n\tif d.ID != \"\" && d.IndexInternalID == nil {\n\t\t// this document may have been sent over as preSearchData and\n\t\t// we need to look up the internal id to visit the doc values for it\n\t\tvar err error\n\t\td.IndexInternalID, err = reader.InternalID(d.ID)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// first visit descendants if any\n\tfor _, descID := range d.Descendants {\n\t\terr := hc.dvReader.VisitDocValues(descID, v)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// now visit the doc values for this document\n\terr := hc.dvReader.VisitDocValues(d.IndexInternalID, v)\n\tif hc.facetsBuilder != nil {\n\t\thc.facetsBuilder.EndDoc()\n\t}\n\n\thc.bytesRead += hc.dvReader.BytesRead()\n\n\treturn err\n}\n\n// SetFacetsBuilder registers a facet builder for this collector\nfunc (hc *TopNCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) {\n\thc.facetsBuilder = facetsBuilder\n\tfieldsRequiredForFaceting := facetsBuilder.RequiredFields()\n\t// for each of these fields, append only if not already there in hc.neededFields.\n\tfor _, field := range fieldsRequiredForFaceting {\n\t\tfound := false\n\t\tfor _, neededField := range hc.neededFields {\n\t\t\tif field == neededField {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\thc.neededFields = append(hc.neededFields, field)\n\t\t}\n\t}\n}\n\n// finalizeResults starts with the heap containing the final top size+skip\n// it now throws away the results to be skipped\n// and does final doc id lookup (if necessary)\nfunc (hc *TopNCollector) finalizeResults(r index.IndexReader) error {\n\tvar err error\n\thc.results, err = hc.store.Final(hc.skip, func(doc *search.DocumentMatch) error {\n\t\tif doc.ID == \"\" {\n\t\t\t// look up the id since we need it for lookup\n\t\t\tvar err error\n\t\t\tdoc.ID, err = r.ExternalID(doc.IndexInternalID)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tdoc.Complete(nil)\n\t\treturn nil\n\t})\n\n\treturn err\n}\n\n// Results returns the collected hits\nfunc (hc *TopNCollector) Results() search.DocumentMatchCollection {\n\treturn hc.results\n}\n\n// Total returns the total number of hits\nfunc (hc *TopNCollector) Total() uint64 {\n\treturn hc.total\n}\n\n// MaxScore returns the maximum score seen across all the hits\nfunc (hc *TopNCollector) MaxScore() float64 {\n\treturn hc.maxScore\n}\n\n// Took returns the time spent collecting hits\nfunc (hc *TopNCollector) Took() time.Duration {\n\treturn hc.took\n}\n\n// FacetResults returns the computed facets results\nfunc (hc *TopNCollector) FacetResults() search.FacetResults {\n\tif hc.facetsBuilder != nil {\n\t\treturn hc.facetsBuilder.Results()\n\t}\n\treturn nil\n}\n\nfunc (hc *TopNCollector) SetKNNHits(knnHits search.DocumentMatchCollection, hybridMergeCallback search.HybridMergeCallbackFn) {\n\thc.knnHits = make(map[string]*search.DocumentMatch, len(knnHits))\n\tfor _, hit := range knnHits {\n\t\thc.knnHits[hit.ID] = hit\n\t}\n\thc.hybridMergeCallback = hybridMergeCallback\n}\n"
  },
  {
    "path": "search/collector/topn_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage collector\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/facet\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestTop10Scores(t *testing.T) {\n\t// a stub search with more than 10 matches\n\t// the top-10 scores are > 10\n\t// everything else is less than 10\n\tsearcher := &stubSearcher{\n\t\tmatches: []*search.DocumentMatch{\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"a\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"b\"),\n\t\t\t\tScore:           9,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"c\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"d\"),\n\t\t\t\tScore:           9,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"e\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"f\"),\n\t\t\t\tScore:           9,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"g\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"h\"),\n\t\t\t\tScore:           9,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"i\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"j\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"k\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"l\"),\n\t\t\t\tScore:           99,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"m\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"n\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t},\n\t}\n\n\tcollector := NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\terr := collector.Collect(context.Background(), searcher, &stubReader{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmaxScore := collector.MaxScore()\n\tif maxScore != 99.0 {\n\t\tt.Errorf(\"expected max score 99.0, got %f\", maxScore)\n\t}\n\n\ttotal := collector.Total()\n\tif total != 14 {\n\t\tt.Errorf(\"expected 14 total results, got %d\", total)\n\t}\n\n\tresults := collector.Results()\n\n\tif len(results) != 10 {\n\t\tt.Logf(\"results: %v\", results)\n\t\tt.Fatalf(\"expected 10 results, got %d\", len(results))\n\t}\n\n\tif results[0].ID != \"l\" {\n\t\tt.Errorf(\"expected first result to have ID 'l', got %s\", results[0].ID)\n\t}\n\n\tif results[0].Score != 99.0 {\n\t\tt.Errorf(\"expected highest score to be 99.0, got %f\", results[0].Score)\n\t}\n\n\tminScore := 1000.0\n\tfor _, result := range results {\n\t\tif result.Score < minScore {\n\t\t\tminScore = result.Score\n\t\t}\n\t}\n\n\tif minScore < 10 {\n\t\tt.Errorf(\"expected minimum score to be higher than 10, got %f\", minScore)\n\t}\n}\n\nfunc TestTop10ScoresSkip10(t *testing.T) {\n\t// a stub search with more than 10 matches\n\t// the top-10 scores are > 10\n\t// everything else is less than 10\n\tsearcher := &stubSearcher{\n\t\tmatches: []*search.DocumentMatch{\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"a\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"b\"),\n\t\t\t\tScore:           9.5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"c\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"d\"),\n\t\t\t\tScore:           9,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"e\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"f\"),\n\t\t\t\tScore:           9,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"g\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"h\"),\n\t\t\t\tScore:           9,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"i\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"j\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"k\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"l\"),\n\t\t\t\tScore:           99,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"m\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"n\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t},\n\t}\n\n\tcollector := NewTopNCollector(10, 10, search.SortOrder{&search.SortScore{Desc: true}})\n\terr := collector.Collect(context.Background(), searcher, &stubReader{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmaxScore := collector.MaxScore()\n\tif maxScore != 99.0 {\n\t\tt.Errorf(\"expected max score 99.0, got %f\", maxScore)\n\t}\n\n\ttotal := collector.Total()\n\tif total != 14 {\n\t\tt.Errorf(\"expected 14 total results, got %d\", total)\n\t}\n\n\tresults := collector.Results()\n\n\tif len(results) != 4 {\n\t\tt.Fatalf(\"expected 4 results, got %d\", len(results))\n\t}\n\n\tif results[0].ID != \"b\" {\n\t\tt.Errorf(\"expected first result to have ID 'b', got %s\", results[0].ID)\n\t}\n\n\tif results[0].Score != 9.5 {\n\t\tt.Errorf(\"expected highest score to be 9.5, got %f\", results[0].Score)\n\t}\n}\n\nfunc TestTop10ScoresSkip10Only9Hits(t *testing.T) {\n\t// a stub search with only 10 matches\n\tsearcher := &stubSearcher{\n\t\tmatches: []*search.DocumentMatch{\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"a\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"c\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"e\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"g\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"i\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"j\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"k\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"m\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"n\"),\n\t\t\t\tScore:           11,\n\t\t\t},\n\t\t},\n\t}\n\n\tcollector := NewTopNCollector(10, 10, search.SortOrder{&search.SortScore{Desc: true}})\n\terr := collector.Collect(context.Background(), searcher, &stubReader{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttotal := collector.Total()\n\tif total != 9 {\n\t\tt.Errorf(\"expected 9 total results, got %d\", total)\n\t}\n\n\tresults := collector.Results()\n\n\tif len(results) != 0 {\n\t\tt.Fatalf(\"expected 0 results, got %d\", len(results))\n\t}\n}\n\nfunc TestPaginationSameScores(t *testing.T) {\n\t// a stub search with more than 10 matches\n\t// all documents have the same score\n\tsearcher := &stubSearcher{\n\t\tmatches: []*search.DocumentMatch{\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"a\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"b\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"c\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"d\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"e\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"f\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"g\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"h\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"i\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"j\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"k\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"l\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"m\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"n\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t},\n\t}\n\n\t// first get first 5 hits\n\tcollector := NewTopNCollector(5, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\terr := collector.Collect(context.Background(), searcher, &stubReader{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttotal := collector.Total()\n\tif total != 14 {\n\t\tt.Errorf(\"expected 14 total results, got %d\", total)\n\t}\n\n\tresults := collector.Results()\n\n\tif len(results) != 5 {\n\t\tt.Fatalf(\"expected 5 results, got %d\", len(results))\n\t}\n\n\tfirstResults := make(map[string]struct{})\n\tfor _, hit := range results {\n\t\tfirstResults[hit.ID] = struct{}{}\n\t}\n\n\t// a stub search with more than 10 matches\n\t// all documents have the same score\n\tsearcher = &stubSearcher{\n\t\tmatches: []*search.DocumentMatch{\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"a\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"b\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"c\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"d\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"e\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"f\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"g\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"h\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"i\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"j\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"k\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"l\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"m\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t\t{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"n\"),\n\t\t\t\tScore:           5,\n\t\t\t},\n\t\t},\n\t}\n\n\t// now get next 5 hits\n\tcollector = NewTopNCollector(5, 5, search.SortOrder{&search.SortScore{Desc: true}})\n\terr = collector.Collect(context.Background(), searcher, &stubReader{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttotal = collector.Total()\n\tif total != 14 {\n\t\tt.Errorf(\"expected 14 total results, got %d\", total)\n\t}\n\n\tresults = collector.Results()\n\n\tif len(results) != 5 {\n\t\tt.Fatalf(\"expected 5 results, got %d\", len(results))\n\t}\n\n\t// make sure that none of these hits repeat ones we saw in the top 5\n\tfor _, hit := range results {\n\t\tif _, ok := firstResults[hit.ID]; ok {\n\t\t\tt.Errorf(\"doc ID %s is in top 5 and next 5 result sets\", hit.ID)\n\t\t}\n\t}\n}\n\n// TestStreamResults verifies the search.DocumentMatchHandler\nfunc TestStreamResults(t *testing.T) {\n\tmatches := []*search.DocumentMatch{\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"a\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"b\"),\n\t\t\tScore:           1,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"c\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"d\"),\n\t\t\tScore:           999,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"e\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"f\"),\n\t\t\tScore:           9,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"g\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"h\"),\n\t\t\tScore:           89,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"i\"),\n\t\t\tScore:           101,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"j\"),\n\t\t\tScore:           112,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"k\"),\n\t\t\tScore:           10,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"l\"),\n\t\t\tScore:           99,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"m\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"n\"),\n\t\t\tScore:           111,\n\t\t},\n\t}\n\n\tsearcher := &stubSearcher{\n\t\tmatches: matches,\n\t}\n\tind := 0\n\tdocMatchHandler := func(hit *search.DocumentMatch) error {\n\t\tif hit == nil {\n\t\t\treturn nil // search completed\n\t\t}\n\t\tif !bytes.Equal(hit.IndexInternalID, matches[ind].IndexInternalID) {\n\t\t\tt.Errorf(\"%d hit IndexInternalID actual: %s, expected: %s\",\n\t\t\t\tind, hit.IndexInternalID, matches[ind].IndexInternalID)\n\t\t}\n\t\tif hit.Score != matches[ind].Score {\n\t\t\tt.Errorf(\"%d hit Score actual: %s, expected: %s\",\n\t\t\t\tind, hit.IndexInternalID, matches[ind].IndexInternalID)\n\t\t}\n\t\tind++\n\t\treturn nil\n\t}\n\n\tvar handlerMaker search.MakeDocumentMatchHandler = func(ctx *search.SearchContext) (search.DocumentMatchHandler, bool, error) {\n\t\treturn docMatchHandler, false, nil\n\t}\n\n\tctx := context.WithValue(context.Background(), search.MakeDocumentMatchHandlerKey, handlerMaker)\n\n\tcollector := NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\terr := collector.Collect(ctx, searcher, &stubReader{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmaxScore := collector.MaxScore()\n\tif maxScore != 999.0 {\n\t\tt.Errorf(\"expected max score 99.0, got %f\", maxScore)\n\t}\n\n\ttotal := collector.Total()\n\tif int(total) != ind {\n\t\tt.Errorf(\"expected 14 total results, got %d\", total)\n\t}\n\n\tresults := collector.Results()\n\n\tif len(results) != 0 {\n\t\tt.Fatalf(\"expected 0 results, got %d\", len(results))\n\t}\n}\n\n// TestCollectorChaining verifies the chaining of collectors.\n// The custom DocumentMatchHandler can process every hit for\n// the search query and then pass the hit to the topn collector\n// to eventually have the sorted top `N` results.\nfunc TestCollectorChaining(t *testing.T) {\n\tmatches := []*search.DocumentMatch{\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"a\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"b\"),\n\t\t\tScore:           1,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"c\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"d\"),\n\t\t\tScore:           999,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"e\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"f\"),\n\t\t\tScore:           9,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"g\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"h\"),\n\t\t\tScore:           89,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"i\"),\n\t\t\tScore:           101,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"j\"),\n\t\t\tScore:           112,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"k\"),\n\t\t\tScore:           10,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"l\"),\n\t\t\tScore:           99,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"m\"),\n\t\t\tScore:           11,\n\t\t},\n\t\t{\n\t\t\tIndexInternalID: index.IndexInternalID(\"n\"),\n\t\t\tScore:           111,\n\t\t},\n\t}\n\n\tsearcher := &stubSearcher{\n\t\tmatches: matches,\n\t}\n\n\tvar topNHandler search.DocumentMatchHandler\n\tind := 0\n\tdocMatchHandler := func(hit *search.DocumentMatch) error {\n\t\tif hit == nil {\n\t\t\treturn nil // search completed\n\t\t}\n\t\tif !bytes.Equal(hit.IndexInternalID, matches[ind].IndexInternalID) {\n\t\t\tt.Errorf(\"%d hit IndexInternalID actual: %s, expected: %s\",\n\t\t\t\tind, hit.IndexInternalID, matches[ind].IndexInternalID)\n\t\t}\n\t\tif hit.Score != matches[ind].Score {\n\t\t\tt.Errorf(\"%d hit Score actual: %s, expected: %s\",\n\t\t\t\tind, hit.IndexInternalID, matches[ind].IndexInternalID)\n\t\t}\n\t\tind++\n\t\t// give the hit back to the topN collector\n\t\terr := topNHandler(hit)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"unexpected err: %v\", err)\n\t\t}\n\t\treturn nil\n\t}\n\n\tvar handlerMaker search.MakeDocumentMatchHandler = func(ctx *search.SearchContext) (search.DocumentMatchHandler, bool, error) {\n\t\ttopNHandler, _, _ = MakeTopNDocumentMatchHandler(ctx)\n\t\treturn docMatchHandler, false, nil\n\t}\n\n\tctx := context.WithValue(context.Background(), search.MakeDocumentMatchHandlerKey,\n\t\thandlerMaker)\n\n\tcollector := NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\terr := collector.Collect(ctx, searcher, &stubReader{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmaxScore := collector.MaxScore()\n\tif maxScore != 999.0 {\n\t\tt.Errorf(\"expected max score 99.0, got %f\", maxScore)\n\t}\n\n\ttotal := collector.Total()\n\tif int(total) != ind {\n\t\tt.Errorf(\"expected 14 total results, got %d\", total)\n\t}\n\n\tresults := collector.Results()\n\n\tif len(results) != 10 { // as it is paged\n\t\tt.Fatalf(\"expected 0 results, got %d\", len(results))\n\t}\n\n\tif results[0].ID != \"d\" {\n\t\tt.Errorf(\"expected first result to have ID 'l', got %s\", results[0].ID)\n\t}\n\n\tif results[0].Score != 999.0 {\n\t\tt.Errorf(\"expected highest score to be 999.0, got %f\", results[0].Score)\n\t}\n\n\tminScore := 1000.0\n\tfor _, result := range results {\n\t\tif result.Score < minScore {\n\t\t\tminScore = result.Score\n\t\t}\n\t}\n\n\tif minScore < 10 {\n\t\tt.Errorf(\"expected minimum score to be higher than 10, got %f\", minScore)\n\t}\n}\n\nfunc setupIndex(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tscorch.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": \"\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n\nfunc TestSetFacetsBuilder(t *testing.T) {\n\t// Field common to both sorting and faceting.\n\tsortFacetsField := \"locations\"\n\n\tcoll := NewTopNCollector(10, 0, search.SortOrder{&search.SortField{Field: sortFacetsField}})\n\n\ti := setupIndex(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfb := search.NewFacetsBuilder(indexReader)\n\tfacetBuilder := facet.NewTermsFacetBuilder(sortFacetsField, 100)\n\tfb.Add(\"locations_facet\", facetBuilder)\n\tcoll.SetFacetsBuilder(fb)\n\n\t// Should not duplicate the \"locations\" field in the collector.\n\tif len(coll.neededFields) != 1 || coll.neededFields[0] != sortFacetsField {\n\t\tt.Errorf(\"expected fields in collector: %v, observed: %v\", []string{sortFacetsField}, coll.neededFields)\n\t}\n}\n\nfunc TestSearchAfterNumeric(t *testing.T) {\n\tidx := setupIndex(t)\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocs := []struct {\n\t\tid   string\n\t\tdata int64\n\t}{\n\t\t{\"a\", 10},\n\t\t{\"b\", 9},\n\t\t{\"c\", 8},\n\t\t{\"d\", 7},\n\t\t{\"e\", 6},\n\t\t{\"f\", 5},\n\t\t{\"g\", 4},\n\t\t{\"h\", 3},\n\t\t{\"i\", 2},\n\t\t{\"j\", 1},\n\t}\n\n\tbatch := index.NewBatch()\n\tfor _, d := range docs {\n\t\tdoc := document.NewDocument(d.id)\n\t\tfield := document.NewNumericFieldWithIndexingOptions(\"data\", []uint64{}, float64(d.data), index.IndexField|index.StoreField|index.IncludeTermVectors)\n\t\tdoc.AddField(field)\n\t\tbatch.Update(doc)\n\t}\n\n\terr := idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tsearcher, err := searcher.NewMatchAllSearcher(context.Background(), reader, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tsortOrder := search.SortOrder{&search.SortField{Field: \"data\", Type: search.SortFieldAsNumber, Desc: true}}\n\n\tafter := []string{\"6\"}\n\n\tcollectorAfter := NewTopNCollectorAfter(5, sortOrder, after)\n\terr = collectorAfter.Collect(context.Background(), searcher, reader)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresultsAfter := collectorAfter.Results()\n\tif len(resultsAfter) != 5 {\n\t\tt.Fatalf(\"expected 5 results, got %d\", len(resultsAfter))\n\t}\n\tfor i := range resultsAfter {\n\t\traID := resultsAfter[i].ID\n\t\tdocID := docs[i+len(resultsAfter)].id\n\t\tif raID != docID {\n\t\t\tt.Errorf(\"expected result '%s', got '%s'\", docID, raID)\n\t\t}\n\t}\n}\n\nfunc TestSearchAfterDateTime(t *testing.T) {\n\tidx := setupIndex(t)\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocs := []struct {\n\t\tid   string\n\t\tdata time.Time\n\t}{\n\t\t{\"a\", time.Unix(10, 0).UTC()},\n\t\t{\"b\", time.Unix(9, 0).UTC()},\n\t\t{\"c\", time.Unix(8, 0).UTC()},\n\t\t{\"d\", time.Unix(7, 0).UTC()},\n\t\t{\"e\", time.Unix(6, 0).UTC()},\n\t\t{\"f\", time.Unix(5, 0).UTC()},\n\t\t{\"g\", time.Unix(4, 0).UTC()},\n\t\t{\"h\", time.Unix(3, 0).UTC()},\n\t\t{\"i\", time.Unix(2, 0).UTC()},\n\t\t{\"j\", time.Unix(1, 0).UTC()},\n\t}\n\n\tbatch := index.NewBatch()\n\tfor _, d := range docs {\n\t\tdoc := document.NewDocument(d.id)\n\t\tfield, err := document.NewDateTimeFieldWithIndexingOptions(\"data\", []uint64{}, d.data, time.RFC3339Nano, index.IndexField|index.StoreField|index.IncludeTermVectors)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tdoc.AddField(field)\n\t\tbatch.Update(doc)\n\t}\n\n\terr := idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tsearcher, err := searcher.NewMatchAllSearcher(context.Background(), reader, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tsortOrder := search.SortOrder{&search.SortField{Field: \"data\", Type: search.SortFieldAsDate, Desc: true}}\n\n\tafterTime := time.Unix(6, 0).UTC()\n\tafter := []string{afterTime.Format(time.RFC3339Nano)}\n\n\tcollectorAfter := NewTopNCollectorAfter(5, sortOrder, after)\n\terr = collectorAfter.Collect(context.Background(), searcher, reader)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresultsAfter := collectorAfter.Results()\n\tif len(resultsAfter) != 5 {\n\t\tt.Fatalf(\"expected 5 results, got %d\", len(resultsAfter))\n\t}\n\tfor i := range resultsAfter {\n\t\traID := resultsAfter[i].ID\n\t\tdocID := docs[i+len(resultsAfter)].id\n\t\tif raID != docID {\n\t\t\tt.Errorf(\"expected result '%s', got '%s'\", docID, raID)\n\t\t}\n\t}\n}\n\nfunc TestSearchAfterGeo(t *testing.T) {\n\tidx := setupIndex(t)\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocs := []struct {\n\t\tid  string\n\t\tlon float64\n\t\tlat float64\n\t}{\n\t\t{\"a\", 1, 0},\n\t\t{\"b\", 2, 0},\n\t\t{\"c\", 3, 0},\n\t\t{\"d\", 4, 0},\n\t\t{\"e\", 5, 0},\n\t\t{\"f\", 6, 0},\n\t\t{\"g\", 7, 0},\n\t\t{\"h\", 8, 0},\n\t\t{\"i\", 9, 0},\n\t\t{\"j\", 10, 0},\n\t}\n\n\tbatch := index.NewBatch()\n\tfor _, d := range docs {\n\t\tdoc := document.NewDocument(d.id)\n\t\tfield := document.NewGeoPointFieldWithIndexingOptions(\"location\", []uint64{}, d.lon, d.lat, index.IndexField|index.StoreField|index.IncludeTermVectors)\n\t\tdoc.AddField(field)\n\t\tbatch.Update(doc)\n\t}\n\n\terr := idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err := idx.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tsearcher, err := searcher.NewMatchAllSearcher(context.Background(), reader, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcenterLon, centerLat := 0.0, 0.0\n\tsortOrder := search.SortOrder{&search.SortGeoDistance{Field: \"location\", Lon: centerLon, Lat: centerLat, Desc: false}}\n\n\t// search after doc \"e\" which has lon 5, lat 0\n\tafterLon, afterLat := 5.0, 0.0\n\tafterDistance := geo.Haversin(centerLon, centerLat, afterLon, afterLat)\n\t// to compensate scaling\n\tafterDistance *= 1000\n\tafter := []string{strconv.FormatFloat(afterDistance, 'f', -1, 64)}\n\n\tcollectorAfter := NewTopNCollectorAfter(5, sortOrder, after)\n\terr = collectorAfter.Collect(context.Background(), searcher, reader)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresultsAfter := collectorAfter.Results()\n\n\tif len(resultsAfter) != 5 {\n\t\tt.Fatalf(\"expected 5 results, got %d\", len(resultsAfter))\n\t}\n\tfor i := range resultsAfter {\n\t\traID := resultsAfter[i].ID\n\t\tdocID := docs[i+len(resultsAfter)].id\n\t\tif raID != docID {\n\t\t\tt.Errorf(\"expected result '%s', got '%s'\", docID, raID)\n\t\t}\n\t}\n}\n\nfunc BenchmarkTop10of0Scores(b *testing.B) {\n\tbenchHelper(0, func() search.Collector {\n\t\treturn NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop10of3Scores(b *testing.B) {\n\tbenchHelper(3, func() search.Collector {\n\t\treturn NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop10of10Scores(b *testing.B) {\n\tbenchHelper(10, func() search.Collector {\n\t\treturn NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop10of25Scores(b *testing.B) {\n\tbenchHelper(25, func() search.Collector {\n\t\treturn NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop10of50Scores(b *testing.B) {\n\tbenchHelper(50, func() search.Collector {\n\t\treturn NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop10of10000Scores(b *testing.B) {\n\tbenchHelper(10000, func() search.Collector {\n\t\treturn NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop100of0Scores(b *testing.B) {\n\tbenchHelper(0, func() search.Collector {\n\t\treturn NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop100of3Scores(b *testing.B) {\n\tbenchHelper(3, func() search.Collector {\n\t\treturn NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop100of10Scores(b *testing.B) {\n\tbenchHelper(10, func() search.Collector {\n\t\treturn NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop100of25Scores(b *testing.B) {\n\tbenchHelper(25, func() search.Collector {\n\t\treturn NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop100of50Scores(b *testing.B) {\n\tbenchHelper(50, func() search.Collector {\n\t\treturn NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop100of10000Scores(b *testing.B) {\n\tbenchHelper(10000, func() search.Collector {\n\t\treturn NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop1000of10000Scores(b *testing.B) {\n\tbenchHelper(10000, func() search.Collector {\n\t\treturn NewTopNCollector(1000, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop10000of100000Scores(b *testing.B) {\n\tbenchHelper(100000, func() search.Collector {\n\t\treturn NewTopNCollector(10000, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop10of100000Scores(b *testing.B) {\n\tbenchHelper(100000, func() search.Collector {\n\t\treturn NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop100of100000Scores(b *testing.B) {\n\tbenchHelper(100000, func() search.Collector {\n\t\treturn NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop1000of100000Scores(b *testing.B) {\n\tbenchHelper(100000, func() search.Collector {\n\t\treturn NewTopNCollector(1000, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n\nfunc BenchmarkTop10000of1000000Scores(b *testing.B) {\n\tbenchHelper(1000000, func() search.Collector {\n\t\treturn NewTopNCollector(10000, 0, search.SortOrder{&search.SortScore{Desc: true}})\n\t}, b)\n}\n"
  },
  {
    "path": "search/collector.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype Collector interface {\n\tCollect(ctx context.Context, searcher Searcher, reader index.IndexReader) error\n\tResults() DocumentMatchCollection\n\tTotal() uint64\n\tMaxScore() float64\n\tTook() time.Duration\n\tSetFacetsBuilder(facetsBuilder *FacetsBuilder)\n\tFacetResults() FacetResults\n}\n\n// DocumentMatchHandler is the type of document match callback\n// bleve will invoke during the search.\n// Eventually, bleve will indicate the completion of an ongoing search,\n// by passing a nil value for the document match callback.\n// The application should take a copy of the hit/documentMatch\n// if it wish to own it or need prolonged access to it.\ntype DocumentMatchHandler func(hit *DocumentMatch) error\n\ntype MakeDocumentMatchHandlerKeyType string\n\nvar MakeDocumentMatchHandlerKey = MakeDocumentMatchHandlerKeyType(\n\t\"MakeDocumentMatchHandlerKey\")\n\nvar MakeKNNDocumentMatchHandlerKey = MakeDocumentMatchHandlerKeyType(\n\t\"MakeKNNDocumentMatchHandlerKey\")\n\n// MakeDocumentMatchHandler is an optional DocumentMatchHandler\n// builder function which the applications can pass to bleve.\n// These builder methods gives a DocumentMatchHandler function\n// to bleve, which it will invoke on every document matches.\ntype MakeDocumentMatchHandler func(ctx *SearchContext) (\n\tcallback DocumentMatchHandler, loadID bool, err error)\n\ntype MakeKNNDocumentMatchHandler func(ctx *SearchContext) (\n\tcallback DocumentMatchHandler, err error)\n"
  },
  {
    "path": "search/explanation.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n)\n\nvar reflectStaticSizeExplanation int\n\nfunc init() {\n\tvar e Explanation\n\treflectStaticSizeExplanation = int(reflect.TypeOf(e).Size())\n}\n\nconst MergedExplMessage = \"sum of merged explanations:\"\n\ntype Explanation struct {\n\tValue        float64        `json:\"value\"`\n\tMessage      string         `json:\"message\"`\n\tPartialMatch bool           `json:\"partial_match,omitempty\"`\n\tChildren     []*Explanation `json:\"children,omitempty\"`\n}\n\nfunc (expl *Explanation) String() string {\n\tjs, err := json.MarshalIndent(expl, \"\", \"  \")\n\tif err != nil {\n\t\treturn fmt.Sprintf(\"error serializing explanation to json: %v\", err)\n\t}\n\treturn string(js)\n}\n\nfunc (expl *Explanation) Size() int {\n\tsizeInBytes := reflectStaticSizeExplanation + size.SizeOfPtr +\n\t\tlen(expl.Message)\n\n\tfor _, entry := range expl.Children {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\n// MergeExpl merges two explanations into one.\n// If either explanation is nil, the other is returned.\n// If the first explanation is already a merged explanation,\n// the second explanation is appended to its children.\n// Otherwise, a new merged explanation is created\n// with the two explanations as its children.\nfunc (expl *Explanation) MergeWith(other *Explanation) *Explanation {\n\tif expl == nil {\n\t\treturn other\n\t}\n\tif other == nil || expl == other {\n\t\treturn expl\n\t}\n\n\tnewScore := expl.Value + other.Value\n\n\t// if both are merged explanations, combine children\n\tif expl.Message == MergedExplMessage && other.Message == MergedExplMessage {\n\t\texpl.Value = newScore\n\t\texpl.Children = append(expl.Children, other.Children...)\n\t\treturn expl\n\t}\n\n\t// atleast one is not a merged explanation see which one it is\n\t// if expl is merged, append other\n\tif expl.Message == MergedExplMessage {\n\t\t// append other as a child to first\n\t\texpl.Value = newScore\n\t\texpl.Children = append(expl.Children, other)\n\t\treturn expl\n\t}\n\n\t// if other is merged, append expl\n\tif other.Message == MergedExplMessage {\n\t\tother.Value = newScore\n\t\tother.Children = append(other.Children, expl)\n\t\treturn other\n\t}\n\t// create a new explanation to hold the merged one\n\trv := &Explanation{\n\t\tValue:    expl.Value + other.Value,\n\t\tMessage:  MergedExplMessage,\n\t\tChildren: []*Explanation{expl, other},\n\t}\n\treturn rv\n}\n"
  },
  {
    "path": "search/facet/benchmark_data.txt",
    "content": "Boiling liquid expanding vapor explosion\nFrom Wikipedia, the free encyclopedia\nSee also: Boiler explosion and Steam explosion\n\nFlames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire.\n\nThis article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013)\nA boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1]\nContents  [hide] \n1 Mechanism\n1.1 Water example\n1.2 BLEVEs without chemical reactions\n2 Fires\n3 Incidents\n4 Safety measures\n5 See also\n6 References\n7 External links\nMechanism[edit]\n\nThis section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013)\nThere are three characteristics of liquids which are relevant to the discussion of a BLEVE:\nIf a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed).\nThe boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example.\nWhen a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did.\nTypically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures.\nIf the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.\nWater example[edit]\nImagine, for example, a tank of pressurized liquid water held at 204.4 °C (400 °F). This vessel would normally be pressurized to 1.7 MPa (250 psi) above atmospheric (\"gauge\") pressure. Were the tank containing the water to split open, there would momentarily exist a volume of liquid water which is\nat atmospheric pressure, and\n204.4 °C (400 °F).\nAt atmospheric pressure the boiling point of water is 100 °C (212 °F) - liquid water at atmospheric pressure cannot exist at temperatures higher than 100 °C (212 °F). It is obvious, then, that 204.4 °C (400 °F) liquid water at atmospheric pressure must immediately flash to gas causing an explosion.\nBLEVEs without chemical reactions[edit]\nIt is important to note that a BLEVE need not be a chemical explosion - nor does there need to be a fire - however if a flammable substance is subject to a BLEVE it may also be subject to intense heating, either from an external source of heat which may have caused the vessel to rupture in the first place or from an internal source of localized heating such as skin friction. This heating can cause a flammable substance to ignite, adding a secondary explosion caused by the primary BLEVE. While blast effects of any BLEVE can be devastating, a flammable substance such as propane can add significantly to the danger.\nBleve explosion.svg\nWhile the term BLEVE is most often used to describe the results of a container of flammable liquid rupturing due to fire, a BLEVE can occur even with a non-flammable substance such as water,[2] liquid nitrogen,[3] liquid helium or other refrigerants or cryogens, and therefore is not usually considered a type of chemical explosion.\nFires[edit]\nBLEVEs can be caused by an external fire near the storage vessel causing heating of the contents and pressure build-up. While tanks are often designed to withstand great pressure, constant heating can cause the metal to weaken and eventually fail. If the tank is being heated in an area where there is no liquid, it may rupture faster without the liquid to absorb the heat. Gas containers are usually equipped with relief valves that vent off excess pressure, but the tank can still fail if the pressure is not released quickly enough.[1] Relief valves are sized to release pressure fast enough to prevent the pressure from increasing beyond the strength of the vessel, but not so fast as to be the cause of an explosion. An appropriately sized relief valve will allow the liquid inside to boil slowly, maintaining a constant pressure in the vessel until all the liquid has boiled and the vessel empties.\nIf the substance involved is flammable, it is likely that the resulting cloud of the substance will ignite after the BLEVE has occurred, forming a fireball and possibly a fuel-air explosion, also termed a vapor cloud explosion (VCE). If the materials are toxic, a large area will be contaminated.[4]\nIncidents[edit]\nThe term \"BLEVE\" was coined by three researchers at Factory Mutual, in the analysis of an accident there in 1957 involving a chemical reactor vessel.[5]\nIn August 1959 the Kansas City Fire Department suffered its largest ever loss of life in the line of duty, when a 25,000 gallon (95,000 litre) gas tank exploded during a fire on Southwest Boulevard killing five firefighters. This was the first time BLEVE was used to describe a burning fuel tank.[citation needed]\nLater incidents included the Cheapside Street Whisky Bond Fire in Glasgow, Scotland in 1960; Feyzin, France in 1966; Crescent City, Illinois in 1970; Kingman, Arizona in 1973; a liquid nitrogen tank rupture[6] at Air Products and Chemicals and Mobay Chemical Company at New Martinsville, West Virginia on January 31, 1978 [1];Texas City, Texas in 1978; Murdock, Illinois in 1983; San Juan Ixhuatepec, Mexico City in 1984; and Toronto, Ontario in 2008.\nSafety measures[edit]\n[icon]\tThis section requires expansion. (July 2013)\nSome fire mitigation measures are listed under liquefied petroleum gas.\nSee also[edit]\nBoiler explosion\nExpansion ratio\nExplosive boiling or phase explosion\nRapid phase transition\nViareggio train derailment\n2008 Toronto explosions\nGas carriers\nLos Alfaques Disaster\nLac-Mégantic derailment\nReferences[edit]\n^ Jump up to: a b Kletz, Trevor (March 1990). Critical Aspects of Safety and Loss Prevention. London: Butterworth–Heinemann. pp. 43–45. ISBN 0-408-04429-2.\nJump up ^ \"Temperature Pressure Relief Valves on Water Heaters: test, inspect, replace, repair guide\". Inspect-ny.com. Retrieved 2011-07-12.\nJump up ^ Liquid nitrogen BLEVE demo\nJump up ^ \"Chemical Process Safety\" (PDF). Retrieved 2011-07-12.\nJump up ^ David F. Peterson, BLEVE: Facts, Risk Factors, and Fallacies, Fire Engineering magazine (2002).\nJump up ^ \"STATE EX REL. VAPOR CORP. v. NARICK\". Supreme Court of Appeals of West Virginia. 1984-07-12. Retrieved 2014-03-16.\nExternal links[edit]\n\tLook up boiling liquid expanding vapor explosion in Wiktionary, the free dictionary.\n\tWikimedia Commons has media related to BLEVE.\nBLEVE Demo on YouTube — video of a controlled BLEVE demo\nhuge explosions on YouTube — video of propane and isobutane BLEVEs from a train derailment at Murdock, Illinois (3 September 1983)\nPropane BLEVE on YouTube — video of BLEVE from the Toronto propane depot fire\nMoscow Ring Road Accident on YouTube - Dozens of LPG tank BLEVEs after a road accident in Moscow\nKingman, AZ BLEVE — An account of the 5 July 1973 explosion in Kingman, with photographs\nPropane Tank Explosions — Description of circumstances required to cause a propane tank BLEVE.\nAnalysis of BLEVE Events at DOE Sites - Details physics and mathematics of BLEVEs.\nHID - SAFETY REPORT ASSESSMENT GUIDE: Whisky Maturation Warehouses - The liquor is aged in wooden barrels that can suffer BLEVE.\nCategories: ExplosivesFirefightingFireTypes of fireGas technologiesIndustrial fires and explosions\nNavigation menu\nCreate accountLog inArticleTalkReadEditView history\n\nMain page\nContents\nFeatured content\nCurrent events\nRandom article\nDonate to Wikipedia\nWikimedia Shop\nInteraction\nHelp\nAbout Wikipedia\nCommunity portal\nRecent changes\nContact page\nTools\nWhat links here\nRelated changes\nUpload file\nSpecial pages\nPermanent link\nPage information\nWikidata item\nCite this page\nPrint/export\nCreate a book\nDownload as PDF\nPrintable version\nLanguages\nCatalà\nDeutsch\nEspañol\nFrançais\nItaliano\nעברית\nNederlands\n日本語\nNorsk bokmål\nPolski\nPortuguês\nРусский\nSuomi\nEdit links\nThis page was last modified on 18 November 2014 at 01:35.\nText is available under the Creative Commons Attribution-ShareAlike License; additional terms may apply. By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia® is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\nPrivacy policyAbout WikipediaDisclaimersContact WikipediaDevelopersMobile viewWikimedia Foundation Powered by MediaWiki\n\n\nThermobaric weapon\nFrom Wikipedia, the free encyclopedia\n\nBlast from a US Navy fuel air explosive used against a decommissioned ship, USS McNulty, 1972.\nA thermobaric weapon is a type of explosive that utilizes oxygen from the surrounding air to generate an intense, high-temperature explosion, and in practice the blast wave such a weapon produces is typically significantly longer in duration than a conventional condensed explosive. The fuel-air bomb is one of the most well-known types of thermobaric weapons.\nMost conventional explosives consist of a fuel-oxidizer premix (gunpowder, for example, contains 25% fuel and 75% oxidizer), whereas thermobaric weapons are almost 100% fuel, so thermobaric weapons are significantly more energetic than conventional condensed explosives of equal weight. Their reliance on atmospheric oxygen makes them unsuitable for use underwater, at high altitude, and in adverse weather. They do, however, cause considerably more destruction when used inside confined environments such as tunnels, caves, and bunkers - partly due to the sustained blast wave, and partly by consuming the available oxygen inside those confined spaces.\nThere are many different types of thermobaric weapons rounds that can be fitted to hand-held launchers.[1]\nContents  [hide] \n1 Terminology\n2 Mechanism\n2.1 Fuel-air explosive\n2.1.1 Effect\n3 Development history\n3.1 Soviet and Russian developments\n3.2 US developments\n4 History\n4.1 Military use\n4.2 Non-military use\n5 See also\n6 References\n7 External links\nTerminology[edit]\nThe term thermobaric is derived from the Greek words for \"heat\" and \"pressure\": thermobarikos (θερμοβαρικός), from thermos (θερμός), hot + baros (βάρος), weight, pressure + suffix -ikos (-ικός), suffix -ic.\nOther terms used for this family of weapons are high-impulse thermobaric weapons (HITs), heat and pressure weapons, vacuum bombs, or fuel-air explosives (FAE or FAX).\nMechanism[edit]\nIn contrast to condensed explosive, where oxidation in a confined region produces a blast front from essentially a point source, a flame front accelerates to a large volume producing pressure fronts both within the mixture of fuel and oxidant and then in the surrounding air.[2]\nThermobaric explosives apply the principles underlying accidental unconfined vapor cloud explosions, which include those from dispersions of flammable dusts and droplets.[3] Previously, such explosions were most often encountered in flour mills and their storage containers, and later in coal mines; but, now, most commonly in discharged oil tankers and refineries, including an incident at Buncefield in the UK in 2005 where the blast wave woke people 150 kilometres (93 mi) from its centre.[4]\nA typical weapon consists of a container packed with a fuel substance, in the center of which is a small conventional-explosive \"scatter charge\". Fuels are chosen on the basis of the exothermicity of their oxidation, ranging from powdered metals, such as aluminium or magnesium, to organic materials, possibly with a self-contained partial oxidant. The most recent development involves the use of nanofuels.[5][6]\nA thermobaric bomb's effective yield requires the most appropriate combination of a number of factors; among these are how well the fuel is dispersed, how rapidly it mixes with the surrounding atmosphere, and the initiation of the igniter and its position relative to the container of fuel. In some designs, strong munitions cases allow the blast pressure to be contained long enough for the fuel to be heated up well above its auto-ignition temperature, so that once the container bursts the super-heated fuel will auto-ignite progressively as it comes into contact with atmospheric oxygen.[7][8][9][10][11][12][13][14][15][16][17]\nConventional upper and lower limits of flammability apply to such weapons. Close in, blast from the dispersal charge, compressing and heating the surrounding atmosphere, will have some influence on the lower limit. The upper limit has been demonstrated strongly to influence the ignition of fogs above pools of oil.[18] This weakness may be eliminated by designs where the fuel is preheated well above its ignition temperature, so that its cooling during its dispersion still results in a minimal ignition delay on mixing. The continual combustion of the outer layer of fuel molecules as they come into contact with the air, generates additional heat which maintains the temperature of the interior of the fireball, and thus sustains the detonation.[19][20][21]\nIn confinement, a series of reflective shock waves are generated,[22][23] which maintain the fireball and can extend its duration to between 10 and 50 ms as exothermic recombination reactions occur.[24] Further damage can result as the gases cool and pressure drops sharply, leading to a partial vacuum. This effect has given rise to the misnomer \"vacuum bomb\". Piston-type afterburning is also believed to occur in such structures, as flame-fronts accelerate through it.[25][26]\nFuel-air explosive[edit]\nA fuel-air explosive (FAE) device consists of a container of fuel and two separate explosive charges. After the munition is dropped or fired, the first explosive charge bursts open the container at a predetermined height and disperses the fuel in a cloud that mixes with atmospheric oxygen (the size of the cloud varies with the size of the munition). The cloud of fuel flows around objects and into structures. The second charge then detonates the cloud, creating a massive blast wave. The blast wave destroys unreinforced buildings and equipment and kills and injures people. The antipersonnel effect of the blast wave is more severe in foxholes, on people with body armor, and in enclosed spaces such as caves, buildings, and bunkers.\nFuel-air explosives were first developed, and used in Vietnam, by the United States. Soviet scientists, however, quickly developed their own FAE weapons, which were reportedly used against China in the Sino-Soviet border conflict and in Afghanistan. Since then, research and development has continued and currently Russian forces field a wide array of third-generation FAE warheads.\nEffect[edit]\nA Human Rights Watch report of 1 February 2000[27] quotes a study made by the US Defense Intelligence Agency:\nThe [blast] kill mechanism against living targets is unique–and unpleasant.... What kills is the pressure wave, and more importantly, the subsequent rarefaction [vacuum], which ruptures the lungs.... If the fuel deflagrates but does not detonate, victims will be severely burned and will probably also inhale the burning fuel. Since the most common FAE fuels, ethylene oxide and propylene oxide, are highly toxic, undetonated FAE should prove as lethal to personnel caught within the cloud as most chemical agents.\nAccording to a U.S. Central Intelligence Agency study,[27] \"the effect of an FAE explosion within confined spaces is immense. Those near the ignition point are obliterated. Those at the fringe are likely to suffer many internal, and thus invisible injuries, including burst eardrums and crushed inner ear organs, severe concussions, ruptured lungs and internal organs, and possibly blindness.\" Another Defense Intelligence Agency document speculates that because the \"shock and pressure waves cause minimal damage to brain tissue…it is possible that victims of FAEs are not rendered unconscious by the blast, but instead suffer for several seconds or minutes while they suffocate.\"[28]\nDevelopment history[edit]\nSoviet and Russian developments[edit]\n\nA RPO-A rocket and launcher.\nThe Soviet armed forces extensively developed FAE weapons,[29] such as the RPO-A, and used them in Chechnya.[30]\nThe Russian armed forces have developed thermobaric ammunition variants for several of their weapons, such as the TGB-7V thermobaric grenade with a lethality radius of 10 metres (33 ft), which can be launched from a RPG-7. The GM-94 is a 43 mm pump-action grenade launcher which is designed mainly to fire thermobaric grenades for close quarters combat. With the grenade weighing 250 grams (8.8 oz) and holding a 160 grams (5.6 oz) explosive mixture, its lethality radius is 3 metres (9.8 ft); however, due to the deliberate \"fragmentation-free\" design of the grenade, 4 metres (13 ft) is already considered a safe distance.[31] The RPO-A and upgraded RPO-M are infantry-portable RPGs designed to fire thermobaric rockets. The RPO-M, for instance, has a thermobaric warhead with a TNT equivalence of 5.5 kilograms (12 lb) of TNT and destructive capabilities similar to a 152 mm High explosive fragmentation artillery shell.[32][33] The RShG-1 and the RShG-2 are thermobaric variants of the RPG-27 and RPG-26 respectively. The RShG-1 is the more powerful variant, with its warhead having a 10 metres (33 ft) lethality radius and producing about the same effect as 6 kg (13 lb) of TNT.[34] The RMG is a further derivative of the RPG-26 that uses a tandem-charge warhead, whereby the precursor HEAT warhead blasts an opening for the main thermobaric charge to enter and detonate inside.[35] The RMG's precursor HEAT warhead can penetrate 300 mm of reinforced concrete or over 100 mm of Rolled homogeneous armour, thus allowing the 105 millimetres (4.1 in) diameter thermobaric warhead to detonate inside.[36]\nThe other examples include the SACLOS or millimeter wave radar-guided thermobaric variants of the 9M123 Khrizantema, the 9M133F-1 thermobaric warhead variant of the 9M133 Kornet, and the 9M131F thermobaric warhead variant of the 9K115-2 Metis-M, all of which are anti-tank missiles. The Kornet has since been upgraded to the Kornet-EM, and its thermobaric variant has a maximum range of 10 kilometres (6.2 mi) and has the TNT equivalent of 7 kilograms (15 lb) of TNT.[37] The 300 mm 9M55S thermobaric cluster warhead rocket was built to be fired from the BM-30 Smerch MLRS. A dedicated carrier of thermobaric weapons is the purpose-built TOS-1, a 24-tube MLRS designed to fire 220 mm caliber thermobaric rockets. A full salvo from the TOS-1 will cover a rectangle 200x400 metres.[38] The Iskander-M theatre ballistic missile can also carry a 700 kilograms (1,500 lb) thermobaric warhead.[39]\n\nThe fireball blast from the Russian Air Force's FOAB, the largest Thermobaric device to be detonated.\nMany Russian Air Force munitions also have thermobaric variants. The 80 mm S-8 rocket has the S-8DM and S-8DF thermobaric variants. The S-8's larger 122 mm brother, the S-13 rocket, has the S-13D and S-13DF thermobaric variants. The S-13DF's warhead weighs only 32 kg (71 lb) but its power is equivalent to 40 kg (88 lb) of TNT. The KAB-500-OD variant of the KAB-500KR has a 250 kg (550 lb) thermobaric warhead. The ODAB-500PM and ODAB-500PMV unguided bombs carry a 190 kg (420 lb) fuel-air explosive each. The KAB-1500S GLONASS/GPS guided 1,500 kg (3,300 lb) bomb also has a thermobaric variant. Its fireball will cover over a 150-metre (490 ft) radius and its lethality zone is a 500-metre (1,600 ft) radius.[40] The 9M120 Ataka-V and the 9K114 Shturm ATGMs both have thermobaric variants.\nIn September 2007 Russia exploded the largest thermobaric weapon ever made. The weapon's yield was reportedly greater than that of the smallest dial-a-yield nuclear weapons at their lowest settings.[41][42] Russia named this particular ordnance the \"Father of All Bombs\" in response to the United States developed \"Massive Ordnance Air Blast\" (MOAB) bomb whose backronym is the \"Mother of All Bombs\", and which previously held the accolade of the most powerful non-nuclear weapon in history.[43] The bomb contains an about 7 tons charge of a liquid fuel such as ethylene oxide, mixed with an energetic nanoparticle such as aluminium, surrounding a high explosive burster[44] that when detonated created an explosion equivalent to 44 metric tons of TNT.\nUS developments[edit]\n\nA BLU-72/B bomb on a USAF A-1E taking off from Nakhon Phanom, in September 1968.\nCurrent US FAE munitions include:\nBLU-73 FAE I\nBLU-95 500-lb (FAE-II)\nBLU-96 2,000-lb (FAE-II)\nCBU-55 FAE I\nCBU-72 FAE I\nThe XM1060 40-mm grenade is a small-arms thermobaric device, which was delivered to U.S. forces in April 2003.[45] Since the 2003 Invasion of Iraq, the US Marine Corps has introduced a thermobaric 'Novel Explosive' (SMAW-NE) round for the Mk 153 SMAW rocket launcher. One team of Marines reported that they had destroyed a large one-story masonry type building with one round from 100 yards (91 m).[46]\nThe AGM-114N Hellfire II, first used by U.S. forces in 2003 in Iraq, uses a Metal Augmented Charge (MAC) warhead that contains a thermobaric explosive fill using fluoridated aluminium layered between the charge casing and a PBXN-112 explosive mixture. When the PBXN-112 detonates, the aluminium mixture is dispersed and rapidly burns. The resultant sustained high pressure is extremely effective against people and structures.[47]\nHistory[edit]\nMilitary use[edit]\n\nUS Navy BLU-118B being prepared for shipping for use in Afghanistan, 5 March 2002.\nThe first experiments with thermobaric weapon were conducted in Germany during World War II and were led by Mario Zippermayr. The German bombs used coal dust as fuel and were extensively tested in 1943 and 1944, but did not reach mass production before the war ended.\nThe TOS-1 system was test fired in Panjshir valley during Soviet war in Afghanistan in the early 1980s.[48]\nUnconfirmed reports suggest that Russian military forces used ground delivered thermobaric weapons in the storming of the Russian parliament during the 1993 Russian constitutional crisis and also during the Battle for Grozny (first and second Chechen wars) to attack dug in Chechen fighters. The use of both TOS-1 heavy MLRS and \"RPO-A Shmel\" shoulder-fired rocket system in the Chechen wars is reported to have occurred.[48][49]\nIt is theorized that a multitude of hand-held thermobaric weapons were used by the Russian Armed Forces in their efforts to retake the school during the 2004 Beslan school hostage crisis. The RPO-A and either the TGB-7V thermobaric rocket from the RPG-7 or rockets from either the RShG-1 or the RShG-2 is claimed to have been used by the Spetsnaz during the initial storming of the school.[50][51][52] At least 3 and as many as 9 RPO-A casings were later found at the positions of the Spetsnaz.[53][54] The Russian Government later admitted to the use of the RPO-A during the crisis.[55]\nAccording to UK Ministry of Defence, British military forces have also used thermobaric weapons in their AGM-114N Hellfire missiles (carried by Apache helicopters and UAVs) against the Taliban in the War in Afghanistan.[56]\nThe US military also used thermobaric weapons in Afghanistan. On 3 March 2002, a single 2,000 lb (910 kg) laser guided thermobaric bomb was used by the United States Army against cave complexes in which Al-Qaeda and Taliban fighters had taken refuge in the Gardez region of Afghanistan.[57][58] The SMAW-NE was used by the US Marines during the First Battle of Fallujah and Second Battle of Fallujah.\nReports by the rebel fighters of the Free Syrian Army claim the Syrian Air Force used such weapons against residential area targets occupied by the rebel fighters, as for instance in the Battle for Aleppo[59] and also in Kafar Batna.[60] A United Nations panel of human rights investigators reported that the Syrian government used thermobaric bombs against the rebellious town of Qusayr in March 2013.[61]\nNon-military use[edit]\nThermobaric and fuel-air explosives have been used in guerrilla warfare since the 1983 Beirut barracks bombing in Lebanon, which used a gas-enhanced explosive mechanism, probably propane, butane or acetylene.[62] The explosive used by the bombers in the 1993 World Trade Center bombing incorporated the FAE principle, using three tanks of bottled hydrogen gas to enhance the blast.[63][64] Jemaah Islamiyah bombers used a shock-dispersed solid fuel charge,[65] based on the thermobaric principle,[66] to attack the Sari nightclub in the 2002 Bali bombings.[67]\nSee also[edit]\nBunker buster\nDust explosion\nFOAB\nFlame fougasse\nMOAB\nRPO-A\nSMAW\nReferences[edit]\nJump up ^ Algeria Isp (2011-10-18). \"Libye – l'Otan utilise une bombe FAE | Politique, Algérie\". Algeria ISP. Retrieved 2013-04-23.\nJump up ^ Nettleton, J. Occ. Accidents, 1, 149 (1976).\nJump up ^ Strehlow, 14th. Symp. (Int.) Comb. 1189, Comb. Inst. (1973).\nJump up ^ Health and Safety Environmental Agency, 5th. and final report, 2008.\nJump up ^ See Nanofuel/Oxidizers For Energetic Compositions – John D. Sullivan and Charles N. Kingery (1994) High explosive disseminator for a high explosive air bomb.\nJump up ^ Slavica Terzić, Mirjana Dakić Kolundžija, Milovan Azdejković and Gorgi Minov (2004) Compatibility Of Thermobaric Mixtures Based On Isopropyl Nitrate And Metal Powders.\nJump up ^ Meyer, Rudolf; Josef Köhler and Axel Homburg (2007). Explosives. Weinheim: Wiley-VCH. pp. 312. ISBN 3-527-31656-6. OCLC 165404124.\nJump up ^ Howard C. Hornig (1998) Non-focusing active warhead.\nJump up ^ Chris Ludwig (Talley Defense) Verifying Performance of Thermobaric Materials for Small to Medium Caliber Rocket Warheads.\nJump up ^ Martin M.West (1982) Composite high explosives for high energy blast applications.\nJump up ^ Raafat H. Guirguis (2005) Reactively Induced Fragmenting Explosives.\nJump up ^ Michael Dunning, William Andrews and Kevin Jaansalu (2005) The Fragmentation of Metal Cylinders Using Thermobaric Explosives.\nJump up ^ David L. Frost, Fan Zhang, Stephen B. Murray and Susan McCahan Critical Conditions For Ignition Of Metal Particles In A Condensed Explosive.\nJump up ^ The Army Doctrine and Training Bulletin (2001) The Threat from Blast Weapons.\nJump up ^ INTERNATIONAL DEFENCE REVIEW (2004) ENHANCED BLAST AND THERMOBARICS.\nJump up ^ F. Winterberg Conjectured Metastable Super-Explosives formed under High Pressure for Thermonuclear Ignition.\nJump up ^ Zhang, Fan (Medicine Hat, CA) Murray, Stephen Burke (Medicine Hat, CA) Higgins, Andrew (Montreal, CA) (2005) Super compressed detonation method and device to effect such detonation.\nJump up ^ Nettleton, arch. combust.,1,131, (1981).\nJump up ^ Stephen B. Murray Fundamental and Applied Studies of Fuel-Air Detonation.\nJump up ^ John H. Lee (1992) Chemical initiation of detonation in fuel-air explosive clouds.\nJump up ^ Frank E. Lowther (1989) Nuclear-sized explosions without radiation.\nJump up ^ Nettleton, Comb. and Flame, 24,65 (1975).\nJump up ^ Fire Prev. Sci. and Tech. No. 19,4 (1976)\nJump up ^ May L.Chan (2001) Advanced Thermobaric Explosive Compositions.\nJump up ^ New Thermobaric Materials and Weapon Concepts.\nJump up ^ Robert C. Morris (2003) Small Thermobaric Weapons An Unnoticed Threat.[dead link]\n^ Jump up to: a b \"Backgrounder on Russian Fuel Air Explosives (\"Vacuum Bombs\") | Human Rights Watch\". Hrw.org. 2000-02-01. Retrieved 2013-04-23.\nJump up ^ Defense Intelligence Agency, \"Future Threat to the Soldier System, Volume I; Dismounted Soldier--Middle East Threat\", September 1993, p. 73. Obtained by Human Rights Watch under the U.S. Freedom of Information Act.\nJump up ^ \"Press | Human Rights Watch\". Hrw.org. 2008-12-27. Retrieved 2009-07-30.\nJump up ^ Lester W. Grau and Timothy L. Thomas(2000)\"Russian Lessons Learned From the Battles For Grozny\"\nJump up ^ \"Modern Firearms – GM-94\". World.guns.ru. 2011-01-24. Retrieved 2011-07-12.\nJump up ^ \"New RPO Shmel-M Infantry Rocket Flamethrower Man-Packable Thermobaric Weapon\". defensereview.com. 2006-07-19. Retrieved 2012-08-27.\nJump up ^ \"Shmel-M: Infantry Rocket-assisted Flamethrower of Enhanced Range and Lethality\". Kbptula.ru. Retrieved 2013-12-28.\nJump up ^ \"Modern Firearms – RShG-1\". World.guns.ru. 2011-01-24. Retrieved 2011-07-12.\nJump up ^ \"Modern Firearms – RMG\". World.guns.ru. 2011-01-24. Retrieved 2011-07-12.\nJump up ^ \"RMG - A new Multi-Purpose Assault Weapon from Bazalt\". defense-update.com. Retrieved 2012-08-27.\nJump up ^ \"Kornet-EM: Multi-purpose Long-range Missile System\". Kbptula.ru. Retrieved 2013-12-28.\nJump up ^ \"TOS-1 Heavy flamethrower system\". military-today.com. Retrieved 2012-08-27.\nJump up ^ \"SS-26\". Missilethreat.com. Retrieved 2013-12-28.\nJump up ^ Air Power Australia (2007-07-04). \"How to Destroy the Australian Defence Force\". Ausairpower.net. Retrieved 2011-07-12.\nJump up ^ \"Russia unveils devastating vacuum bomb\". ABC News. 2007. Retrieved 2007-09-12.\nJump up ^ \"Video of test explosion\". BBC News. 2007. Retrieved 2007-09-12.\nJump up ^ Harding, Luke (2007-09-12). \"Russia unveils the father of all bombs\". London: The Guardian. Retrieved 2007-09-12.\nJump up ^ Berhie, Saba. \"Dropping the Big One | Popular Science\". Popsci.com. Retrieved 2011-07-12.\nJump up ^ John Pike (2003-04-22). \"XM1060 40mm Thermobaric Grenade\". Globalsecurity.org. Retrieved 2011-07-12.\nJump up ^ David Hambling (2005) \"Marines Quiet About Brutal New Weapon\"\nJump up ^ John Pike (2001-09-11). \"AGM-114N Metal Augmented Charge (MAC) Thermobaric Hellfire\". Globalsecurity.org. Retrieved 2011-07-12.\n^ Jump up to: a b John Pike. \"TOS-1 Buratino 220mm Multiple Rocket Launcher\". Globalsecurity.org. Retrieved 2013-04-23.\nJump up ^ \"Foreign Military Studies Office Publications - A 'Crushing' Victory: Fuel-Air Explosives and Grozny 2000\". Fmso.leavenworth.army.mil. Retrieved 2013-04-23.\nJump up ^ \"Russian forces faulted in Beslan school tragedy\". Christian Science Monitor. 1 September 2006. Retrieved 14 February 2007.\nJump up ^ Russia: Independent Beslan Investigation Sparks Controversy, The Jamestown Foundation, 29 August 2006\nJump up ^ Beslan still a raw nerve for Russia, BBC News, 1 September 2006\nJump up ^ ACHING TO KNOW, Los Angeles Times, 27 August 2005\nJump up ^ Searching for Traces of “Shmel” in Beslan School, Kommersant, 12 September 2005\nJump up ^ A Reversal Over Beslan Only Fuels Speculation, The Moscow Times, 21 July 2005\nJump up ^ \"MoD's Controversial Thermobaric Weapons Use in Afghanistan\". Armedforces-int.com. 2008-06-23. Retrieved 2013-04-23.\nJump up ^ \"US Uses Bunker-Busting 'Thermobaric' Bomb for First Time\". Commondreams.org. 2002-03-03. Retrieved 2013-04-23.\nJump up ^ John Pike. \"BLU-118/B Thermobaric Weapon Demonstration / Hard Target Defeat Program\". Globalsecurity.org. Retrieved 2013-04-23.\nJump up ^ \"Syria rebels say Assad using 'mass-killing weapons' in Aleppo\". October 10, 2012. Retrieved November 11, 2012.\nJump up ^ \"Dropping Thermobaric Bombs on Residential Areas in Syria_ Nov. 5. 2012\". First Post. November 11, 2012. Retrieved November 11, 2012.\nJump up ^ Cumming-Bruce, Nick (2013-06-04). \"U.N. Panel Reports Increasing Brutality by Both Sides in Syria\". The New York Times.\nJump up ^ Richard J. Grunawalt. Hospital Ships In The War On Terror: Sanctuaries or Targets? (PDF), Naval War College Review, Winter 2005, pp. 110–11.\nJump up ^ Paul Rogers (2000) \"Politics in the Next 50 Years: The Changing Nature of International Conflict\"\nJump up ^ J. Gilmore Childers, Henry J. DePippo (February 24, 1998). \"Senate Judiciary Committee, Subcommittee on Technology, Terrorism, and Government Information hearing on \"Foreign Terrorists in America: Five Years After the World Trade Center\"\". Fas.org. Retrieved 2011-07-12.\nJump up ^ P. Neuwald, H. Reichenbach, A. L. Kuhl (2003). \"Shock-Dispersed-Fuel Charges-Combustion in Chambers and Tunnels\".\nJump up ^ David Eshel (2006). \"Is the world facing Thermobaric Terrorism?\".[dead link]\nJump up ^ Wayne Turnbull (2003). \"Bali:Preparations\".\nExternal links[edit]\nFuel/Air Explosive (FAE)\nThermobaric Explosive (Global Security)\nAspects of thermobaric weaponry (PDF) – Dr. Anna E Wildegger-Gaissmaier, Australian Defence Force Health\nThermobaric warhead for RPG-7\nXM1060 40 mm Thermobaric Grenade (Global Security)\nDefense Update: Fuel-Air Explosive Mine Clearing System\nForeign Military Studies Office – A 'Crushing' Victory: Fuel-Air Explosives and Grozny 2000\nSoon to make a comeback in Afghanistan\nRussia claims to have tested the most powerful \"Vacuum\" weapon\nCategories: Explosive weaponsAmmunitionThermobaric weaponsAnti-personnel weapons\nNavigation menu\nCreate accountLog inArticleTalkReadEditView history\n\nMain page\nContents\nFeatured content\nCurrent events\nRandom article\nDonate to Wikipedia\nWikimedia Shop\nInteraction\nHelp\nAbout Wikipedia\nCommunity portal\nRecent changes\nContact page\nTools\nWhat links here\nRelated changes\nUpload file\nSpecial pages\nPermanent link\nPage information\nWikidata item\nCite this page\nPrint/export\nCreate a book\nDownload as PDF\nPrintable version\nLanguages\nالعربية\nБеларуская\nБългарски\nČeština\nDeutsch\nEspañol\nفارسی\nFrançais\nहिन्दी\nItaliano\nעברית\nLatviešu\nМакедонски\nNederlands\n日本語\nPolski\nРусский\nSuomi\nSvenska\nTürkçe\nУкраїнська\nTiếng Việt\n粵語\n中文\nEdit links\nThis page was last modified on 28 November 2014 at 10:32.\nText is available under the Creative Commons Attribution-ShareAlike License; additional terms may apply. By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia® is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\nPrivacy policyAbout WikipediaDisclaimersContact WikipediaDevelopersMobile viewWikimedia Foundation Powered by MediaWiki\n\n\nGunpowder\nFrom Wikipedia, the free encyclopedia\nFor other uses, see Gunpowder (disambiguation).\nIn American English, the term gunpowder also refers broadly to any gun propellant.[1] Gunpowder (black powder) as described in this article is not normally used in modern firearms, which instead use smokeless powders.\n\nBlack powder for muzzleloading rifles and pistols in FFFG granulation size. American Quarter (diameter 24 mm) for comparison.\nGunpowder, also known as black powder, is a chemical explosive—the earliest known. It is a mixture of sulfur, charcoal, and potassium nitrate (saltpeter). The sulfur and charcoal act as fuels, and the saltpeter is an oxidizer.[2][3] Because of its burning properties and the amount of heat and gas volume that it generates, gunpowder has been widely used as a propellant in firearms and as a pyrotechnic composition in fireworks.\nGunpowder is assigned the UN number UN0027 and has a hazard class of 1.1D. It has a flash point of approximately 427–464 °C (801–867 °F). The specific flash point may vary based on the specific composition of the gunpowder. Gunpowder's gravity is 1.70–1.82 (mercury method) orŠ 1.92–2.08 (pycnometer), and it has a pH of 6.0–8.0. It is also considered to be an insoluble material.[4]\nGunpowder was, according to prevailing academic consensus, invented in the 9th century in China,[5][6] and the earliest record of a written formula for gunpowder appears in the 11th century Song Dynasty text, Wujing Zongyao.[7] This discovery led to the invention of fireworks and the earliest gunpowder weapons in China. In the centuries following the Chinese discovery, gunpowder weapons began appearing in the Muslim world, Europe, and India. The technology spread from China through the Middle East or Central Asia, and then into Europe.[8] The earliest Western accounts of gunpowder appear in texts written by English philosopher Roger Bacon in the 13th century.[9]\nGunpowder is classified as a low explosive because of its relatively slow decomposition rate and consequently low brisance. Low explosives deflagrate (i.e., burn) at subsonic speeds, whereas high explosives detonate, producing a supersonic wave. Gunpowder's burning rate increases with pressure, so it bursts containers if contained but otherwise just burns in the open. Ignition of the powder packed behind a bullet must generate enough pressure to force it from the muzzle at high speed, but not enough to rupture the gun barrel. Gunpowder thus makes a good propellant, but is less suitable for shattering rock or fortifications. Gunpowder was widely used to fill artillery shells and in mining and civil engineering to blast rock roughly until the second half of the 19th century, when the first high explosives (nitro-explosives) were discovered. Gunpowder is no longer used in modern explosive military warheads, nor is it used as main explosive in mining operations due to its cost relative to that of newer alternatives such as ammonium nitrate/fuel oil (ANFO).[10] Black powder is still used as a delay element in various munitions where its slow-burning properties are valuable.\nFormulations used in blasting rock (such as in quarrying) are called blasting powder.\nContents  [hide] \n1 History\n1.1 China\n1.2 Middle East\n1.3 Mainland Europe\n1.4 Britain and Ireland\n1.5 India\n1.6 Indonesia\n2 Manufacturing technology\n3 Composition and characteristics\n4 Serpentine\n5 Corning\n6 Modern types\n7 Other types of gunpowder\n8 Sulfur-free gunpowder\n9 Combustion characteristics\n9.1 Advantages\n9.2 Disadvantages\n9.3 Transportation\n10 Other uses\n11 See also\n12 References\n13 External links\nHistory[edit]\n\nEarly Chinese rocket\n\nA Mongol bomb thrown against a charging Japanese samurai during the Mongol invasions of Japan after founding the Yuan Dynasty, 1281.\nMain article: History of gunpowder\nGunpowder was invented in China while taoists attempted to create a potion of immortality. Chinese military forces used gunpowder-based weapons (i.e. rockets, guns, cannons) and explosives (i.e. grenades and different types of bombs) against the Mongols when the Mongols attempted to invade and breach city fortifications on China's northern borders. After the Mongols conquered China and founded the Yuan Dynasty, they used the Chinese gunpowder-based weapons technology in their attempted invasion of Japan; they also used gunpowder to fuel rockets.\nThe mainstream scholarly consensus is that gunpowder was invented in China, spread through the Middle East, and then into Europe,[8] although there is a dispute over how much the Chinese advancements in gunpowder warfare influenced later advancements in the Middle East and Europe.[11][12] The spread of gunpowder across Asia from China is widely attributed to the Mongols. One of the first examples of Europeans encountering gunpowder and firearms is at the Battle of Mohi in 1241. At this battle the Mongols not only used gunpowder in early Chinese firearms but in the earliest grenades as well.\nA major problem confronting the study of the early history of gunpowder is ready access to sources close to the events described. Often enough, the first records potentially describing use of gunpowder in warfare were written several centuries after the fact, and may well have been colored by the contemporary experiences of the chronicler.[13] It is also difficult to accurately translate original alchemy texts, especially medieval Chinese texts that try to explain phenomena through metaphor, into modern scientific language with rigidly defined terminology. The translation difficulty has led to errors or loose interpretations bordering on artistic licence.[14][15] Early writings potentially mentioning gunpowder are sometimes marked by a linguistic process where old words acquired new meanings.[16] For instance, the Arabic word naft transitioned from denoting naphtha to denoting gunpowder, and the Chinese word pao evolved from meaning catapult to referring to cannon.[17] According to science and technology historian Bert S. Hall: \"It goes without saying, however, that historians bent on special pleading, or simply with axes of their own to grind, can find rich material in these terminological thickets.\"[18]\nChina[edit]\nFurther information: Wujing Zongyao, Four Great Inventions and List of Chinese inventions\n\nChinese Ming Dynasty (1368-1644) matchlock firearms\nSaltpeter was known to the Chinese by the mid-1st century AD and there is strong evidence of the use of saltpeter and sulfur in various largely medicinal combinations.[19] A Chinese alchemical text dated 492 noted saltpeter burnt with a purple flame, providing a practical and reliable means of distinguishing it from other inorganic salts, thus enabling alchemists to evaluate and compare purification techniques; the earliest Latin accounts of saltpeter purification are dated after 1200.[20]\n\nYuan Dynasty bronze hand cannon from 1332 at th (c. 808); it describes mixing six parts sulfur to six parts saltpeter to one part birthwort herb (which would provide carbon).[21]\nThe first reference to the incendiary properties of such mixtures is the passage of the Zhenyuan miaodao yaolüe, a Taoist text tentatively dated to the mid-9th century AD:[20] \"Some have heated together sulfur, realgar and saltpete with honey; smoke and flames result, so that their hands and faces have been burnt, and even the whole house where they were working burned down.\"[22] The Chinese word for \"gunpowder\" is Chinese: 火药/火藥; pinyin: huŏ yào /xuou yɑʊ/, which literally means \"Fire Medicine\";[23] however this name only came into use some centuries after the mixture's discovery.[24] During the 9th century, Taoist monks or alchemists searching for an elixir of immortality had serendipitously stumbled upon gunpowder.[8][25] The Chinese wasted little time in applying gunpowder to the development of weapons, and in the centuries that followed, they produced a variety of gunpowder weapons, including flamethrowers, rockets, bombs, and land mines, before inventing guns as a projectile weapon.[26] Archaeological evidence of a hand cannon has been excavated in Manchuria dated from the late 1200s[27] and the shells of explosive bombs have been discovered in a shipwreck off the shore of Japan dated from 1281, during the Mongol invasions of Japan.[28]\nThe Chinese \"Wu Ching Tsung Yao\" (Complete Essentials from the Military Classics), written by Tseng Kung-Liang between 1040–1044, provides encyclopedia references to a variety of mixtures that included petrochemicals—as well as garlic and honey. A slow match for flame throwing mechanisms using the siphon principle and for fireworks and rockets are mentioned. The mixture formulas in this book do not contain enough saltpeter to create an explosive however; being limited to at most 50% saltpeter, they produce an incendiary.[29] The Essentials was however written by a Song Dynasty court bureaucrat, and there's little evidence that it had any immediate impact on warfare; there is no mention of gunpowder use in the chronicles of the wars against the Tanguts in the eleventh century, and China was otherwise mostly at peace during this century. The first chronicled use of \"fire spears\" (or \"fire lances\") is at the siege of De'an in 1132.[30]\n\nFormula for gunpowder in 1044 Wujing zongyao part I vol 12\n \n\nInstruction for fire bomb in Wujing zongyao\n \n\nFire bomb\n \n\nFire grenade\n \n\nProto-cannon from the Ming Dynasty text Huolongjing\n \n\nLand mine from the Ming Dynasty text Huolongjing\n \n\nFire arrow rocket launcher from the Wujing zongyao\nMiddle East[edit]\nMain articles: Inventions in the Islamic world and Alchemy and chemistry in Islam\n\nThe Sultani Cannon, a very heavy bronze breech-loading cannon of type used by Ottoman Empire in the conquest of Constantinople, in 1453.\nThe Muslims acquired knowledge of gunpowder some time between 1240 and 1280, by which time the Syrian Hasan al-Rammah had written, in Arabic, recipes for gunpowder, instructions for the purification of saltpeter, and descriptions of gunpowder incendiaries. Gunpowder arrived in the Middle East, possibly through India, from China. This is implied by al-Rammah's usage of \"terms that suggested he derived his knowledge from Chinese sources\" and his references to saltpeter as \"Chinese snow\" Arabic: ثلج الصين‎ thalj al-ṣīn, fireworks as \"Chinese flowers\" and rockets as \"Chinese arrows\".[31] However, because al-Rammah attributes his material to \"his father and forefathers\", al-Hassan argues that gunpowder became prevalent in Syria and Egypt by \"the end of the twelfth century or the beginning of the thirteenth\".[32] Persians called saltpeter \"Chinese salt\" [33][34][35][36][37] or \"salt from Chinese salt marshes\" (namak shūra chīnī Persian: نمک شوره چيني‎).[38][39]\n\nA picture of a 15th-century Granadian cannon from the book Al-izz wal rifa'a.\nAl-Hassan claims that in the Battle of Ain Jalut of 1260, the Mamluks used against the Mongols in \"the first cannon in history\" gunpowder formula with near-identical ideal composition ratios for explosive gunpowder.[32] Other historians urge caution regarding claims of Islamic firearms use in the 1204-1324 period as late medieval Arabic texts used the same word for gunpowder, naft, that they used for an earlier incendiary, naphtha.[13][17] Khan claims that it was invading Mongols who introduced gunpowder to the Islamic world[40] and cites Mamluk antagonism towards early musketeers in their infantry as an example of how gunpowder weapons were not always met with open acceptance in the Middle East.[41] Similarly, the refusal of their Qizilbash forces to use firearms contributed to the Safavid rout at Chaldiran in 1514.[41]\nThe earliest surviving documentary evidence for the use of the hand cannon, considered the oldest type of portable firearm and a forerunner of the handgun, are from several Arabic manuscripts dated to the 14th century.[42] Al-Hassan argues that these are based on earlier originals and that they report hand-held cannons being used by the Mamluks at the Battle of Ain Jalut in 1260.[32]\nHasan al-Rammah included 107 gunpowder recipes in his text al-Furusiyyah wa al-Manasib al-Harbiyya (The Book of Military Horsemanship and Ingenious War Devices), 22 of which are for rockets. If one takes the median of 17 of these 22 compositions for rockets (75% nitrates, 9.06% sulfur, and 15.94% carbon), it is nearly identical to the modern reported ideal gunpowder recipe of 75% potassium nitrate, 10% sulfur, and 15% carbon.[32]\nThe state-controlled manufacture of gunpowder by the Ottoman Empire through early supply chains to obtain nitre, sulfur and high-quality charcoal from oaks in Anatolia contributed significantly to its expansion the 15th and 18th century. It was not until later in the 19th century when the syndicalist production of Turkish gunpowder was greatly reduced, which coincided with the decline of its military might.[43]\nMainland Europe[edit]\nSeveral sources mention Chinese firearms and gunpowder weapons being deployed by the Mongols against European forces at the Battle of Mohi in 1241.[44][45][46] Professor Kenneth Warren Chase credits the Mongols for introducing into Europe gunpowder and its associated weaponry.[47]\nC. F. Temler interprets Peter, Bishop of Leon, as reporting the use of cannons in Seville in 1248.[48]\nIn Europe, one of the first mentions of gunpowder use appears in a passage found in Roger Bacon's Opus Maius and Opus Tertium in what has been interpreted as being firecrackers. The most telling passage reads: \"We have an example of these things (that act on the senses) in [the sound and fire of] that children's toy which is made in many [diverse] parts of the world; i.e., a device no bigger than one's thumb. From the violence of that salt called saltpeter [together with sulfur and willow charcoal, combined into a powder] so horrible a sound is made by the bursting of a thing so small, no more than a bit of parchment [containing it], that we find [the ear assaulted by a noise] exceeding the roar of strong thunder, and a flash brighter than the most brilliant lightning.\"[9] In the early 20th century, British artillery officer Henry William Lovett Hime proposed that another work tentatively attributed to Bacon, Epistola de Secretis Operibus Artis et Naturae, et de Nullitate Magiae contained an encrypted formula for gunpowder. This claim has been disputed by historians of science including Lynn Thorndike, John Maxson Stillman and George Sarton and by Bacon's editor Robert Steele, both in terms of authenticity of the work, and with respect to the decryption method.[9] In any case, the formula claimed to have been decrypted (7:5:5 saltpeter:charcoal:sulfur) is not useful for firearms use or even firecrackers, burning slowly and producing mostly smoke.[49][50]\n\nCannon forged in 1667 at the Fortín de La Galera, Nueva Esparta, Venezuela.\nThe Liber Ignium, or Book of Fires, attributed to Marcus Graecus, is a collection of incendiary recipes, including some gunpowder recipes. Partington dates the gunpowder recipes to approximately 1300.[51] One recipe for \"flying fire\" (ingis volatilis) involves saltpeter, sulfur, and colophonium, which, when inserted into a reed or hollow wood, \"flies away suddenly and burns up everything.\" Another recipe, for artificial \"thunder\", specifies a mixture of one pound native sulfur, two pounds linden or willow charcoal, and six pounds of saltpeter.[52] Another specifies a 1:3:9 ratio.[52]\nSome of the gunpowder recipes of De Mirabilibus Mundi of Albertus Magnus are identical to the recipes of the Liber Ignium, and according to Partington, \"may have been taken from that work, rather than conversely.\"[53] Partington suggests that some of the book may have been compiled by Albert's students, \"but since it is found in thirteenth century manuscripts, it may well be by Albert.\"[53] Albertus Magnus died in 1280.\nA common German folk-tale is of the German priest/monk named Berthold Schwarz who independently invented gunpowder, thus earning it the German name Schwarzpulver or in English Schwarz's powder. Schwarz is also German for black so this folk-tale, while likely containing elements of truth, is considered problematic.\nA major advance in manufacturing began in Europe in the late 14th century when the safety and thoroughness of incorporation was improved by wet grinding; liquid, such as distilled spirits or perhaps the urine of wine-drinking bishops[54] was added during the grinding-together of the ingredients and the moist paste dried afterwards. (The principle of wet mixing to prevent the separation of dry ingredients, invented for gunpowder, is used today in the pharmaceutical industry.[55]) It was also discovered that if the paste was rolled into balls before drying the resulting gunpowder absorbed less water from the air during storage and traveled better. The balls were then crushed in a mortar by the gunner immediately before use, with the old problem of uneven particle size and packing causing unpredictable results.\nIf the right size particles were chosen, however, the result was a great improvement in power. Forming the damp paste into corn-sized clumps by hand or with the use of a sieve instead of larger balls produced a product after drying that loaded much better, as each tiny piece provided its own surrounding air space that allowed much more rapid combustion than a fine powder. This \"corned\" gunpowder was from 30% to 300% more powerful. An example is cited where 34 pounds of serpentine was needed to shoot a 47 pound ball, but only 18 pounds of corned powder.[54] The optimum size of the grain depended on its use; larger for large cannon, finer for small arms. Larger cast cannons were easily muzzle-loaded with corned powder using a long-handled ladle. Corned powder also retained the advantage of low moisture absorption, as even tiny grains still had much less surface area to attract water than a floury powder.\nDuring this time, European manufacturers also began regularly purifying saltpeter, using wood ashes containing potassium carbonate to precipitate calcium from their dung liquor, and using ox blood, alum, and slices of turnip to clarify the solution.[54]\nGunpowder-making and metal-smelting and casting for shot and cannon fee was closely held by skilled military tradesmen, who formed guilds that collected dues, tested apprentices, and gave pensions. \"Fire workers\" were also required to craft fireworks for celebrations of victory or peace. During the Renaissance, two European schools of pyrotechnic thought emerged, one in Italy and the other at Nuremberg, Germany. Vannoccio Biringuccio, born in 1480, was a member of the guild Fraternita di Santa Barbara but broke with the tradition of secrecy by setting down everything he knew in a book titled De la pirotechnia, written in vernacular. The first printed book on either gunpowder or metalworking, it was published posthumously in 1540, with 9 editions over 138 years, and also reprinted by MIT Press in 1966.[54] By the mid-17th century fireworks were used for entertainment on an unprecedented scale in Europe, being popular even at resorts and public gardens.[56]\nIn 1774 Louis XVI ascended to the throne of France at age 20. After he discovered that France was not self-sufficient in gunpowder, a Gunpowder Administration was established; to head it, the lawyer Antoine Lavoisier was appointed. Although from a bourgeois family, after his degree in law Lavoisier became wealthy from a company set up to collect taxes for the Crown; this allowed him to pursue experimental natural science as a hobby.[57]\nWithout access to cheap Indian saltpeter (controlled by the British), for hundreds of years France had relied on saltpetermen with royal warrants, the droit de fouille or \"right to dig\", to seize nitrous-containing soil and demolished walls of barnyards, without compensation to the owners.[58] This caused farmers, the wealthy, or entire villages to bribe the petermen and the associated bureaucracy to leave their buildings alone and the saltpeter uncollected. Lavoisier instituted a crash program to increase saltpeter production, revised (and later eliminated) the droit de fouille, researched best refining and powder manufacturing methods, instituted management and record-keeping, and established pricing that encouraged private investment in works. Although saltpeter from new Prussian-style putrefaction works had not been produced yet (the process taking about 18 months), in only a year France had gunpowder to export. A chief beneficiary of this surplus was the American Revolution. By careful testing and adjusting the proportions and grinding time, powder from mills such as at Essonne outside Paris became the best in the world by 1788, and inexpensive.[58] [59]\nBritain and Ireland[edit]\n\nThe old Powder or Pouther magazine dating from 1642, built by order of Charles I. Irvine, North Ayrshire, Scotland\nGunpowder production in Britain appears to have started in the mid 14th century AD with the aim of supplying the English Crown.[60] Records show that gunpowder was being made, in England, in 1346, at the Tower of London; a powder house existed at the Tower in 1461; and in 1515 three King's gunpowder makers worked there.[60] Gunpowder was also being made or stored at other Royal castles, such as Portchester. By the early 14th century, according to N.J.G. Pounds's study The Medieval Castle in England and Wales, many English castles had been deserted and others were crumbling. Their military significance faded except on the borders. Gunpowder had made smaller castles useless.[61]\nHenry VIII of England was short of gunpowder when he invaded France in 1544 and England needed to import gunpowder via the port of Antwerp in what is now Belgium.[60]\nThe English Civil War (1642–1645) led to an expansion of the gunpowder industry, with the repeal of the Royal Patent in August 1641.[60]\nTwo British physicists, Andrew Noble and Frederick Abel, worked to improve the properties of black powder during the late 19th century. This formed the basis for the Noble-Abel gas equation for internal ballistics.[62]\nThe introduction of smokeless powder in the late 19th century led to a contraction of the gunpowder industry. After the end of World War I, the majority of the United Kingdom gunpowder manufacturers merged into a single company, \"Explosives Trades limited\"; and number of sites were closed down, including those in Ireland. This company became Nobel Industries Limited; and in 1926 became a founding member of Imperial Chemical Industries. The Home Office removed gunpowder from its list of Permitted Explosives; and shortly afterwards, on 31 December 1931, the former Curtis & Harvey's Glynneath gunpowder factory at Pontneddfechan, in Wales, closed down, and it was demolished by fire in 1932.[63]\n\nGunpowder storing barrels at Martello tower in Point Pleasant Park\nThe last remaining gunpowder mill at the Royal Gunpowder Factory, Waltham Abbey was damaged by a German parachute mine in 1941 and it never reopened.[64] This was followed by the closure of the gunpowder section at the Royal Ordnance Factory, ROF Chorley, the section was closed and demolished at the end of World War II; and ICI Nobel's Roslin gunpowder factory, which closed in 1954.[64][65]\nThis left the sole United Kingdom gunpowder factory at ICI Nobel's Ardeer site in Scotland; it too closed in October 1976.[64] Since then gunpowder has been imported into the United Kingdom. In the late 1970s/early 1980s gunpowder was bought from eastern Europe, particularly from what was then the German Democratic Republic and former Yugoslavia.\nIndia[edit]\n\nIn the year 1780 the British began to annex the territories of the Sultanate of Mysore, during the Second Anglo-Mysore War. The British battalion was defeated during the Battle of Guntur, by the forces of Hyder Ali, who effectively utilized Mysorean rockets and Rocket artillery against the closely massed British forces.\n\nMughal Emperor Shah Jahan, hunting deer using a Matchlock as the sun sets in the horizon.\nGunpowder and gunpowder weapons were transmitted to India through the Mongol invasions of India.[66][67] The Mongols were defeated by Alauddin Khilji of the Delhi Sultanate, and some of the Mongol soldiers remained in northern India after their conversion to Islam.[67] It was written in the Tarikh-i Firishta (1606–1607) that Nasir ud din Mahmud the ruler of the Delhi Sultanate presented the envoy of the Mongol ruler Hulegu Khan with a dazzling pyrotechnics display upon his arrival in Delhi in 1258 AD. Nasir ud din Mahmud tried to express his strength as a ruler and tried to ward off any Mongol attempt similar to the Siege of Baghdad (1258).[68] Firearms known as top-o-tufak also existed in many Muslim kingdoms in India by as early as 1366 AD.[68] From then on the employment of gunpowder warfare in India was prevalent, with events such as the \"Siege of Belgaum\" in 1473 by Sultan Muhammad Shah Bahmani.[69]\nThe shipwrecked Ottoman Admiral Seydi Ali Reis is known to have introduced the earliest type of Matchlock weapons, which the Ottomans used against the Portuguese during the Siege of Diu (1531). After that, a diverse variety of firearms; large guns in particular, became visible in Tanjore, Dacca, Bijapur, and Murshidabad.[70] Guns made of bronze were recovered from Calicut (1504)- the former capital of the Zamorins[71]\nThe Mughal Emperor Akbar mass-produced matchlocks for the Mughal Army. Akbar is personally known to have shot a leading Rajput commander during the Siege of Chittorgarh.[72] The Mughals began to use Bamboo rockets (mainly for signalling) and employ Sappers: special units that undermined heavy stone fortifications to plant gunpowder charges.\nThe Mughal Emperor Shah Jahan is known to have introduced much more advanced Matchlocks, their designs were a combination of Ottoman and Mughal designs. Shah Jahan also countered the British and other Europeans in his province of Gujarāt, which supplied Europe saltpeter for use in gunpowder warfare during the 17th century.[73] Bengal and Mālwa participated in saltpeter production.[73] The Dutch, French, Portuguese, and English used Chhapra as a center of saltpeter refining.[73]\nEver since the founding of the Sultanate of Mysore by Hyder Ali, French military officers were employed to train the Mysore Army. Hyder Ali and his son Tipu Sultan were the first to introduce modern Cannons and Muskets, their army was also the first in India to have official uniforms. During the Second Anglo-Mysore War Hyder Ali and his son Tipu Sultan unleashed the Mysorean rockets at their British opponents effectively defeating them on various occasions. The Mysorean rockets inspired the development of the Congreve rocket, which the British widely utilized during the Napoleonic Wars and the War of 1812.[74]\nIndonesia[edit]\nThe Javanese Majapahit Empire was arguably able to encompass much of modern day Indonesia due to its unique mastery of bronze smithing and use of a central arsenal fed by a large number of cottage industries within the immediate region. Documentary and archeological evidence indicate that Arab or Indian traders introduced gunpowder, gonnes, muskets, blunderbusses, and cannons to the Javanese, Acehnese, and Batak via long established commercial trade routes around the early to mid 14th century CE.[75] Portuguese and Spanish invaders were unpleasantly surprised and occasionally even outgunned on occasion.[76] The resurgent Singhasari Empire overtook Sriwijaya and later emerged as the Majapahit whose warfare featured the use of fire-arms and cannonade.[77] Circa 1540 CE the Javanese, always alert for new weapons found the newly arrived Portuguese weaponry superior to that of the locally made variants. Javanese bronze breech-loaded swivel-guns, known as meriam, or erroneously as lantaka, was used widely by the Majapahit navy as well as by pirates and rival lords. The demise of the Majapahit empire and the dispersal of disaffected skilled bronze cannon-smiths to Brunei, modern Sumatra, Malaysia and the Philippines lead to widespread use, especially in the Makassar Strait.\nSaltpeter harvesting was recorded by Dutch and German travelers as being common in even the smallest villages and was collected from the decomposition process of large dung hills specifically piled for the purpose. The Dutch punishment for possession of non-permitted gunpowder appears to have been amputation.[78] Ownership and manufacture of gunpowder was later prohibited by the colonial Dutch occupiers.[75] According to a colonel McKenzie quoted in Sir Thomas Stamford Raffles, The History of Java (1817), the purest sulfur was supplied from a crater from a mountain near the straits of Bali.[77]\nManufacturing technology[edit]\n\nEdge-runner mill in a restored mill, at Eleutherian Mills\nFor the most powerful black powder meal, a wood charcoal is used. The best wood for the purpose is Pacific willow,[79] but others such as alder or buckthorn can be used. In Great Britain between the 15th to 19th centuries charcoal from alder buckthorn was greatly prized for gunpowder manufacture; cottonwood was used by the American Confederate States.[80] The ingredients are reduced in particle size and mixed as intimately as possible. Originally this was with a mortar-and-pestle or a similarly operating stamping-mill, using copper, bronze or other non-sparking materials, until supplanted by the rotating ball mill principle with non-sparking bronze or lead. Historically, a marble or limestone edge runner mill, running on a limestone bed was used in Great Britain; however, by the mid 19th century AD this had changed to either an iron shod stone wheel or a cast iron wheel running on an iron bed.[81] The mix was dampened with alcohol or water during grinding to prevent accidental ignition. This also helps the extremely soluble saltpeter mix into the microscopic nooks and crannies of the very high surface-area charcoal.\nAround the late 14th century AD, European powdermakers first began adding liquid during grinding to improve mixing, reduce dust, and with it the risk of explosion.[82] The powder-makers would then shape the resulting paste of dampened gunpowder, known as mill cake, into corns, or grains, to dry. Not only did corned powder keep better because of its reduced surface area, gunners also found that it was more powerful and easier to load into guns. Before long, powder-makers standardized the process by forcing mill cake through sieves instead of corning powder by hand.\nThe improvement was based on reducing the surface area of a higher density composition. At the beginning of the 19th century, makers increased density further by static pressing. They shoveled damp mill cake into a two-foot square box, placed this beneath a screw press and reduced it to 1/2 its volume. \"Presscake\" had the hardness of slate. They broke the dried slabs with hammers or rollers, and sorted the granules with sieves into different grades. In the United States, Irenee du Pont, who had learned the trade from Lavoisier, tumbled the dried grains in rotating barrels to round the edges and increase durability during shipping and handling. (Sharp grains rounded off in transport, producing fine \"meal dust\" that changed the burning properties.)\nAnother advance was the manufacture of kiln charcoal by distilling wood in heated iron retorts instead of burning it in earthen pits. Controlling the temperature influenced the power and consistency of the finished gunpowder. In 1863, in response to high prices for Indian saltpeter, DuPont chemists developed a process using potash or mined potassium chloride to convert plentiful Chilean sodium nitrate to potassium nitrate.[83]\nDuring the 18th century gunpowder factories became increasingly dependent on mechanical energy.[84] Despite mechanization, production difficulties related to humidity control, especially during the pressing, were still present in the late 19th century. A paper from 1885 laments that \"Gunpowder is such a nervous and sensitive spirit, that in almost every process of manufacture it changes under our hands as the weather changes.\" Pressing times to the desired density could vary by factor of three depending on the atmospheric humidity.[85]\nComposition and characteristics[edit]\nThe term black powder was coined in the late 19th century, primarily in the United States, to distinguish prior gunpowder formulations from the new smokeless powders and semi-smokeless powders, in cases where these are not referred to as cordite. Semi-smokeless powders featured bulk volume properties that approximated black powder, but had significantly reduced amounts of smoke and combustion products. Smokeless powder has different burning properties (pressure vs. time) and can generate higher pressures and work per gram. This can rupture older weapons designed for black powder. Smokeless powders ranged in color from brownish tan to yellow to white. Most of the bulk semi-smokeless powders ceased to be manufactured in the 1920s.[86][87][88]\nBlack powder is a granular mixture of\na nitrate, typically potassium nitrate (KNO3), which supplies oxygen for the reaction;\ncharcoal, which provides carbon and other fuel for the reaction, simplified as carbon (C);\nsulfur (S), which, while also serving as a fuel, lowers the temperature required to ignite the mixture, thereby increasing the rate of combustion.\nPotassium nitrate is the most important ingredient in terms of both bulk and function because the combustion process releases oxygen from the potassium nitrate, promoting the rapid burning of the other ingredients.[89] To reduce the likelihood of accidental ignition by static electricity, the granules of modern black powder are typically coated with graphite, which prevents the build-up of electrostatic charge.\nCharcoal does not consist of pure carbon; rather, it consists of partially pyrolyzed cellulose, in which the wood is not completely decomposed. Carbon differs from charcoal. Whereas charcoal's autoignition temperature is relatively low, carbon's is much greater. Thus, a black powder composition containing pure carbon would burn similarly to a match head, at best.[90]\nThe current standard composition for the black powders that are manufactured by pyrotechnicians was adopted as long ago as 1780. Proportions by weight are 75% potassium nitrate (known as saltpeter or saltpetre), 15% softwood charcoal, and 10% sulfur.[81] These ratios have varied over the centuries and by country, and can be altered somewhat depending on the purpose of the powder. For instance, power grades of black powder, unsuitable for use in firearms but adequate for blasting rock in quarrying operations, is called blasting powder rather than gunpowder with standard proportions of 70% nitrate, 14% charcoal, and 16% sulfur; blasting powder may be made with the cheaper sodium nitrate substituted for potassium nitrate and proportions may be as low as 40% nitrate, 30% charcoal, and 30% sulfur.[91] In 1857, Lamont DuPont solved the main problem of using cheaper sodium nitrate formulations when he patented DuPont \"B\" Blasting powder. After manufacturing grains from press-cake in the usual way, his process tumbled the powder with graphite dust for 12 hours. This formed a graphite coating on each grain that reduced its ability to absorb moisture.[92]\nFrench war powder in 1879 used the ratio 75% saltpeter, 12.5% charcoal, 12.5% sulfur. English war powder in 1879 used the ratio 75% saltpeter, 15% charcoal, 10% sulfur.[93] The British Congreve rockets used 62.4% saltpeter, 23.2% charcoal and 14.4% sulfur, but the British Mark VII gunpowder was changed to 65% saltpeter, 20% charcoal and 15% sulfur.[94] The explanation for the wide variety in formulation relates to usage. Powder used for rocketry can use a slower burn rate since it accelerates the projectile for a much longer time—whereas powders for weapons such as flintlocks, cap-locks, or matchlocks need a higher burn rate to accelerate the projectile in a much shorter distance. Cannons usually used lower burn rate powders, because most would burst with higher burn rate powders.\nSerpentine[edit]\nThe original dry-compounded powder used in fifteenth-century Europe was known as \"Serpentine\", either a reference to Satan[95] or to a common artillery piece that used it.[96] The ingredients were ground together with a mortar and pestle, perhaps for 24 hours,[96] resulting in a fine flour. Vibration during transportation could cause the components to separate again, requiring remixing in the field. Also if the quality of the saltpeter was low (for instance if it was contaminated with highly hygroscopic calcium nitrate), or if the powder was simply old (due to the mildly hygroscopic nature of potassium nitrate), in humid weather it would need to be re-dried. The dust from \"repairing\" powder in the field was a major hazard.\nLoading cannons or bombards before the powder-making advances of the Renaissance was a skilled art. Fine powder loaded haphazardly or too tightly would burn incompletely or too slowly. Typically, the breech-loading powder chamber in the rear of the piece was filled only about half full, the serpentine powder neither too compressed nor too loose, a wooden bung pounded in to seal the chamber from the barrel when assembled, and the projectile placed on. A carefully determined empty space was necessary for the charge to burn effectively. When the cannon was fired through the touchhole, turbulence from the initial surface combustion caused the rest of the powder to be rapidly exposed to the flame.[96]\nThe advent of much more powerful and easy to use corned powder changed this procedure, but serpentine was used with older guns into the seventeenth century.[97]\nCorning[edit]\nFor gunpowder to explode effectively, the combustible ingredients must be reduced to the smallest possible particle sizes, and thoroughly mixed as possible. Once mixed, however, for better results in a gun, makers discovered that the final product should be in the form of individual, dense, grains that spread the fire quickly from grain to grain, much as straw or twigs catch fire more quickly than a pile of sawdust.\nPrimarily for safety reasons, size reduction and mixing is done while the ingredients are damp, usually with water. After 1800, instead of forming grains by hand or with sieves, the damp mill-cake was pressed in molds to increase its density and extract the liquid, forming press-cake. The pressing took varying amounts of time, depending on conditions such as atmospheric humidity. The hard, dense product was broken again into tiny pieces, which were separated with sieves to produce a uniform product for each purpose: coarse powders for cannons, finer grained powders for muskets, and the finest for small hand guns and priming.[97] Inappropriately fine-grained powder often caused cannons to burst before the projectile could move down the barrel, due to the high initial spike in pressure.[98] Mammoth powder with large grains made for Rodman's 15-inch cannon reduced the pressure to only 20 percent as high as ordinary cannon powder would have produced.[99]\nIn the mid-nineteenth century, measurements were made determining that the burning rate within a grain of black powder (or a tightly packed mass) is about 0.20 fps, while the rate of ignition propagation from grain to grain is around 30 fps, over two orders of magnitude faster.[97]\nModern types[edit]\nModern corning first compresses the fine black powder meal into blocks with a fixed density (1.7 g/cm³).[100] In the United States, gunpowder grains were designated F (for fine) or C (for coarse). Grain diameter decreased with a larger number of Fs and increased with a larger number of Cs, ranging from about 2 mm for 7F to 15 mm for 7C. Even larger grains were produced for artillery bore diameters greater than about 17 cm (6.7 in). The standard DuPont Mammoth powder developed by Thomas Rodman and Lammot du Pont for use during the American Civil War had grains averaging 0.6 inches diameter, with edges rounded in a glazing barrel.[99] Other versions had grains the size of golf and tennis balls for use in 20-inch (50-cm) Rodman guns.[101] In 1875 DuPont introduced Hexagonal powder for large artillery, which was pressed using shaped plates with a small center core—about 1.5 inches diameter, like a wagon wheel nut, the center hole widened as the grain burned.[102] By 1882 German makers also produced hexagonal grained powders of a similar size for artillery.[102]\nBy the late 19th century manufacturing focused on standard grades of black powder from Fg used in large bore rifles and shotguns, through FFg (medium and small-bore arms such as muskets and fusils), FFFg (small-bore rifles and pistols), and FFFFg (extreme small bore, short pistols and most commonly for priming flintlocks).[103] A coarser grade for use in military artillery blanks was designated A-1. These grades were sorted on a system of screens with oversize retained on a mesh of 6 wires per inch, A-1 retained on 10 wires per inch, Fg retained on 14, FFg on 24, FFFg on 46, and FFFFg on 60. Fines designated FFFFFg were usually reprocessed to minimize explosive dust hazards.[104] In the United Kingdom, the main service gunpowders were classified RFG (rifle grained fine) with diameter of one or two millimeters and RLG (rifle grained large) for grain diameters between two and six millimeters.[101] Gunpowder grains can alternatively be categorized by mesh size: the BSS sieve mesh size, being the smallest mesh size, which retains no grains. Recognized grain sizes are Gunpowder G 7, G 20, G 40, and G 90.\nOwing to the large market of antique and replica black-powder firearms in the US, modern gunpowder substitutes like Pyrodex, Triple Seven and Black Mag3[105] pellets have been developed since the 1970s. These products, which should not be confused with smokeless powders, aim to produce less fouling (solid residue), while maintaining the traditional volumetric measurement system for charges. Claims of less corrosiveness of these products have been controversial however. New cleaning products for black-powder guns have also been developed for this market.[103]\nOther types of gunpowder[edit]\nBesides black powder, there are other historically important types of gunpowder. \"Brown gunpowder\" is cited as composed of 79% nitre, 3% sulfur, and 18% charcoal per 100 of dry powder, with about 2% moisture. Prismatic Brown Powder is a large-grained product the Rottweil Company introduced in 1884 in Germany, which was adopted by the British Royal Navy shortly thereafter. The French navy adopted a fine, 3.1 millimeter, not prismatic grained product called Slow Burning Cocoa (SBC) or \"cocoa powder\". These brown powders reduced burning rate even further by using as little as 2 percent sulfur and using charcoal made from rye straw that had not been completely charred, hence the brown color.[102]\nLesmok powder was a product developed by DuPont in 1911[106] one of several semi-smokeless products in the industry containing a mixture of black and nitrocellulose powder. It was sold to Winchester and others primarily for .22 and .32 small calibers. Its advantage was that it was believed at the time to be less corrosive than smokeless powders then in use. It was not understood in the U.S. until the 1920s that the actual source of corrosion was the potassium chloride residue from potassium chlorate sensitized primers. The bulkier black powder fouling better disperses primer residue. Failure to mitigate primer corrosion by dispersion caused the false impression that nitrocellulose-based powder caused corrosion.[107] Lesmok had some of the bulk of black powder for dispersing primer residue, but somewhat less total bulk than straight black powder, thus requiring less frequent bore cleaning.[108] It was last sold by Winchester in 1947.\nSulfur-free gunpowder[edit]\n\nBurst barrel of a muzzle loader pistol replica, which was loaded with nitrocellulose powder instead of black powder and couldn't withstand the higher pressures of the modern propellant\nThe development of smokeless powders, such as cordite, in the late 19th century created the need for a spark-sensitive priming charge, such as gunpowder. However, the sulfur content of traditional gunpowders caused corrosion problems with Cordite Mk I and this led to the introduction of a range of sulfur-free gunpowders, of varying grain sizes.[64] They typically contain 70.5 parts of saltpeter and 29.5 parts of charcoal.[64] Like black powder, they were produced in different grain sizes. In the United Kingdom, the finest grain was known as sulfur-free mealed powder (SMP). Coarser grains were numbered as sulfur-free gunpowder (SFG n): 'SFG 12', 'SFG 20', 'SFG 40' and 'SFG 90', for example; where the number represents the smallest BSS sieve mesh size, which retained no grains.\nSulfur's main role in gunpowder is to decrease the ignition temperature. A sample reaction for sulfur-free gunpowder would be\n6 KNO3 + C7H4O → 3 K2CO3 + 4 CO2 + 2 H2O + 3 N2\nCombustion characteristics[edit]\nA simple, commonly cited, chemical equation for the combustion of black powder is\n2 KNO3 + S + 3 C → K2S + N2 + 3 CO2.\nA balanced, but still simplified, equation is[109]\n10 KNO3 + 3 S + 8 C → 2 K2CO3 + 3 K2SO4 + 6 CO2 + 5 N2.\nAlthough charcoal's chemical formula varies, it can be best summed up by its empirical formula: C7H4O.\nTherefore, an even more accurate equation of the decomposition of regular black powder with the use of sulfur can be described as:\n6 KNO3 + C7H4O + 2 S → K2CO3 + K2SO4 + K2S + 4 CO2 + 2 CO + 2 H2O + 3 N2\nBlack powder without the use of sulfur:\n10 KNO3 + 2 C7H4O → 5 K2CO3 + 4 CO2 + 5 CO + 4 H2O + 5 N2\nThe burning of gunpowder does not take place as a single reaction, however, and the byproducts are not easily predicted. One study's results showed that it produced (in order of descending quantities) 55.91% solid products: potassium carbonate, potassium sulfate, potassium sulfide, sulfur, potassium nitrate, potassium thiocyanate, carbon, ammonium carbonate and 42.98% gaseous products: carbon dioxide, nitrogen, carbon monoxide, hydrogen sulfide, hydrogen, methane, 1.11% water.\nBlack powder made with less-expensive and more plentiful sodium nitrate (in appropriate proportions) works just as well but is more hygroscopic than powders made from Potassium nitrate—popularly known as saltpeter. Because corned black powder grains made with saltpeter are less affected by moisture in the air, they can be stored unsealed without degradation by humidity. Muzzleloaders have been known to fire after hanging on a wall for decades in a loaded state, provided they remained dry. By contrast, black powder made with sodium nitrate must be kept sealed to remain stable.\nGunpowder contains 3 megajoules per kilogram, and contains its own oxidant. For comparison, the energy density of TNT is 4.7 megajoules per kilogram, and the energy density of gasoline is 47.2 megajoules per kilogram. Gunpowder is a low explosive and as such it does not detonate; rather it deflagrates. Since it contains its own oxidizer and additionally burns faster under pressure, its combustion is capable of rupturing containers such as shell, grenade, or improvised \"pipe bomb\" or \"pressure cooker\" casings, forming shrapnel.\nAdvantages[edit]\nIn quarrying, high explosives are generally preferred for shattering rock. However, because of its low brisance, black powder causes fewer fractures and results in more usable stone compared to other explosives, making black powder useful for blasting monumental stone such as granite and marble. Black powder is well suited for blank rounds, signal flares, burst charges, and rescue-line launches. Black powder is also used in fireworks for lifting shells, in rockets as fuel, and in certain special effects.\nDisadvantages[edit]\nBlack powder has a low energy density compared to modern \"smokeless\" powders, and thus to achieve high energy loadings, large amounts of black powder are needed with heavy projectiles. Black powder also produces thick smoke as a byproduct, which in military applications may give a soldier's location away to an enemy observer and may also impair aiming for additional shots.\nCombustion converts less than half the mass of black powder to gas. The rest ends up as a thick layer of soot inside the barrel. In addition to being a nuisance, the residue from burnt black powder is hygroscopic and with the addition of moisture absorbed from the air, this residue forms a caustic substance. The soot contains potassium oxide or sodium oxide that turns into potassium hydroxide, or sodium hydroxide, which corrodes wrought iron or steel gun barrels. Black powder arms must be well cleaned both inside and out to remove the residue. The matchlock musket or pistol (an early gun ignition system), as well as the flintlock would often be unusable in wet weather, due to powder in the pan being exposed and dampened. Because of this unreliability, soldiers carrying muskets, known as musketeers, were armed with additional weapons such as swords or pikes. The bayonet was developed to allow the musket to be used as a pike, thus eliminating the need for the soldier to carry a secondary weapon.\nTransportation[edit]\nThe United Nations Model Regulations on the Transportation of Dangerous Goods and national transportation authorities, such as United States Department of Transportation, have classified gunpowder (black powder) as a Group A: Primary explosive substance for shipment because it ignites so easily. Complete manufactured devices containing black powder are usually classified as Group D: Secondary detonating substance, or black powder, or article containing secondary detonating substance, such as firework, class D model rocket engine, etc., for shipment because they are harder to ignite than loose powder. As explosives, they all fall into the category of Class 1.\nOther uses[edit]\nBesides its use as an explosive, gunpowder has been occasionally employed for other purposes; after the Battle of Aspern-Essling (1809), the surgeon of the Napoleonic Army Larrey combated the lack of food for the wounded under his care by preparing a bouillon of horse meat seasoned with gunpowder for lack of salt.[110][111] It was also used for sterilizing on ships when there was no alcohol.\nJack Tars (British sailors) used gunpowder to create tattoos when ink wasn't available, by pricking the skin and rubbing the powder into the wound in a method known as traumatic tattooing.[112]\nChristiaan Huygens experimented with gunpowder in 1673 in an early attempt to build an internal combustion engine, but he did not succeed. Modern attempts to recreate his invention were similarly unsuccessful.\nFireworks use gunpowder as lifting and burst charges, although sometimes other more powerful compositions are added to the burst charge to improve performance in small shells or provide a louder report. Most modern firecrackers no longer contain black powder.\nBeginning in the 1930s, gunpowder or smokeless powder was used in rivet guns, stun guns for animals, cable splicers and other industrial construction tools.[113] The \"stud gun\" drove nails or screws into solid concrete, a function not possible with hydraulic tools. See Powder-actuated tool. Shotguns have been used to eliminate persistent material rings in operating rotary kilns (such as those for cement, lime, phosphate, etc.) and clinker in operating furnaces, and commercial tools make the method more reliable.[114]\nNear London in 1853, Captain Shrapnel demonstrated a method for crushing gold-bearing ores by firing them from a cannon into an iron chamber, and \"much satisfaction was expressed by all present\". He hoped it would be useful on the goldfields of California and Australia. Nothing came of the invention, as continuously-operating crushing machines that achieved more reliable comminution were already coming into use.[115]\nSee also[edit]\nBallistics\nBlack powder substitute\nFaversham explosives industry\nBulk loaded liquid propellants\nGunpowder magazine\nGunpowder Plot\nBerthold Schwarz\nGunpowder warfare\nHistory of gunpowder\nTechnology of the Song Dynasty\nReferences[edit]\nJump up ^ http://www.merriam-webster.com/dictionary/gunpowder\nJump up ^ Jai Prakash Agrawal (2010). High Energy Materials: Propellants, Explosives and Pyrotechnics. Wiley-VCH. p. 69. ISBN 978-3-527-32610-5.\nJump up ^ David Cressy, Saltpeter: The Mother of Gunpowder (Oxford University Press, 2013)\nJump up ^ Owen Compliance Services. \"Black Powder\". Material Safety Data Sheet. Retrieved 31 August 2014.\nJump up ^ http://www.history.com/shows/ancient-discoveries/articles/who-built-it-first-2\nJump up ^ http://chemistry.about.com/od/historyofchemistry/a/gunpowder.htm\nJump up ^ Chase 2003:31 : \"the earliest surviving formulas for gunpowder can be found in the Wujing zongyao, a military work from around 1040\"\n^ Jump up to: a b c Buchanan 2006, p. 2 \"With its ninth century AD origins in China, the knowledge of gunpowder emerged from the search by alchemists for the secrets of life, to filter through the channels of Middle Eastern culture, and take root in Europe with consequences that form the context of the studies in this volume.\"\n^ Jump up to: a b c Joseph Needham; Gwei-Djen Lu; Ling Wang (1987). Science and civilisation in China, Volume 5, Part 7. Cambridge University Press. pp. 48–50. ISBN 978-0-521-30358-3.\nJump up ^ Hazel Rossotti (2002). Fire: Servant, Scourge, and Enigma. Courier Dover Publications. pp. 132–137. ISBN 978-0-486-42261-9.\nJump up ^ Jack Kelly Gunpowder: Alchemy, Bombards, and Pyrotechnics: The History of the Explosive that Changed the World, Perseus Books Group: 2005, ISBN 0-465-03722-4, ISBN 978-0-465-03722-3: 272 pages\nJump up ^ St. C. Easton: \"Roger Bacon and his Search for a Universal Science\", Oxford (1962)\n^ Jump up to: a b Gábor Ágoston (2005). Guns for the sultan: military power and the weapons industry in the Ottoman Empire. Cambridge University Press. p. 15. ISBN 978-0-521-84313-3.\nJump up ^ Ingham-Brown, George (1989) The Big Bang: A History of Explosives, Sutton Publishers, ISBN 0-7509-1878-0, ISBN 978-0-7509-1878-7, page vi\nJump up ^ Kelly, Jack (2005) Gunpowder: Alchemy, Bombards, and Pyrotechnics: The History of the Explosive that Changed the World, Perseus Books Group, ISBN 0-465-03722-4, ISBN 978-0-465-03722-3, page 22\nJump up ^ Bert S. Hall, \"Introduction, 1999\" pp. xvi–xvii to the reprinting of James Riddick Partington (1960). A history of Greek fire and gunpowder. JHU Press. ISBN 978-0-8018-5954-0.\n^ Jump up to: a b Peter Purton (2009). A History of the Late Medieval Siege, 1200–1500. Boydell & Brewer. pp. 108–109. ISBN 978-1-84383-449-6.\nJump up ^ Bert S. Hall, \"Introduction, 1999\" p. xvii to the reprinting of James Riddick Partington (1960). A history of Greek fire and gunpowder. JHU Press. ISBN 978-0-8018-5954-0.\nJump up ^ Buchanan. \"Editor's Introduction: Setting the Context\", in Buchanan 2006.\n^ Jump up to: a b Chase 2003:31–32\nJump up ^ Lorge, Peter A. (2008). The Asian military revolution, 1300-2000 : from gunpowder to the bomb (1. publ. ed.). Cambridge: Cambridge University Press. p. 32. ISBN 978052160954-8.\nJump up ^ Kelly 2004:4\nJump up ^ The Big Book of Trivia Fun, Kidsbooks, 2004\nJump up ^ Peter Allan Lorge (2008), The Asian military revolution: from gunpowder to the bomb, Cambridge University Press, p. 18, ISBN 978-0-521-60954-8\nJump up ^ Needham 1986, p. 7 \"Without doubt it was in the previous century, around +850, that the early alchemical experiments on the constituents of gunpowder, with its self-contained oxygen, reached their climax in the appearance of the mixture itself.\"\nJump up ^ Chase 2003:1 \"The earliest known formula for gunpowder can be found in a Chinese work dating probably from the 800s. The Chinese wasted little time in applying it to warfare, and they produced a variety of gunpowder weapons, including flamethrowers, rockets, bombs, and land mines, before inventing firearms.\"\nJump up ^ Chase 2003:1\nJump up ^ Delgado, James (February 2003). \"Relics of the Kamikaze\". Archaeology (Archaeological Institute of America) 56 (1).\nJump up ^ Chase 2003:31\nJump up ^ Peter Allan Lorge (2008), The Asian military revolution: from gunpowder to the bomb, Cambridge University Press, pp. 33–34, ISBN 978-0-521-60954-8\nJump up ^ Kelly 2004:22 'Around year 1240, Arabs acquired knowledge of saltpeter (\"Chinese snow\") from the East, perhaps through India. They knew of gunpowder soon afterward. They also learned about fireworks (\"Chinese flowers\") and rockets (\"Chinese arrows\"). Arab warriors had acquired fire lances before year 1280. Around that same year, a Syrian named Hasan al-Rammah wrote a book that, as he put it, \"treats of machines of fire to be used for amusement or for useful purposes.\" He talked of rockets, fireworks, fire lances, and other incendiaries, using terms that suggested he derived his knowledge from Chinese sources. He gave instructions for the purification of saltpeter and recipes for making different types of gunpowder.'\n^ Jump up to: a b c d Hassan, Ahmad Y. \"Transfer of Islamic Technology to the West: Part III\". History of Science and Technology in Islam.\nJump up ^ Peter Watson (2006). Ideas: A History of Thought and Invention, from Fire to Freud. HarperCollins. p. 304. ISBN 978-0-06-093564-1. The first use of a metal tube in this context was made around 1280 in the wars between the Song and the Mongols, where a new term, chong, was invented to describe the new horror...Like paper, it reached the West via the Muslims, in this case the writings of the Andalusian botanist Ibn al-Baytar, who died in Damascus in 1248. The Arabic term for saltpetre is 'Chinese snow' while the Persian usage is 'Chinese salt'.28\nJump up ^ Cathal J. Nolan (2006). The age of wars of religion, 1000–1650: an encyclopedia of global warfare and civilization. Volume 1 of Greenwood encyclopedias of modern world wars. Greenwood Publishing Group. p. 365. ISBN 0-313-33733-0. Retrieved 2011-11-28. In either case, there is linguistic evidence of Chinese origins of the technology: in Damascus, Arabs called the saltpeter used in making gunpowder \" Chinese snow,\" while in Iran it was called \"Chinese salt.\" Whatever the migratory route\nJump up ^ Oliver Frederick Gillilan Hogg (1970). Artillery: its origin, heyday, and decline. Archon Books. p. 123. The Chinese were certainly acquainted with saltpetre, the essential ingredient of gunpowder. They called it Chinese Snow and employed it early in the Christian era in the manufacture of fireworks and rockets.\nJump up ^ Oliver Frederick Gillilan Hogg (1963). English artillery, 1326–1716: being the history of artillery in this country prior to the formation of the Royal Regiment of Artillery. Royal Artillery Institution. p. 42. The Chinese were certainly acquainted with saltpetre, the essential ingredient of gunpowder. They called it Chinese Snow and employed it early in the Christian era in the manufacture of fireworks and rockets.\nJump up ^ Oliver Frederick Gillilan Hogg (1993). Clubs to cannon: warfare and weapons before the introduction of gunpowder (reprint ed.). Barnes & Noble Books. p. 216. ISBN 1-56619-364-8. Retrieved 2011-11-28. The Chinese were certainly acquainted with saltpetre, the essential ingredient of gunpowder. They called it Chinese snow and used it early in the Christian era in the manufacture of fireworks and rockets.\nJump up ^ Partington, J. R. (1960). A History of Greek Fire and Gunpowder (illustrated, reprint ed.). JHU Press. p. 335. ISBN 0801859549. Retrieved 2014-11-21.\nJump up ^ Needham, Joseph; Yu, Ping-Yu (1980). Needham, Joseph, ed. Science and Civilisation in China: Volume 5, Chemistry and Chemical Technology, Part 4, Spagyrical Discovery and Invention: Apparatus, Theories and Gifts. Volume 5 (Issue 4 of Science and Civilisation in China). Contributors Joseph Needham, Lu Gwei-Djen, Nathan Sivin (illustrated, reprint ed.). Cambridge University Press. p. 194. ISBN 052108573X. Retrieved 2014-11-21.\nJump up ^ Khan 1996\n^ Jump up to: a b Khan 2004:6\nJump up ^ Ancient Discoveries, Episode 12: Machines of the East, History Channel, 2007 (Part 4 and Part 5)\nJump up ^ Nelson, Cameron Rubaloff (2010-07). Manufacture and transportation of gunpowder in the Ottoman Empire: 1400-1800 M.A. Thesis.\nJump up ^ William H. McNeill (1992). The Rise of the West: A History of the Human Community. University of Chicago Press. p. 492. ISBN 0-226-56141-0. Retrieved 29 July 2011.\nJump up ^ Michael Kohn (2006), Dateline Mongolia: An American Journalist in Nomad's Land, RDR Books, p. 28, ISBN 1-57143-155-1, retrieved 29 July 2011\nJump up ^ Robert Cowley (1993). Robert Cowley, ed. Experience of War (reprint ed.). Random House Inc. p. 86. ISBN 0-440-50553-4. Retrieved 29 July 2011.\nJump up ^ Kenneth Warren Chase (2003). Firearms: a global history to 1700 (illustrated ed.). Cambridge University Press. p. 58. ISBN 0-521-82274-2. Retrieved 29 July 2011.\nJump up ^ C. F. Temler, Historische Abhandlungen der Koniglichen Gesellschaft der Wissenschaften zu Kopenhagen ... ubersetzt ... von V. A. Heinze, Kiel, Dresden and Leipzig, 1782, i, 168, as cited in Partington, p. 228, footnote 6.\nJump up ^ Joseph Needham; Gwei-Djen Lu; Ling Wang (1987). Science and civilisation in China, Volume 5, Part 7. Cambridge University Press. p. 358. ISBN 978-0-521-30358-3.\nJump up ^ Bert S. Hall, \"Introduction, 1999\" p. xxiv to the reprinting of James Riddick Partington (1960). A history of Greek fire and gunpowder. JHU Press. ISBN 978-0-8018-5954-0.\nJump up ^ Partington 1960:60\n^ Jump up to: a b Partington 1960:48–49, 54\n^ Jump up to: a b Partington 1960:82–83\n^ Jump up to: a b c d Kelly 2004, p.61\nJump up ^ Molerus, Otto. \"History of Civilization in the Western Hemisphere from the Point of View of Particulate Technology, Part 2,\" Advanced Powder Technology 7 (1996): 161-66\nJump up ^ Microsoft Encarta Online Encyclopedia 2007 Archived 31 October 2009.\nJump up ^ In 1777 Lavoisier named oxygen, which had earlier been isolated by Priestley; the realization that saltpeter contained this substance was fundamental to understanding gunpowder.\n^ Jump up to: a b Kelly 2004, p.164\nJump up ^ Metzner, Paul (1998), Crescendo of the Virtuoso: Spectacle, Skill, and Self-Promotion in Paris during the Age of Revolution, University of California Press\n^ Jump up to: a b c d Cocroft 2000, \"Success to the Black Art!\". Chapter 1\nJump up ^ Ross, Charles. The Custom of the Castle: From Malory to Macbeth. Berkeley: University of California Press, c1997. [1] pages 131-130\nJump up ^ The Noble-Abel Equation of State: Thermodynamic Derivations for Ballistics Modelling\nJump up ^ Pritchard, Tom; Evans, Jack; Johnson, Sydney (1985), The Old Gunpowder Factory at Glynneath, Merthyr Tydfil: Merthyr Tydfil & District Naturalists' Society\n^ Jump up to: a b c d e Cocroft 2000, \"The demise of gunpowder\". Chapter 4\nJump up ^ MacDougall, Ian (2000). 'Oh, ye had to be careful' : personal recollections by Roslin gunpowder mill and bomb factory workers. East Linton, Scotland: Tuckwell Press in association with the European Ethnological Research Centre and the Scottish Working People's History Trust. ISBN 1-86232-126-4.\nJump up ^ Iqtidar Alam Khan (2004). Gunpowder And Firearms: Warfare In Medieval India. Oxford University Press. ISBN 978-0-19-566526-0.\n^ Jump up to: a b Iqtidar Alam Khan (25 April 2008). Historical Dictionary of Medieval India. Scarecrow Press. p. 157. ISBN 978-0-8108-5503-8.\n^ Jump up to: a b Khan 2004:9–10\nJump up ^ Khan 2004:10\nJump up ^ Partington (Johns Hopkins University Press edition, 1999), 225\nJump up ^ Partington (Johns Hopkins University Press edition, 1999), 226\nJump up ^ http://www.youtube.com/watch?v=DTfEDaWMj4o\n^ Jump up to: a b c \"India.\" Encyclopædia Britannica. Encyclopedia Britannica 2008 Ultimate Reference Suite. Chicago: Encyclopedia Britannica, 2008.\nJump up ^ \"rocket and missile system.\" Encyclopædia Britannica. Encyclopædia Britannica 2008 Ultimate Reference Suite. Chicago: Encyclopædia Britannica, 2008.\n^ Jump up to: a b Dipanegara, P. B. R. Carey, Babad Dipanagara: an account of the outbreak of the Java war, 1825-30 : the Surakarta court version of the Babad Dipanagara with translations into English and Indonesian volume 9: Council of the M.B.R.A.S. by Art Printing Works: 1981.\nJump up ^ Atsushi, Ota (2006). Changes of regime and social dynamics in West Java : society, state, and the outer world of Banten, 1750-1830. Leiden: Brill. ISBN 90-04-15091-9.\n^ Jump up to: a b Thomas Stamford Raffles, The History of Java, Oxford University Press, 1965 (originally published in 1817), ISBN 0-19-580347-7\nJump up ^ Raffles, Thomas Stamford (1978). The History of Java ([Repr.]. ed.). Kuala Lumpur: Oxford University Press. ISBN 0-19-580347-7.\nJump up ^ US Department of Agriculture (1917). Department Bulleting No. 316: Willows: Their growth, use, and importance. The Department. p. 31.\nJump up ^ Kelly 2004, p.200\n^ Jump up to: a b Earl 1978, Chapter 2: The Development of Gunpowder\nJump up ^ Kelly 2004:60–63\nJump up ^ Kelly 2004, p.199\nJump up ^ Frangsmyr, Tore, J. L. Heilbron, and Robin E. Rider, editors The Quantifying Spirit in the Eighteenth Century. Berkeley: University of California Press, c1990. http://ark.cdlib.org/ark:/13030/ft6d5nb455/ p. 292.\nJump up ^ C.E. Munroe (1885) \"Notes on the literature of explosives no. VIII\", Proceedings of the US Naval Institute, no. XI, p. 285\nJump up ^ The History of the 10.4×38 Swiss Cartridge\nJump up ^ Blackpowder to Pyrodex and Beyond by Randy Wakeman at Chuck Hawks\nJump up ^ The History and Art of Shotshells by Jon Farrar, Nebraskaland Magazine\nJump up ^ Buchanan. \"Editor's Introduction: Setting the Context\", in Buchanan 2006, p. 4.\nJump up ^ Black Powder Recipes, Ulrich Bretscher\nJump up ^ Julian S. Hatcher, Hatcher's Notebook, Military Service Publishing Company, 1947. Chapter XIII Notes on Gunpowder, pages 300-305.\nJump up ^ Kelly 2004, p.218\nJump up ^ Book title Workshop Receipts Publisher William Clowes and Son limited Author Ernest Spon. Date 1 August 1873.\nJump up ^ GunpowderTranslation. Academic. Retrieved 2014-08-31.\nJump up ^ Cathal J. Nolan (2006), The age of wars of religion, 1000-1650: an encyclopedia of global warfare and civilization, Greenwood Publishing Group, p. 365, ISBN 978-0-313-33733-8\n^ Jump up to: a b c Kelly 2004, p58\n^ Jump up to: a b c John Francis Guilmartin (2003). Gunpowder & galleys: changing technology & Mediterranean warfare at sea in the 16th century. Conway Maritime Press. pp. 109–110 and 298–300. ISBN 0851779514.\nJump up ^ T.J. Rodman (1861), Reports of experiments on the properties of metals for cannon and the qualities of cannon powder, p. 270\n^ Jump up to: a b Kelly 2004, p.195\nJump up ^ Tenney L. Davis (1943). The Chemistry of Powder and Explosives (PDF). p. 139.\n^ Jump up to: a b Brown, G.I. (1998) The Big Bang: A history of Explosives Sutton Publishing pp.22&32 ISBN 0-7509-1878-0\n^ Jump up to: a b c Kelly 2004, p.224\n^ Jump up to: a b Rodney James (2011). The ABCs of Reloading: The Definitive Guide for Novice to Expert (9 ed.). Krause Publications. pp. 53–59. ISBN 978-1-4402-1396-0.\nJump up ^ Sharpe, Philip B. (1953) Complete Guide to Handloading Funk & Wagnalls p.137\nJump up ^ Wakeman, Randy. \"Blackpowder to Pyrodex and Beyond\". Retrieved 31 August 2014.\nJump up ^ \"LESMOK POWDER\".\nJump up ^ Julian S. Hatcher, Hatcher's Notebook, Stackpole Books, 1962. Chapter XIV, Gun Corrosion and Ammunition Developments, pages 346-349.\nJump up ^ Wakeman, Randy. \"Blackpowder to Pyrodex and Beyond\".\nJump up ^ Flash! Bang! Whiz!, University of Denver\nJump up ^ Parker, Harold T. (1983). Three Napoleonic battles. (Repr., Durham, 1944. ed.). Durham, NC: Duke Univ. Pr. p. 83. ISBN 0-8223-0547-X.\nJump up ^ Larrey is quoted in French at Dr Béraud, Études Hygiéniques de la chair de cheval comme aliment, Musée des Familles (1841-42).\nJump up ^ Rediker, Marcus (1989). Between the devil and the deep blue sea : merchant seamen, pirates, and the Anglo-American maritime world, 1700-1750 (1st pbk. ed. ed.). Cambridge: Cambridge University Press. p. 12. ISBN 9780521379830.\nJump up ^ \"Gunpowder Now Used To Drive Rivets And Splice Cables\", April 1932, Popular Science\nJump up ^ \"MasterBlaster System\". Remington Products.\nJump up ^ Mining Journal 22 January 1853, p. 61\nBenton, Captain James G. (1862). A Course of Instruction in Ordnance and Gunnery (2 ed.). West Point, New York: Thomas Publications. ISBN 1-57747-079-6..\nBrown, G. I. (1998). The Big Bang: A History of Explosives. Sutton Publishing. ISBN 0-7509-1878-0..\nBuchanan, Brenda J., ed. (2006). Gunpowder, Explosives and the State: A Technological History. Aldershot: Ashgate. ISBN 0-7546-5259-9..\nChase, Kenneth (2003). Firearms: A Global History to 1700. Cambridge University Press. ISBN 0-521-82274-2..\nCocroft, Wayne (2000). Dangerous Energy: The archaeology of gunpowder and military explosives manufacture. Swindon: English Heritage. ISBN 1-85074-718-0..\nCrosby, Alfred W. (2002). Throwing Fire: Projectile Technology Through History. Cambridge University Press. ISBN 0-521-79158-8..\nEarl, Brian (1978). Cornish Explosives. Cornwall: The Trevithick Society. ISBN 0-904040-13-5..\nal-Hassan, Ahmad Y.. \"History of Science and Technology in Islam\". |chapter= ignored (help).\nJohnson, Norman Gardner. \"explosive\". Encyclopædia Britannica. Chicago: Encyclopædia Britannica Online..\nKelly, Jack (2004). Gunpowder: Alchemy, Bombards, & Pyrotechnics: The History of the Explosive that Changed the World. Basic Books. ISBN 0-465-03718-6..\nKhan, Iqtidar Alam (1996). \"Coming of Gunpowder to the Islamic World and North India: Spotlight on the Role of the Mongols\". Journal of Asian History 30: 41–5..\nKhan, Iqtidar Alam (2004). \"Gunpowder and Firearms: Warfare in Medieval India\". Oxford University Press. doi:10.1086/ahr.111.3.817..\nNeedham, Joseph (1986). \"Science & Civilisation in China\". V:7: The Gunpowder Epic. Cambridge University Press. ISBN 0-521-30358-3..\nNorris, John (2003). Early Gunpowder Artillery: 1300-1600. Marlborough: The Crowood Press. ISBN 9781861266156..\nPartington, J.R. (1960). A History of Greek Fire and Gunpowder. Cambridge, UK: W. Heffer & Sons..\nPartington, James Riddick; Hall, Bert S. (1999). A History of Greek Fire and Gunpowder. Baltimore: Johns Hopkins University Press. doi:10.1353/tech.2000.0031. ISBN 0-8018-5954-9.\nUrbanski, Tadeusz (1967). \"Chemistry and Technology of Explosives\" III. New York: Pergamon Press..\nExternal links[edit]\n\tWikimedia Commons has media related to Gunpowder.\n\tLook up gunpowder in Wiktionary, the free dictionary.\nGun and Gunpowder\nThe Origins of Gunpowder\nCannons and Gunpowder\nOare Gunpowder Works, Kent, UK\nRoyal Gunpowder Mills\nThe DuPont Company on the Brandywine A digital exhibit produced by the Hagley Library that covers the founding and early history of the DuPont Company powder yards in Delaware\n\"Ulrich Bretschler's Gunpowder Chemistry page\".\nVideo Demonstration of the Medieval Siege Society's Guns, Including showing ignition of gunpowder\nBlack Powder Recipes\n\"Dr. Sasse's investigations (and others) found via search at US DTIC.MIL These contain scientific studies of BP properties and details of measurement techniques.\".\nCategories: GunpowderChinese inventionsExplosivesFirearm propellantsPyrotechnic compositionsRocket fuelsSolid fuels\nNavigation menu\nCreate accountLog inArticleTalkReadEditView history\n\nMain page\nContents\nFeatured content\nCurrent events\nRandom article\nDonate to Wikipedia\nWikimedia Shop\nInteraction\nHelp\nAbout Wikipedia\nCommunity portal\nRecent changes\nContact page\nTools\nWhat links here\nRelated changes\nUpload file\nSpecial pages\nPermanent link\nPage information\nWikidata item\nCite this page\nPrint/export\nCreate a book\nDownload as PDF\nPrintable version\nLanguages\nAfrikaans\nالعربية\nAragonés\nAsturianu\nAzərbaycanca\nБашҡортса\nБеларуская\nБеларуская (тарашкевіца)‎\nБългарски\nBosanski\nBrezhoneg\nБуряад\nCatalà\nЧӑвашла\nČeština\nCorsu\nCymraeg\nDansk\nDeutsch\nEesti\nΕλληνικά\nEspañol\nEsperanto\nEuskara\nفارسی\nFrançais\nGaeilge\nGalego\n贛語\nХальмг\n한국어\nहिन्दी\nHrvatski\nIlokano\nBahasa Indonesia\nÍslenska\nItaliano\nעברית\nKapampangan\nKiswahili\nKurdî\nLatina\nLatviešu\nLietuvių\nLimburgs\nMagyar\nМакедонски\nമലയാളം\nمصرى\nМонгол\nNederlands\nनेपाली\nनेपाल भाषा\n日本語\nНохчийн\nNorsk bokmål\nNorsk nynorsk\nOccitan\nOʻzbekcha\nپنجابی\nPolski\nPortuguês\nRomână\nRuna Simi\nРусский\nСаха тыла\nScots\nShqip\nSicilianu\nSimple English\nSlovenčina\nSlovenščina\nکوردی\nСрпски / srpski\nSrpskohrvatski / српскохрватски\nSuomi\nSvenska\nTagalog\nதமிழ்\nТатарча/tatarça\nไทย\nTürkçe\nУкраїнська\nاردو\nTiếng Việt\nVõro\nWinaray\nייִדיש\n粵語\nŽemaitėška\n中文\nEdit links\nThis page was last modified on 28 November 2014 at 05:37.\nText is available under the Creative Commons Attribution-ShareAlike License; additional terms may apply. By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia® is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\nPrivacy policyAbout WikipediaDisclaimersContact WikipediaDevelopersMobile viewWikimedia Foundation Powered by MediaWiki\n\n\nSmokeless powder\nFrom Wikipedia, the free encyclopedia\n\nFinnish smokeless powder\nSmokeless powder is the name given to a number of propellants used in firearms and artillery that produce negligible smoke when fired, unlike the black powder they replaced. The term is unique to the United States and is generally not used in other English-speaking countries, which initially used proprietary names such as \"Ballistite\" and \"Cordite\" but gradually shifted to \"propellant\" as the generic term.\nThe basis of the term smokeless is that the combustion products are mainly gaseous, compared to around 55% solid products (mostly potassium carbonate, potassium sulfate, and potassium sulfide) for black powder.[1] Despite its name, smokeless powder is not completely smoke-free;[2] while there may be little noticeable smoke from small-arms ammunition, smoke from artillery fire can be substantial. This article focuses on nitrocellulose formulations, but the term smokeless powder was also used to describe various picrate mixtures with nitrate, chlorate, or dichromate oxidizers during the late 19th century, before the advantages of nitrocellulose became evident.[3]\nSince the 14th century[4] gunpowder was not actually a physical \"powder,\" and smokeless powder can only be produced as a pelletized or extruded granular material. Smokeless powder allowed the development of modern semi- and fully automatic firearms and lighter breeches and barrels for artillery. Burnt black powder leaves a thick, heavy fouling that is hygroscopic and causes rusting of the barrel. The fouling left by smokeless powder exhibits none of these properties (though some primer compounds can leave hygroscopic salts that have a similar effect; non-corrosive primer compounds were introduced in the 1920s[5][6]). This makes an autoloading firearm with many moving parts feasible (which would otherwise jam or seize under heavy black powder fouling).\nSmokeless powders are classified as, typically, division 1.3 explosives under the UN Recommendations on the transportation of Dangerous goods – Model Regulations, regional regulations (such as ADR) and national regulations (such the United States' ATF). However, they are used as solid propellants; in normal use, they undergo deflagration rather than detonation.\nContents  [hide] \n1 Background\n2 Nitroglycerine and guncotton\n3 Propellant improvements\n4 Chemical formulations\n5 Instability and stabilization\n6 Physical variations\n7 Smokeless propellant components\n8 Manufacturing\n9 Flashless propellant\n10 See also\n11 References\n11.1 Notes\n11.2 Sources\n12 External links\nBackground[edit]\nMilitary commanders had been complaining since the Napoleonic Wars about the problems of giving orders on a battlefield obscured by the smoke of firing. Verbal commands could not be heard above the noise of the guns, and visual signals could not be seen through the thick smoke from the gunpowder used by the guns. Unless there was a strong wind, after a few shots, soldiers using black powder ammunition would have their view obscured by a huge cloud of smoke. Snipers or other concealed shooters were given away by a cloud of smoke over the firing position. Black powder is also corrosive, making cleaning mandatory after every use. Likewise, black powder's tendency to produce severe fouling caused actions to jam and often made reloading difficult.\nNitroglycerine and guncotton[edit]\nNitroglycerine was synthesized by the Italian chemist Ascanio Sobrero in 1847.[7] It was subsequently developed and manufactured by Alfred Nobel as an industrial explosive, but even then it was unsuitable as a propellant: despite its energetic and smokeless qualities, it detonates instead of deflagrating smoothly, making it more amenable to shattering a gun than propelling a projectile out of it. Nitroglycerine per se is also highly unstable, making it unfit to be carried in battlefield conditions.\nA major step forward was the discovery of guncotton, a nitrocellulose-based material, by Swiss chemist Christian Friedrich Schönbein in 1846. He promoted its use as a blasting explosive[8] and sold manufacturing rights to the Austrian Empire. Guncotton was more powerful than gunpowder, but at the same time was once again somewhat more unstable. John Taylor obtained an English patent for guncotton; and John Hall & Sons began manufacture in Faversham.\nEnglish interest languished after an explosion destroyed the Faversham factory in 1847. Austrian Baron Wilhelm Lenk von Wolfsberg built two guncotton plants producing artillery propellent, but it too was dangerous under field conditions, and guns that could fire thousands of rounds using gunpowder would reach their service life after only a few hundred shots with the more powerful guncotton. Small arms could not withstand the pressures generated by guncotton at all.\nAfter one of the Austrian factories blew up in 1862, Thomas Prentice & Company began manufacturing guncotton in Stowmarket in 1863; and British War Office chemist Sir Frederick Abel began thorough research at Waltham Abbey Royal Gunpowder Mills leading to a manufacturing process that eliminated the impurities in nitrocellulose making it safer to produce and a stable product safer to handle. Abel patented this process in 1865, when the second Austrian guncotton factory exploded. After the Stowmarket factory exploded in 1871, Waltham Abbey began production of guncotton for torpedo and mine warheads.[9]\nPropellant improvements[edit]\nIn 1863, Prussian artillery captain Johann F. E. Schultze patented a small arms propellent of nitrated hardwood impregnated with saltpetre or barium nitrate. Prentice received an 1866 patent for a sporting powder of nitrated paper manufactured at Stowmarket, but ballistic uniformity suffered as the paper absorbed atmospheric moisture. In 1871, Frederick Volkmann received an Austrian patent for a colloided version of Schultze powder called Collodin, which he manufactured near Vienna for use in sporting firearms. Austrian patents were not published at the time, and the Austrian Empire considered the operation a violation of the government monopoly on explosives manufacture and closed the Volkmann factory in 1875.[9] In 1882, the Explosives Company at Stowmarket patented an improved formulation of nitrated cotton gelatinised by ether-alcohol with nitrates of potassium and barium. These propellants were suitable for shotguns but not rifles.[10]\n\nPoudre B single-base smokeless powder flakes\nIn 1884, Paul Vieille invented a smokeless powder called Poudre B (short for poudre blanche—white powder, as distinguished from black powder)[11] made from 68.2% insoluble nitrocellulose, 29.8% soluble nitrocellusose gelatinized with ether and 2% paraffin. This was adopted for the Lebel rifle.[12] It was passed through rollers to form paper thin sheets, which were cut into flakes of the desired size.[11] The resulting propellant, today known as pyrocellulose, contains somewhat less nitrogen than guncotton and is less volatile. A particularly good feature of the propellant is that it will not detonate unless it is compressed, making it very safe to handle under normal conditions.\nVieille's powder revolutionized the effectiveness of small guns, because it gave off almost no smoke and was three times more powerful than black powder. Higher muzzle velocity meant a flatter trajectory and less wind drift and bullet drop, making 1000 meter shots practicable. Since less powder was needed to propel a bullet, the cartridge could be made smaller and lighter. This allowed troops to carry more ammunition for the same weight. Also, it would burn even when wet. Black powder ammunition had to be kept dry and was almost always stored and transported in watertight cartridges.\nOther European countries swiftly followed and started using their own versions of Poudre B, the first being Germany and Austria, which introduced new weapons in 1888. Subsequently Poudre B was modified several times with various compounds being added and removed. Krupp began adding diphenylamine as a stabilizer in 1888.[9]\nMeanwhile, in 1887, Alfred Nobel obtained an English patent for a smokeless gunpowder he called Ballistite. In this propellant the fibrous structure of cotton (nitro-cellulose) was destroyed by a nitro-glycerine solution instead of a solvent.[13] In England in 1889, a similar powder was patented by Hiram Maxim, and in the USA in 1890 by Hudson Maxim.[14] Ballistite was patented in the United States in 1891.\nThe Germans adopted ballistite for naval use in 1898, calling it WPC/98. The Italians adopted it as filite, in cord instead of flake form, but realising its drawbacks changed to a formulation with nitroglycerine they called solenite. In 1891 the Russians tasked the chemist Mendeleef with finding a suitable propellant, he created nitrocellulose gelatinised by ether-alcohol, which produced more nitrogen and more uniform colloidal structure than the French use of nitro-cottons in Poudre B. He called it pyro-collodion.[13]\nBritain conducted trials on all the various types of propellant brought to their attention, but were dissatisfied with them all and sought something superior to all existing types. In 1889, Sir Frederick Abel, James Dewar and Dr W Kellner patented (Nos 5614 and 11,664 in the names of Abel and Dewar) a new formulation that was manufactured at the Royal Gunpowder Factory at Waltham Abbey. It entered British service in 1891 as Cordite Mark 1. Its main composition was 58% Nitro-glycerine, 37% Guncotton and 3% mineral jelly. A modified version, Cordite MD, entered service in 1901, this increased guncotton to 65% and reduced nitro-glycerine to 30%, this change reduced the combustion temperature and hence erosion and barrel wear. Cordite's advantages over gunpowder were reduced maximum pressure in the chamber (hence lighter breeches, etc.) but longer high pressure. Cordite could be made in any desired shape or size.[15] The creation of cordite led to a lengthy court battle between Nobel, Maxim, and another inventor over alleged British patent infringement.\nThe Anglo-American Explosives Company began manufacturing its shotgun powder in Oakland, New Jersey in 1890. DuPont began producing guncotton at Carneys Point Township, New Jersey in 1891.[3] Charles E. Munroe of the Naval Torpedo Station in Newport, Rhode Island patented a formulation of guncotton colloided with nitrobenzene, called Indurite, in 1891.[16] Several United States firms began producing smokeless powder when Winchester Repeating Arms Company started loading sporting cartridges with Explosives Company powder in 1893. California Powder Works began producing a mixture of nitroglycerine and nitrocellulose with ammonium picrate as Peyton Powder, Leonard Smokeless Powder Company began producing nitroglycerine-nitrocellulose Ruby powders, Laflin & Rand negotiated a license to produce Ballistite, and DuPont started producing smokeless shotgun powder. The United States Army evaluated 25 varieties of smokeless powder and selected Ruby and Peyton Powders as the most suitable for use in the Krag-Jørgensen service rifle. Ruby was preferred, because tin-plating was required to protect brass cartridge cases from picric acid in the Peyton Powder. Rather than paying the required royalties for Ballistite, Laflin & Rand financed Leonard's reorganization as the American Smokeless Powder Company. United States Army Lieutenant Whistler assisted American Smokeless Powder Company factory superintendent Aspinwall in formulating an improved powder named W.A. for their efforts. W.A. smokeless powder was the standard for United States military service rifles from 1897 until 1908.[3]\nIn 1897, United States Navy Lieutenant John Bernadou patented a nitrocellulose powder colloided with ether-alcohol.[16] The Navy licensed or sold patents for this formulation to DuPont and the California Powder Works while retaining manufacturing rights for the Naval Powder Factory, Indian Head, Maryland constructed in 1900. The United States Army adopted the Navy single-base formulation in 1908 and began manufacture at Picatinny Arsenal.[3] By that time Laflin & Rand had taken over the American Powder Company to protect their investment, and Laflin & Rand had been purchased by DuPont in 1902.[17] Upon securing a 99-year lease of the Explosives Company in 1903, DuPont enjoyed use of all significant smokeless powder patents in the United States, and was able to optimize production of smokeless powder.[3] When government anti-trust action forced divestiture in 1912, DuPont retained the nitrocellulose smokeless powder formulations used by the United States military and released the double-base formulations used in sporting ammunition to the reorganized Hercules Powder Company. These newer propellants were more stable and thus safer to handle than Poudre B, and also more powerful.\nChemical formulations[edit]\n\"Double base\" redirects here. For the musical instrument, see double bass.\nCurrently, propellants using nitrocellulose (detonation velocity 7,300 m/s (23,950 ft/s)) (typically an ether-alcohol colloid of nitrocellulose) as the sole explosive propellant ingredient are described as single-base powder.[18]\nPropellants mixtures containing nitrocellulose and nitroglycerin (detonation velocity 7,700 m/s (25,260 ft/s)) as explosive propellant ingredients are known as double-base powder.[19]\nDuring the 1930s triple-base propellant containing nitrocellulose, nitroglycerin, and a substantial quantity of nitroguanidine (detonation velocity 8,200 m/s (26,900 ft/s)) as explosive propellant ingredients was developed. These propellant mixtures have reduced flash and flame temperature without sacrificing chamber pressure compared to single and double base propellants, albeit at the cost of more smoke.\nIn practice, triple base propellants are reserved mainly for large caliber ammunition such as used in (naval) artillery and tank guns. During World War II it had some use by British artillery. After that war it became the standard propellant in all British large caliber ammunition designs except small-arms. Most western nations, except the United States, followed a similar path.\nIn the late 20th century new propellant formulations started to appear. These are based on nitroguanidine and high explosives of the RDX (detonation velocity 8,750 m/s (28,710 ft/s)) type.\nInstability and stabilization[edit]\nNitrocellulose deteriorates with time, yielding acidic byproducts. Those byproducts catalyze the further deterioration, increasing its rate. The released heat, in case of bulk storage of the powder, or too large blocks of solid propellant, can cause self-ignition of the material. Single-base nitrocellulose propellants are hygroscopic and most susceptible to degradation; double-base and triple-base propellants tend to deteriorate more slowly. To neutralize the decomposition products, which could otherwise cause corrosion of metals of the cartridges and gun barrels, calcium carbonate is added to some formulations.\nTo prevent buildup of the deterioration products, stabilizers are added. Diphenylamine is one of the most common stabilizers used. Nitrated analogs of diphenylamine formed in the process of stabilizing decomposing powder are sometimes used as stabilizers themselves.[20][21] The stabilizers are added in the amount of 0.5–2% of the total amount of the formulation; higher amounts tend to degrade its ballistic properties. The amount of the stabilizer is depleted with time. Propellants in storage should be periodically tested for the amount of stabilizer remaining, as its depletion may lead to auto-ignition of the propellant.\nPhysical variations[edit]\n\nAmmunition handloading powders\nSmokeless powder may be corned into small spherical balls or extruded into cylinders or strips with many cross-sectional shapes (strips with various rectangular proportions, single or multi-hole cylinders, slotted cylinders) using solvents such as ether. These extrusions can be cut into short ('flakes') or long pieces ('cords' many inches long). Cannon powder has the largest pieces.\nThe properties of the propellant are greatly influenced by the size and shape of its pieces. The specific surface area of the propellant influences the speed of burning, and the size and shape of the particles determine the specific surface area. By manipulation of the shape it is possible to influence the burning rate and hence the rate at which pressure builds during combustion. Smokeless powder burns only on the surfaces of the pieces. Larger pieces burn more slowly, and the burn rate is further controlled by flame-deterrent coatings that retard burning slightly. The intent is to regulate the burn rate so that a more or less constant pressure is exerted on the propelled projectile as long as it is in the barrel so as to obtain the highest velocity. The perforations stabilize the burn rate because as the outside burns inward (thus shrinking the burning surface area) the inside is burning outward (thus increasing the burning surface area, but faster, so as to fill up the increasing volume of barrel presented by the departing projectile).[22] Fast-burning pistol powders are made by extruding shapes with more area such as flakes or by flattening the spherical granules. Drying is usually performed under a vacuum. The solvents are condensed and recycled. The granules are also coated with graphite to prevent static electricity sparks from causing undesired ignitions.[23]\nFaster-burning propellants generate higher temperatures and higher pressures, however they also increase wear on gun barrels.\nSmokeless propellant components[edit]\nThe propellant formulations may contain various energetic and auxiliary components:\nPropellants:\nNitrocellulose, an energetic component of most smokeless propellants[24]\nNitroglycerin, an energetic component of double-base and triple-base formulations[24]\nNitroguanidine, a component of triple-base formulations[24]\nD1NA (bis-nitroxyethylnitramine)[25]\nFivonite (tetramethylolcyclopentanone)[25]\nDGN (di-ethylene glycol dinitrate)[26]\nAcetyl cellulose[27]\nDeterrents, (or moderants), to slow the burning rate\nCentralites (symmetrical diphenyl urea—primarily diethyl or dimethyl)[28][29]\nDibutyl phthalate[24][29]\nDinitrotoluene (toxic, carcinogenic, and obsolete)[24][30]\nAkardite (asymmetrical diphenyl urea)[26]\northo-tolyl urethane[31]\nPolyester adipate\nCamphor (obsolete)[29]\nStabilizers, to prevent or slow down self-decomposition[32]\nDiphenylamine[33]\nPetroleum jelly[34]\nCalcium carbonate[24]\nMagnesium oxide[26]\nSodium bicarbonate[27]\nbeta-naphthol methyl ether[31]\nAmyl alcohol (obsolete)[35]\nAniline (obsolete)[36]\nDecoppering additives, to hinder the buildup of copper residues from the gun barrel rifling\nTin metal and compounds (e.g., tin dioxide)[24][37]\nBismuth metal and compounds (e.g., bismuth trioxide, bismuth subcarbonate, bismuth nitrate, bismuth antimonide); the bismuth compounds are favored as copper dissolves in molten bismuth, forming brittle and easily removable alloy\nLead foil and lead compounds, phased out due to toxicity[25]\nFlash reducers, to reduce the brightness of the muzzle flash (all have a disadvantage: the production of smoke)[38]\nPotassium chloride[39]\nPotassium nitrate\nPotassium sulfate[24][37]\nPotassium hydrogen tartarate (a byproduct of wine production formerly used by French artillery)[39]\nWear reduction additives, to lower the wear of the gun barrel liners[40]\nWax\nTalc\nTitanium dioxide\nPolyurethane jackets over the powder bags, in large guns\nOther additives\nEthyl acetate, a solvent for manufacture of spherical powder[34]\nRosin, a surfactant to hold the grain shape of spherical powder\nGraphite, a lubricant to cover the grains and prevent them from sticking together, and to dissipate static electricity[23]\nManufacturing[edit]\nThis section describes procedures used in the United States. See Cordite for alternative procedures formerly used in the United Kingdom.\nThe United States Navy manufactured single-base tubular powder for naval artillery at Indian Head, Maryland, beginning in 1900. Similar procedures were used for United States Army production at Picatinny Arsenal beginning in 1907[18] and for manufacture of smaller grained Improved Military Rifle (IMR) powders after 1914. Short-fiber cotton linter was boiled in a solution of sodium hydroxide to remove vegetable waxes, and then dried before conversion to nitrocellulose by mixing with concentrated nitric and sulfuric acids. Nitrocellulose still resembles fibrous cotton at this point in the manufacturing process, and was typically identified as pyrocellulose because it would spontaneously ignite in air until unreacted acid was removed. The term guncotton was also used; although some references identify guncotton as a more extensively nitrated and refined product used in torpedo and mine warheads prior to use of TNT.[41]\nUnreacted acid was removed from pyrocellulose pulp by a multistage draining and water washing process similar to that used in paper mills during production of chemical woodpulp. Pressurized alcohol removed remaining water from drained pyrocellulose prior to mixing with ether and diphenylamine. The mixture was then fed through a press extruding a long turbular cord form to be cut into grains of the desired length.[42]\nAlcohol and ether were then evaporated from \"green\" powder grains to a remaining solvent concentration between 3 percent for rifle powders and 7 percent for large artillery powder grains. Burning rate is inversely proportional to solvent concentration. Grains were coated with electrically conductive graphite to minimize generation of static electricity during subsequent blending. \"Lots\" containing more than ten tonnes of powder grains were mixed through a tower arrangement of blending hoppers to minimize ballistic differences. Each blended lot was then subjected to testing to determine the correct loading charge for the desired performance.[43][44]\nMilitary quantities of old smokeless powder were sometimes reworked into new lots of propellants.[45] Through the 1920s Dr. Fred Olsen worked at Picatinny Arsenal experimenting with ways to salvage tons of single-base cannon powder manufactured for World War I. Dr. Olsen was employed by Western Cartridge Company in 1929 and developed a process for manufacturing spherical smokeless powder by 1933.[46] Reworked powder or washed pyrocellulose can be dissolved in ethyl acetate containing small quantities of desired stabilizers and other additives. The resultant syrup, combined with water and surfactants, can be heated and agitated in a pressurized container until the syrup forms an emulsion of small spherical globules of the desired size. Ethyl acetate distills off as pressure is slowly reduced to leave small spheres of nitrocellulose and additives. The spheres can be subsequently modified by adding nitroglycerine to increase energy, flattening between rollers to a uniform minimum dimension, coating with phthalate deterrents to retard ignition, and/or glazing with graphite to improve flow characteristics during blending.[47][48]\nModern smokeless powder is produced in the United States by St. Marks Powder, Inc. owned by General Dynamics.[49]\nFlashless propellant[edit]\nMuzzle flash is the light emitted in the vicinity of the muzzle by the hot propellant gases and the chemical reactions that follow as the gases mix with the surrounding air. Before projectiles exit a slight pre-flash may occur from gases leaking past the projectiles. Following muzzle exit the heat of gases is usually sufficient to emit visible radiation – the primary flash. The gases expand but as they pass through the Mach disc they are re-compressed to produce an intermediate flash. Hot combustible gases (e.g. hydrogen and carbon-monoxide) may follow when they mix with oxygen in the surrounding air to produce the secondary flash, the brightest. The secondary flash does not usually occur with small-arms.[50]\nNitrocellulose contains insufficient oxygen to completely oxidize its carbon and hydrogen. The oxygen deficit is increased by addition of graphite and organic stabilizers. Products of combustion within the gun barrel include flammable gasses like hydrogen and carbon monoxide. At high temperature, these flammable gasses will ignite when turbulently mixed with atmospheric oxygen beyond the muzzle of the gun. During night engagements the flash produced by ignition can reveal the location of the gun to enemy forces[51] and cause temporary night-blindness among the gun crew by photo-bleaching visual purple.[52]\nFlash suppressors are commonly used on small arms to reduce the flash signature, but this approach is not practical for artillery. Artillery muzzle flash up to 150 feet (46 m) from the muzzle has been observed, and can be reflected off clouds and be visible for distances up to 30 miles (48 km).[51] For artillery the most effective method is a propellant that produces a large proportion of inert nitrogen at relatively low temperatures that dilutes the combustible gases. Triple based propellants are used for this because of the nitrogen in the nitroguandine.[53]\nBefore the use of triple based propellants the usual method of flash reduction was to add inorganic salts like potassium chloride so their specific heat capacity might reduce the temperature of combustion gasses and their finely divided particulate smoke might block visible wavelengths of radiant energy of combustion.[39]\nSee also[edit]\nPortal icon\tPyrotechnics portal\nAntique guns\nBallistite\nCordite\nFirearms\nGunpowder\nNitrocellulose\nSmall arms\nBrown-brown – a drug created by mixing cocaine with cartridge powder\nReferences[edit]\nNotes[edit]\nJump up ^ Hatcher, Julian S. and Barr, Al Handloading Hennage Lithograph Company (1951) p.34\nJump up ^ Fairfield, A. P., CDR USN Naval Ordnance Lord Baltimore Press (1921) p.44\n^ Jump up to: a b c d e Sharpe, Philip B. Complete Guide to Handloading 3rd Edition (1953) Funk & Wagnalls pp.146-149\nJump up ^ seegunpowder\nJump up ^ Sharpe, Philip B. Complete Guide To Handloading (1953) Funk & Wagnalls p.60\nJump up ^ Davis, William C., Jr. Handloading (1981) National Rifle Association p.21\nJump up ^ Davis, Tenney L. The Chemistry of Powder & Explosives (1943) page 195\nJump up ^ Davis, William C., Jr. Handloading National Rifle Association of America (1981) p.28\n^ Jump up to: a b c Sharpe, Philip B. Complete Guide to Handloading 3rd Edition (1953) Funk & Wagnalls pp.141-144\nJump up ^ Hogg, Oliver F. G. Artillery: Its Origin, Heyday and Decline (1969) p.138-139\n^ Jump up to: a b Davis, Tenney L. The Chemistry of Powder & Explosives (1943) pages 289–292\nJump up ^ Hogg, Oliver F. G. Artillery: Its Origin, Heyday and Decline (1969) p.139\n^ Jump up to: a b Hogg, Oliver F. G. Artillery: Its Origin, Heyday and Decline (1969) p.140\nJump up ^ U.S. Patent 430,212 – Manufacture of explosive – H. S. Maxim\nJump up ^ Hogg, Oliver F. G. Artillery: Its Origin, Heyday and Decline (1969) p.141\n^ Jump up to: a b Davis, Tenney L. The Chemistry of Powder & Explosives (1943) pages 296-297\nJump up ^ \"Laflin & Rand Powder Company\". DuPont. Retrieved 2012-02-24.\n^ Jump up to: a b Davis, Tenny L. The Chemistry of Powder & Explosives (1943) p.297\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) p.298\nJump up ^ Fairfield, A. P., CDR USN Naval Ordnance Lord Baltimore Press (1921) p.28\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) p. 310\nJump up ^ Fairfield, A. P., CDR USN Naval Ordnance Lord Baltimore Press (1921) pp.41–43\n^ Jump up to: a b Davis, Tenny L. The Chemistry of Powder & Explosives (1943) p.306\n^ Jump up to: a b c d e f g h Campbell, John Naval Weapons of World War Two (1985) p. 5\n^ Jump up to: a b c Campbell, John Naval Weapons of World War Two (1985) p. 104\n^ Jump up to: a b c Campbell, John Naval Weapons of World War Two (1985) p. 221\n^ Jump up to: a b Campbell, John Naval Weapons of World War Two (1985) p. 318\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) pages 317–320\n^ Jump up to: a b c Davis, William C., Jr. Handloading National Rifle Association of America (1981) p.30\nJump up ^ Davis, William C., Jr. Handloading National Rifle Association of America (1981) p.31\n^ Jump up to: a b Campbell, John Naval Weapons of World War Two (1985) p. 174\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) pages 307–311\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) p. 302\n^ Jump up to: a b Davis, Tenny L. The Chemistry of Powder & Explosives (1943) p. 296\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) p. 307\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) p. 308\n^ Jump up to: a b Davis, William C., Jr. Handloading National Rifle Association of America (1981) p.32\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) pages 322–327\n^ Jump up to: a b c Davis, Tenny L. The Chemistry of Powder & Explosives (1943) pages 323–327\nJump up ^ \"USA 16\"/50 (40.6 cm) Mark 7\". NavWeaps. 2008-11-03. Retrieved 2008-12-05.\nJump up ^ Fairfield, A. P., CDR USN Naval Ordnance Lord Baltimore Press (1921) pages 28–31\nJump up ^ Fairfield, A. P., CDR USN Naval Ordnance Lord Baltimore Press (1921) pages 31–35\nJump up ^ Fairfield, A. P., CDR USN Naval Ordnance Lord Baltimore Press (1921) pages 35–41\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) pages 293 & 306\nJump up ^ Fairfield, A. P., CDR USN Naval Ordnance Lord Baltimore Press (1921) p.39\nJump up ^ Matunas, E. A. Winchester-Western Ball Powder Loading Data Olin Corporation (1978) p.3\nJump up ^ Davis, Tenny L. The Chemistry of Powder & Explosives (1943) pages 328–330\nJump up ^ Wolfe, Dave Propellant Profiles Volume 1 Wolfe Publishing Company (1982) pages 136–137\nJump up ^ General Dynamics Commercial Powder Applications.\nJump up ^ Moss G. M., Leeming D. W., Farrar C. L. Military Ballisitcs (1969) pages 55–56\n^ Jump up to: a b Davis, Tenny L. The Chemistry of Powder & Explosives (1943) pages 322–323\nJump up ^ Milner p.68\nJump up ^ Moss G. M., Leeming D. W., Farrar C. L. Military Ballisitcs (1969) pages 59–60\nSources[edit]\nCampbell, John (1985). Naval Weapons of World War Two. Naval Institute Press. ISBN 0-87021-459-4.\nDavis, Tenney L. (1943). The Chemistry of Powder & Explosives (Angriff Press [1992] ed.). John Wiley & Sons Inc. ISBN 0-913022-00-4.\nDavis, William C., Jr. (1981). Handloading. National Rifle Association of America. ISBN 0-935998-34-9.\nFairfield, A. P., CDR USN (1921). Naval Ordnance. Lord Baltimore Press.\nHatcher, Julian S. and Barr, Al (1951). Handloading. Hennage Lithograph Company.\nMatunas, E. A. (1978). Winchester-Western Ball Powder Loading Data. Olin Corporation.\nMilner, Marc (1985). North Atlantic Run. Naval Institute Press. ISBN 0-87021-450-0.\nWolfe, Dave (1982). Propellant Profiles Volume 1. Wolfe Publishing Company. ISBN 0-935632-10-7.\nExternal links[edit]\nThe Manufacture of Smokeless Powders and their Forensic Analysis: A Brief Review – Robert M. Heramb, Bruce R. McCord\nHudson Maxim papers (1851-1925) at Hagley Museum and Library. Collection includes material relating to Maxim's patent on the process of making smokeless powder.\nCategories: CorditeExplosivesFirearm propellantsSolid fuels\nNavigation menu\nCreate accountLog inArticleTalkReadEditView history\n\nMain page\nContents\nFeatured content\nCurrent events\nRandom article\nDonate to Wikipedia\nWikimedia Shop\nInteraction\nHelp\nAbout Wikipedia\nCommunity portal\nRecent changes\nContact page\nTools\nWhat links here\nRelated changes\nUpload file\nSpecial pages\nPermanent link\nPage information\nWikidata item\nCite this page\nPrint/export\nCreate a book\nDownload as PDF\nPrintable version\nLanguages\nالعربية\nБългарски\nDansk\nDeutsch\nEspañol\nفارسی\nFrançais\nBahasa Indonesia\nÍslenska\nItaliano\nעברית\nNederlands\n日本語\nPolski\nPortuguês\nРусский\nSvenska\nதமிழ்\n中文\nEdit links\nThis page was last modified on 25 July 2014 at 22:33.\nText is available under the Creative Commons Attribution-ShareAlike License; additional terms may apply. By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia® is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\nPrivacy policyAbout WikipediaDisclaimersContact WikipediaDevelopersMobile viewWikimedia Foundation Powered by MediaWiki\n\n\nDeflagration\nFrom Wikipedia, the free encyclopedia\n\n[hide]This article has multiple issues. Please help improve it or discuss these issues on the talk page.\nThis article needs additional citations for verification. (April 2011)\nThis article may be too technical for most readers to understand. (December 2013)\n\nA log in a fireplace.\nDeflagration [1] (Lat: de + flagrare, \"to burn down\") is a term describing subsonic combustion propagating through heat transfer; hot burning material heats the next layer of cold material and ignites it. Most \"fire\" found in daily life, from flames to explosions, is deflagration. Deflagration is different from detonation, which is supersonic and propagates through shock.\nContents  [hide] \n1 Applications\n2 Oil/wax fire and water\n3 Flame physics\n4 Damaging deflagration events\n5 See also\n6 References\nApplications[edit]\nIn engineering applications, deflagrations are easier to control than detonations. Consequently, they are better suited when the goal is to move an object (a bullet in a gun, or a piston in an internal combustion engine) with the force of the expanding gas. Typical examples of deflagrations are the combustion of a gas-air mixture in a gas stove or a fuel-air mixture in an internal combustion engine, and the rapid burning of gunpowder in a firearm or of pyrotechnic mixtures in fireworks. Deflagration systems and products can also be used in mining, demolition and stone quarrying via gas pressure blasting as a beneficial alternative to high explosives.\nOil/wax fire and water[edit]\nAdding water to a burning hydrocarbon such as oil or wax produces a deflagration. The water boils rapidly and ejects the burning material as a fine spray of droplets. A deflagration then occurs as the fine mist of oil ignites and burns extremely rapidly. These are particularly common in chip pan fires, which are responsible for one in five household fires in Britain.[2]\nFlame physics[edit]\nThe underlying flame physics can be understood with the help of an idealized model consisting of a uniform one-dimensional tube of unburnt and burned gaseous fuel, separated by a thin transitional region of width \\delta\\;  in which the burning occurs. The burning region is commonly referred to as the flame or flame front. In equilibrium, thermal diffusion across the flame front is balanced by the heat supplied by burning.\nThere are two characteristic timescales which are important here. The first is the thermal diffusion timescale \\tau_d\\;, which is approximately equal to\n\\tau_d \\simeq \\delta^2 / \\kappa,\nwhere \\kappa \\; is the thermal diffusivity. The second is the burning timescale \\tau_b that strongly decreases with temperature, typically as\n\\tau_b\\propto \\exp[\\Delta U/(k_B T_f)],\nwhere \\Delta U\\; is the activation barrier for the burning reaction and T_f\\; is the temperature developed as the result of burning; the value of this so-called \"flame temperature\" can be determined from the laws of thermodynamics.\nFor a stationary moving deflagration front, these two timescales must be equal: the heat generated by burning is equal to the heat carried away by heat transfer. This makes it possible to calculate the characteristic width \\delta\\; of the flame front:\n\\tau_b = \\tau_d\\;,\nthus\n \\delta \\simeq \\sqrt {\\kappa \\tau_b} .\nNow, the thermal flame front propagates at a characteristic speed S_l\\;, which is simply equal to the flame width divided by the burn time:\nS_l \\simeq \\delta / \\tau_b \\simeq \\sqrt {\\kappa  / \\tau_b} .\nThis simplified model neglects the change of temperature and thus the burning rate across the deflagration front. This model also neglects the possible influence of turbulence. As a result, this derivation gives only the laminar flame speed -- hence the designation S_l\\;.\nDamaging deflagration events[edit]\nDamage to buildings, equipment and people can result from a large-scale, short-duration deflagration. The potential damage is primarily a function of the total amount of fuel burned in the event (total energy available), the maximum flame velocity that is achieved, and the manner in which the expansion of the combustion gases is contained.\nIn free-air deflagrations, there is a continuous variation in deflagration effects relative to the maximum flame velocity. When flame velocities are low, the effect of a deflagration is to release heat. Some authors use the term flash fire to describe these low-speed deflagrations. At flame velocities near the speed of sound, the energy released is in the form of pressure and the results resemble a detonation. Between these extremes both heat and pressure are released.\nWhen a low-speed deflagration occurs within a closed vessel or structure, pressure effects can produce damage due to expansion of gases as a secondary effect. The heat released by the deflagration causes the combustion gases and excess air to expand thermally. The net result is that the volume of the vessel or structure must expand to accommodate the hot combustion gases, or the vessel must be strong enough to withstand the additional internal pressure, or it fails, allowing the gases to escape. The risks of deflagration inside waste storage drums is a growing concern in storage facilities.\nSee also[edit]\n\tLook up deflagration in Wiktionary, the free dictionary.\nPressure piling\nReferences[edit]\nJump up ^ \"Glossary D-H\". Hutchisonrodway.co.nz. Retrieved 2013-12-29.\nJump up ^ UK Fire Service advice on chip pan fires\nCategories: Explosives\nNavigation menu\nCreate accountLog inArticleTalkReadEditView history\n\nMain page\nContents\nFeatured content\nCurrent events\nRandom article\nDonate to Wikipedia\nWikimedia Shop\nInteraction\nHelp\nAbout Wikipedia\nCommunity portal\nRecent changes\nContact page\nTools\nWhat links here\nRelated changes\nUpload file\nSpecial pages\nPermanent link\nPage information\nWikidata item\nCite this page\nPrint/export\nCreate a book\nDownload as PDF\nPrintable version\nLanguages\nCatalà\nČeština\nDeutsch\nEspañol\nFrançais\nItaliano\nLietuvių\nNederlands\nNorsk bokmål\nPolski\nPortuguês\nРусский\nСрпски / srpski\nSvenska\nEdit links\nThis page was last modified on 2 October 2014 at 16:44.\nText is available under the Creative Commons Attribution-ShareAlike License; additional terms may apply. By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia® is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\nPrivacy policyAbout WikipediaDisclaimersContact WikipediaDevelopersMobile viewWikimedia Foundation Powered by MediaWiki\n\n\nUnited Kingdom\nFrom Wikipedia, the free encyclopedia\nThis article is about the sovereign state. For the island, see Great Britain. For other uses, see United Kingdom (disambiguation) and UK (disambiguation).\nPage semi-protected\nUnited Kingdom of Great\nBritain and Northern Ireland[show]\n\nA flag featuring both cross and saltire in red, white and blue\tCoat of arms containing shield and crown in centre, flanked by lion and unicorn\nFlag\tRoyal coat of arms[nb 1]\nAnthem: \"God Save the Queen\"[nb 2]\nMENU0:00\nTwo islands to the north-west of continental Europe. Highlighted are the larger island and the north-eastern fifth of the smaller island to the west.\nLocation of the  United Kingdom  (dark green)\n– in Europe  (green & dark grey)\n– in the European Union  (green)\nCapital\nand largest city\tLondon\n51°30′N 0°7′W\nOfficial language\nand national language\tEnglish\nRecognised regional\nlanguages\tCornish, Irish, Scots, Scottish Gaelic, Ulster-Scots, Welsh[nb 3]\nEthnic groups (2011)\t87.1% White\n7.0% Asian\n3.0% Black\n2.0% Mixed\n0.9% Other\nDemonym\tBritish, Briton\nGovernment\tUnitary parliamentary constitutional monarchy\n - \tMonarch\tElizabeth II\n - \tPrime Minister\tDavid Cameron\nLegislature\tParliament\n - \tUpper house\tHouse of Lords\n - \tLower house\tHouse of Commons\nFormation\n - \tActs of Union 1707\t1 May 1707 \n - \tActs of Union 1800\t1 January 1801 \n - \tIrish Free State Constitution Act\t5 December 1922 \nArea\n - \tTotal\t243,610 km2 (80th)\n94,060 sq mi\n - \tWater (%)\t1.34\nPopulation\n - \t2013 estimate\t64,100,000[3] (22nd)\n - \t2011 census\t63,181,775[4] (22nd)\n - \tDensity\t255.6/km2 (51st)\n661.9/sq mi\nGDP (PPP)\t2014 estimate\n - \tTotal\t$2.435 trillion[5] (10th)\n - \tPer capita\t$37,744[5] (27th)\nGDP (nominal)\t2014 estimate\n - \tTotal\t$2.848 trillion[5] (6th)\n - \tPer capita\t$44,141[5] (22nd)\nGini (2012)\tpositive decrease 32.8[6]\nmedium · 33rd\nHDI (2013)\tSteady 0.892[7]\nvery high · 14th\nCurrency\tPound sterling (GBP)\nTime zone\tGMT (UTC​)\n - \tSummer (DST)\tBST (UTC+1)\nDate format\tdd/mm/yyyy (AD)\nDrives on the\tleft\nCalling code\t+44\nISO 3166 code\tGB\nInternet TLD\t.uk\nThe United Kingdom of Great Britain and Northern Ireland Listeni/ɡreɪt ˈbrɪt(ə)n ənd ˈnɔːð(ə)n ˈʌɪələnd/, commonly known as the United Kingdom (UK) or Britain, is a sovereign state in Europe. Lying off the north-western coast of the European mainland, the country includes the island of Great Britain (a term also applied loosely to refer to the whole country),[8] the north-eastern part of the island of Ireland, and many smaller islands. Northern Ireland is the only part of the UK that shares a land border with another state: the Republic of Ireland.[nb 4] Apart from this land border, the UK is surrounded by the Atlantic Ocean, with the North Sea in the east and the English Channel in the south. The Irish Sea lies between Great Britain and Ireland. The UK has an area of 243,610 square kilometres (94,060 sq mi), making it the 78th-largest sovereign state in the world and the 11th-largest in Europe.\nThe United Kingdom is the 22nd-most populous country, with an estimated 64.1 million inhabitants.[3] It is a constitutional monarchy with a parliamentary system of governance.[9][10] Its capital city is London, an important global city and financial centre with the fourth-largest urban area in Europe.[11] The current monarch—since 6 February 1952—is Queen Elizabeth II. The UK consists of four countries: England, Scotland, Wales, and Northern Ireland.[12] The latter three have devolved administrations,[13] each with varying powers,[14][15] based in their capitals, Edinburgh, Cardiff, and Belfast, respectively. Guernsey, Jersey, and the Isle of Man are not part of the United Kingdom, being Crown dependencies with the British Government responsible for defence and international representation.[16] The UK has fourteen Overseas Territories,[17] including the disputed Falkland Islands, Gibraltar, and Indian Ocean Territory.\nThe relationships among the countries of the United Kingdom have changed over time. Wales was annexed by the Kingdom of England under the Acts of Union of 1536 and 1543. A treaty between England and Scotland resulted in a unified Kingdom of Great Britain in 1707, which in 1801, merged with the Kingdom of Ireland to form the United Kingdom of Great Britain and Ireland. In 1922, five-sixths of Ireland seceded from the country, leaving the present formulation of the United Kingdom of Great Britain and Northern Ireland.[nb 5] British Overseas Territories, formerly colonies, are the remnants of the British Empire which, at its height in the late 19th and early 20th centuries, encompassed almost a quarter of the world's land mass and was the largest empire in history. British influence can be observed in the language, culture, and legal systems of many of its former colonies.\nThe United Kingdom is a developed country and has the world's sixth-largest economy by nominal GDP and tenth-largest by purchasing power parity. The country is considered to have a high-income economy and is categorised as very high in the Human Development Index, currently ranking 14th in the world. It was the world's first industrialised country and the world's foremost power during the 19th and early 20th centuries.[18][19] The UK remains a great power with considerable economic, cultural, military, scientific, and political influence internationally.[20][21] It is a recognised nuclear weapons state and its military expenditure ranks fifth or sixth in the world.[22][23] The UK has been a permanent member of the United Nations Security Council since its first session in 1946. It has been a member state of the European Union (EU) and its predecessor, the European Economic Community (EEC), since 1973; it is also a member of the Commonwealth of Nations, the Council of Europe, the G7, the G8, the G20, NATO, the Organisation for Economic Co-operation and Development (OECD), and the World Trade Organization (WTO).\nContents  [hide] \n1 Etymology and terminology\n2 History\n2.1 Before 1707\n2.2 Since the Acts of Union of 1707\n3 Geography\n3.1 Climate\n3.2 Administrative divisions\n4 Dependencies\n5 Politics\n5.1 Government\n5.2 Devolved administrations\n5.3 Law and criminal justice\n5.4 Foreign relations\n5.5 Military\n6 Economy\n6.1 Science and technology\n6.2 Transport\n6.3 Energy\n7 Demographics\n7.1 Ethnic groups\n7.2 Languages\n7.3 Religion\n7.4 Migration\n7.5 Education\n7.6 Healthcare\n8 Culture\n8.1 Literature\n8.2 Music\n8.3 Visual art\n8.4 Cinema\n8.5 Media\n8.6 Philosophy\n8.7 Sport\n8.8 Symbols\n9 See also\n10 Notes\n11 References\n12 Further reading\n13 External links\nEtymology and terminology\nSee also: Britain (placename) and Terminology of the British Isles\nThe 1707 Acts of Union declared that the kingdoms of England and Scotland were \"United into One Kingdom by the Name of Great Britain\", though the new state is also referred to in the Acts as the \"Kingdom of Great Britain\", \"United Kingdom of Great Britain\" and \"United Kingdom\".[24][25][nb 6] However, the term \"united kingdom\" is only found in informal use during the 18th century and the country was only occasionally referred to as he \"United Kingdom of Great Britain\".[26] The Acts of Union 1800 united the Kingdom of Great Britain and the Kingdom of Ireland in 1801, forming the United Kingdom of Great Britain and Ireland. The name \"United Kingdom of Great Britain and Northern Ireland\" was adopted following the independence of the Irish Free State, and the partition of Ireland, in 1922, which left Northern Ireland as the only part of the island of Ireland within the UK.[27]\nAlthough the United Kingdom, as a sovereign state, is a country, England, Scotland, Wales, and to a lesser degree, Northern Ireland, are also regarded as countries, though they are not sovereign states.[28][29] Scotland, Wales and Northern Ireland have devolved self-government.[30][31] The British Prime Minister's website has used the phrase \"countries within a country\" to describe the United Kingdom.[12] Some statistical summaries, such as those for the twelve NUTS 1 regions of the UK, also refer to Scotland, Wales and Northern Ireland as \"regions\".[32][33] Northern Ireland is also referred to as a \"province\".[28][34] With regard to Northern Ireland, the descriptive name used \"can be controversial, with the choice often revealing one's political preferences.\"[35]\nThe term Britain is often used as synonym for the United Kingdom. The term Great Britain, by contrast, refers conventionally to the island of Great Britain, or politically to England, Scotland and Wales in combination.[36][37][38] However, it is sometimes used as a loose synonym for the United Kingdom as a whole.[39][40] GB and GBR are the standard country codes for the United Kingdom (see ISO 3166-2 and ISO 3166-1 alpha-3) and are consequently used by international organisations to refer to the United Kingdom. Additionally, the United Kingdom's Olympic team competes under the name \"Great Britain\" or \"Team GB\".[41][42]\nThe adjective British is commonly used to refer to matters relating to the United Kingdom. The term has no definite legal connotation, but is used in law to refer to UK citizenship and matters to do with nationality.[43] People of the United Kingdom use a number of different terms to describe their national identity and may identify themselves as being British; or as being English, Scottish, Welsh, Northern Irish, or Irish;[44] or as being both.[45]\nIn 2006, a new design of British passport was introduced. Its first page shows the long form name of the state in English, Welsh and Scottish Gaelic.[46] In Welsh, the long form name of the state is \"Teyrnas Unedig Prydain Fawr a Gogledd Iwerddon\" with \"Teyrnas Unedig\" being used as a short form name on government websites.[47] In Scottish Gaelic, the long form is \"Rìoghachd Aonaichte Bhreatainn is Èireann a Tuath\" and the short form \"Rìoghachd Aonaichte\".\nHistory\nSee also: History of the British Isles\nBefore 1707\n\nStonehenge, in Wiltshire, was erected around 2500 BC.\nMain articles: History of England, History of Wales, History of Scotland, History of Ireland and History of the formation of the United Kingdom\nSettlement by anatomically modern humans of what was to become the United Kingdom occurred in waves beginning by about 30,000 years ago.[48] By the end of the region's prehistoric period, the population is thought to have belonged, in the main, to a culture termed Insular Celtic, comprising Brythonic Britain and Gaelic Ireland.[49] The Roman conquest, beginning in 43 AD, and the 400-year rule of southern Britain, was followed by an invasion by Germanic Anglo-Saxon settlers, reducing the Brythonic area mainly to what was to become Wales and the historic Kingdom of Strathclyde.[50] Most of the region settled by the Anglo-Saxons became unified as the Kingdom of England in the 10th century.[51] Meanwhile, Gaelic-speakers in north west Britain (with connections to the north-east of Ireland and traditionally supposed to have migrated from there in the 5th century)[52][53] united with the Picts to create the Kingdom of Scotland in the 9th century.[54]\nIn 1066, the Normans invaded England from France and after its conquest, seized large parts of Wales, conquered much of Ireland and were invited to settle in Scotland, bringing to each country feudalism on the Northern French model and Norman-French culture.[55] The Norman elites greatly influenced, but eventually assimilated with, each of the local cultures.[56] Subsequent medieval English kings completed the conquest of Wales and made an unsuccessful attempt to annex Scotland. Thereafter, Scotland maintained its independence, albeit in near-constant conflict with England. The English monarchs, through inheritance of substantial territories in France and claims to the French crown, were also heavily involved in conflicts in France, most notably the Hundred Years War, while the Kings of Scots were in an alliance with the French during this period.[57]\n\nThe Bayeux Tapestry depicts the Battle of Hastings and the events leading to it.\nThe early modern period saw religious conflict resulting from the Reformation and the introduction of Protestant state churches in each country.[58] Wales was fully incorporated into the Kingdom of England,[59] and Ireland was constituted as a kingdom in personal union with the English crown.[60] In what was to become Northern Ireland, the lands of the independent Catholic Gaelic nobility were confiscated and given to Protestant settlers from England and Scotland.[61]\nIn 1603, the kingdoms of England, Scotland and Ireland were united in a personal union when James VI, King of Scots, inherited the crowns of England and Ireland and moved his court from Edinburgh to London; each country nevertheless remained a separate political entity and retained its separate political, legal, and religious institutions.[62][63]\nIn the mid-17th century, all three kingdoms were involved in a series of connected wars (including the English Civil War) which led to the temporary overthrow of the monarchy and the establishment of the short-lived unitary republic of the Commonwealth of England, Scotland and Ireland.[64][65]\nAlthough the monarchy was restored, it ensured (with the Glorious Revolution of 1688) that, unlike much of the rest of Europe, royal absolutism would not prevail, and a professed Catholic could never accede to the throne. The British constitution would develop on the basis of constitutional monarchy and the parliamentary system.[66] During this period, particularly in England, the development of naval power (and the interest in voyages of discovery) led to the acquisition and settlement of overseas colonies, particularly in North America.[67][68]\nSince the Acts of Union of 1707\nMain article: History of the United Kingdom\n\nThe Treaty of Union led to a single united kingdom encompassing all Great Britain.\nOn 1 May 1707, the united kingdom of Great Britain came into being, the result of Acts of Union being passed by the parliaments of England and Scotland to ratify the 1706 Treaty of Union and so unite the two kingdoms.[69][70][71]\nIn the 18th century, cabinet government developed under Robert Walpole, in practice the first prime minister (1721–1742). A series of Jacobite Uprisings sought to remove the Protestant House of Hanover from the British throne and restore the Catholic House of Stuart. The Jacobites were finally defeated at the Battle of Culloden in 1746, after which the Scottish Highlanders were brutally suppressed. The British colonies in North America that broke away from Britain in the American War of Independence became the United States of America in 1782. British imperial ambition turned elsewhere, particularly to India.[72]\nDuring the 18th century, Britain was involved in the Atlantic slave trade. British ships transported an estimated 2 million slaves from Africa to the West Indies before banning the trade in 1807.[73] The term 'United Kingdom' became official in 1801 when the parliaments of Britain and Ireland each passed an Act of Union, uniting the two kingdoms and creating the United Kingdom of Great Britain and Ireland.[74]\nIn the early 19th century, the British-led Industrial Revolution began to transform the country. It slowly led to a shift in political power away from the old Tory and Whig landowning classes towards the new industrialists. An alliance of merchants and industrialists with the Whigs would lead to a new party, the Liberals, with an ideology of free trade and laissez-faire. In 1832 Parliament passed the Great Reform Act, which began the transfer of political power from the aristocracy to the middle classes. In the countryside, enclosure of the land was driving small farmers out. Towns and cities began to swell with a new urban working class. Few ordinary workers had the vote, and they created their own organisations in the form of trade unions.\nPainting of a bloody battle. Horses and infantry fight or lie on grass.\nThe Battle of Waterloo marked the end of the Napoleonic Wars and the start of Pax Britannica.\nAfter the defeat of France in the Revolutionary and Napoleonic Wars (1792–1815), the UK emerged as the principal naval and imperial power of the 19th century (with London the largest city in the world from about 1830).[75] Unchallenged at sea, British dominance was later described as Pax Britannica.[76][77] By the time of the Great Exhibition of 1851, Britain was described as the \"workshop of the world\".[78] The British Empire was expanded to include India, large parts of Africa and many other territories throughout the world. Alongside the formal control it exerted over its own colonies, British dominance of much of world trade meant that it effectively controlled the economies of many countries, such as China, Argentina and Siam.[79][80] Domestically, political attitudes favoured free trade and laissez-faire policies and a gradual widening of the voting franchise. During the century, the population increased at a dramatic rate, accompanied by rapid urbanisation, causing significant social and economic stresses.[81] After 1875, the UK's industrial monopoly was challenged by Germany and the USA. To seek new markets and sources of raw materials, the Conservative Party under Disraeli launched a period of imperialist expansion in Egypt, South Africa and elsewhere. Canada, Australia and New Zealand became self-governing dominions.[82]\nSocial reform and home rule for Ireland were important domestic issues after 1900. The Labour Party emerged from an alliance of trade unions and small Socialist groups in 1900, and suffragettes campaigned for women's right to vote before 1914.\nBlack-and-white photo of two dozen men in military uniforms and metal helmets sitting or standing in a muddy trench.\nInfantry of the Royal Irish Rifles during the Battle of the Somme. More than 885,000 British soldiers died on the battlefields of World War I.\nThe UK fought with France, Russia and (after 1917) the US, against Germany and its allies in World War I (1914–18).[83] The UK armed forces were engaged across much of the British Empire and in several regions of Europe, particularly on the Western front.[84] The high fatalities of trench warfare caused the loss of much of a generation of men, with lasting social effects in the nation and a great disruption in the social order.\nAfter the war, the UK received the League of Nations mandate over a number of former German and Ottoman colonies. The British Empire reached its greatest extent, covering a fifth of the world's land surface and a quarter of its population.[85] However, the UK had suffered 2.5 million casualties and finished the war with a huge national debt.[84] The rise of Irish Nationalism and disputes within Ireland over the terms of Irish Home Rule led eventually to the partition of the island in 1921,[86] and the Irish Free State became independent with Dominion status in 1922. Northern Ireland remained part of the United Kingdom.[87] A wave of strikes in the mid-1920s culminated in the UK General Strike of 1926. The UK had still not recovered from the effects of the war when the Great Depression (1929–32) occurred. This led to considerable unemployment and hardship in the old industrial areas, as well as political and social unrest in the 1930s. A coalition government was formed in 1931.[88]\nThe UK entered World War II by declaring war on Germany in 1939, after it had invaded Poland and Czechoslovakia. In 1940, Winston Churchill became prime minister and head of a coalition government. Despite the defeat of its European allies in the first year of the war, the UK continued the fight alone against Germany. In 1940, the RAF defeated the German Luftwaffe in a struggle for control of the skies in the Battle of Britain. The UK suffered heavy bombing during the Blitz. There were also eventual hard-fought victories in the Battle of the Atlantic, the North Africa campaign and Burma campaign. UK forces played an important role in the Normandy landings of 1944, achieved with its ally the US. After Germany's defeat, the UK was one of the Big Three powers who met to plan the post-war world; it was an original signatory to the Declaration of the United Nations. The UK became one of the five permanent members of the United Nations Security Council. However, the war left the UK severely weakened and depending financially on Marshall Aid and loans from the United States.[89]\nMap of the world. Canada, the eastern United States, countries in east Africa, India, most of Australasia and some other countries are highlighted in pink.\nTerritories that were at one time part of the British Empire. Current British Overseas Territories are underlined in red.\nIn the immediate post-war years, the Labour government initiated a radical programme of reforms, which had a significant effect on British society in the following decades.[90] Major industries and public utilities were nationalised, a Welfare State was established, and a comprehensive, publicly funded healthcare system, the National Health Service, was created.[91] The rise of nationalism in the colonies coincided with Britain's now much-diminished economic position, so that a policy of decolonisation was unavoidable. Independence was granted to India and Pakistan in 1947.[92] Over the next three decades, most colonies of the British Empire gained their independence. Many became members of the Commonwealth of Nations.[93]\nAlthough the UK was the third country to develop a nuclear weapons arsenal (with its first atomic bomb test in 1952), the new post-war limits of Britain's international role were illustrated by the Suez Crisis of 1956. The international spread of the English language ensured the continuing international influence of its literature and culture. From the 1960s onward, its popular culture was also influential abroad. As a result of a shortage of workers in the 1950s, the UK government encouraged immigration from Commonwealth countries. In the following decades, the UK became a multi-ethnic society.[94] Despite rising living standards in the late 1950s and 1960s, the UK's economic performance was not as successful as many of its competitors, such as West Germany and Japan. In 1973, the UK joined the European Economic Community (EEC), and when the EEC became the European Union (EU) in 1992, it was one of the 12 founding members.\n\nAfter the two vetos of France in 1961 and 1967, the UK entered in the European Union in 1973. In 1975, 67% of Britons voted yes to the permanence in the European Union.\nFrom the late 1960s, Northern Ireland suffered communal and paramilitary violence (sometimes affecting other parts of the UK) conventionally known as the Troubles. It is usually considered to have ended with the Belfast \"Good Friday\" Agreement of 1998.[95][96][97]\nFollowing a period of widespread economic slowdown and industrial strife in the 1970s, the Conservative Government of the 1980s initiated a radical policy of monetarism, deregulation, particularly of the financial sector (for example, Big Bang in 1986) and labour markets, the sale of state-owned companies (privatisation), and the withdrawal of subsidies to others.[98] This resulted in high unemployment and social unrest, but ultimately also economic growth, particularly in the services sector. From 1984, the economy was helped by the inflow of substantial North Sea oil revenues.[99]\nAround the end of the 20th century there were major changes to the governance of the UK with the establishment of devolved administrations for Scotland, Wales and Northern Ireland.[13][100] The statutory incorporation followed acceptance of the European Convention on Human Rights. The UK is still a key global player diplomatically and militarily. It plays leading roles in the EU, UN and NATO. However, controversy surrounds some of Britain's overseas military deployments, particularly in Afghanistan and Iraq.[101]\nThe 2008 global financial crisis severely affected the UK economy. The coalition government of 2010 introduced austerity measures intended to tackle the substantial public deficits which resulted.[102] In 2014 the Scottish Government held a referendum on Scottish independence, with the majority of voters rejecting the independence proposal and opting to remain within the United Kingdom.[103]\nGeography\nMain article: Geography of the United Kingdom\nMap of United Kingdom showing hilly regions to north and west, and flattest region in the south-east.\nThe topography of the UK\nThe total area of the United Kingdom is approximately 243,610 square kilometres (94,060 sq mi). The country occupies the major part of the British Isles[104] archipelago and includes the island of Great Britain, the northeastern one-sixth of the island of Ireland and some smaller surrounding islands. It lies between the North Atlantic Ocean and the North Sea with the south-east coast coming within 22 miles (35 km) of the coast of northern France, from which it is separated by the English Channel.[105] In 1993 10% of the UK was forested, 46% used for pastures and 25% cultivated for agriculture.[106] The Royal Greenwich Observatory in London is the defining point of the Prime Meridian.[107]\nThe United Kingdom lies between latitudes 49° to 61° N, and longitudes 9° W to 2° E. Northern Ireland shares a 224-mile (360 km) land boundary with the Republic of Ireland.[105] The coastline of Great Britain is 11,073 miles (17,820 km) long.[108] It is connected to continental Europe by the Channel Tunnel, which at 31 miles (50 km) (24 miles (38 km) underwater) is the longest underwater tunnel in the world.[109]\nEngland accounts for just over half of the total area of the UK, covering 130,395 square kilometres (50,350 sq mi).[110] Most of the country consists of lowland terrain,[106] with mountainous terrain north-west of the Tees-Exe line; including the Cumbrian Mountains of the Lake District, the Pennines and limestone hills of the Peak District, Exmoor and Dartmoor. The main rivers and estuaries are the Thames, Severn and the Humber. England's highest mountain is Scafell Pike (978 metres (3,209 ft)) in the Lake District. Its principal rivers are the Severn, Thames, Humber, Tees, Tyne, Tweed, Avon, Exe and Mersey.[106]\nScotland accounts for just under a third of the total area of the UK, covering 78,772 square kilometres (30,410 sq mi)[111] and including nearly eight hundred islands,[112] predominantly west and north of the mainland; notably the Hebrides, Orkney Islands and Shetland Islands. The topography of Scotland is distinguished by the Highland Boundary Fault – a geological rock fracture – which traverses Scotland from Arran in the west to Stonehaven in the east.[113] The faultline separates two distinctively different regions; namely the Highlands to the north and west and the lowlands to the south and east. The more rugged Highland region contains the majority of Scotland's mountainous land, including Ben Nevis which at 1,343 metres (4,406 ft) is the highest point in the British Isles.[114] Lowland areas – especially the narrow waist of land between the Firth of Clyde and the Firth of Forth known as the Central Belt – are flatter and home to most of the population including Glasgow, Scotland's largest city, and Edinburgh, its capital and political centre.\nA view of Ben Nevis in the distance, fronted by rolling plains\nBen Nevis, in Scotland, is the highest point in the British Isles\nWales accounts for less than a tenth of the total area of the UK, covering 20,779 square kilometres (8,020 sq mi).[115] Wales is mostly mountainous, though South Wales is less mountainous than North and mid Wales. The main population and industrial areas are in South Wales, consisting of the coastal cities of Cardiff, Swansea and Newport, and the South Wales Valleys to their north. The highest mountains in Wales are in Snowdonia and include Snowdon (Welsh: Yr Wyddfa) which, at 1,085 metres (3,560 ft), is the highest peak in Wales.[106] The 14, or possibly 15, Welsh mountains over 3,000 feet (914 m) high are known collectively as the Welsh 3000s. Wales has over 2,704 kilometres (1,680 miles) of coastline.[116] Several islands lie off the Welsh mainland, the largest of which is Anglesey (Ynys Môn) in the northwest.\nNorthern Ireland, separated from Great Britain by the Irish Sea and North Channel, has an area of 14,160 square kilometres (5,470 sq mi) and is mostly hilly. It includes Lough Neagh which, at 388 square kilometres (150 sq mi), is the largest lake in the British Isles by area.[117] The highest peak in Northern Ireland is Slieve Donard in the Mourne Mountains at 852 metres (2,795 ft).[106]\nClimate\nMain article: Climate of the United Kingdom\nThe United Kingdom has a temperate climate, with plentiful rainfall all year round.[105] The temperature varies with the seasons seldom dropping below −11 °C (12 °F) or rising above 35 °C (95 °F).[118] The prevailing wind is from the south-west and bears frequent spells of mild and wet weather from the Atlantic Ocean,[105] although the eastern parts are mostly sheltered from this wind since the majority of the rain falls over the western regions the eastern parts are therefore the driest. Atlantic currents, warmed by the Gulf Stream, bring mild winters; especially in the west where winters are wet and even more so over high ground. Summers are warmest in the south-east of England, being closest to the European mainland, and coolest in the north. Heavy snowfall can occur in winter and early spring on high ground, and occasionally settles to great depth away from the hills.\nAdministrative divisions\nMain article: Administrative geography of the United Kingdom\nEach country of the United Kingdom has its own system of administrative and geographic demarcation, whose origins often pre-date the formation of the United Kingdom. Thus there is \"no common stratum of administrative unit encompassing the United Kingdom\".[119] Until the 19th century there was little change to those arrangements, but there has since been a constant evolution of role and function.[120] Change did not occur in a uniform manner and the devolution of power over local government to Scotland, Wales and Northern Ireland means that future changes are unlikely to be uniform either.\nThe organisation of local government in England is complex, with the distribution of functions varying according to local arrangements. Legislation concerning local government in England is the responsibility of the UK parliament and the Government of the United Kingdom, as England has no devolved parliament. The upper-tier subdivisions of England are the nine Government office regions or European Union government office regions.[121] One region, Greater London, has had a directly elected assembly and mayor since 2000 following popular support for the proposal in a referendum.[122] It was intended that other regions would also be given their own elected regional assemblies, but a proposed assembly in the North East region was rejected by a referendum in 2004.[123] Below the regional tier, some parts of England have county councils and district councils and others have unitary authorities; while London consists of 32 London boroughs and the City of London. Councillors are elected by the first-past-the-post system in single-member wards or by the multi-member plurality system in multi-member wards.[124]\nFor local government purposes, Scotland is divided into 32 council areas, with wide variation in both size and population. The cities of Glasgow, Edinburgh, Aberdeen and Dundee are separate council areas, as is the Highland Council which includes a third of Scotland's area but only just over 200,000 people. Local councils are made up of elected councillors, of whom there are currently 1,222;[125] they are paid a part-time salary. Elections are conducted by single transferable vote in multi-member wards that elect either three or four councillors. Each council elects a Provost, or Convenor, to chair meetings of the council and to act as a figurehead for the area. Councillors are subject to a code of conduct enforced by the Standards Commission for Scotland.[126] The representative association of Scotland's local authorities is the Convention of Scottish Local Authorities (COSLA).[127]\nLocal government in Wales consists of 22 unitary authorities. These include the cities of Cardiff, Swansea and Newport which are unitary authorities in their own right.[128] Elections are held every four years under the first-past-the-post system.[129] The most recent elections were held in May 2012, except for the Isle of Anglesey. The Welsh Local Government Association represents the interests of local authorities in Wales.[130]\nLocal government in Northern Ireland has since 1973 been organised into 26 district councils, each elected by single transferable vote. Their powers are limited to services such as collecting waste, controlling dogs and maintaining parks and cemeteries.[131] On 13 March 2008 the executive agreed on proposals to create 11 new councils and replace the present system.[132] The next local elections were postponed until 2016 to facilitate this.[133]\nDependencies\n\nA view of the Caribbean Sea from the Cayman Islands, one of the world's foremost international financial centres[134] and tourist destinations.[135]\nMain articles: British Overseas Territories, Crown dependencies and British Islands\nThe United Kingdom has sovereignty over seventeen territories which do not form part of the United Kingdom itself: fourteen British Overseas Territories[136] and three Crown dependencies.[137]\nThe fourteen British Overseas Territories are: Anguilla; Bermuda; the British Antarctic Territory; the British Indian Ocean Territory; the British Virgin Islands; the Cayman Islands; the Falkland Islands; Gibraltar; Montserrat; Saint Helena, Ascension and Tristan da Cunha; the Turks and Caicos Islands; the Pitcairn Islands; South Georgia and the South Sandwich Islands; and Sovereign Base Areas on Cyprus.[138] British claims in Antarctica are not universally recognised.[139] Collectively Britain's overseas territories encompass an approximate land area of 1,727,570 square kilometres (667,018 sq mi) and a population of approximately 260,000 people.[140] They are the remnants of the British Empire and several have specifically voted to remain British territories (Bermuda in 1995, Gibraltar in 2002 and the Falkland Islands in 2013).[141]\nThe Crown dependencies are possessions of the Crown, as opposed to overseas territories of the UK.[142] They comprise three independently administered jurisdictions: the Channel Islands of Jersey and Guernsey in the English Channel, and the Isle of Man in the Irish Sea. By mutual agreement, the British Government manages the islands' foreign affairs and defence and the UK Parliament has the authority to legislate on their behalf. However, internationally, they are regarded as \"territories for which the United Kingdom is responsible\".[143] The power to pass legislation affecting the islands ultimately rests with their own respective legislative assemblies, with the assent of the Crown (Privy Council or, in the case of the Isle of Man, in certain circumstances the Lieutenant-Governor).[144] Since 2005 each Crown dependency has had a Chief Minister as its head of government.[145]\nPolitics\nMain articles: Politics of the United Kingdom, Monarchy of the United Kingdom and Elections in the United Kingdom\nElderly lady with a yellow hat and grey hair is smiling in outdoor setting.\nElizabeth II, Queen of the United Kingdom and the other Commonwealth realms\nThe United Kingdom is a unitary state under a constitutional monarchy. Queen Elizabeth II is the head of state of the UK as well as monarch of fifteen other independent Commonwealth countries. The monarch has \"the right to be consulted, the right to encourage, and the right to warn\".[146] The United Kingdom is one of only four countries in the world to have an uncodified constitution.[147][nb 7] The Constitution of the United Kingdom thus consists mostly of a collection of disparate written sources, including statutes, judge-made case law and international treaties, together with constitutional conventions. As there is no technical difference between ordinary statutes and \"constitutional law\", the UK Parliament can perform \"constitutional reform\" simply by passing Acts of Parliament, and thus has the political power to change or abolish almost any written or unwritten element of the constitution. However, no Parliament can pass laws that future Parliaments cannot change.[148]\nGovernment\nMain article: Government of the United Kingdom\nThe UK has a parliamentary government based on the Westminster system that has been emulated around the world: a legacy of the British Empire. The parliament of the United Kingdom that meets in the Palace of Westminster has two houses; an elected House of Commons and an appointed House of Lords. All bills passed are given Royal Assent before becoming law.\nThe position of prime minister,[nb 8] the UK's head of government,[149] belongs to the person most likely to command the confidence of the House of Commons; this individual is typically the leader of the political party or coalition of parties that holds the largest number of seats in that chamber. The prime minister chooses a cabinet and they are formally appointed by the monarch to form Her Majesty's Government. By convention, the Queen respects the prime minister's decisions of government.[150]\nLarge sand-coloured building of Gothic design beside brown river and road bridge. The building has several large towers, including large clock-tower.\nThe Palace of Westminster, seat of both houses of the Parliament of the United Kingdom\nThe cabinet is traditionally drawn from members of a prime minister's party or coalition and mostly from the House of Commons but always from both legislative houses, the cabinet being responsible to both. Executive power is exercised by the prime minister and cabinet, all of whom are sworn into the Privy Council of the United Kingdom, and become Ministers of the Crown. The current Prime Minister is David Cameron, who has been in office since 11 May 2010.[151] Cameron is the leader of the Conservative Party and heads a coalition with the Liberal Democrats. For elections to the House of Commons, the UK is currently divided into 650 constituencies,[152] each electing a single member of parliament (MP) by simple plurality. General elections are called by the monarch when the prime minister so advises. The Parliament Acts 1911 and 1949 require that a new election must be called no later than five years after the previous general election.[153]\nThe UK's three major political parties are the Conservative Party (Tories), the Labour Party and the Liberal Democrats, representing the British traditions of conservatism, socialism and social liberalism, respectively. During the 2010 general election these three parties won 622 out of 650 seats available in the House of Commons.[154][155] Most of the remaining seats were won by parties that contest elections only in one part of the UK: the Scottish National Party (Scotland only); Plaid Cymru (Wales only); and the Alliance Party, Democratic Unionist Party, Social Democratic and Labour Party and Sinn Féin (Northern Ireland only[nb 9]). In accordance with party policy, no elected Sinn Féin members of parliament have ever attended the House of Commons to speak on behalf of their constituents because of the requirement to take an oath of allegiance to the monarch.\nDevolved administrations\nMain articles: Devolution in the United Kingdom, Northern Ireland Executive, Scottish Government and Welsh Government\nModern one-story building with grass on roof and large sculpted grass area in front. Behind are residential buildings in a mixture of styles.\nThe Scottish Parliament Building in Holyrood is the seat of the Scottish Parliament.\nScotland, Wales and Northern Ireland each have their own government or executive, led by a First Minister (or, in the case of Northern Ireland, a diarchal First Minister and deputy First Minister), and a devolved unicameral legislature. England, the largest country of the United Kingdom, has no such devolved executive or legislature and is administered and legislated for directly by the UK government and parliament on all issues. This situation has given rise to the so-called West Lothian question which concerns the fact that members of parliament from Scotland, Wales and Northern Ireland can vote, sometimes decisively,[156] on matters that only affect England.[157] The McKay Commission reported on this matter in March 2013 recommending that laws affecting only England should need support from a majority of English members of parliament.[158]\nThe Scottish Government and Parliament have wide-ranging powers over any matter that has not been specifically reserved to the UK parliament, including education, healthcare, Scots law and local government.[159] At the 2011 elections the Scottish National Party won re-election and achieved an overall majority in the Scottish parliament, with its leader, Alex Salmond, as First Minister of Scotland.[160][161] In 2012, the UK and Scottish governments signed the Edinburgh Agreement setting out the terms for a referendum on Scottish independence in 2014, which was defeated 55% to 45%.\nThe Welsh Government and the National Assembly for Wales have more limited powers than those devolved to Scotland.[162] The Assembly is able to legislate on devolved matters through Acts of the Assembly, which require no prior consent from Westminster. The 2011 elections resulted in a minority Labour administration led by Carwyn Jones.[163]\nThe Northern Ireland Executive and Assembly have powers similar to those devolved to Scotland. The Executive is led by a diarchy representing unionist and nationalist members of the Assembly. Currently, Peter Robinson (Democratic Unionist Party) and Martin McGuinness (Sinn Féin) are First Minister and deputy First Minister respectively.[164] Devolution to Northern Ireland is contingent on participation by the Northern Ireland administration in the North-South Ministerial Council, where the Northern Ireland Executive cooperates and develops joint and shared policies with the Government of Ireland. The British and Irish governments co-operate on non-devolved matters affecting Northern Ireland through the British–Irish Intergovernmental Conference, which assumes the responsibilities of the Northern Ireland administration in the event of its non-operation.\nThe UK does not have a codified constitution and constitutional matters are not among the powers devolved to Scotland, Wales or Northern Ireland. Under the doctrine of parliamentary sovereignty, the UK Parliament could, in theory, therefore, abolish the Scottish Parliament, Welsh Assembly or Northern Ireland Assembly.[165][166] Indeed, in 1972, the UK Parliament unilaterally prorogued the Parliament of Northern Ireland, setting a precedent relevant to contemporary devolved institutions.[167] In practice, it would be politically difficult for the UK Parliament to abolish devolution to the Scottish Parliament and the Welsh Assembly, given the political entrenchment created by referendum decisions.[168] The political constraints placed upon the UK Parliament's power to interfere with devolution in Northern Ireland are even greater than in relation to Scotland and Wales, given that devolution in Northern Ireland rests upon an international agreement with the Government of Ireland.[169]\nLaw and criminal justice\nMain article: Law of the United Kingdom\n\nThe Royal Courts of Justice of England and Wales\nThe United Kingdom does not have a single legal system, as Article 19 of the 1706 Treaty of Union provided for the continuation of Scotland's separate legal system.[170] Today the UK has three distinct systems of law: English law, Northern Ireland law and Scots law. A new Supreme Court of the United Kingdom came into being in October 2009 to replace the Appellate Committee of the House of Lords.[171][172] The Judicial Committee of the Privy Council, including the same members as the Supreme Court, is the highest court of appeal for several independent Commonwealth countries, the British Overseas Territories and the Crown Dependencies.[173]\nBoth English law, which applies in England and Wales, and Northern Ireland law are based on common-law principles.[174] The essence of common law is that, subject to statute, the law is developed by judges in courts, applying statute, precedent and common sense to the facts before them to give explanatory judgements of the relevant legal principles, which are reported and binding in future similar cases (stare decisis).[175] The courts of England and Wales are headed by the Senior Courts of England and Wales, consisting of the Court of Appeal, the High Court of Justice (for civil cases) and the Crown Court (for criminal cases). The Supreme Court is the highest court in the land for both criminal and civil appeal cases in England, Wales and Northern Ireland and any decision it makes is binding on every other court in the same jurisdiction, often having a persuasive effect in other jurisdictions.[176]\n\nThe High Court of Justiciary – the supreme criminal court of Scotland.\nScots law is a hybrid system based on both common-law and civil-law principles. The chief courts are the Court of Session, for civil cases,[177] and the High Court of Justiciary, for criminal cases.[178] The Supreme Court of the United Kingdom serves as the highest court of appeal for civil cases under Scots law.[179] Sheriff courts deal with most civil and criminal cases including conducting criminal trials with a jury, known as sheriff solemn court, or with a sheriff and no jury, known as sheriff summary Court.[180] The Scots legal system is unique in having three possible verdicts for a criminal trial: \"guilty\", \"not guilty\" and \"not proven\". Both \"not guilty\" and \"not proven\" result in an acquittal.[181]\nCrime in England and Wales increased in the period between 1981 and 1995, though since that peak there has been an overall fall of 48% in crime from 1995 to 2007/08,[182] according to crime statistics. The prison population of England and Wales has almost doubled over the same period, to over 80,000, giving England and Wales the highest rate of incarceration in Western Europe at 147 per 100,000.[183] Her Majesty's Prison Service, which reports to the Ministry of Justice, manages most of the prisons within England and Wales. Crime in Scotland fell to its lowest recorded level for 32 years in 2009/10, falling by ten per cent.[184] At the same time Scotland's prison population, at over 8,000,[185] is at record levels and well above design capacity.[186] The Scottish Prison Service, which reports to the Cabinet Secretary for Justice, manages Scotland's prisons.\nForeign relations\nMain article: Foreign relations of the United Kingdom\n\nThe Prime Minister of the United Kingdom, David Cameron, and the President of the United States, Barack Obama, during the 2010 G-20 Toronto summit.\nThe UK is a permanent member of the United Nations Security Council, a member of NATO, the Commonwealth of Nations, G7, G8, G20, the OECD, the WTO, the Council of Europe, the OSCE, and is a member state of the European Union. The UK is said to have a \"Special Relationship\" with the United States and a close partnership with France—the \"Entente cordiale\"—and shares nuclear weapons technology with both countries.[187][188] The UK is also closely linked with the Republic of Ireland; the two countries share a Common Travel Area and co-operate through the British-Irish Intergovernmental Conference and the British-Irish Council. Britain's global presence and influence is further amplified through its trading relations, foreign investments, official development assistance and military engagements.[189]\nMilitary\n\nTroopers of the Blues and Royals during the 2007 Trooping the Colour ceremony\nMain article: British Armed Forces\nThe armed forces of the United Kingdom—officially, Her Majesty's Armed Forces—consist of three professional service branches: the Royal Navy and Royal Marines (forming the Naval Service), the British Army and the Royal Air Force.[190] The forces are managed by the Ministry of Defence and controlled by the Defence Council, chaired by the Secretary of State for Defence. The Commander-in-Chief is the British monarch, Elizabeth II, to whom members of the forces swear an oath of allegiance.[191] The Armed Forces are charged with protecting the UK and its overseas territories, promoting the UK's global security interests and supporting international peacekeeping efforts. They are active and regular participants in NATO, including the Allied Rapid Reaction Corps, as well as the Five Power Defence Arrangements, RIMPAC and other worldwide coalition operations. Overseas garrisons and facilities are maintained in Ascension Island, Belize, Brunei, Canada, Cyprus, Diego Garcia, the Falkland Islands, Germany, Gibraltar, Kenya and Qatar.[192]\nThe British armed forces played a key role in establishing the British Empire as the dominant world power in the 18th, 19th and early 20th centuries. Throughout its unique history the British forces have seen action in a number of major wars, such as the Seven Years' War, the Napoleonic Wars, the Crimean War, World War I and World War II—as well as many colonial conflicts. By emerging victorious from such conflicts, Britain has often been able to decisively influence world events. Since the end of the British Empire, the UK has nonetheless remained a major military power. Following the end of the Cold War, defence policy has a stated assumption that \"the most demanding operations\" will be undertaken as part of a coalition.[193] Setting aside the intervention in Sierra Leone, recent UK military operations in Bosnia, Kosovo, Afghanistan, Iraq and, most recently, Libya, have followed this approach. The last time the British military fought alone was the Falklands War of 1982.\nAccording to various sources, including the Stockholm International Peace Research Institute and the International Institute for Strategic Studies, the United Kingdom has the fifth- or sixth-highest military expenditure in the world. Total defence spending currently accounts for around 2.4% of total national GDP.[22][23]\nEconomy\nMain article: Economy of the United Kingdom\n\nThe Bank of England – the central bank of the United Kingdom\nThe UK has a partially regulated market economy.[194] Based on market exchange rates the UK is today the sixth-largest economy in the world and the third-largest in Europe after Germany and France, having fallen behind France for the first time in over a decade in 2008.[195] HM Treasury, led by the Chancellor of the Exchequer, is responsible for developing and executing the British government's public finance policy and economic policy. The Bank of England is the UK's central bank and is responsible for issuing notes and coins in the nation's currency, the pound sterling. Banks in Scotland and Northern Ireland retain the right to issue their own notes, subject to retaining enough Bank of England notes in reserve to cover their issue. Pound sterling is the world's third-largest reserve currency (after the US Dollar and the Euro).[196] Since 1997 the Bank of England's Monetary Policy Committee, headed by the Governor of the Bank of England, has been responsible for setting interest rates at the level necessary to achieve the overall inflation target for the economy that is set by the Chancellor each year.[197]\nThe UK service sector makes up around 73% of GDP.[198] London is one of the three \"command centres\" of the global economy (alongside New York City and Tokyo),[199] it is the world's largest financial centre alongside New York,[200][201][202] and it has the largest city GDP in Europe.[203] Edinburgh is also one of the largest financial centres in Europe.[204] Tourism is very important to the British economy and, with over 27 million tourists arriving in 2004, the United Kingdom is ranked as the sixth major tourist destination in the world and London has the most international visitors of any city in the world.[205][206] The creative industries accounted for 7% GVA in 2005 and grew at an average of 6% per annum between 1997 and 2005.[207]\n\nThe Airbus A350 has its wings and engines manufactured in the UK.\nThe Industrial Revolution started in the UK with an initial concentration on the textile industry,[208] followed by other heavy industries such as shipbuilding, coal mining and steelmaking.[209][210]\nThe empire was exploited as an overseas market for British products, allowing the UK to dominate international trade in the 19th century. As other nations industrialised, coupled with economic decline after two world wars, the United Kingdom began to lose its competitive advantage and heavy industry declined, by degrees, throughout the 20th century. Manufacturing remains a significant part of the economy but accounted for only 16.7% of national output in 2003.[211]\nThe automotive industry is a significant part of the UK manufacturing sector and employs over 800,000 people, with a turnover of some £52 billion, generating £26.6 billion of exports.[212]\nThe aerospace industry of the UK is the second- or third-largest national aerospace industry in the world depending upon the method of measurement and has an annual turnover of around £20 billion. The wings for the Airbus A380 and the A350 XWB are designed and manufactured at Airbus UK's world-leading Broughton facility, whilst over a quarter of the value of the Boeing 787 comes from UK manufacturers including Eaton (fuel subsystem pumps), Messier-Bugatti-Dowty (the landing gear) and Rolls-Royce (the engines). Other key names include GKN Aerospace – an expert in metallic and composite aerostructures that's involved in almost every civil and military fixed and rotary wing aircraft in production and development today.[213][214][215][216]\nBAE Systems - plays a critical role on some of the world's biggest defence aerospace projects. The company makes large sections of the Typhoon Eurofighter at its sub-assembly plant in Salmesbury and assembles the aircraft for the RAF at its Warton Plant, near Preston. It is also a principal subcontractor on the F35 Joint Strike Fighter - the world's largest single defence project - for which it designs and manufactures a range of components including the aft fuselage, vertical and horizontal tail and wing tips and fuel system. As well as this it manufactures the Hawk, the world's most successful jet training aircraft.[216] Airbus UK also manufactures the wings for the A400m military transporter. Rolls-Royce, is the world's second-largest aero-engine manufacturer. Its engines power more than 30 types of commercial aircraft and it has more than 30,000 engines currently in service across both the civil and defence sectors. Agusta Westland designs and manufactures complete helicopters in the UK.[216]\nThe UK space industry is growing very fast. Worth £9.1bn in 2011 and employing 29,000 people, it is growing at a rate of some 7.5 per cent annually, according to its umbrella organisation, the UK Space Agency. Government strategy is for the space industry to be a £40bn business for the UK by 2030, capturing a 10 per cent share of the $250bn world market for commercial space technology.[216] On 16 July 2013, the British government pledged £60m to the Skylon project: this investment will provide support at a \"crucial stage\" to allow a full-scale prototype of the SABRE engine to be built.\nThe pharmaceutical industry plays an important role in the UK economy and the country has the third-highest share of global pharmaceutical R&D expenditures (after the United States and Japan).[217][218]\nAgriculture is intensive, highly mechanised and efficient by European standards, producing about 60% of food needs with less than 1.6% of the labour force (535,000 workers).[219] Around two-thirds of production is devoted to livestock, one-third to arable crops. Farmers are subsidised by the EU's Common Agricultural Policy. The UK retains a significant, though much reduced fishing industry. It is also rich in a number of natural resources including coal, petroleum, natural gas, tin, limestone, iron ore, salt, clay, chalk, gypsum, lead, silica and an abundance of arable land.\n\nThe City of London is the world's largest financial centre alongside New York[200][201][202]\nIn the final quarter of 2008 the UK economy officially entered recession for the first time since 1991.[220] Unemployment increased from 5.2% in May 2008 to 7.6% in May 2009 and by January 2012 the unemployment rate among 18 to 24-year-olds had risen from 11.9% to 22.5%, the highest since current records began in 1992.[221][222] Total UK government debt rose from 44.4% of GDP in 2007 to 82.9% of GDP in 2011.[223] In February 2013, the UK lost its top AAA credit rating for the first time since 1978.[224]\nInflation-adjusted wages in the UK fell by 3.2% between the third quarter of 2010 and the third quarter of 2012.[225] Since the 1980s, economic inequality has grown faster in the UK than in any other developed country.[226]\nThe poverty line in the UK is commonly defined as being 60% of the median household income.[nb 10] In 2007–2008 13.5 million people, or 22% of the population, lived below this line. This is a higher level of relative poverty than all but four other EU members.[227] In the same year 4.0 million children, 31% of the total, lived in households below the poverty line after housing costs were taken into account. This is a decrease of 400,000 children since 1998–1999.[228] The UK imports 40% of its food supplies.[229] The Office for National Statistics has estimated that in 2011, 14 million people were at risk of poverty or social exclusion, and that one person in 20 (5.1%) was now experiencing \"severe material depression,\"[230] up from 3 million people in 1977.[231][232]\nScience and technology\nMain article: Science and technology in the United Kingdom\n\nCharles Darwin (1809–82), whose theory of evolution by natural selection is the foundation of modern biological sciences\nEngland and Scotland were leading centres of the Scientific Revolution from the 17th century[233] and the United Kingdom led the Industrial Revolution from the 18th century,[208] and has continued to produce scientists and engineers credited with important advances.[234] Major theorists from the 17th and 18th centuries include Isaac Newton, whose laws of motion and illumination of gravity have been seen as a keystone of modern science;[235] from the 19th century Charles Darwin, whose theory of evolution by natural selection was fundamental to the development of modern biology, and James Clerk Maxwell, who formulated classical electromagnetic theory; and more recently Stephen Hawking, who has advanced major theories in the fields of cosmology, quantum gravity and the investigation of black holes.[236] Major scientific discoveries from the 18th century include hydrogen by Henry Cavendish;[237] from the 20th century penicillin by Alexander Fleming,[238] and the structure of DNA, by Francis Crick and others.[239] Major engineering projects and applications by people from the UK in the 18th century include the steam locomotive, developed by Richard Trevithick and Andrew Vivian;[240] from the 19th century the electric motor by Michael Faraday, the incandescent light bulb by Joseph Swan,[241] and the first practical telephone, patented by Alexander Graham Bell;[242] and in the 20th century the world's first working television system by John Logie Baird and others,[243] the jet engine by Frank Whittle, the basis of the modern computer by Alan Turing, and the World Wide Web by Tim Berners-Lee.[244]\nScientific research and development remains important in British universities, with many establishing science parks to facilitate production and co-operation with industry.[245] Between 2004 and 2008 the UK produced 7% of the world's scientific research papers and had an 8% share of scientific citations, the third and second highest in the world (after the United States and China, and the United States, respectively).[246] Scientific journals produced in the UK include Nature, the British Medical Journal and The Lancet.[247]\nTransport\nMain article: Transport in the United Kingdom\n\nHeathrow Terminal 5 building. London Heathrow Airport has the most international passenger traffic of any airport in the world.[248][249]\nA radial road network totals 29,145 miles (46,904 km) of main roads, 2,173 miles (3,497 km) of motorways and 213,750 miles (344,000 km) of paved roads.[105] In 2009 there were a total of 34 million licensed vehicles in Great Britain.[250]\nThe UK has a railway network of 10,072 miles (16,209 km) in Great Britain and 189 miles (304 km) in Northern Ireland. Railways in Northern Ireland are operated by NI Railways, a subsidiary of state-owned Translink. In Great Britain, the British Rail network was privatised between 1994 and 1997. Network Rail owns and manages most of the fixed assets (tracks, signals etc.). About 20 privately owned (and foreign state-owned railways including: Deutsche Bahn; SNCF and Nederlandse Spoorwegen) Train Operating Companies (including state-owned East Coast), operate passenger trains and carry over 18,000 passenger trains daily. There are also some 1,000 freight trains in daily operation.[105] The UK government is to spend £30 billion on a new high-speed railway line, HS2, to be operational by 2025.[251] Crossrail, under construction in London, Is Europe's largest construction project with a £15 billion projected cost.[252][253]\nIn the year from October 2009 to September 2010 UK airports handled a total of 211.4 million passengers.[254] In that period the three largest airports were London Heathrow Airport (65.6 million passengers), Gatwick Airport (31.5 million passengers) and London Stansted Airport (18.9 million passengers).[254] London Heathrow Airport, located 15 miles (24 km) west of the capital, has the most international passenger traffic of any airport in the world[248][249] and is the hub for the UK flag carrier British Airways, as well as for BMI and Virgin Atlantic.[255]\nEnergy\nMain article: Energy in the United Kingdom\n\nAn oil platform in the North Sea\nIn 2006, the UK was the world's ninth-largest consumer of energy and the 15th-largest producer.[256] The UK is home to a number of large energy companies, including two of the six oil and gas \"supermajors\" – BP and Royal Dutch Shell – and BG Group.[257][258] In 2011, 40% of the UK's electricity was produced by gas, 30% by coal, 19% by nuclear power and 4.2% by wind, hydro, biofuels and wastes.[259]\nIn 2009, the UK produced 1.5 million barrels per day (bbl/d) of oil and consumed 1.7 million bbl/d.[260] Production is now in decline and the UK has been a net importer of oil since 2005.[260] In 2010 the UK had around 3.1 billion barrels of proven crude oil reserves, the largest of any EU member state.[260] In 2009, 66.5% of the UK's oil supply was imported.[261]\nIn 2009, the UK was the 13th-largest producer of natural gas in the world and the largest producer in the EU.[262] Production is now in decline and the UK has been a net importer of natural gas since 2004.[262] In 2009, half of British gas was supplied from imports and this is expected to increase to at least 75% by 2015, as domestic reserves are depleted.[259]\nCoal production played a key role in the UK economy in the 19th and 20th centuries. In the mid-1970s, 130 million tonnes of coal was being produced annually, not falling below 100 million tonnes until the early 1980s. During the 1980s and 1990s the industry was scaled back considerably. In 2011, the UK produced 18.3 million tonnes of coal.[263] In 2005 it had proven recoverable coal reserves of 171 million tons.[263] The UK Coal Authority has stated there is a potential to produce between 7 billion tonnes and 16 billion tonnes of coal through underground coal gasification (UCG) or 'fracking',[264] and that, based on current UK coal consumption, such reserves could last between 200 and 400 years.[265] However, environmental and social concerns have been raised over chemicals getting into the water table and minor earthquakes damaging homes.[266][267]\nIn the late 1990s, nuclear power plants contributed around 25% of total annual electricity generation in the UK, but this has gradually declined as old plants have been shut down and ageing-related problems affect plant availability. In 2012, the UK had 16 reactors normally generating about 19% of its electricity. All but one of the reactors will be retired by 2023. Unlike Germany and Japan, the UK intends to build a new generation of nuclear plants from about 2018.[259]\nDemographics\nMain article: Demographics of the United Kingdom\n\nMap of population density in the UK as at the 2011 census.\nA census is taken simultaneously in all parts of the UK every ten years.[268] The Office for National Statistics is responsible for collecting data for England and Wales, the General Register Office for Scotland and the Northern Ireland Statistics and Research Agency each being responsible for censuses in their respective countries.[269] In the 2011 census the total population of the United Kingdom was 63,181,775.[270] It is the third-largest in the European Union, the fifth-largest in the Commonwealth and the 21st-largest in the world. 2010 was the third successive year in which natural change contributed more to population growth than net long-term international migration.[271][271] Between 2001 and 2011 the population increased by an average annual rate of approximately 0.7 per cent.[270] This compares to 0.3 per cent per year in the period 1991 to 2001 and 0.2 per cent in the decade 1981 to 1991.[271] The 2011 census also confirmed that the proportion of the population aged 0–14 has nearly halved (31 per cent in 1911 compared to 18 in 2011) and the proportion of older people aged 65 and over has more than trebled (from 5 to 16 per cent).[270] It has been estimated that the number of people aged 100 or over will rise steeply to reach over 626,000 by 2080.[272]\nEngland's population in 2011 was found to be 53 million.[273] It is one of the most densely populated countries in the world, with 383 people resident per square kilometre in mid-2003,[274] with a particular concentration in London and the south-east.[275] The 2011 census put Scotland's population at 5.3 million,[276] Wales at 3.06 million and Northern Ireland at 1.81 million.[273] In percentage terms England has had the fastest growing population of any country of the UK in the period from 2001 to 2011, with an increase of 7.9%.\nIn 2012 the average total fertility rate (TFR) across the UK was 1.92 children per woman.[277] While a rising birth rate is contributing to current population growth, it remains considerably below the 'baby boom' peak of 2.95 children per woman in 1964,[278] below the replacement rate of 2.1, but higher than the 2001 record low of 1.63.[277] In 2012, Scotland had the lowest TFR at only 1.67, followed by Wales at 1.88, England at 1.94, and Northern Ireland at 2.03.[277] In 2011, 47.3% of births in the UK were to unmarried women.[279] A government figure estimated that there are 3.6 million homosexual people in Britain comprising 6 per cent of the population.[280]\nview talk edit\nview talk edit\nLargest urban areas of the United Kingdom\nUnited Kingdom 2011 census Built-up areas[281][282][283]\nRank\tUrban area\tPop.\tPrincipal settlement\tRank\tUrban area\tPop.\tPrincipal settlement\t\nGreater London Urban Area\nGreater London Urban Area\nGreater Manchester Urban Area\nGreater Manchester Urban Area\n1\tGreater London Urban Area\t9,787,426\tLondon\t11\tBristol Urban Area\t617,280\tBristol\tWest Midlands Urban Area\nWest Midlands Urban Area\nWest Yorkshire Urban Area\nWest Yorkshire Urban Area\n2\tGreater Manchester Urban Area\t2,553,379\tManchester\t12\tBelfast Metropolitan Urban Area\t579,236\tBelfast\n3\tWest Midlands Urban Area\t2,440,986\tBirmingham\t13\tLeicester Urban Area\t508,916\tLeicester\n4\tWest Yorkshire Urban Area\t1,777,934\tLeeds\t14\tEdinburgh\t488,610\tEdinburgh\n5\tGreater Glasgow\t976,970\tGlasgow\t15\tBrighton/Worthing/Littlehampton\t474,485\tBrighton\n6\tLiverpool Urban Area\t864,122\tLiverpool\t16\tSouth East Dorset conurbation\t466,266\tBournemouth\n7\tSouth Hampshire\t855,569\tSouthampton\t17\tCardiff Urban Area\t390,214\tCardiff\n8\tTyneside\t774,891\tNewcastle\t18\tTeesside\t376,633\tMiddlesbrough\n9\tNottingham Urban Area\t729,977\tNottingham\t19\tThe Potteries Urban Area\t372,775\tStoke-on-Trent\n10\tSheffield Urban Area\t685,368\tSheffield\t20\tCoventry and Bedworth Urban Area\t359,262\tCoventry\n\nEthnic groups\n\nMap showing the percentage of the population who are not white according to the 2011 census.\nEthnic group\t2011\npopulation\t2011\n%\nWhite\t55,010,359\t87.1\nWhite: Irish Traveller\t63,193\t0.1\nAsian or Asian British: Indian\t1,451,862\t\n2.3\nAsian or Asian British: Pakistani\t1,173,892\t\n1.9\nAsian or Asian British: Bangladeshi\t451,529\t\n0.7\nAsian or Asian British: Chinese\t433,150\t\n0.7\nAsian or Asian British: Asian Other\t861,815\t\n1.4\nAsian or Asian British: Total\t4,373,339\t\n7.0\nBlack or Black British\t1,904,684\t\n3.0\nBritish Mixed\t1,250,229\t\n2.0\nOther: Total\t580,374\t\n0.9\nTotal[284]\t63,182,178\t\n100\nHistorically, indigenous British people were thought to be descended from the various ethnic groups that settled there before the 11th century: the Celts, Romans, Anglo-Saxons, Norse and the Normans. Welsh people could be the oldest ethnic group in the UK.[285] A 2006 genetic study shows that more than 50 per cent of England's gene pool contains Germanic Y chromosomes.[286] Another 2005 genetic analysis indicates that \"about 75 per cent of the traceable ancestors of the modern British population had arrived in the British isles by about 6,200 years ago, at the start of the British Neolithic or Stone Age\", and that the British broadly share a common ancestry with the Basque people.[287][288][289]\nThe UK has a history of small-scale non-white immigration, with Liverpool having the oldest Black population in the country dating back to at least the 1730s during the period of the African slave trade,[290] and the oldest Chinese community in Europe, dating to the arrival of Chinese seamen in the 19th century.[291] In 1950 there were probably fewer than 20,000 non-white residents in Britain, almost all born overseas.[292]\nSince 1948 substantial immigration from Africa, the Caribbean and South Asia has been a legacy of ties forged by the British Empire. Migration from new EU member states in Central and Eastern Europe since 2004 has resulted in growth in these population groups but, as of 2008, the trend is reversing. Many of these migrants are returning to their home countries, leaving the size of these groups unknown.[293] In 2011, 86% of the population identified themselves as White, meaning 12.9% of the UK population identify themselves as of mixed ethnic minority.\nEthnic diversity varies significantly across the UK. 30.4% of London's population and 37.4% of Leicester's was estimated to be non-white in 2005,[294][295] whereas less than 5% of the populations of North East England, Wales and the South West were from ethnic minorities, according to the 2001 census.[296] In 2011, 26.5% of primary and 22.2% of secondary pupils at state schools in England were members of an ethnic minority.[297]\nThe non-white British population of England and Wales increased by 38% from 6.6 million in 2001 to 9.1 million in 2009.[298] The fastest-growing group was the mixed-ethnicity population, which doubled from 672,000 in 2001 to 986,600 in 2009. Also in the same period, a decrease of 36,000 white British people was recorded.[299]\nLanguages\nMain article: Languages of the United Kingdom\n\nThe English-speaking world. Countries in dark blue have a majority of native speakers; countries where English is an official but not a majority language are shaded in light blue. English is one of the official languages of the European Union[300] and the United Nations[301]\nThe UK's de facto official language is English.[302][303] It is estimated that 95% of the UK's population are monolingual English speakers.[304] 5.5% of the population are estimated to speak languages brought to the UK as a result of relatively recent immigration.[304] South Asian languages, including Bengali, Tamil, Punjabi, Hindi and Gujarati, are the largest grouping and are spoken by 2.7% of the UK population.[304] According to the 2011 census, Polish has become the second-largest language spoken in England and has 546,000 speakers.[305]\nFour Celtic languages are spoken in the UK: Welsh; Irish; Scottish Gaelic; and Cornish. All are recognised as regional or minority languages, subject to specific measures of protection and promotion under the European Charter for Regional or Minority Languages[2][306] and the Framework Convention for the Protection of National Minorities.[307] In the 2001 Census over a fifth (21%) of the population of Wales said they could speak Welsh,[308] an increase from the 1991 Census (18%).[309] In addition it is estimated that about 200,000 Welsh speakers live in England.[310] In the same census in Northern Ireland 167,487 people (10.4%) stated that they had \"some knowledge of Irish\" (see Irish language in Northern Ireland), almost exclusively in the nationalist (mainly Catholic) population. Over 92,000 people in Scotland (just under 2% of the population) had some Gaelic language ability, including 72% of those living in the Outer Hebrides.[311] The number of schoolchildren being taught through Welsh, Scottish Gaelic and Irish is increasing.[312] Among emigrant-descended populations some Scottish Gaelic is still spoken in Canada (principally Nova Scotia and Cape Breton Island),[313] and Welsh in Patagonia, Argentina.[314]\nScots, a language descended from early northern Middle English, has limited recognition alongside its regional variant, Ulster Scots in Northern Ireland, without specific commitments to protection and promotion.[2][315]\nIt is compulsory for pupils to study a second language up to the age of 14 in England,[316] and up to age 16 in Scotland. French and German are the two most commonly taught second languages in England and Scotland. All pupils in Wales are taught Welsh as a second language up to age 16, or are taught in Welsh.[317]\nReligion\nMain article: Religion in the United Kingdom\n\nWestminster Abbey is used for the coronation of British monarchs\nForms of Christianity have dominated religious life in what is now the United Kingdom for over 1,400 years.[318] Although a majority of citizens still identify with Christianity in many surveys, regular church attendance has fallen dramatically since the middle of the 20th century,[319] while immigration and demographic change have contributed to the growth of other faiths, most notably Islam.[320] This has led some commentators to variously describe the UK as a multi-faith,[321] secularised,[322] or post-Christian society.[323]\nIn the 2001 census 71.6% of all respondents indicated that they were Christians, with the next largest faiths (by number of adherents) being Islam (2.8%), Hinduism (1.0%), Sikhism (0.6%), Judaism (0.5%), Buddhism (0.3%) and all other religions (0.3%).[324] 15% of respondents stated that they had no religion, with a further 7% not stating a religious preference.[325] A Tearfund survey in 2007 showed only one in ten Britons actually attend church weekly.[326] Between the 2001 and 2011 census there was a decrease in the amount of people who identified as Christian by 12%, whilst the percentage of those reporting no religious affiliation doubled. This contrasted with growth in the other main religious group categories, with the number of Muslims increasing by the most substantial margin to a total of about 5%.[327]\nThe Church of England is the established church in England.[328] It retains a representation in the UK Parliament and the British monarch is its Supreme Governor.[329] In Scotland the Presbyterian Church of Scotland is recognised as the national church. It is not subject to state control, and the British monarch is an ordinary member, required to swear an oath to \"maintain and preserve the Protestant Religion and Presbyterian Church Government\" upon his or her accession.[330][331] The (Anglican) Church in Wales was disestablished in 1920 and, as the (Anglican) Church of Ireland was disestablished in 1870 before the partition of Ireland, there is no established church in Northern Ireland.[332] Although there are no UK-wide data in the 2001 census on adherence to individual Christian denominations, it has been estimated that 62% of Christians are Anglican, 13.5% Catholic, 6% Presbyterian, 3.4% Methodist with small numbers of other Protestant denominations such as Open Brethren, and Orthodox churches.[333]\nMigration\nMain article: Immigration to the United Kingdom since 1922\nSee also: Foreign-born population of the United Kingdom\n\nEstimated foreign-born population by country of birth, April 2007 – March 2008\nThe United Kingdom has experienced successive waves of migration. The Great Famine in Ireland, then part of the United Kingdom, resulted in perhaps a million people migrating to Great Brtain.[334] Unable to return to Poland at the end of World War II, over 120,000 Polish veterans remained in the UK permanently.[335] After World War II, there was significant immigration from the colonies and newly independent former colonies, partly as a legacy of empire and partly driven by labour shortages. Many of these migrants came from the Caribbean and the Indian subcontinent.[336] The British Asian population has increased from 2.2 million in 2001 to over 4.2 million in 2011.[337]\nOne of the more recent trends in migration has been the arrival of workers from the new EU member states in Eastern Europe. In 2010, there were 7.0 million foreign-born residents in the UK, corresponding to 11.3% of the total population. Of these, 4.76 million (7.7%) were born outside the EU and 2.24 million (3.6%) were born in another EU Member State.[338] The proportion of foreign-born people in the UK remains slightly below that of many other European countries.[339] However, immigration is now contributing to a rising population[340] with arrivals and UK-born children of migrants accounting for about half of the population increase between 1991 and 2001. Analysis of Office for National Statistics (ONS) data shows that a net total of 2.3 million migrants moved to the UK in the 15 years from 1991 to 2006.[341][342] In 2008 it was predicted that migration would add 7 million to the UK population by 2031,[343] though these figures are disputed.[344] The ONS reported that net migration rose from 2009 to 2010 by 21 per cent to 239,000.[345] In 2011 the net increase was 251,000: immigration was 589,000, while the number of people emigrating (for more than 12 months) was 338,000.[346][347]\n195,046 foreign nationals became British citizens in 2010,[348] compared to 54,902 in 1999.[348][349] A record 241,192 people were granted permanent settlement rights in 2010, of whom 51 per cent were from Asia and 27 per cent from Africa.[350] 25.5 per cent of babies born in England and Wales in 2011 were born to mothers born outside the UK, according to official statistics released in 2012.[351]\nCitizens of the European Union, including those of the UK, have the right to live and work in any EU member state.[352] The UK applied temporary restrictions to citizens of Romania and Bulgaria, which joined the EU in January 2007.[353] Research conducted by the Migration Policy Institute for the Equality and Human Rights Commission suggests that, between May 2004 and September 2009, 1.5 million workers migrated from the new EU member states to the UK, two-thirds of them Polish, but that many subsequently returned home, resulting in a net increase in the number of nationals of the new member states in the UK of some 700,000 over that period.[354][355] The late-2000s recession in the UK reduced the economic incentive for Poles to migrate to the UK,[356] the migration becoming temporary and circular.[357] In 2009, for the first time since enlargement, more nationals of the eight central and eastern European states that had joined the EU in 2004 left the UK than arrived.[358] In 2011, citizens of the new EU member states made up 13% of the immigrants entering the country.[346]\n\nEstimated number of British citizens living overseas by country, 2006\nThe UK government has introduced a points-based immigration system for immigration from outside the European Economic Area to replace former schemes, including the Scottish Government's Fresh Talent Initiative.[359] In June 2010 the UK government introduced a temporary limit of 24,000 on immigration from outside the EU, aiming to discourage applications before a permanent cap was imposed in April 2011.[360] The cap has caused tension within the coalition: business secretary Vince Cable has argued that it is harming British businesses.[361]\nEmigration was an important feature of British society in the 19th century. Between 1815 and 1930 around 11.4 million people emigrated from Britain and 7.3 million from Ireland. Estimates show that by the end of the 20th century some 300 million people of British and Irish descent were permanently settled around the globe.[362] Today, at least 5.5 million UK-born people live abroad,[363][364][365] mainly in Australia, Spain, the United States and Canada.[363][366]\nEducation\nMain article: Education in the United Kingdom\nSee also: Education in England, Education in Northern Ireland, Education in Scotland and Education in Wales\n\nKing's College, part of the University of Cambridge, which was founded in 1209\nEducation in the United Kingdom is a devolved matter, with each country having a separate education system.\nWhilst education in England is the responsibility of the Secretary of State for Education, the day-to-day administration and funding of state schools is the responsibility of local authorities.[367] Universally free of charge state education was introduced piecemeal between 1870 and 1944.[368][369] Education is now mandatory from ages five to sixteen (15 if born in late July or August). In 2011, the Trends in International Mathematics and Science Study (TIMSS) rated 13–14-year-old pupils in England and Wales 10th in the world for maths and 9th for science.[370] The majority of children are educated in state-sector schools, a small proportion of which select on the grounds of academic ability. Two of the top ten performing schools in terms of GCSE results in 2006 were state-run grammar schools. Over half of students at the leading universities of Cambridge and Oxford had attended state schools.[371] Despite a fall in actual numbers the proportion of children in England attending private schools has risen to over 7%.[372] In 2010, more than 45% of places at the University of Oxford and 40% at the University of Cambridge were taken by students from private schools, even though they educate just 7% of the population.[373] England has the two oldest universities in English-speaking world, Universities of Oxford and Cambridge (jointly known as \"Oxbridge\") with history of over eight centuries. The United Kingdom has 9 universities featured in the Times Higher Education top 100 rankings, making it second to the United States in terms of representation.[374]\n\nQueen's University Belfast, built in 1849[375]\nEducation in Scotland is the responsibility of the Cabinet Secretary for Education and Lifelong Learning, with day-to-day administration and funding of state schools the responsibility of Local Authorities. Two non-departmental public bodies have key roles in Scottish education. The Scottish Qualifications Authority is responsible for the development, accreditation, assessment and certification of qualifications other than degrees which are delivered at secondary schools, post-secondary colleges of further education and other centres.[376] The Learning and Teaching Scotland provides advice, resources and staff development to education professionals.[377] Scotland first legislated for compulsory education in 1496.[378] The proportion of children in Scotland attending private schools is just over 4%, and it has been rising slowly in recent years.[379] Scottish students who attend Scottish universities pay neither tuition fees nor graduate endowment charges, as fees were abolished in 2001 and the graduate endowment scheme was abolished in 2008.[380]\nThe Welsh Government has responsibility for education in Wales. A significant number of Welsh students are taught either wholly or largely in the Welsh language; lessons in Welsh are compulsory for all until the age of 16.[381] There are plans to increase the provision of Welsh-medium schools as part of the policy of creating a fully bilingual Wales.\nEducation in Northern Ireland is the responsibility of the Minister of Education and the Minister for Employment and Learning, although responsibility at a local level is administered by five education and library boards covering different geographical areas. The Council for the Curriculum, Examinations & Assessment (CCEA) is the body responsible for advising the government on what should be taught in Northern Ireland's schools, monitoring standards and awarding qualifications.[382]\nA government commission's report in 2014 found that privately educated people comprise 7% of the general population of the UK but much larger percentages of the top professions, the most extreme case quoted being 71% of senior judges.[383][384]\nHealthcare\nMain article: Healthcare in the United Kingdom\n\nThe Royal Aberdeen Children's Hospital, an NHS Scotland specialist children's hospital\nHealthcare in the United Kingdom is a devolved matter and each country has its own system of private and publicly funded health care, together with alternative, holistic and complementary treatments. Public healthcare is provided to all UK permanent residents and is mostly free at the point of need, being paid for from general taxation. The World Health Organization, in 2000, ranked the provision of healthcare in the United Kingdom as fifteenth best in Europe and eighteenth in the world.[385][386]\nRegulatory bodies are organised on a UK-wide basis such as the General Medical Council, the Nursing and Midwifery Council and non-governmental-based, such as the Royal Colleges. However, political and operational responsibility for healthcare lies with four national executives; healthcare in England is the responsibility of the UK Government; healthcare in Northern Ireland is the responsibility of the Northern Ireland Executive; healthcare in Scotland is the responsibility of the Scottish Government; and healthcare in Wales is the responsibility of the Welsh Assembly Government. Each National Health Service has different policies and priorities, resulting in contrasts.[387][388]\nSince 1979 expenditure on healthcare has been increased significantly to bring it closer to the European Union average.[389] The UK spends around 8.4 per cent of its gross domestic product on healthcare, which is 0.5 percentage points below the Organisation for Economic Co-operation and Development average and about one percentage point below the average of the European Union.[390]\nCulture\nMain article: Culture of the United Kingdom\nThe culture of the United Kingdom has been influenced by many factors including: the nation's island status; its history as a western liberal democracy and a major power; as well as being a political union of four countries with each preserving elements of distinctive traditions, customs and symbolism. As a result of the British Empire, British influence can be observed in the language, culture and legal systems of many of its former colonies including Australia, Canada, India, Ireland, New Zealand, South Africa and the United States. The substantial cultural influence of the United Kingdom has led it to be described as a \"cultural superpower.\"[391][392]\nLiterature\nMain article: British literature\n\nThe Chandos portrait, believed to depict William Shakespeare\n'British literature' refers to literature associated with the United Kingdom, the Isle of Man and the Channel Islands. Most British literature is in the English language. In 2005, some 206,000 books were published in the United Kingdom and in 2006 it was the largest publisher of books in the world.[393]\nThe English playwright and poet William Shakespeare is widely regarded as the greatest dramatist of all time,[394][395][396] and his contemporaries Christopher Marlowe and Ben Jonson have also been held in continuous high esteem. More recently the playwrights Alan Ayckbourn, Harold Pinter, Michael Frayn, Tom Stoppard and David Edgar have combined elements of surrealism, realism and radicalism.\nNotable pre-modern and early-modern English writers include Geoffrey Chaucer (14th century), Thomas Malory (15th century), Sir Thomas More (16th century), John Bunyan (17th century) and John Milton (17th century). In the 18th century Daniel Defoe (author of Robinson Crusoe) and Samuel Richardson were pioneers of the modern novel. In the 19th century there followed further innovation by Jane Austen, the gothic novelist Mary Shelley, the children's writer Lewis Carroll, the Brontë sisters, the social campaigner Charles Dickens, the naturalist Thomas Hardy, the realist George Eliot, the visionary poet William Blake and romantic poet William Wordsworth. 20th-century English writers include the science-fiction novelist H. G. Wells; the writers of children's classics Rudyard Kipling, A. A. Milne (the creator of Winnie-the-Pooh), Roald Dahl and Enid Blyton; the controversial D. H. Lawrence; the modernist Virginia Woolf; the satirist Evelyn Waugh; the prophetic novelist George Orwell; the popular novelists W. Somerset Maugham and Graham Greene; the crime writer Agatha Christie (the best-selling novelist of all time);[397] Ian Fleming (the creator of James Bond); the poets T.S. Eliot, Philip Larkin and Ted Hughes; the fantasy writers J. R. R. Tolkien, C. S. Lewis and J. K. Rowling; the graphic novelist Alan Moore, whose novel Watchmen is often cited by critics as comic's greatest series and graphic novel[398] and one of the best-selling graphic novels ever published.[399]\n\nA photograph of Victorian era novelist Charles Dickens\nScotland's contributions include the detective writer Arthur Conan Doyle (the creator of Sherlock Holmes), romantic literature by Sir Walter Scott, the children's writer J. M. Barrie, the epic adventures of Robert Louis Stevenson and the celebrated poet Robert Burns. More recently the modernist and nationalist Hugh MacDiarmid and Neil M. Gunn contributed to the Scottish Renaissance. A more grim outlook is found in Ian Rankin's stories and the psychological horror-comedy of Iain Banks. Scotland's capital, Edinburgh, was UNESCO's first worldwide City of Literature.[400]\nBritain's oldest known poem, Y Gododdin, was composed in Yr Hen Ogledd (The Old North), most likely in the late 6th century. It was written in Cumbric or Old Welsh and contains the earliest known reference to King Arthur.[401] From around the seventh century, the connection between Wales and the Old North was lost, and the focus of Welsh-language culture shifted to Wales, where Arthurian legend was further developed by Geoffrey of Monmouth.[402] Wales's most celebrated medieval poet, Dafydd ap Gwilym (fl.1320–1370), composed poetry on themes including nature, religion and especially love. He is widely regarded as one of the greatest European poets of his age.[403] Until the late 19th century the majority of Welsh literature was in Welsh and much of the prose was religious in character. Daniel Owen is credited as the first Welsh-language novelist, publishing Rhys Lewis in 1885. The best-known of the Anglo-Welsh poets are both Thomases. Dylan Thomas became famous on both sides of the Atlantic in the mid-20th century. He is remembered for his poetry – his \"Do not go gentle into that good night; Rage, rage against the dying of the light.\" is one of the most quoted couplets of English language verse – and for his 'play for voices', Under Milk Wood. The influential Church in Wales 'poet-priest' and Welsh nationalist R. S. Thomas was nominated for the Nobel Prize in Literature in 1996. Leading Welsh novelists of the twentieth century include Richard Llewellyn and Kate Roberts.[404][405]\nAuthors of other nationalities, particularly from Commonwealth countries, the Republic of Ireland and the United States, have lived and worked in the UK. Significant examples through the centuries include Jonathan Swift, Oscar Wilde, Bram Stoker, George Bernard Shaw, Joseph Conrad, T.S. Eliot, Ezra Pound and more recently British authors born abroad such as Kazuo Ishiguro and Sir Salman Rushdie.[406][407]\nMusic\nMain article: Music of the United Kingdom\nSee also: British rock\n\nThe Beatles are the most commercially successful and critically acclaimed band in the history of music, selling over a billion records internationally.[408][409][410]\nVarious styles of music are popular in the UK from the indigenous folk music of England, Wales, Scotland and Northern Ireland to heavy metal. Notable composers of classical music from the United Kingdom and the countries that preceded it include William Byrd, Henry Purcell, Sir Edward Elgar, Gustav Holst, Sir Arthur Sullivan (most famous for working with the librettist Sir W. S. Gilbert), Ralph Vaughan Williams and Benjamin Britten, pioneer of modern British opera. Sir Peter Maxwell Davies is one of the foremost living composers and current Master of the Queen's Music. The UK is also home to world-renowned symphonic orchestras and choruses such as the BBC Symphony Orchestra and the London Symphony Chorus. Notable conductors include Sir Simon Rattle, John Barbirolli and Sir Malcolm Sargent. Some of the notable film score composers include John Barry, Clint Mansell, Mike Oldfield, John Powell, Craig Armstrong, David Arnold, John Murphy, Monty Norman and Harry Gregson-Williams. George Frideric Handel, although born German, was a naturalised British citizen[411] and some of his best works, such as Messiah, were written in the English language.[412] Andrew Lloyd Webber has achieved enormous worldwide commercial success and is a prolific composer of musical theatre, works which have dominated London's West End for a number of years and have travelled to Broadway in New York.[413]\nThe Beatles have international sales of over one billion units and are the biggest-selling and most influential band in the history of popular music.[408][409][410][414] Other prominent British contributors to have influenced popular music over the last 50 years include; The Rolling Stones, Led Zeppelin, Pink Floyd, Queen, the Bee Gees, and Elton John, all of whom have world wide record sales of 200 million or more.[415][416][417][418][419][420] The Brit Awards are the BPI's annual music awards, and some of the British recipients of the Outstanding Contribution to Music award include; The Who, David Bowie, Eric Clapton, Rod Stewart and The Police.[421] More recent UK music acts that have had international success include Coldplay, Radiohead, Oasis, Spice Girls, Robbie Williams, Amy Winehouse and Adele.[422]\nA number of UK cities are known for their music. Acts from Liverpool have had more UK chart number one hit singles per capita (54) than any other city worldwide.[423] Glasgow's contribution to music was recognised in 2008 when it was named a UNESCO City of Music, one of only three cities in the world to have this honour.[424]\nVisual art\nMain article: Art of the United Kingdom\n\nJ. M. W. Turner self-portrait, oil on canvas, c. 1799\nThe history of British visual art forms part of western art history. Major British artists include: the Romantics William Blake, John Constable, Samuel Palmer and J.M.W. Turner; the portrait painters Sir Joshua Reynolds and Lucian Freud; the landscape artists Thomas Gainsborough and L. S. Lowry; the pioneer of the Arts and Crafts Movement William Morris; the figurative painter Francis Bacon; the Pop artists Peter Blake, Richard Hamilton and David Hockney; the collaborative duo Gilbert and George; the abstract artist Howard Hodgkin; and the sculptors Antony Gormley, Anish Kapoor and Henry Moore. During the late 1980s and 1990s the Saatchi Gallery in London helped to bring to public attention a group of multi-genre artists who would become known as the \"Young British Artists\": Damien Hirst, Chris Ofili, Rachel Whiteread, Tracey Emin, Mark Wallinger, Steve McQueen, Sam Taylor-Wood and the Chapman Brothers are among the better-known members of this loosely affiliated movement.\nThe Royal Academy in London is a key organisation for the promotion of the visual arts in the United Kingdom. Major schools of art in the UK include: the six-school University of the Arts London, which includes the Central Saint Martins College of Art and Design and Chelsea College of Art and Design; Goldsmiths, University of London; the Slade School of Fine Art (part of University College London); the Glasgow School of Art; the Royal College of Art; and The Ruskin School of Drawing and Fine Art (part of the University of Oxford). The Courtauld Institute of Art is a leading centre for the teaching of the history of art. Important art galleries in the United Kingdom include the National Gallery, National Portrait Gallery, Tate Britain and Tate Modern (the most-visited modern art gallery in the world, with around 4.7 million visitors per year).[425]\nCinema\nMain article: Cinema of the United Kingdom\n\nFilm director Alfred Hitchcock\nThe United Kingdom has had a considerable influence on the history of the cinema. The British directors Alfred Hitchcock, whose film Vertigo is considered by some critics as the best film of all time,[426] and David Lean are among the most critically acclaimed of all-time.[427] Other important directors including Charlie Chaplin,[428] Michael Powell,[429] Carol Reed[430] and Ridley Scott.[431] Many British actors have achieved international fame and critical success, including: Julie Andrews,[432] Richard Burton,[433] Michael Caine,[434] Charlie Chaplin,[435] Sean Connery,[436] Vivien Leigh,[437] David Niven,[438] Laurence Olivier,[439] Peter Sellers,[440] Kate Winslet,[441] and Daniel Day-Lewis, the only person to win an Oscar in the best actor category three times.[442] Some of the most commercially successful films of all time have been produced in the United Kingdom, including the two highest-grossing film franchises (Harry Potter and James Bond).[443] Ealing Studios has a claim to being the oldest continuously working film studio in the world.[444]\nDespite a history of important and successful productions, the industry has often been characterised by a debate about its identity and the level of American and European influence. British producers are active in international co-productions and British actors, directors and crew feature regularly in American films. Many successful Hollywood films have been based on British people, stories or events, including Titanic, The Lord of the Rings, Pirates of the Caribbean.\nIn 2009, British films grossed around $2 billion worldwide and achieved a market share of around 7% globally and 17% in the United Kingdom.[445] UK box-office takings totalled £944 million in 2009, with around 173 million admissions.[445] The British Film Institute has produced a poll ranking of what it considers to be the 100 greatest British films of all time, the BFI Top 100 British films.[446] The annual British Academy Film Awards, hosted by the British Academy of Film and Television Arts, are the British equivalent of the Oscars.[447]\nMedia\nMain article: Media of the United Kingdom\n\nBroadcasting House in London, headquarters of the BBC, the oldest and largest broadcaster in the world.[448][449][450]\nThe BBC, founded in 1922, is the UK's publicly funded radio, television and Internet broadcasting corporation, and is the oldest and largest broadcaster in the world.[448][449][450] It operates numerous television and radio stations in the UK and abroad and its domestic services are funded by the television licence.[451][452] Other major players in the UK media include ITV plc, which operates 11 of the 15 regional television broadcasters that make up the ITV Network,[453] and News Corporation, which owns a number of national newspapers through News International such as the most popular tabloid The Sun and the longest-established daily \"broadsheet\" The Times,[454] as well as holding a large stake in satellite broadcaster British Sky Broadcasting.[455] London dominates the media sector in the UK: national newspapers and television and radio are largely based there, although Manchester is also a significant national media centre. Edinburgh and Glasgow, and Cardiff, are important centres of newspaper and broadcasting production in Scotland and Wales respectively.[456] The UK publishing sector, including books, directories and databases, journals, magazines and business media, newspapers and news agencies, has a combined turnover of around £20 billion and employs around 167,000 people.[457]\nIn 2009, it was estimated that individuals viewed a mean of 3.75 hours of television per day and 2.81 hours of radio. In that year the main BBC public service broadcasting channels accounted for an estimated 28.4% of all television viewing; the three main independent channels accounted for 29.5% and the increasingly important other satellite and digital channels for the remaining 42.1%.[458] Sales of newspapers have fallen since the 1970s and in 2009 42% of people reported reading a daily national newspaper.[459] In 2010 82.5% of the UK population were Internet users, the highest proportion amongst the 20 countries with the largest total number of users in that year.[460]\nPhilosophy\nMain article: British philosophy\nThe United Kingdom is famous for the tradition of 'British Empiricism', a branch of the philosophy of knowledge that states that only knowledge verified by experience is valid, and 'Scottish Philosophy', sometimes referred to as the 'Scottish School of Common Sense'.[461] The most famous philosophers of British Empiricism are John Locke, George Berkeley and David Hume; while Dugald Stewart, Thomas Reid and William Hamilton were major exponents of the Scottish \"common sense\" school. Two Britons are also notable for a theory of moral philosophy utilitarianism, first used by Jeremy Bentham and later by John Stuart Mill in his short work Utilitarianism.[462][463] Other eminent philosophers from the UK and the unions and countries that preceded it include Duns Scotus, John Lilburne, Mary Wollstonecraft, Sir Francis Bacon, Adam Smith, Thomas Hobbes, William of Ockham, Bertrand Russell and A.J. \"Freddie\" Ayer. Foreign-born philosophers who settled in the UK include Isaiah Berlin, Karl Marx, Karl Popper and Ludwig Wittgenstein.\nSport\nMain article: Sport in the United Kingdom\n\nWembley Stadium, London, home of the England national football team, is one of the most expensive stadia ever built.[464]\nMajor sports, including association football, tennis, rugby union, rugby league, golf, boxing, rowing and cricket, originated or were substantially developed in the UK and the states that preceded it. With the rules and codes of many modern sports invented and codified in late 19th-century Victorian Britain, in 2012, the President of the IOC, Jacques Rogge, stated; \"This great, sports-loving country is widely recognized as the birthplace of modern sport. It was here that the concepts of sportsmanship and fair play were first codified into clear rules and regulations. It was here that sport was included as an educational tool in the school curriculum\".[465][466]\nIn most international competitions, separate teams represent England, Scotland and Wales. Northern Ireland and the Republic of Ireland usually field a single team representing all of Ireland, with notable exceptions being association football and the Commonwealth Games. In sporting contexts, the English, Scottish, Welsh and Irish / Northern Irish teams are often referred to collectively as the Home Nations. There are some sports in which a single team represents the whole of United Kingdom, including the Olympics, where the UK is represented by the Great Britain team. The 1908, 1948 and 2012 Summer Olympics were held in London, making it the first city to host the games three times. Britain has participated in every modern Olympic Games to date and is third in the medal count.\nA 2003 poll found that football is the most popular sport in the United Kingdom.[467] Each of the Home Nations has its own football association, national team and league system. The English top division, the Premier League, is the most watched football league in the world.[468] The first-ever international football match was contested by England and Scotland on 30 November 1872.[469] England, Scotland, Wales and Northern Ireland compete as separate countries in international competitions.[470] A Great Britain Olympic football team was assembled for the first time to compete in the London 2012 Olympic Games. However, the Scottish, Welsh and Northern Irish football associations declined to participate, fearing that it would undermine their independent status – a fear confirmed by FIFA president Sepp Blatter.[471]\n\nThe Millennium Stadium, Cardiff, opened for the 1999 Rugby World Cup.\nCricket was invented in England. The England cricket team, controlled by the England and Wales Cricket Board,[472] is the only national team in the UK with Test status. Team members are drawn from the main county sides, and include both English and Welsh players. Cricket is distinct from football and rugby where Wales and England field separate national teams, although Wales had fielded its own team in the past. Irish and Scottish players have played for England because neither Scotland nor Ireland have Test status and have only recently started to play in One Day Internationals.[473][474] Scotland, England (and Wales), and Ireland (including Northern Ireland) have competed at the Cricket World Cup, with England reaching the finals on three occasions. There is a professional league championship in which clubs representing 17 English counties and 1 Welsh county compete.[475]\nRugby league is a popular sport in some regions of the UK. It originated in Huddersfield and is generally played in Northern England.[476] A single 'Great Britain Lions' team had competed in the Rugby League World Cup and Test match games, but this changed in 2008 when England, Scotland and Ireland competed as separate nations.[477] Great Britain is still being retained as the full national team for Ashes tours against Australia, New Zealand and France. Super League is the highest level of professional rugby league in the UK and Europe. It consists of 11 teams from Northern England, 1 from London, 1 from Wales and 1 from France.\nIn rugby union, England, Scotland, Wales, Ireland, France and Italy compete in the Six Nations Championship; the premier international tournament in the northern hemisphere. Sport governing bodies in England, Scotland, Wales and Ireland organise and regulate the game separately.[478] If any of the British teams or the Irish team beat the other three in a tournament, then it is awarded the Triple Crown.[479]\n\nThe Wimbledon Championships, a Grand Slam tennis tournament, is held in Wimbledon, London every June or July.\nThoroughbred racing, which originated under Charles II of England as the \"sport of kings\", is popular throughout the UK with world-famous races including the Grand National, the Epsom Derby, Royal Ascot and the Cheltenham National Hunt Festival (including the Cheltenham Gold Cup). The UK has proved successful in the international sporting arena in rowing.\nThe UK is closely associated with motorsport. Many teams and drivers in Formula One (F1) are based in the UK, and the country has won more drivers' and constructors' titles than any other. The UK hosted the very first F1 Grand Prix in 1950 at Silverstone, the current location of the British Grand Prix held each year in July. The country also hosts legs of the Grand Prix motorcycle racing, World Rally Championship and FIA World Endurance Championship. The premier national auto racing event is the British Touring Car Championship (BTCC). Motorcycle road racing has a long tradition with races such as the Isle of Man TT and the North West 200.\nGolf is the sixth-most popular sport, by participation, in the UK. Although The Royal and Ancient Golf Club of St Andrews in Scotland is the sport's home course,[480] the world's oldest golf course is actually Musselburgh Links' Old Golf Course.[481]\nSnooker is one of the UK's popular sporting exports, with the world championships held annually in Sheffield.[482] The modern game of lawn tennis first originated in the city of Birmingham between 1859 and 1865.[483] The Championships, Wimbledon are international tennis events held in Wimbledon in south London every summer and are regarded as the most prestigious event of the global tennis calendar. In Northern Ireland Gaelic football and hurling are popular team sports, both in terms of participation and spectating, and Irish expatriates in the UK and the US also play them.[484] Shinty (or camanachd) is popular in the Scottish Highlands.[485]\nSymbols\nMain article: Symbols of the United Kingdom, the Channel Islands and the Isle of Man\n\nThe Statue of Britannia in Plymouth. Britannia is a national personification of the UK.\nThe flag of the United Kingdom is the Union Flag (also referred to as the Union Jack). It was created in 1606 by the superimposition of the Flag of England on the Flag of Scotland and updated in 1801 with the addition of Saint Patrick's Flag. Wales is not represented in the Union Flag, as Wales had been conquered and annexed to England prior to the formation of the United Kingdom. The possibility of redesigning the Union Flag to include representation of Wales has not been completely ruled out.[486] The national anthem of the United Kingdom is \"God Save the King\", with \"King\" replaced with \"Queen\" in the lyrics whenever the monarch is a woman.\nBritannia is a national personification of the United Kingdom, originating from Roman Britain.[487] Britannia is symbolised as a young woman with brown or golden hair, wearing a Corinthian helmet and white robes. She holds Poseidon's three-pronged trident and a shield, bearing the Union Flag. Sometimes she is depicted as riding on the back of a lion. Since the height of the British Empire in the late 19th century, Britannia has often been associated with British maritime dominance, as in the patriotic song \"Rule, Britannia!\". Up until 2008, the lion symbol was depicted behind Britannia on the British fifty pence coin and on the back of the British ten pence coin. It is also used as a symbol on the non-ceremonial flag of the British Army. The bulldog is sometimes used as a symbol of the United Kingdom and has been associated with Winston Churchill's defiance of Nazi Germany.[488]\nSee also\nOutline of the United Kingdom\n United Kingdom – Wikipedia book\nWalking in the United Kingdom\nFlag of the United Kingdom.svgUnited Kingdom portal Flag of Europe.svgEuropean Union portal Europe green light.pngEurope portal\nNotes\nJump up ^ The Royal coat of arms used in Scotland:\n Royal Coat of Arms of the United Kingdom (Scotland).svg\nJump up ^ There is no authorised version of the national anthem as the words are a matter of tradition; only the first verse is usually sung.[1] No law was passed making \"God Save the Queen\" the official anthem. In the English tradition, such laws are not necessary; proclamation and usage are sufficient to make it the national anthem. \"God Save the Queen\" also serves as the Royal anthem for several other countries, namely certain Commonwealth realms.\nJump up ^ Under the Council of Europe's European Charter for Regional or Minority Languages, Scots, Ulster-Scots, Welsh, Cornish, Irish and Scottish Gaelic, are officially recognised as regional or minority languages by the British government for the purposes of the Charter. See also Languages of the United Kingdom.[2]\nJump up ^ Although Northern Ireland is the only part of the UK that shares a land border with another state, two of its Overseas Territories also share land borders with other states. Gibraltar shares a border with Spain, while the Sovereign Base Areas of Akrotiri and Dhekelia share borders with the Republic of Cyprus, Turkish Republic of Northern Cyprus and UN buffer zone separating the two Cypriot polities.\nJump up ^ The Anglo-Irish Treaty was signed on 6 December 1921 to resolve the Irish War of Independence. Effective one year later, it established the Irish Free State as a separate dominion within the Commonwealth. The UK's current name was adopted in 1927 to reflect the change.\nJump up ^ Compare to section 1 of both of the 1800 Acts of Union which reads: the Kingdoms of Great Britain and Ireland shall...be united into one Kingdom, by the Name of \"The United Kingdom of Great Britain and Ireland\"\nJump up ^ New Zealand, Israel and San Marino are the other countries with uncodified constitutions.\nJump up ^ Since the early twentieth century the prime minister has held the office of First Lord of the Treasury, and in recent decades has also held the office of Minister for the Civil Service.\nJump up ^ Sinn Féin, an Irish republican party, also contests elections in the Republic of Ireland.\nJump up ^ In 2007–2008, this was calculated to be £115 per week for single adults with no dependent children; £199 per week for couples with no dependent children; £195 per week for single adults with two dependent children under 14; and £279 per week for couples with two dependent children under 14.\nReferences\nJump up ^ National Anthem, British Monarchy official website. Retrieved 16 November 2013.\n^ Jump up to: a b c \"List of declarations made with respect to treaty No. 148\". Council of Europe. Retrieved 12 December 2013.\n^ Jump up to: a b \"Population Estimates for UK, England and Wales, Scotland and Northern Ireland, Mid-2013\". Office for National Statistics. Retrieved 26 June 2014.\nJump up ^ \"2011 UK censuses\". Office for National Statistics. Retrieved 17 December 2012.\n^ Jump up to: a b c d \"United Kingdom\". International Monetary Fund. Retrieved 1 November 2014.\nJump up ^ \"Gini coefficient of equivalised disposable income (source: SILC)\". Eurostat Data Explorer. Retrieved 13 August 2013.\nJump up ^ \"2014 Human Development Report\". 14 March 2013. pp. 22–25. Retrieved 27 July 2014.\nJump up ^ \"Definition of Great Britain in English\". Oxford University Press. Retrieved 29 October 2014. Great Britain is the name for the island that comprises England, Scotland, and Wales, although the term is also used loosely to refer to the United Kingdom.\nJump up ^ The British Monarchy, What is constitutional monarchy?. Retrieved 17 July 2013\nJump up ^ CIA, The World Factbook. Retrieved 17 July 2013\nJump up ^ \"The World Factbook\". Central Intelligence Agency. 1 February 2014. Retrieved 23 February 2014.\n^ Jump up to: a b \"Countries within a country\". Prime Minister's Office. 10 January 2003.\n^ Jump up to: a b \"Devolution of powers to Scotland, Wales, and Northern Ireland\". United Kingdom Government. Retrieved 17 April 2013. In a similar way to how the government is formed from members from the two Houses of Parliament, members of the devolved legislatures nominate ministers from among themselves to comprise an executive, known as the devolved administrations...\nJump up ^ \"Fall in UK university students\". BBC News. 29 January 2009.\nJump up ^ \"Country Overviews: United Kingdom\". Transport Research Knowledge Centre. Retrieved 28 March 2010.\nJump up ^ \"Key facts about the United Kingdom\". Directgov. Retrieved 3 May 2011. The full title of this country is 'the United Kingdom of Great Britain and Northern Ireland'. 'The UK' is made up of England, Scotland, Wales and Northern Ireland. 'Britain' is used informally, usually meaning the United Kingdom. 'Great Britain' is made up of England, Scotland and Wales. The Channel Islands and the Isle of Man are not part of the UK.[dead link]\nJump up ^ \"Working with Overseas Territories\". Foreign and Commonwealth Office. Retrieved 3 May 2011.\nJump up ^ Mathias, P. (2001). The First Industrial Nation: the Economic History of Britain, 1700–1914. London: Routledge. ISBN 0-415-26672-6.\nJump up ^ Ferguson, Niall (2004). Empire: The rise and demise of the British world order and the lessons for global power. New York: Basic Books. ISBN 0-465-02328-2.\nJump up ^ Sheridan, Greg (15 May 2010). \"Cameron has chance to make UK great again\". The Australian (Sydney). Retrieved 23 May 2011.\nJump up ^ Dugan, Emily (18 November 2012). \"Britain is now most powerful nation on earth\". The Independent (London). Retrieved 18 November 2012.\n^ Jump up to: a b \"The 15 countries with the highest military expenditure in 2013 (table)\" (PDF). Stockholm International Peace Research Institute. Retrieved 4 May 2014.\n^ Jump up to: a b The Military Balance 2014: Top 15 Defence Budgets 2013 (IISS)\nJump up ^ \"Treaty of Union, 1706\". Scots History Online. Retrieved 23 August 2011.\nJump up ^ Barnett, Hilaire; Jago, Robert (2011). Constitutional & Administrative Law (8th ed.). Abingdon: Routledge. p. 165. ISBN 978-0-415-56301-7.\nJump up ^ Gascoigne, Bamber. \"History of Great Britain (from 1707)\". History World. Retrieved 18 July 2011.\nJump up ^ Cottrell, P. (2008). The Irish Civil War 1922–23. p. 85. ISBN 1-84603-270-9.\n^ Jump up to: a b S. Dunn; H. Dawson (2000), An Alphabetical Listing of Word, Name and Place in Northern Ireland and the Living Language of Conflict, Lampeter: Edwin Mellen Press, One specific problem - in both general and particular senses - is to know what to call Northern Ireland itself: in the general sense, it is not a country, or a province, or a state - although some refer to it contemptuously as a statelet: the least controversial word appears to be jurisdiction, but this might change.\nJump up ^ \"Changes in the list of subdivision names and code elements\". ISO 3166-2. International Organization for Standardization. 15 December 2011. Retrieved 28 May 2012.\nJump up ^ Population Trends, Issues 75–82, p.38, 1994, UK Office of Population Censuses and Surveys\nJump up ^ Life in the United Kingdom: a journey to citizenship, p. 7, United Kingdom Home Office, 2007, ISBN 978-0-11-341313-3.\nJump up ^ \"Statistical bulletin: Regional Labour Market Statistics\". Retrieved 5 March 2014.\nJump up ^ \"13.4% Fall In Earnings Value During Recession\". Retrieved 5 March 2014.\nJump up ^ Murphy, Dervla (1979). A Place Apart. London: Penguin. ISBN 978-0-14-005030-1.\nJump up ^ Whyte, John; FitzGerald, Garret (1991). Interpreting Northern Ireland. Oxford: Clarendon Press. ISBN 978-0-19-827380-6.\nJump up ^ \"Guardian Unlimited Style Guide\". London: Guardian News and Media Limited. 19 December 2008. Retrieved 23 August 2011.\nJump up ^ \"BBC style guide (Great Britain)\". BBC News. 19 August 2002. Retrieved 23 August 2011.\nJump up ^ \"Key facts about the United Kingdom\". Government, citizens and rights. HM Government. Retrieved 24 August 2011.[dead link]\nJump up ^ \"Merriam-Webster Dictionary Online Definition of ''Great Britain''\". Merriam Webster. 31 August 2012. Retrieved 9 April 2013.\nJump up ^ New Oxford American Dictionary: \"Great Britain: England, Wales, and Scotland considered as a unit. The name is also often used loosely to refer to the United Kingdom.\"\nJump up ^ \"Great Britain\". International Olympic Committee. Retrieved 10 May 2011.\nJump up ^ \"Team GB – Our Greatest Team\". British Olympic Association. Retrieved 10 May 2011.[dead link]\nJump up ^ Bradley, Anthony Wilfred; Ewing, Keith D. (2007). Constitutional and administrative law 1 (14th ed.). Harlow: Pearson Longman. p. 36. ISBN 978-1-4058-1207-8.\nJump up ^ \"Which of these best describes the way you think of yourself?\". Northern Ireland Life and Times Survey 2010. ARK – Access Research Knowledge. 2010. Retrieved 1 July 2010.\nJump up ^ Schrijver, Frans (2006). Regionalism after regionalisation: Spain, France and the United Kingdom. Amsterdam University Press. pp. 275–277. ISBN 978-90-5629-428-1.\nJump up ^ Jack, Ian (11 December 2010). \"Why I'm saddened by Scotland going Gaelic\". The Guardian (London).\nJump up ^ Ffeithiau allweddol am y Deyrnas Unedig : Directgov – Llywodraeth, dinasyddion a hawliau[dead link]\nJump up ^ \"Ancient skeleton was 'even older'\". BBC News. 30 October 2007. Retrieved 27 April 2011.\nJump up ^ Koch, John T. (2006). Celtic culture: A historical encyclopedia. Santa Barbara, CA: ABC-CLIO. p. 973. ISBN 978-1-85109-440-0.\nJump up ^ Davies, John; Jenkins, Nigel; Baines, Menna; Lynch, Peredur I., eds. (2008). The Welsh Academy Encyclopaedia of Wales. Cardiff: University of Wales Press. p. 915. ISBN 978-0-7083-1953-6.\nJump up ^ \"Short Athelstan biography\". BBC History. Retrieved 9 April 2013.\nJump up ^ Mackie, J.D. (1991). A History of Scotland. London: Penguin. pp. 18–19. ISBN 978-0-14-013649-4.\nJump up ^ Campbell, Ewan (1999). Saints and Sea-kings: The First Kingdom of the Scots. Edinburgh: Canongate. pp. 8–15. ISBN 0-86241-874-7.\nJump up ^ Haigh, Christopher (1990). The Cambridge Historical Encyclopedia of Great Britain and Ireland. Cambridge University Press. p. 30. ISBN 978-0-521-39552-6.\nJump up ^ Ganshof, F.L. (1996). Feudalism. University of Toronto. p. 165. ISBN 978-0-8020-7158-3.\nJump up ^ Chibnall, Marjorie (1999). The debate on the Norman Conquest. Manchester University Press. pp. 115–122. ISBN 978-0-7190-4913-2.\nJump up ^ Keen, Maurice. \"The Hundred Years War\". BBC History.\nJump up ^ The Reformation in England and Scotland and Ireland: The Reformation Period & Ireland under Elizabth I, Encyclopædia Britannica Online.\nJump up ^ \"British History in Depth – Wales under the Tudors\". BBC History. 5 November 2009. Retrieved 21 September 2010.\nJump up ^ Nicholls, Mark (1999). A history of the modern British Isles, 1529–1603: The two kingdoms. Oxford: Blackwell. pp. 171–172. ISBN 978-0-631-19334-0.\nJump up ^ Canny, Nicholas P. (2003). Making Ireland British, 1580–1650. Oxford University Press. pp. 189–200. ISBN 978-0-19-925905-2.\nJump up ^ Ross, D. (2002). Chronology of Scottish History. Glasgow: Geddes & Grosset. p. 56. ISBN 1-85534-380-0\nJump up ^ Hearn, J. (2002). Claiming Scotland: National Identity and Liberal Culture. Edinburgh University Press. p. 104. ISBN 1-902930-16-9\nJump up ^ \"English Civil Wars\". Encyclopaedia Britannica. Retrieved 28 April 2013.\nJump up ^ \"Scotland and the Commonwealth: 1651–1660\". Archontology.org. 14 March 2010. Retrieved 20 April 2010.\nJump up ^ Lodge, Richard (2007) [1910]. The History of England – From the Restoration to the Death of William III (1660–1702). Read Books. p. 8. ISBN 978-1-4067-0897-4.\nJump up ^ \"Tudor Period and the Birth of a Regular Navy\". Royal Navy History. Institute of Naval History. Retrieved 24 December 2010.[dead link]\nJump up ^ Canny, Nicholas (1998). The Origins of Empire, The Oxford History of the British Empire Volume I. Oxford University Press. ISBN 0-19-924676-9.\nJump up ^ \"Articles of Union with Scotland 1707\". UK Parliament. Retrieved 19 October 2008.\nJump up ^ \"Acts of Union 1707\". UK Parliament. Retrieved 6 January 2011.\nJump up ^ \"Treaty (act) of Union 1706\". Scottish History online. Retrieved 3 February 2011.\nJump up ^ Library of Congress, The Impact of the American Revolution Abroad, p. 73.\nJump up ^ Loosemore, Jo (2007). Sailing against slavery. BBC Devon. 2007.\nJump up ^ \"The Act of Union\". Act of Union Virtual Library. Retrieved 15 May 2006.\nJump up ^ Tellier, L.-N. (2009). Urban World History: an Economic and Geographical Perspective. Quebec: PUQ. p. 463. ISBN 2-7605-1588-5.\nJump up ^ Sondhaus, L. (2004). Navies in Modern World History. London: Reaktion Books. p. 9. ISBN 1-86189-202-0.\nJump up ^ Porter, Andrew (1998). The Nineteenth Century, The Oxford History of the British Empire Volume III. Oxford University Press. p. 332. ISBN 0-19-924678-5.\nJump up ^ \"The Workshop of the World\". BBC History. Retrieved 28 April 2013.\nJump up ^ Porter, Andrew (1998). The Nineteenth Century, The Oxford History of the British Empire Volume III. Oxford University Press. p. 8. ISBN 0-19-924678-5.\nJump up ^ Marshall, P.J. (1996). The Cambridge Illustrated History of the British Empire. Cambridge University Press. pp. 156–57. ISBN 0-521-00254-0.\nJump up ^ Tompson, Richard S. (2003). Great Britain: a reference guide from the Renaissance to the present. New York: Facts on File. p. 63. ISBN 978-0-8160-4474-0.\nJump up ^ Hosch, William L. (2009). World War I: People, Politics, and Power. America at War. New York: Britannica Educational Publishing. p. 21. ISBN 978-1-61530-048-8.\nJump up ^ Turner, John (1988). Britain and the First World War. London: Unwin Hyman. pp. 22–35. ISBN 978-0-04-445109-9.\n^ Jump up to: a b Westwell, I.; Cove, D. (eds) (2002). History of World War I, Volume 3. London: Marshall Cavendish. pp. 698 and 705. ISBN 0-7614-7231-2.\nJump up ^ Turner, J. (1988). Britain and the First World War. Abingdon: Routledge. p. 41. ISBN 0-04-445109-1.\nJump up ^ SR&O 1921, No. 533 of 3 May 1921.\nJump up ^ \"The Anglo-Irish Treaty, 6 December 1921\". CAIN. Retrieved 15 May 2006.\nJump up ^ Rubinstein, W. D. (2004). Capitalism, Culture, and Decline in Britain, 1750–1990. Abingdon: Routledge. p. 11. ISBN 0-415-03719-0.\nJump up ^ \"Britain to make its final payment on World War II loan from U.S.\". The New York Times. 28 December 2006. Retrieved 25 August 2011.\nJump up ^ Francis, Martin (1997). Ideas and policies under Labour, 1945–1951: Building a new Britain. Manchester University Press. pp. 225–233. ISBN 978-0-7190-4833-3.\nJump up ^ Lee, Stephen J. (1996). Aspects of British political history, 1914–1995. London; New York: Routledge. pp. 173–199. ISBN 978-0-415-13103-2.\nJump up ^ Larres, Klaus (2009). A companion to Europe since 1945. Chichester: Wiley-Blackwell. p. 118. ISBN 978-1-4051-0612-2.\nJump up ^ \"Country List\". Commonwealth Secretariat. 19 March 2009. Retrieved 11 September 2012.[dead link]\nJump up ^ Julios, Christina (2008). Contemporary British identity: English language, migrants, and public discourse. Studies in migration and diaspora. Aldershot: Ashgate. p. 84. ISBN 978-0-7546-7158-9.\nJump up ^ Aughey, Arthur (2005). The Politics of Northern Ireland: Beyond the Belfast Agreement. London: Routledge. p. 7. ISBN 978-0-415-32788-6.\nJump up ^ \"The troubles were over, but the killing continued. Some of the heirs to Ireland's violent traditions refused to give up their inheritance.\" Holland, Jack (1999). Hope against History: The Course of Conflict in Northern Ireland. New York: Henry Holt. p. 221. ISBN 978-0-8050-6087-4.\nJump up ^ Elliot, Marianne (2007). The Long Road to Peace in Northern Ireland: Peace Lectures from the Institute of Irish Studies at Liverpool University. University of Liverpool Institute of Irish Studies, Liverpool University Press. p. 2. ISBN 1-84631-065-2.\nJump up ^ Dorey, Peter (1995). British politics since 1945. Making contemporary Britain. Oxford: Blackwell. pp. 164–223. ISBN 978-0-631-19075-2.\nJump up ^ Griffiths, Alan; Wall, Stuart (2007). Applied Economics (11th ed.). Harlow: Financial Times Press. p. 6. ISBN 978-0-273-70822-3. Retrieved 26 December 2010.\nJump up ^ Keating, Michael (1 January 1998). \"Reforging the Union: Devolution and Constitutional Change in the United Kingdom\". Publius: the Journal of Federalism 28 (1): 217. doi:10.1093/oxfordjournals.pubjof.a029948. Retrieved 4 February 2009.\nJump up ^ Jackson, Mike (3 April 2011). \"Military action alone will not save Libya\". Financial Times (London).\nJump up ^ \"United Kingdom country profile\". BBC. 24 January 2013. Retrieved 9 April 2013.\nJump up ^ \"Scotland to hold independence poll in 2014 – Salmond\". BBC News. 10 January 2012. Retrieved 10 January 2012.\nJump up ^ Oxford English Dictionary: \"British Isles: a geographical term for the islands comprising Great Britain and Ireland with all their offshore islands including the Isle of Man and the Channel Islands.\"\n^ Jump up to: a b c d e f \"United Kingdom\". The World Factbook. Central Intelligence Agency. Retrieved 23 September 2008.\n^ Jump up to: a b c d e Latimer Clarke Corporation Pty Ltd. \"United Kingdom – Atlapedia Online\". Atlapedia.com. Retrieved 26 October 2010.\nJump up ^ ROG Learing Team (23 August 2002). \"The Prime Meridian at Greenwich\". Royal Museums Greenwich. Royal Museums Greenwich. Retrieved 11 September 2012.\nJump up ^ Neal, Clare. \"How long is the UK coastline?\". British Cartographic Society. Retrieved 26 October 2010.\nJump up ^ \"The Channel Tunnel\". Eurotunnel. Retrieved 29 November 2010.[dead link]\nJump up ^ \"England – Profile\". BBC News. 11 February 2010.\nJump up ^ \"Scotland Facts\". Scotland Online Gateway. Archived from the original on 21 June 2008. Retrieved 16 July 2008.\nJump up ^ Winter, Jon (19 May 2001). \"The complete guide to Scottish Islands\". The Independent (London).\nJump up ^ \"Overview of Highland Boundary Fault\". Gazetteer for Scotland. University of Edinburgh. Retrieved 27 December 2010.\nJump up ^ \"Ben Nevis Weather\". Ben Nevis Weather. Retrieved 26 October 2008.\nJump up ^ \"Profile: Wales\". BBC News. 9 June 2010. Retrieved 7 November 2010.\nJump up ^ Giles Darkes (26 April 2014). \"How long is the UK coastline?\". The British Cartographic Society.\nJump up ^ \"Geography of Northern Ireland\". University of Ulster. Retrieved 22 May 2006.\nJump up ^ \"UK climate summaries\". Met Office. Retrieved 1 May 2011.\nJump up ^ United Nations Economic and Social Council (August 2007). \"Ninth UN Conference on the standardization of Geographical Names\". UN Statistics Division. Archived from the original on 1 December 2009. Retrieved 21 October 2008.\nJump up ^ Barlow, I.M. (1991). Metropolitan Government. London: Routledge. ISBN 978-0-415-02099-2.\nJump up ^ \"Welcome to the national site of the Government Office Network\". Government Offices. Archived from the original on 15 June 2009. Retrieved 3 July 2008.\nJump up ^ \"A short history of London government\". Greater London Authority. Archived from the original on 21 April 2008. Retrieved 4 October 2008.\nJump up ^ Sherman, Jill; Norfolk, Andrew (5 November 2004). \"Prescott's dream in tatters as North East rejects assembly\". The Times (London). Retrieved 15 February 2008. The Government is now expected to tear up its twelve-year-old plan to create eight or nine regional assemblies in England to mirror devolution in Scotland and Wales. (subscription required)\nJump up ^ \"Local Authority Elections\". Local Government Association. Retrieved 3 October 2008.[dead link]\nJump up ^ \"STV in Scotland: Local Government Elections 2007\". Political Studies Association. Archived from the original on 20 March 2011. Retrieved 2 August 2008.\nJump up ^ Ethical Standards in Public Life framework: \"Ethical Standards in Public Life\". The Scottish Government. Retrieved 3 October 2008.\nJump up ^ \"Who we are\". Convention of Scottish Local Authorities. Retrieved 5 July 2011.\nJump up ^ \"Local Authorities\". The Welsh Assembly Government. Retrieved 31 July 2008.\nJump up ^ \"Local government elections in Wales\". The Electoral Commission. 2008. Retrieved 8 April 2011.\nJump up ^ \"Welsh Local Government Association\". Welsh Local Government Association. Retrieved 20 March 2008.\nJump up ^ Devenport, Mark (18 November 2005). \"NI local government set for shake-up\". BBC News. Retrieved 15 November 2008.\nJump up ^ \"Foster announces the future shape of local government\" (Press release). Northern Ireland Executive. 13 March 2008. Retrieved 20 October 2008.\nJump up ^ \"Local Government elections to be aligned with review of public administration\" (Press release). Northern Ireland Office. 25 April 2008. Retrieved 2 August 2008.[dead link]\nJump up ^ \"CIBC PWM Global – Introduction to The Cayman Islands\". Cibc.com. 11 July 2012. Retrieved 17 August 2012.\nJump up ^ Rappeport, Laurie. \"Cayman Islands Tourism\". Washington DC: USA Today Travel Tips. Retrieved 9 April 2013.\nJump up ^ \"Working with Overseas Territories\". Foreign & Commonwealth Office. 6 October 2010. Retrieved 5 November 2010.\nJump up ^ http://www.justice.gov.uk/downloads/about/moj/our-responsibilities/Background_Briefing_on_the_Crown_Dependencies2.pdf\nJump up ^ \"Overseas Territories\". Foreign & Commonwealth Office. Retrieved 6 September 2010.\nJump up ^ \"The World Factbook\". CIA. Retrieved 26 December 2010.\nJump up ^ \"Country profiles\". Foreign & Commonwealth Office. 21 February 2008. Retrieved 6 September 2010.[dead link]\nJump up ^ Davison, Phil (18 August 1995). \"Bermudians vote to stay British\". The Independent (London). Retrieved 11 September 2012.\nJump up ^ The Committee Office, House of Commons. \"House of Commons – Crown Dependencies – Justice Committee\". Publications.parliament.uk. Retrieved 7 November 2010.\nJump up ^ Fact sheet on the UK's relationship with the Crown Dependencies – gov.uk, Ministry of Justice. Retrieved 25 August 2014.\nJump up ^ \"Profile of Jersey\". States of Jersey. Retrieved 31 July 2008. The legislature passes primary legislation, which requires approval by The Queen in Council, and enacts subordinate legislation in many areas without any requirement for Royal Sanction and under powers conferred by primary legislation.\nJump up ^ \"Chief Minister to meet Channel Islands counterparts – Isle of Man Public Services\" (Press release). Isle of Man Government. 29 May 2012. Retrieved 9 April 2013.[dead link]\nJump up ^ Bagehot, Walter (1867). The English Constitution. London: Chapman and Hall. p. 103.\nJump up ^ Carter, Sarah. \"A Guide To the UK Legal System\". University of Kent at Canterbury. Retrieved 16 May 2006.\nJump up ^ \"Parliamentary sovereignty\". UK Parliament. n.d. Archived from the original on 27 May 2012.\nJump up ^ \"The Government, Prime Minister and Cabinet\". Public services all in one place. Directgov. Retrieved 12 February 2010.\nJump up ^ \"Brown is UK's new prime minister\". BBC News. 27 June 2007. Retrieved 23 January 2008.\nJump up ^ \"David Cameron is UK's new prime minister\". BBC News. 11 May 2010. Retrieved 11 May 2010.\nJump up ^ November 2010 \"Elections and voting\". UK Parliament. Archived from the original on 14 November 2010. Retrieved 14 November 2010.\nJump up ^ November 2010 \"The Parliament Acts\". UK Parliament. Archived from the original on 14 November 2010.\nJump up ^ \"United Kingdom\". European Election Database. Norwegian Social Science Data Services. Retrieved 3 July 2010.\nJump up ^ Wainwright, Martin (28 May 2010). \"Thirsk and Malton: Conservatives take final seat in parliament\". The Guardian (London). Retrieved 3 July 2010.\nJump up ^ \"Scots MPs attacked over fees vote\". BBC News. 27 January 2004. Retrieved 21 October 2008.\nJump up ^ Taylor, Brian (1 June 1998). \"Talking Politics: The West Lothian Question\". BBC News. Retrieved 21 October 2008.\nJump up ^ \"England-only laws 'need majority from English MPs'\". BBC News. 25 March 2013. Retrieved 28 April 2013.\nJump up ^ \"Scotland's Parliament – powers and structures\". BBC News. 8 April 1999. Retrieved 21 October 2008.\nJump up ^ \"Salmond elected as first minister\". BBC News. 16 May 2007. Retrieved 21 October 2008.\nJump up ^ \"Scottish election: SNP wins election\". BBC News. 6 May 2011.\nJump up ^ \"Structure and powers of the Assembly\". BBC News. 9 April 1999. Retrieved 21 October 2008.\nJump up ^ \"Carwyn Jones clinches leadership in Wales\". WalesOnline (Media Wales). 1 December 2009. Retrieved 1 December 2009.\nJump up ^ \"Devolved Government – Ministers and their departments\". Northern Ireland Executive. Archived from the original on 22 August 2007.\nJump up ^ Burrows, N. (1999). \"Unfinished Business: The Scotland Act 1998\". The Modern Law Review 62 (2): 241–60 [p. 249]. doi:10.1111/1468-2230.00203. The UK Parliament is sovereign and the Scottish Parliament is subordinate. The White Paper had indicated that this was to be the approach taken in the legislation. The Scottish Parliament is not to be seen as a reflection of the settled will of the people of Scotland or of popular sovereignty but as a reflection of its subordination to a higher legal authority. Following the logic of this argument, the power of the Scottish Parliament to legislate can be withdrawn or overridden...\nJump up ^ Elliot, M. (2004). \"United Kingdom: Parliamentary sovereignty under pressure\". International Journal of Constitutional Law 2 (3): 545–627 [pp. 553–554]. doi:10.1093/icon/2.3.545. Notwithstanding substantial differences among the schemes, an important common factor is that the U.K. Parliament has not renounced legislative sovereignty in relation to the three nations concerned. For example, the Scottish Parliament is empowered to enact primary legislation on all matters, save those in relation to which competence is explicitly denied ... but this power to legislate on what may be termed \"devolved matters\" is concurrent with the Westminster Parliament's general power to legislate for Scotland on any matter at all, including devolved matters ... In theory, therefore, Westminster may legislate on Scottish devolved matters whenever it chooses...\nJump up ^ Walker, G. (2010). \"Scotland, Northern Ireland, and Devolution, 1945–1979\". Journal of British Studies 39 (1): 124 & 133. doi:10.1086/644536.\nJump up ^ Gamble, A. \"The Constitutional Revolution in the United Kingdom\". Publius 36 (1): 19–35 [p. 29]. doi:10.1093/publius/pjj011. The British parliament has the power to abolish the Scottish parliament and the Welsh assembly by a simple majority vote in both houses, but since both were sanctioned by referenda, it would be politically difficult to abolish them without the sanction of a further vote by the people. In this way several of the constitutional measures introduced by the Blair government appear to be entrenched and not subject to a simple exercise of parliamentary sovereignty at Westminster.\nJump up ^ Meehan, E. (1999). \"The Belfast Agreement—Its Distinctiveness and Points of Cross-Fertilization in the UK's Devolution Programme\". Parliamentary Affairs 52 (1): 19–31 [p. 23]. doi:10.1093/pa/52.1.19. [T]he distinctive involvement of two governments in the Northern Irish problem means that Northern Ireland's new arrangements rest upon an intergovernmental agreement. If this can be equated with a treaty, it could be argued that the forthcoming distribution of power between Westminster and Belfast has similarities with divisions specified in the written constitutions of federal states... Although the Agreement makes the general proviso that Westminster's 'powers to make legislation for Northern Ireland' remains 'unaffected', without an explicit categorical reference to reserved matters, it may be more difficult than in Scotland or Wales for devolved powers to be repatriated. The retraction of devolved powers would not merely entail consultation in Northern Ireland backed implicitly by the absolute power of parliamentary sovereignty but also the renegotiation of an intergovernmental agreement.\nJump up ^ \"The Treaty (act) of the Union of Parliament 1706\". Scottish History Online. Retrieved 5 October 2008.\nJump up ^ \"UK Supreme Court judges sworn in\". BBC News. 1 October 2009.\nJump up ^ \"Constitutional reform: A Supreme Court for the United Kingdom\". Department for Constitutional Affairs. July 2003. Retrieved 13 May 2013.\nJump up ^ \"Role of the JCPC\". Judicial Committee of the Privy Council. Retrieved 28 April 2013.\nJump up ^ Bainham, Andrew (1998). The international survey of family law: 1996. The Hague: Martinus Nijhoff. p. 298. ISBN 978-90-411-0573-8.\nJump up ^ Adeleye, Gabriel; Acquah-Dadzie, Kofi; Sienkewicz, Thomas; McDonough, James (1999). World dictionary of foreign expressions. Waucojnda, IL: Bolchazy-Carducci. p. 371. ISBN 978-0-86516-423-9.\nJump up ^ \"The Australian courts and comparative law\". Australian Law Postgraduate Network. Retrieved 28 December 2010.\nJump up ^ \"Court of Session – Introduction\". Scottish Courts. Retrieved 5 October 2008.[dead link]\nJump up ^ \"High Court of Justiciary – Introduction\". Scottish Courts. Retrieved 5 October 2008.[dead link]\nJump up ^ \"House of Lords – Practice Directions on Permission to Appeal\". UK Parliament. Retrieved 22 June 2009.\nJump up ^ \"Introduction\". Scottish Courts. Retrieved 5 October 2008.[dead link]\nJump up ^ Samuel Bray (2005). \"Not proven: introducing a third verdict\". The University of Chicago Law Review 72 (4): 1299. Retrieved 30 November 2013.\nJump up ^ \"Police-recorded crime down by 9%\". BBC News. 17 July 2008. Retrieved 21 October 2008.\nJump up ^ \"New record high prison population\". BBC News. 8 February 2008. Retrieved 21 October 2008.\nJump up ^ \"Crime falls to 32 year low\" (Press release). Scottish Government. 7 September 2010. Retrieved 21 April 2011.\nJump up ^ \"Prisoner Population at Friday 22 August 2008\". Scottish Prison Service. Retrieved 28 August 2008.\nJump up ^ \"Scots jail numbers at record high\". BBC News. 29 August 2008. Retrieved 21 October 2008.\nJump up ^ Swaine, Jon (13 January 2009). \"Barack Obama presidency will strengthen special relationship, says Gordon Brown\". The Daily Telegraph (London). Retrieved 3 May 2011.\nJump up ^ Kirchner, E. J.; Sperling, J. (2007). Global Security Governance: Competing Perceptions of Security in the 21st Century. London: Taylor & Francis. p. 100. ISBN 0-415-39162-8\nJump up ^ The Committee Office, House of Commons (19 February 2009). \"DFID's expenditure on development assistance\". UK Parliament. Retrieved 28 April 2013.\nJump up ^ \"Ministry of Defence\". Ministry of Defence. Retrieved 21 February 2012.\nJump up ^ \"Speaker addresses Her Majesty Queen Elizabeth II\". UK Parliament. 30 March 2012. Retrieved 28 April 2013.\nJump up ^ \"House of Commons Hansard\". UK Parliament. Retrieved 23 October 2008.\nJump up ^ UK 2005: The Official Yearbook of the United Kingdom of Great Britain and Northern Ireland. Office for National Statistics. p. 89.\nJump up ^ \"Principles for Economic Regulation\". Department for Business, Innovation & Skills. April 2011. Retrieved 1 May 2011.\nJump up ^ \"United Kingdom\". International Monetary Fund. Retrieved 1 October 2009.\nJump up ^ Chavez-Dreyfuss, Gertrude (1 April 2008). \"Global reserves, dollar share up at end of 2007-IMF\". Reuters. Retrieved 21 December 2009.\nJump up ^ \"More About the Bank\". Bank of England. n.d. Archived from the original on 12 March 2008.\nJump up ^ \"Index of Services (experimental)\". Office for National Statistics. 7 May 2006. Archived from the original on 7 May 2006.\nJump up ^ Sassen, Saskia (2001). The Global City: New York, London, Tokyo (2nd ed.). Princeton University Press. ISBN 0-691-07866-1.\n^ Jump up to: a b \"Global Financial Centres 7\". Z/Yen. 2010. Retrieved 21 April 2010.\n^ Jump up to: a b \"Worldwide Centres of Commerce Index 2008\". Mastercard. Retrieved 5 July 2011.\n^ Jump up to: a b Zumbrun, Joshua (15 July 2008). \"\"World's Most Economically Powerful Cities\".\". Forbes (New York). Archived from the original on 19 May 2011. Retrieved 3 October 2010.\nJump up ^ \"Global city GDP rankings 2008–2025\". PricewaterhouseCoopers. Archived from the original on 19 May 2011. Retrieved 16 November 2010.\nJump up ^ Lazarowicz, Mark (Labour MP) (30 April 2003). \"Financial Services Industry\". UK Parliament. Retrieved 17 October 2008.\nJump up ^ International Tourism Receipts[dead link]. UNWTO Tourism Highlights, Edition 2005. page 12. World Tourism Organisation. Retrieved 24 May 2006.\nJump up ^ Bremner, Caroline (10 January 2010). \"Euromonitor International's Top City Destination Ranking\". Euromonitor International. Archived from the original on 19 May 2011. Retrieved 31 May 2011.\nJump up ^ \"From the Margins to the Mainstream – Government unveils new action plan for the creative industries\". DCMS. 9 March 2007. Retrieved 9 March 2007.[dead link]\n^ Jump up to: a b \"European Countries – United Kingdom\". Europa (web portal). Retrieved 15 December 2010.\nJump up ^ Harrington, James W.; Warf, Barney (1995). Industrial location: Principles, practices, and policy. London: Routledge. p. 121. ISBN 978-0-415-10479-1.\nJump up ^ Spielvogel, Jackson J. (2008). Western Civilization: Alternative Volume: Since 1300. Belmont, CA: Thomson Wadsworth. ISBN 978-0-495-55528-5.\nJump up ^ Hewitt, Patricia (15 July 2004). \"TUC Manufacturing Conference\". Department of Trade and Industry. Retrieved 16 May 2006.\nJump up ^ \"Industry topics\". Society of Motor Manufacturers and Traders. 2011. Retrieved 5 July 2011.\nJump up ^ Robertson, David (9 January 2009). \"The Aerospace industry has thousands of jobs in peril\". The Times (London). Retrieved 9 June 2011. (subscription required)\nJump up ^ \"Facts & Figures – 2009\". Aerospace & Defence Association of Europe. Retrieved 9 June 2011.[dead link]\nJump up ^ \"UK Aerospace Industry Survey – 2010\". ADS Group. Retrieved 9 June 2011.\n^ Jump up to: a b c d http://www.theengineer.co.uk/aerospace/in-depth/reasons-to-be-cheerful-about-the-uk-aerospace-sector/1017274.article\nJump up ^ \"The Pharmaceutical sector in the UK\". Department for Business, Innovation & Skills. Retrieved 9 June 2011.\nJump up ^ \"Ministerial Industry Strategy Group – Pharmaceutical Industry: Competitiveness and Performance Indicators\". Department of Health. Retrieved 9 June 2011.[dead link]\nJump up ^ [1][dead link]\nJump up ^ \"UK in recession as economy slides\". BBC News. 23 January 2009. Retrieved 23 January 2009.\nJump up ^ \"UK youth unemployment at its highest in two decades: 22.5%\". MercoPress. 15 April 2012.\nJump up ^ Groom, Brian (19 January 2011). \"UK youth unemployment reaches record\". Financial Times (London).\nJump up ^ \"Release: EU Government Debt and Deficit returns\". Office for National Statistics. March 2012. Retrieved 17 August 2012.\nJump up ^ \"UK loses top AAA credit rating for first time since 1978\". BBC News. 23 February 2013. Retrieved 23 February 2013.\nJump up ^ \"Britain sees real wages fall 3.2%\". Daily Express (London). 2 March 2013.\nJump up ^ Beckford, Martin (5 December 2011). \"Gap between rich and poor growing fastest in Britain\". The Daily Telegraph (London).\nJump up ^ \"United Kingdom: Numbers in low income\". The Poverty Site. Retrieved 25 September 2009.\nJump up ^ \"United Kingdom: Children in low income households\". The Poverty Site. Retrieved 25 September 2009.\nJump up ^ \"Warning of food price hike crisis\". BBC News. 4 April 2009.\nJump up ^ Andrews, J. (16 January 2013). \"How poor is Britain now\". Yahoo! Finance UK\nJump up ^ Glynn, S.; Booth, A. (1996). Modern Britain: An Economic and Social History. London: Routledge.\nJump up ^ \"Report highlights 'bleak' poverty levels in the UK\" Phys.org, 29 March 2013\nJump up ^ Gascoin, J. \"A reappraisal of the role of the universities in the Scientific Revolution\", in Lindberg, David C. and Westman, Robert S., eds (1990), Reappraisals of the Scientific Revolution. Cambridge University Press. p. 248. ISBN 0-521-34804-8.\nJump up ^ Reynolds, E.E.; Brasher, N.H. (1966). Britain in the Twentieth Century, 1900–1964. Cambridge University Press. p. 336. OCLC 474197910\nJump up ^ Burtt, E.A. (2003) [1924].The Metaphysical Foundations of Modern Science. Mineola, NY: Courier Dover. p. 207. ISBN 0-486-42551-7.\nJump up ^ Hatt, C. (2006). Scientists and Their Discoveries. London: Evans Brothers. pp. 16, 30 and 46. ISBN 0-237-53195-X.\nJump up ^ Jungnickel, C.; McCormmach, R. (1996). Cavendish. American Philosophical Society. ISBN 0-87169-220-1.\nJump up ^ \"The Nobel Prize in Physiology or Medicine 1945: Sir Alexander Fleming, Ernst B. Chain, Sir Howard Florey\". The Nobel Foundation. Archived from the original on 21 June 2011.\nJump up ^ Hatt, C. (2006). Scientists and Their Discoveries. London: Evans Brothers. p. 56. ISBN 0-237-53195-X.\nJump up ^ James, I. (2010). Remarkable Engineers: From Riquet to Shannon. Cambridge University Press. pp. 33–6. ISBN 0-521-73165-8.\nJump up ^ Bova, Ben (2002) [1932]. The Story of Light. Naperville, IL: Sourcebooks. p. 238. ISBN 978-1-4022-0009-0.\nJump up ^ \"Alexander Graham Bell (1847–1922)\". Scottish Science Hall of Fame. Archived from the original on 21 June 2011.\nJump up ^ \"John Logie Baird (1888–1946)\". BBC History. Archived from the original on 21 June 2011.\nJump up ^ Cole, Jeffrey (2011). Ethnic Groups of Europe: An Encyclopedia. Santa Barbara, CA: ABC-CLIO. p. 121. ISBN 1-59884-302-8.\nJump up ^ Castells, M.; Hall, P.; Hall, P.G. (2004). Technopoles of the World: the Making of Twenty-First-Century Industrial Complexes. London: Routledge. pp. 98–100. ISBN 0-415-10015-1.\nJump up ^ \"Knowledge, networks and nations: scientific collaborations in the twenty-first century\". Royal Society. 2011. Archived from the original on 22 June 2011.\nJump up ^ McCook, Alison. \"Is peer review broken?\". Reprinted from the Scientist 20(2) 26, 2006. Archived from the original on 21 June 2011.\n^ Jump up to: a b \"Heathrow 'needs a third runway'\". BBC News. 25 June 2008. Retrieved 17 October 2008.\n^ Jump up to: a b \"Statistics: Top 30 World airports\" (Press release). Airports Council International. July 2008. Retrieved 15 October 2008.\nJump up ^ \"Transport Statistics Great Britain: 2010\". Department for Transport. Archived from the original on 16 December 2010.\nJump up ^ \"Major new rail lines considered\". BBC News. 21 June 2008. Archived from the original on 9 October 2010.\nJump up ^ \"Crossrail's giant tunnelling machines unveiled\". BBC News. 2 January 2012.\nJump up ^ Leftly, Mark (29 August 2010). \"Crossrail delayed to save £1bn\". The Independent on Sunday (London).\n^ Jump up to: a b \"Size of Reporting Airports October 2009 – September 2010\". Civil Aviation Authority. Retrieved 5 December 2010.\nJump up ^ \"BMI being taken over by Lufthansa\". BBC News. 29 October 2008. Retrieved 23 December 2009.\nJump up ^ \"United Kingdom Energy Profile\". U.S. Energy Information Administration. Retrieved 4 November 2010.\nJump up ^ Mason, Rowena (24 October 2009). \"Let the battle begin over black gold\". The Daily Telegraph (London). Retrieved 26 November 2010.\nJump up ^ Heath, Michael (26 November 2010). \"RBA Says Currency Containing Prices, Rate Level 'Appropriate' in Near Term\". Bloomberg (New York). Retrieved 26 November 2010.\n^ Jump up to: a b c \"Nuclear Power in the United Kingdom\". World Nuclear Association. April 2013. Retrieved 9 April 2013.\n^ Jump up to: a b c \"United Kingdom – Oil\". U.S. Energy Information Administration. Retrieved 4 November 2010.[dead link]\nJump up ^ \"Diminishing domestic reserves, escalating imports\". EDF Energy. Retrieved 9 April 2013.\n^ Jump up to: a b \"United Kingdom – Natural Gas\". U.S. Energy Information Administration. Retrieved 4 November 2010.[dead link]\n^ Jump up to: a b \"United Kingdom – Quick Facts Energy Overview\". U.S. Energy Information Administration. Retrieved 4 November 2010.[dead link]\nJump up ^ The Coal Authority (10 April 2006). \"Coal Reserves in the United Kingdom\". The Coal Authority. Archived from the original on 4 January 2009. Retrieved 5 July 2011.\nJump up ^ \"England Expert predicts 'coal revolution'\". BBC News. 16 October 2007. Retrieved 23 September 2008.\nJump up ^ Watts, Susan (20 March 2012). \"Fracking: Concerns over gas extraction regulations\". BBC News. Retrieved 9 April 2013.\nJump up ^ \"Quit fracking aboot\". Friends of the Earth Scotland. Retrieved 9 April 2013.\nJump up ^ \"Census Geography\". Office for National Statistics. 30 October 2007. Archived from the original on 4 June 2011. Retrieved 14 April 2012.\nJump up ^ \"Welcome to the 2011 Census for England and Wales\". Office for National Statistics. n.d. Retrieved 11 October 2008.\n^ Jump up to: a b c \"2011 Census: Population Estimates for the United Kingdom\". Office for National Statistics. 27 March 2011. Retrieved 18 December 2012.\n^ Jump up to: a b c \"Annual Mid-year Population Estimates, 2010\". Office for National Statistics. 2011. Retrieved 14 April 2012.\nJump up ^ Batty, David (30 December 2010). \"One in six people in the UK today will live to 100, study says\". The Guardian (London).\n^ Jump up to: a b \"2011 UK censuses\". Office for National Statistics. Retrieved 18 December 2012.\nJump up ^ \"Population: UK population grows to 59.6 million\" (Press release). Office for National Statistics. 24 June 2004. Archived from the original on 22 July 2004. Retrieved 14 April 2012.\nJump up ^ Khan, Urmee (16 September 2008). \"England is most crowded country in Europe\". The Daily Telegraph (London). Retrieved 5 September 2009.\nJump up ^ Carrell, Severin (17 December 2012). \"Scotland's population at record high\". The Guardian. London. Retrieved 18 December 2012.\n^ Jump up to: a b c \"Vital Statistics: Population and Health Reference Tables (February 2014 Update): Annual Time Series Data\". ONS. Retrieved 27 April 2014.\nJump up ^ Boseley, Sarah (14 July 2008). \"The question: What's behind the baby boom?\". The Guardian (London). p. 3. Retrieved 28 August 2009.\nJump up ^ Tables, Graphs and Maps Interface (TGM) table. Eurostat (26 February 2013). Retrieved 12 July 2013.\nJump up ^ Campbell, Denis (11 December 2005). \"3.6m people in Britain are gay – official\". The Observer (London). Retrieved 28 April 2013.\nJump up ^ \"2011 Census - Built-up areas\". ONS. Retrieved 1 July 2013.\nJump up ^ Mid-2012 Population Estimates for Settlements and Localities in Scotland General Register Office for Scotland\nJump up ^ \"Belfast Metropolitan Urban Area NISRA 2005\". Retrieved 28 April 2013.\nJump up ^ 2011 Census: KS201UK Ethnic group, local authorities in the United Kingdom, Accessed 21 February 2014\nJump up ^ \"Welsh people could be most ancient in UK, DNA suggests\". BBC News. 19 June 2012. Retrieved 28 April 2013.\nJump up ^ Thomas, Mark G. et al. \"Evidence for a segregated social structure in early Anglo-Saxon England\". Proceedings of the Royal Society B: Biological Sciences 273(1601): 2651–2657.\nJump up ^ Owen, James (19 July 2005). \"Review of 'The Tribes of Britain'\". National Geographic (Washington DC).\nJump up ^ Oppenheimer, Stephen (October 2006). \"Myths of British ancestry\" at the Wayback Machine (archived 26 September 2006). Prospect (London). Retrieved 5 November 2010.\nJump up ^ Henderson, Mark (23 October 2009). \"Scientist – Griffin hijacked my work to make race claim about 'British aborigines'\". The Times (London). Retrieved 26 October 2009. (subscription required)\nJump up ^ Costello, Ray (2001). Black Liverpool: The Early History of Britain's Oldest Black Community 1730–1918. Liverpool: Picton Press. ISBN 1-873245-07-6.\nJump up ^ \"Culture and Ethnicity Differences in Liverpool – Chinese Community\". Chambré Hardman Trust. Retrieved 26 October 2009.\nJump up ^ Coleman, David; Compton, Paul; Salt, John (2002). \"The demographic characteristics of immigrant populations\", Council of Europe, p.505. ISBN 92-871-4974-7.\nJump up ^ Mason, Chris (30 April 2008). \"'Why I left UK to return to Poland'\". BBC News.\nJump up ^ \"Resident population estimates by ethnic group (percentages): London\". Office for National Statistics. Retrieved 23 April 2008.\nJump up ^ \"Resident population estimates by ethnic group (percentages): Leicester\". Office for National Statistics. Retrieved 23 April 2008.\nJump up ^ \"Census 2001 – Ethnicity and religion in England and Wales\". Office for National Statistics. Retrieved 23 April 2008.\nJump up ^ Loveys, Kate (22 June 2011). \"One in four primary school pupils are from an ethnic minority and almost a million schoolchildren do not speak English as their first language\". Daily Mail (London). Retrieved 28 June 2011.\nJump up ^ Rogers, Simon (19 May 2011). \"Non-white British population reaches 9.1 million\". The Guardian (London).\nJump up ^ Wallop, Harry (18 May 2011). \"Population growth of last decade driven by non-white British\". The Daily Telegraph (London).\nJump up ^ \"Official EU languages\". European Commission. 8 May 2009. Retrieved 16 October 2009.\nJump up ^ \"Language Courses in New York\". United Nations. 2006. Retrieved 29 November 2010.\nJump up ^ \"English language – Government, citizens and rights\". Directgov. Retrieved 23 August 2011.\nJump up ^ \"Commonwealth Secretariat – UK\". Commonwealth Secretariat. Retrieved 23 August 2011.\n^ Jump up to: a b c \"Languages across Europe: United Kingdom\". BBC. Retrieved 4 February 2013.\nJump up ^ Booth, Robert (30 January 2013). \"Polish becomes England's second language\". The Guardian (London). Retrieved 4 February 2012.\nJump up ^ European Charter for Regional or Minority Languages, Strasbourg, 5.XI.1992 - http://conventions.coe.int/treaty/en/Treaties/Html/148.htm\nJump up ^ Framework Convention for the Protection of National Minorities, Strasbourg, 1.II.1995 - http://conventions.coe.int/Treaty/en/Treaties/Html/157.htm\nJump up ^ National Statistics Online – Welsh Language[dead link]. National Statistics Office.\nJump up ^ \"Differences in estimates of Welsh Language Skills\". Office for National Statistics. Archived from the original on 12 January 2010. Retrieved 30 December 2008.\nJump up ^ Wynn Thomas, Peter (March 2007). \"Welsh today\". Voices. BBC. Retrieved 5 July 2011.\nJump up ^ \"Scotland's Census 2001 – Gaelic Report\". General Register Office for Scotland. Retrieved 28 April 2013.\nJump up ^ \"Local UK languages 'taking off'\". BBC News. 12 February 2009.\nJump up ^ Edwards, John R. (2010). Minority languages and group identity: cases and categories. John Benjamins. pp. 150–158. ISBN 978-90-272-1866-7. Retrieved 12 March 2011.\nJump up ^ Koch, John T. (2006). Celtic culture: a historical encyclopedia. ABC-CLIO. p. 696. ISBN 978-1-85109-440-0.\nJump up ^ \"Language Data – Scots\". European Bureau for Lesser-Used Languages. Archived from the original on 23 June 2007. Retrieved 2 November 2008.\nJump up ^ \"Fall in compulsory language lessons\". BBC News. 4 November 2004.\nJump up ^ \"The School Gate for parents in Wales\". BBC. Retrieved 28 April 2013.\nJump up ^ Cannon, John, ed. (2nd edn., 2009). A Dictionary of British History. Oxford University Press. p. 144. ISBN 0-19-955037-9.\nJump up ^ Field, Clive D. (November 2009). \"British religion in numbers\"[dead link]. BRIN Discussion Series on Religious Statistics, Discussion Paper 001. Retrieved 3 June 2011.\nJump up ^ Yilmaz, Ihsan (2005). Muslim Laws, Politics and Society in Modern Nation States: Dynamic Legal Pluralisms in England, Turkey, and Pakistan. Aldershot: Ashgate Publishing. pp. 55–6. ISBN 0-7546-4389-1.\nJump up ^ Brown, Callum G. (2006). Religion and Society in Twentieth-Century Britain. Harlow: Pearson Education. p. 291. ISBN 0-582-47289-X.\nJump up ^ Norris, Pippa; Inglehart, Ronald (2004). Sacred and Secular: Religion and Politics Worldwide. Cambridge University Press. p. 84. ISBN 0-521-83984-X.\nJump up ^ Fergusson, David (2004). Church, State and Civil Society. Cambridge University Press. p. 94. ISBN 0-521-52959-X.\nJump up ^ \"UK Census 2001\". National Office for Statistics. Archived from the original on 12 March 2007. Retrieved 22 April 2007.\nJump up ^ \"Religious Populations\". Office for National Statistics. 11 October 2004. Archived from the original on 6 June 2011.\nJump up ^ \"United Kingdom: New Report Finds Only One in 10 Attend Church\". News.adventist.org. 4 April 2007. Retrieved 12 September 2010.\nJump up ^ Philby, Charlotte (12 December 2012). \"Less religious and more ethnically diverse: Census reveals a picture of Britain today\". The Independent (London).\nJump up ^ The History of the Church of England. The Church of England. Retrieved 23 November 2008.\nJump up ^ \"Queen and Church of England\". British Monarchy Media Centre. Archived from the original on 8 October 2006. Retrieved 5 June 2010.\nJump up ^ \"Queen and the Church\". The British Monarchy (Official Website). Archived from the original on 7 June 2011.\nJump up ^ \"How we are organised\". Church of Scotland. Archived from the original on 7 June 2011.\nJump up ^ Weller, Paul (2005). Time for a Change: Reconfiguring Religion, State, and Society. London: Continuum. pp. 79–80. ISBN 0567084876.\nJump up ^ Peach, Ceri, \"United Kingdom, a major transformation of the religious landscape\", in H. Knippenberg. ed. (2005). The Changing Religious Landscape of Europe. Amsterdam: Het Spinhuis. pp. 44–58. ISBN 90-5589-248-3.\nJump up ^ Richards, Eric (2004). Britannia's children: Emigration from England, Scotland, Wales and Ireland since 1600. London: Hambledon, p. 143. ISBN 978-1-85285-441-6.\nJump up ^ Gibney, Matthew J.; Hansen, Randall (2005). Immigration and asylum: from 1900 to the present, ABC-CLIO, p. 630. ISBN 1-57607-796-9\nJump up ^ \"Short history of immigration\". BBC. 2005. Retrieved 28 August 2010.\nJump up ^ Rogers, Simon (11 December 2012). \"Census 2011 mapped and charted: England & Wales in religion, immigration and race\". London: Guardian. Retrieved 11 December 2012.\nJump up ^ 6.5% of the EU population are foreigners and 9.4% are born abroad, Eurostat, Katya Vasileva, 34/2011.\nJump up ^ Muenz, Rainer (June 2006). \"Europe: Population and Migration in 2005\". Migration Policy Institute. Retrieved 2 April 2007.\nJump up ^ \"Immigration and births to non-British mothers pushes British population to record high\". London Evening Standard. 21 August 2008.\nJump up ^ Doughty, Steve; Slack, James (3 June 2008). \"Third World migrants behind our 2.3m population boom\". Daily Mail (London).\nJump up ^ Bentham, Martin (20 October 2008). \"Tories call for tougher control of immigration\". London Evening Standard.\nJump up ^ \"Minister rejects migrant cap plan\". BBC News. 8 September 2008. Retrieved 26 April 2011.\nJump up ^ Johnston, Philip (5 January 2007). \"Immigration 'far higher' than figures say\". The Daily Telegraph (London). Retrieved 20 April 2007.\nJump up ^ Travis, Alan (25 August 2011). \"UK net migration rises 21%\". The Guardian (London).\n^ Jump up to: a b \"Migration Statistics Quarterly Report May 2012\". Office for National Statistics. 24 May 2012.\nJump up ^ \"Migration to UK more than double government target\". BBC News. 24 May 2012.\n^ Jump up to: a b \"Citizenship\". Home Office. August 2011. Retrieved 24 October 2011.[dead link]\nJump up ^ Bamber, David (20 December 2000). \"Migrant squad to operate in France\". The Daily Telegraph (London).\nJump up ^ \"Settlement\". Home Office. August 2011. Retrieved 24 October 2011.[dead link]\nJump up ^ \"Births in England and Wales by parents' country of birth, 2011\". Office for National Statistics. 30 August 2012. Retrieved 28 April 2013.\nJump up ^ \"Right of Union citizens and their family members to move and reside freely within the territory of the Member States\". European Commission. Retrieved 28 April 2013.\nJump up ^ Doward, Jamie; Temko, Ned (23 September 2007). \"Home Office shuts the door on Bulgaria and Romania\". The Observer (London). p. 2. Retrieved 23 August 2008.\nJump up ^ Sumption, Madeleine; Somerville, Will (January 2010). The UK's new Europeans: Progress and challenges five years after accession. Policy Report (London: Equality and Human Rights Commission). p. 13. ISBN 978-1-84206-252-4. Retrieved 19 January 2010.\nJump up ^ Doward, Jamie; Rogers, Sam (17 January 2010). \"Young, self-reliant, educated: portrait of UK's eastern European migrants\". The Observer (London). Retrieved 19 January 2010.\nJump up ^ Hopkirk, Elizabeth (20 October 2008). \"Packing up for home: Poles hit by UK's economic downturn\". London Evening Standard.\nJump up ^ \"Migrants to UK 'returning home'\". BBC News. 8 September 2009. Retrieved 8 September 2009.\nJump up ^ \"UK sees shift in migration trend\". BBC News. 27 May 2010. Retrieved 28 May 2010.\nJump up ^ \"Fresh Talent: Working in Scotland\". London: UK Border Agency. Retrieved 30 October 2010.\nJump up ^ Boxell, James (28 June 2010). \"Tories begin consultation on cap for migrants\". Financial Times (London). Retrieved 17 September 2010.\nJump up ^ \"Vince Cable: Migrant cap is hurting economy\". The Guardian (London). Press Association. 17 September 2010. Retrieved 17 September 2010.\nJump up ^ Richards (2004), pp. 6–7.\n^ Jump up to: a b Sriskandarajah, Dhananjayan; Drew, Catherine (11 December 2006). \"Brits Abroad: Mapping the scale and nature of British emigration\". Institute for Public Policy Research. Retrieved 20 January 2007.\nJump up ^ \"Brits Abroad: world overview\". BBC. n.d. Retrieved 20 April 2007.\nJump up ^ Casciani, Dominic (11 December 2006). \"5.5 m Britons 'opt to live abroad'\". BBC News. Retrieved 20 April 2007.\nJump up ^ \"Brits Abroad: Country-by-country\". BBC News. 11 December 2006.\nJump up ^ \"Local Authorities\". Department for Children, Schools and Families. Retrieved 21 December 2008.\nJump up ^ Gordon, J.C.B. (1981). Verbal Deficit: A Critique. London: Croom Helm. p. 44 note 18. ISBN 978-0-85664-990-5.\nJump up ^ Section 8 ('Duty of local education authorities to secure provision of primary and secondary schools'), Sections 35–40 ('Compulsory attendance at Primary and Secondary Schools') and Section 61 ('Prohibition of fees in schools maintained by local education authorities ...'), Education Act 1944.\nJump up ^ \"England's pupils in global top 10\". BBC News. 10 December 2008.\nJump up ^ \"More state pupils in universities\". BBC News. 19 July 2007.\nJump up ^ MacLeod, Donald (9 November 2007). \"Private school pupil numbers in decline\". The Guardian (London). Retrieved 31 March 2010.\nJump up ^ Frankel, Hannah (3 September 2010). \"Is Oxbridge still a preserve of the posh?\". TES (London). Retrieved 9 April 2013.\nJump up ^ \"World's top 100 universities 2013: their reputations ranked by Times Higher Education\". The Guardian (London). 2013. Retrieved 23 October 2014.\nJump up ^ Davenport, F.; Beech, C.; Downs, T.; Hannigan, D. (2006). Ireland. Lonely Planet, 7th edn. ISBN 1-74059-968-3. p. 564.\nJump up ^ \"About SQA\". Scottish Qualifications Authority. 10 April 2013. Retrieved 28 April 2013.\nJump up ^ \"About Learning and Teaching Scotland\". Learning and Teaching Scotland. Retrieved 28 April 2013.\nJump up ^ \"Brain drain in reverse\". Scotland Online Gateway. July 2002. Archived from the original on 4 December 2007.\nJump up ^ \"Increase in private school intake\". BBC News. 17 April 2007.\nJump up ^ \"MSPs vote to scrap endowment fee\". BBC News. 28 February 2008.\nJump up ^ What will your child learn?[dead link] The Welsh Assembly Government. Retrieved 22 January 2010.\nJump up ^ CCEA. \"About Us – What we do\". Council for the Curriculum Examinations & Assessment. Retrieved 28 April 2013.\nJump up ^ Elitist Britain?, Social Mobility and Child Poverty Commission, 28 August 2014\nJump up ^ Arnett, George (28 August 2014). \"Elitism in Britain - breakdown by profession\". The Guardian: Datablog.\nJump up ^ Haden, Angela; Campanini, Barbara, eds. (2000). The world health report 2000 – Health systems: improving performance. Geneva: World Health Organisation. ISBN 92-4-156198-X. Retrieved 5 July 2011.\nJump up ^ World Health Organization. \"Measuring overall health system performance for 191 countries\". New York University. Retrieved 5 July 2011.\nJump up ^ \"'Huge contrasts' in devolved NHS\". BBC News. 28 August 2008.\nJump up ^ Triggle, Nick (2 January 2008). \"NHS now four different systems\". BBC News.\nJump up ^ Fisher, Peter. \"The NHS from Thatcher to Blair\". NHS Consultants Association (International Association of Health Policy). The Budget ... was even more generous to the NHS than had been expected amounting to an annual rise of 7.4% above the rate of inflation for the next 5 years. This would take us to 9.4% of GDP spent on health ie around EU average.\nJump up ^ \"OECD Health Data 2009 – How Does the United Kingdom Compare\". Paris: Organisation for Economic Co-operation and Development. Retrieved 28 April 2013.[dead link]\nJump up ^ \"The cultural superpower: British cultural projection abroad\". Journal of the British Politics Society, Norway. Volume 6. No. 1. Winter 2011\nJump up ^ Sheridan, Greg (15 May 2010). \"Cameron has chance to make UK great again\". The Australian (Sydney). Retrieved 20 May 2012.\nJump up ^ Goldfarb, Jeffrey (10 May 2006). \"Bookish Britain overtakes America as top publisher\". RedOrbit (Texas). Reuters.\nJump up ^ \"William Shakespeare (English author)\". Britannica Online encyclopedia. Retrieved 26 February 2006.\nJump up ^ MSN Encarta Encyclopedia article on Shakespeare. Archived from the original on 9 February 2006. Retrieved 26 February 2006.\nJump up ^ William Shakespeare. Columbia Electronic Encyclopedia. Retrieved 26 February 2006.\nJump up ^ \"Mystery of Christie's success is solved\". The Daily Telegraph (London). 19 December 2005. Retrieved 14 November 2010.\nJump up ^ \"All-Time Essential Comics\". IGN. Retrieved 15 August 2013.\nJump up ^ Johnston, Rich.\"Before Watchmen To Double Up For Hardcover Collections\". Bleeding Cool. 10 December 2012. Retrieved 15 August 2013.\nJump up ^ \"Edinburgh, UK appointed first UNESCO City of Literature\". Unesco. 2004. Retrieved 28 April 2013.[dead link]\nJump up ^ \"Early Welsh poetry\". BBC Wales. Retrieved 29 December 2010.\nJump up ^ Lang, Andrew (2003) [1913]. History of English Literature from Beowulf to Swinburne. Holicong, PA: Wildside Press. p. 42. ISBN 978-0-8095-3229-2.\nJump up ^ \"Dafydd ap Gwilym\". Academi website. Academi. 2011. Retrieved 3 January 2011. Dafydd ap Gwilym is widely regarded as one of the greatest Welsh poets of all time, and amongst the leading European poets of the Middle Ages.\nJump up ^ True birthplace of Wales's literary hero. BBC News. Retrieved 28 April 2012\nJump up ^ Kate Roberts: Biography at the Wayback Machine. BBC Wales. Retrieved 28 April 2012\nJump up ^ Swift, Jonathan; Fox, Christopher (1995). Gulliver's travels: complete, authoritative text with biographical and historical contexts, critical history, and essays from five contemporary critical perspectives. Basingstoke: Macmillan. p. 10. ISBN 978-0-333-63438-7.\nJump up ^ \"Bram Stoker.\" (PDF). The New York Times. 23 April 1912. Retrieved 1 January 2011.\n^ Jump up to: a b \"1960–1969\". EMI Group. Retrieved 31 May 2008.\n^ Jump up to: a b \"Paul At Fifty\". Time (New York). 8 June 1992.\n^ Jump up to: a b Most Successful Group The Guinness Book of Records 1999, p. 230. Retrieved 19 March 2011.\nJump up ^ \"British Citizen by Act of Parliament: George Frideric Handel\". UK Parliament. 20 July 2009. Retrieved 11 September 2009.[dead link]\nJump up ^ Andrews, John (14 April 2006). \"Handel all'inglese\". Playbill (New York). Retrieved 11 September 2009.\nJump up ^ Citron, Stephen (2001). Sondheim and Lloyd-Webber: The new musical. London: Chatto & Windus. ISBN 978-1-85619-273-6.\nJump up ^ \"Beatles a big hit with downloads\". Belfast Telegraph. 25 November 2010. Retrieved 16 May 2011.\nJump up ^ \"British rock legends get their own music title for PlayStation3 and PlayStation2\" (Press release). EMI. 2 February 2009.\nJump up ^ Khan, Urmee (17 July 2008). \"Sir Elton John honoured in Ben and Jerry ice cream\". The Daily Telegraph (London).\nJump up ^ Alleyne, Richard (19 April 2008). \"Rock group Led Zeppelin to reunite\". The Daily Telegraph (London). Retrieved 31 March 2010.\nJump up ^ Fresco, Adam (11 July 2006). \"Pink Floyd founder Syd Barrett dies at home\". The Times (London). Retrieved 31 March 2010. (subscription required)\nJump up ^ Holton, Kate (17 January 2008). \"Rolling Stones sign Universal album deal\". Reuters. Retrieved 26 October 2008.\nJump up ^ Walker, Tim (12 May 2008). \"Jive talkin': Why Robin Gibb wants more respect for the Bee Gees\". The Independent (London). Retrieved 26 October 2008.\nJump up ^ \"Brit awards winners list 2012: every winner since 1977\". The Guardian (London). Retrieved 28 February 2012.\nJump up ^ Corner, Lewis (16 February 2012). \"Adele, Coldplay biggest-selling UK artists worldwide in 2011\". Digital Spy. Retrieved 22 March 2012.\nJump up ^ Hughes, Mark (14 January 2008). \"A tale of two cities of culture: Liverpool vs Stavanger\". The Independent (London). Retrieved 2 August 2009.\nJump up ^ \"Glasgow gets city of music honour\". BBC News. 20 August 2008. Retrieved 2 August 2009.\nJump up ^ Bayley, Stephen (24 April 2010). \"The startling success of Tate Modern\". The Times (London). Retrieved 19 January 2011. (subscription required)\nJump up ^ \"Vertigo is named 'greatest film of all time'\". BBC News. 2 August 2012. Retrieved 18 August 2012.\nJump up ^ \"The Directors' Top Ten Directors\". British Film Institute. Archived from the original on 27 May 2012.\nJump up ^ \"Chaplin, Charles (1889–1977)\". British Film Institute. Retrieved 25 January 2011.\nJump up ^ \"Powell, Michael (1905–1990)\". British Film Institute. Retrieved 25 January 2011.\nJump up ^ \"Reed, Carol (1906–1976)\". British Film Institute. Retrieved 25 January 2011.\nJump up ^ \"Scott, Sir Ridley (1937–)\". British Film Institute. Retrieved 25 January 2011.\nJump up ^ \"Andrews, Julie (1935–)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Burton, Richard (1925–1984)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Caine, Michael (1933–)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Chaplin, Charles (1889–1977)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Connery, Sean (1930–)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Leigh, Vivien (1913–1967)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Niven, David (1910–1983)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Olivier, Laurence (1907–1989)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Sellers, Peter (1925–1980)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Winslet, Kate (1975–)\". British Film Institute. Retrieved 11 December 2010.\nJump up ^ \"Daniel Day-Lewis makes Oscar history with third award\"'. BBC News. Retrieved 15 August 2013\nJump up ^ \"Harry Potter becomes highest-grossing film franchise\". The Guardian (London). 11 September 2007. Retrieved 2 November 2010.\nJump up ^ \"History of Ealing Studios\". Ealing Studios. Retrieved 5 June 2010.\n^ Jump up to: a b \"UK film – the vital statistics\". UK Film Council. Retrieved 22 October 2010.[dead link]\nJump up ^ \"The BFI 100\". British Film Institute. 6 September 2006. Archived from the original on 1 April 2011.\nJump up ^ \"Baftas fuel Oscars race\". BBC News. 26 February 2001. Retrieved 14 February 2011.\n^ Jump up to: a b \"BBC: World's largest broadcaster & Most trusted media brand\". Media Newsline. Archived from the original on 5 October 2010. Retrieved 23 September 2010.\n^ Jump up to: a b \"Digital licence\". Prospect. Retrieved 23 September 2010.\n^ Jump up to: a b \"About the BBC – What is the BBC\". BBC Online. Retrieved 23 September 2010.\nJump up ^ Newswire7 (13 August 2009). \"BBC: World's largest broadcaster & Most trusted media brand\". Media Newsline. Archived from the original on 17 June 2011.\nJump up ^ \"TV Licence Fee: facts & figures\". BBC Press Office. April 2010. Archived from the original on 17 June 2011.\nJump up ^ \"Publications & Policies: The History of ITV\". ITV.com. Archived from the original on 17 June 2011.\nJump up ^ \"Publishing\". News Corporation. Archived from the original on 17 June 2011.\nJump up ^ \"Direct Broadcast Satellite Television\". News Corporation. Archived from the original on 17 June 2011.\nJump up ^ William, D. (2010). UK Cities: A Look at Life and Major Cities in England, Scotland, Wales and Northern Ireland. Eastbourne: Gardners Books. ISBN 978-9987-16-021-1, pp. 22, 46, 109 and 145.\nJump up ^ \"Publishing\". Department of Culture, Media and Sport. Archived from the original on 17 June 2011.\nJump up ^ Ofcom \"Communication Market Report 2010\", 19 August 2010, pp. 97, 164 and 191\nJump up ^ \"Social Trends: Lifestyles and social participation\". Office for National Statistics. 16 February 2010. Archived from the original on 17 June 2011.\nJump up ^ \"Top 20 countries with the highest number of Internet users\". Internet World Stats. Archived from the original on 17 June 2011.\nJump up ^ Fieser, James, ed. (2000). A bibliography of Scottish common sense philosophy: Sources and origins. Bristol: Thoemmes Press. Retrieved 17 December 2010.\nJump up ^ Palmer, Michael (1999). Moral Problems in Medicine: A Practical Coursebook. Cambridge: Lutterworth Press. p. 66. ISBN 978-0-7188-2978-0.\nJump up ^ Scarre, Geoffrey (1995). Utilitarianism. London: Routledge. p. 82. ISBN 978-0-415-12197-2.\nJump up ^ Gysin, Christian (9 March 2007). \"Wembley kick-off: Stadium is ready and England play first game in fortnight\". Daily Mail (London). Retrieved 19 March 2007.\nJump up ^ \"Opening ceremony of the games of the XXX Olympiad\". Olympic.org. Retrieved 30 November 2013\nJump up ^ \"Unparalleled Sporting History\" . Reuters. Retrieved 30 November 2013\nJump up ^ \"Rugby Union 'Britain's Second Most Popular Sport'\". Ipsos-Mori. 22 December 2003. Retrieved 28 April 2013.\nJump up ^ Ebner, Sarah (2 July 2013). \"History and time are key to power of football, says Premier League chief\". The Times (London). Retrieved 30 November 2013.\nJump up ^ Mitchell, Paul (November 2005). \"The first international football match\". BBC Sport Scotland. Retrieved 15 December 2013.\nJump up ^ \"Why is there no GB Olympics football team?\". BBC Sport. 5 August 2008. Retrieved 31 December 2010.\nJump up ^ \"Blatter against British 2012 team\". BBC News. 9 March 2008. Retrieved 2 April 2008.\nJump up ^ \"About ECB\". England and Wales Cricket Board. n.d. Retrieved 28 April 2013.\nJump up ^ McLaughlin, Martyn (4 August 2009). \"Howzat happen? England fields a Gaelic-speaking Scotsman in Ashes\". The Scotsman (Edinburgh). Retrieved 30 December 2010.\nJump up ^ \"Uncapped Joyce wins Ashes call up\". BBC Sport. 15 November 2006. Retrieved 30 December 2010.\nJump up ^ \"Glamorgan\". BBC South East Wales. August 2009. Retrieved 30 December 2010.\nJump up ^ Ardener, Shirley (2007). Professional identities: policy and practice in business and bureaucracy. New York: Berghahn. p. 27. ISBN 978-1-84545-054-0.\nJump up ^ \"Official Website of Rugby League World Cup 2008\". Archived from the original on 16 October 2007.\nJump up ^ Louw, Jaco; Nesbit, Derrick (2008). The Girlfriends Guide to Rugby. Johannesburg: South Publishers. ISBN 978-0-620-39541-0.\nJump up ^ \"Triple Crown\". RBS 6 Nations. Retrieved 6 March 2011.\nJump up ^ \"Tracking the Field\". Ipsos MORI. Archived from the original on 5 February 2009. Retrieved 17 October 2008.\nJump up ^ \"Links plays into the record books\". BBC News. 17 March 2009.\nJump up ^ Chowdhury, Saj (22 January 2007). \"China in Ding's hands\". BBC Sport. Retrieved 2 January 2011.\nJump up ^ \"Lawn Tennis and Major T.Gem\". The Birmingham Civic Society. Archived from the original on 18 August 2011. Retrieved 31 December 2010.\nJump up ^ Gould, Joe (10 April 2007). \"The ancient Irish sport of hurling catches on in America\". Columbia News Service (Columbia Journalism School). Retrieved 17 May 2011.\nJump up ^ \"Shinty\". Scottishsport.co.uk. Retrieved 28 April 2013.\nJump up ^ \"Welsh dragon call for Union flag\". BBC News. 27 November 2007. Retrieved 17 October 2008.\nJump up ^ \"Britannia on British Coins\". Chard. Retrieved 25 June 2006.\nJump up ^ Baker, Steve (2001). Picturing the Beast. University of Illinois Press. p. 52. ISBN 0-252-07030-5.\nFurther reading\nHitchens, Peter (2000). The Abolition of Britain: from Winston Churchill to Princess Diana. Second ed. San Francisco, Calif.: Encounter Books. xi, 332 p. ISBN 1-893554-18-X.\nLambert, Richard S. (1964). The Great Heritage: a History of Britain for Canadians. House of Grant, 1964 (and earlier editions and/or printings).\nExternal links\nFind more about\nUnited Kingdom\nat Wikipedia's sister projects\nSearch Wiktionary\tDefinitions from Wiktionary\nSearch Commons\tMedia from Commons\nSearch Wikinews\tNews stories from Wikinews\nSearch Wikiquote\tQuotations from Wikiquote\nSearch Wikisource\tSource texts from Wikisource\nSearch Wikibooks\tTextbooks from Wikibooks\nSearch Wikivoyage\tTravel guide from Wikivoyage\nSearch Wikiversity\tLearning resources from Wikiversity\nGovernment\nOfficial website of HM Government\nOfficial website of the British Monarchy\nOfficial Yearbook of the United Kingdom statistics\nThe official site of the British Prime Minister's Office\nGeneral information\nUnited Kingdom from the BBC News\nUnited Kingdom entry at The World Factbook\nUnited Kingdom from UCB Libraries GovPubs\nUnited Kingdom at DMOZ\nUnited Kingdom Encyclopædia Britannica entry\nUnited Kingdom from the OECD\nUnited Kingdom at the EU\n Wikimedia Atlas of United Kingdom\n Geographic data related to United Kingdom at OpenStreetMap\nKey Development Forecasts for the United Kingdom from International Futures\nTravel\nOfficial tourist guide to Britain\n[hide] v t e\nUnited Kingdom topics\nHistory\t\nChronology\t\nFormation Georgian era Victorian era Edwardian era World War I Interwar World War II UK since 1945 (Postwar Britain)\nBy topic\t\nEconomic Empire Maritime Military\nGeography\t\nAdministrative\t\nCountries of the United Kingdom Crown dependencies Overseas territories City status Towns Former colonies\nPhysical\t\nBritish Isles terminology Great Britain Geology Northern Ireland Lakes and lochs Mountains Rivers Volcanoes\nResources\t\nEnergy/Renewable energy Biodiesel Coal Geothermal Hydraulic frac. Hydroelectricity Marine North Sea oil Solar Wind Food Agriculture Fishing English Scottish Hunting Materials Flora Forestry Mining\nPolitics\t\nConstitution Courts Elections Foreign relations Judiciary Law Law enforcement Legislation Monarchy monarchs Nationality Parliament House of Commons House of Lords Political parties\nGovernment\t\nCabinet list Civil service Departments Prime Minister list\nMilitary\t\nRoyal Navy Army Royal Air Force Weapons of mass destruction\nEconomy\t\nBanks Bank of England Budget Economic geography Pound (currency) Stock Exchange Taxation Telecommunications Tourism Transport\nSociety\t\nAffordability of housing Crime Demography Drug policy Education Ethnic groups Health care Immigration Languages Poverty Food banks Prostitution Public holidays Social care Social structure\nCulture\t\nArt Cinema Cuisine Identity Literature Media television Music Religion Sport Symbols Theatre\n[show] \nCountries of the United Kingdom\nOutline Index\nBook Category Portal WikiProject\n[show] \nGnome-globe.svg Geographic locale\n[show] v t e\nMember states of the European Union\n[show] \nInternational organisations\n[show] v t e\nEnglish-speaking world\n[show] v t e\nNational personifications\nCoordinates: 55°N 3°W\nCategories: United KingdomBritish IslandsConstitutional monarchiesCountries in EuropeEnglish-speaking countries and territoriesG20 nationsG7 nationsG8 nationsIsland countriesLiberal democraciesMember states of NATOMember states of the Commonwealth of NationsMember states of the Council of EuropeMember states of the European UnionMember states of the Union for the MediterraneanMember states of the United NationsNorthern EuropeWestern Europe\nNavigation menu\nCreate accountLog inArticleTalkReadView sourceView history\n\nMain page\nContents\nFeatured content\nCurrent events\nRandom article\nDonate to Wikipedia\nWikimedia Shop\nInteraction\nHelp\nAbout Wikipedia\nCommunity portal\nRecent changes\nContact page\nTools\nWhat links here\nRelated changes\nUpload file\nSpecial pages\nPermanent link\nPage information\nWikidata item\nCite this page\nPrint/export\nCreate a book\nDownload as PDF\nPrintable version\nLanguages\nАдыгэбзэ\nAfrikaans\nAkan\nAlemannisch\nአማርኛ\nÆnglisc\nАҧсшәа\nالعربية\nAragonés\nܐܪܡܝܐ\nArmãneashti\nArpetan\nAsturianu\nAvañe'ẽ\nАвар\nAzərbaycanca\nবাংলা\nBahasa Banjar\nBân-lâm-gú\nБашҡортса\nБеларуская\nБеларуская (тарашкевіца)‎\nभोजपुरी\nBikol Central\nBislama\nБългарски\nBoarisch\nབོད་ཡིག\nBosanski\nBrezhoneg\nБуряад\nCatalà\nЧӑвашла\nCebuano\nČeština\nChavacano de Zamboanga\nChiShona\nCorsu\nCymraeg\nDansk\nDeutsch\nދިވެހިބަސް\nDiné bizaad\nDolnoserbski\nཇོང་ཁ\nEesti\nΕλληνικά\nEmiliàn e rumagnòl\nEspañol\nEsperanto\nEstremeñu\nEuskara\nفارسی\nFiji Hindi\nFøroyskt\nFrançais\nFrysk\nFurlan\nGaeilge\nGaelg\nGagauz\nGàidhlig\nGalego\n贛語\nગુજરાતી\n客家語/Hak-kâ-ngî\nХальмг\n한국어\nHausa\nHawaii\nՀայերեն\nहिन्दी\nHornjoserbsce\nHrvatski\nIdo\nIgbo\nIlokano\nবিষ্ণুপ্রিয়া মণিপুরী\nBahasa Indonesia\nInterlingua\nInterlingue\nИрон\nIsiZulu\nÍslenska\nItaliano\nעברית\nBasa Jawa\nKalaallisut\nಕನ್ನಡ\nKapampangan\nКъарачай-малкъар\nქართული\nKaszëbsczi\nҚазақша\nKernowek\nKinyarwanda\nKiswahili\nКоми\nKongo\nKreyòl ayisyen\nKurdî\nКыргызча\nКырык мары\nLadino\nЛезги\nລາວ\nLatgaļu\nLatina\nLatviešu\nLëtzebuergesch\nLietuvių\nLigure\nLimburgs\nLingála\nLojban\nLumbaart\nMagyar\nМакедонски\nMalagasy\nമലയാളം\nMalti\nMāori\nमराठी\nმარგალური\nمصرى\nمازِرونی\nBahasa Melayu\nMìng-dĕ̤ng-ngṳ̄\nMirandés\nМонгол\nမြန်မာဘာသာ\nNāhuatl\nDorerin Naoero\nNederlands\nNedersaksies\nनेपाली\nनेपाल भाषा\n日本語\nNapulitano\nНохчийн\nNordfriisk\nNorfuk / Pitkern\nNorsk bokmål\nNorsk nynorsk\nNouormand\nNovial\nOccitan\nОлык марий\nଓଡ଼ିଆ\nOromoo\nOʻzbekcha\nਪੰਜਾਬੀ\nPangasinan\nپنجابی\nPapiamentu\nپښتو\nПерем Коми\nភាសាខ្មែរ\nPicard\nPiemontèis\nTok Pisin\nPlattdüütsch\nPolski\nΠοντιακά\nPortuguês\nQırımtatarca\nReo tahiti\nRipoarisch\nRomână\nRomani\nRumantsch\nRuna Simi\nРусиньскый\nРусский\nСаха тыла\nSámegiella\nसंस्कृतम्\nSardu\nScots\nSeeltersk\nShqip\nSicilianu\nසිංහල\nSimple English\nSiSwati\nSlovenčina\nSlovenščina\nСловѣньскъ / ⰔⰎⰑⰂⰡⰐⰠⰔⰍⰟ\nŚlůnski\nSoomaaliga\nکوردی\nSranantongo\nСрпски / srpski\nSrpskohrvatski / српскохрватски\nBasa Sunda\nSuomi\nSvenska\nTagalog\nதமிழ்\nTaqbaylit\nTarandíne\nТатарча/tatarça\nతెలుగు\nTetun\nไทย\nТоҷикӣ\nᏣᎳᎩ\nTsetsêhestâhese\nTürkçe\nTwi\nУдмурт\nᨅᨔ ᨕᨘᨁᨗ\nУкраїнська\nاردو\nئۇيغۇرچە / Uyghurche\nVahcuengh\nVèneto\nVepsän kel’\nTiếng Việt\nVolapük\nVõro\nWalon\n文言\nWest-Vlams\nWinaray\nWolof\n吴语\nייִדיש\nYorùbá\n粵語\nZazaki\nZeêuws\nŽemaitėška\n中文\nEdit links\nThis page was last modified on 22 November 2014 at 11:19.\nText is available under the Creative Commons Attribution-ShareAlike License; additional terms may apply. By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia® is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\nPrivacy policyAbout WikipediaDisclaimersContact WikipediaDevelopersMobile viewWikimedia Foundation Powered by MediaWiki\n\n\nWorld Trade Organization\nFrom Wikipedia, the free encyclopedia\n\"WTO\" redirects here. For other uses, see WTO (disambiguation).\nWorld Trade Organization (English)\nOrganisation mondiale du commerce (French)\nOrganización Mundial del Comercio (Spanish)\nWorld Trade Organization (logo and wordmark).svg\nOfficial logo of WTO\nWTO members and observers.svg\n  Members\n  Members, dually represented by the EU\n  Observers\n  Non-members\nAbbreviation\tWTO\nFormation\t1 January 1995; 19 years ago\nType\tInternational trade organization\nPurpose\tLiberalize international trade\nHeadquarters\tCentre William Rappard, Geneva, Switzerland\nCoordinates\t46.12°N 6.09°ECoordinates: 46.12°N 6.09°E\nRegion served\tWorldwide\nMembership\t160 member states[1]\nOfficial language\tEnglish, French, Spanish[2]\nDirector-General\tRoberto Azevêdo\nBudget\t196 million Swiss francs (approx. 209 million US$) in 2011.[3]\nStaff\t640[4]\nWebsite\twww.wto.org\nThe World Trade Organization (WTO) is an organization that intends to supervise and liberalize international trade. The organization officially commenced on 1 January 1995 under the Marrakech Agreement, replacing the General Agreement on Tariffs and Trade (GATT), which commenced in 1948.[5] The organization deals with regulation of trade between participating countries by providing a framework for negotiating and formalizing trade agreements and a dispute resolution process aimed at enforcing participants' adherence to WTO agreements, which are signed by representatives of member governments[6]:fol.9–10 and ratified by their parliaments.[7] Most of the issues that the WTO focuses on derive from previous trade negotiations, especially from the Uruguay Round (1986–1994).\nThe organization is attempting to complete negotiations on the Doha Development Round, which was launched in 2001 with an explicit focus on addressing the needs of developing countries. As of June 2012, the future of the Doha Round remained uncertain: the work programme lists 21 subjects in which the original deadline of 1 January 2005 was missed, and the round is still incomplete.[8] The conflict between free trade on industrial goods and services but retention of protectionism on farm subsidies to domestic agricultural sector (requested by developed countries) and the substantiation of the international liberalization of fair trade on agricultural products (requested by developing countries) remain the major obstacles. These points of contention have hindered any progress to launch new WTO negotiations beyond the Doha Development Round. As a result of this impasse, there has been an increasing number of bilateral free trade agreements signed.[9] As of July 2012, there were various negotiation groups in the WTO system for the current agricultural trade negotiation which is in the condition of stalemate.[10]\nWTO's current Director-General is Roberto Azevêdo,[11][12] who leads a staff of over 600 people in Geneva, Switzerland.[13] A trade facilitation agreement known as the Bali Package was reached by all members on 7 December 2013, the first comprehensive agreement in the organization's history.[14][15]\nContents  [hide] \n1 History\n1.1 GATT rounds of negotiations\n1.1.1 From Geneva to Tokyo\n1.1.2 Uruguay Round\n1.2 Ministerial conferences\n1.3 Doha Round (Doha Agenda)\n2 Functions\n3 Principles of the trading system\n4 Organizational structure\n5 Decision-making\n6 Dispute settlement\n7 Accession and membership\n7.1 Accession process\n7.2 Members and observers\n8 Agreements\n9 Office of director-general\n9.1 List of directors-general\n10 See also\n11 Notes and references\n12 External links\nHistory\n\nThe economists Harry White (left) and John Maynard Keynes at the Bretton Woods Conference. Both had been strong advocates of a central-controlled international trade environment and recommended the establishment of three institutions: the IMF (for fiscal and monetary issues); the World Bank (for financial and structural issues); and the ITO (for international economic cooperation).[16]\nThe WTO's predecessor, the General Agreement on Tariffs and Trade (GATT), was established after World War II in the wake of other new multilateral institutions dedicated to international economic cooperation – notably the Bretton Woods institutions known as the World Bank and the International Monetary Fund. A comparable international institution for trade, named the International Trade Organization was successfully negotiated. The ITO was to be a United Nations specialized agency and would address not only trade barriers but other issues indirectly related to trade, including employment, investment, restrictive business practices, and commodity agreements. But the ITO treaty was not approved by the U.S. and a few other signatories and never went into effect.[17][18][19]\nIn the absence of an international organization for trade, the GATT would over the years \"transform itself\" into a de facto international organization.[20]\nGATT rounds of negotiations\nSee also: General Agreement on Tariffs and Trade\nThe GATT was the only multilateral instrument governing international trade from 1946 until the WTO was established on 1 January 1995.[21] Despite attempts in the mid-1950s and 1960s to create some form of institutional mechanism for international trade, the GATT continued to operate for almost half a century as a semi-institutionalized multilateral treaty regime on a provisional basis.[22]\nFrom Geneva to Tokyo\nSeven rounds of negotiations occurred under GATT. The first real GATT trade rounds concentrated on further reducing tariffs. Then, the Kennedy Round in the mid-sixties brought about a GATT anti-dumping Agreement and a section on development. The Tokyo Round during the seventies was the first major attempt to tackle trade barriers that do not take the form of tariffs, and to improve the system, adopting a series of agreements on non-tariff barriers, which in some cases interpreted existing GATT rules, and in others broke entirely new ground. Because these plurilateral agreements were not accepted by the full GATT membership, they were often informally called \"codes\". Several of these codes were amended in the Uruguay Round, and turned into multilateral commitments accepted by all WTO members. Only four remained plurilateral (those on government procurement, bovine meat, civil aircraft and dairy products), but in 1997 WTO members agreed to terminate the bovine meat and dairy agreements, leaving only two.[21]\nUruguay Round\nMain article: Uruguay Round\n\nDuring the Doha Round, the US government blamed Brazil and India for being inflexible and the EU for impeding agricultural imports.[23] The then-President of Brazil, Luiz Inácio Lula da Silva (above right), responded to the criticisms by arguing that progress would only be achieved if the richest countries (especially the US and countries in the EU) made deeper cuts in agricultural subsidies and further opened their markets for agricultural goods.[24]\nWell before GATT's 40th anniversary, its members concluded that the GATT system was straining to adapt to a new globalizing world economy.[25][26] In response to the problems identified in the 1982 Ministerial Declaration (structural deficiencies, spill-over impacts of certain countries' policies on world trade GATT could not manage etc.), the eighth GATT round – known as the Uruguay Round – was launched in September 1986, in Punta del Este, Uruguay.[25]\nIt was the biggest negotiating mandate on trade ever agreed: the talks were going to extend the trading system into several new areas, notably trade in services and intellectual property, and to reform trade in the sensitive sectors of agriculture and textiles; all the original GATT articles were up for review.[26] The Final Act concluding the Uruguay Round and officially establishing the WTO regime was signed 15 April 1994, during the ministerial meeting at Marrakesh, Morocco, and hence is known as the Marrakesh Agreement.[27]\nThe GATT still exists as the WTO's umbrella treaty for trade in goods, updated as a result of the Uruguay Round negotiations (a distinction is made between GATT 1994, the updated parts of GATT, and GATT 1947, the original agreement which is still the heart of GATT 1994).[25] GATT 1994 is not however the only legally binding agreement included via the Final Act at Marrakesh; a long list of about 60 agreements, annexes, decisions and understandings was adopted. The agreements fall into a structure with six main parts:\nThe Agreement Establishing the WTO\nGoods and investment – the Multilateral Agreements on Trade in Goods including the GATT 1994 and the Trade Related Investment Measures (TRIMS)\nServices — the General Agreement on Trade in Services\nIntellectual property – the Agreement on Trade-Related Aspects of Intellectual Property Rights (TRIPS)\nDispute settlement (DSU)\nReviews of governments' trade policies (TPRM)[28]\nIn terms of the WTO's principle relating to tariff \"ceiling-binding\" (No. 3), the Uruguay Round has been successful in increasing binding commitments by both developed and developing countries, as may be seen in the percentages of tariffs bound before and after the 1986–1994 talks.[29]\nMinisterial conferences\n\nThe World Trade Organization Ministerial Conference of 1998, in the Palace of Nations (Geneva, Switzerland).\nThe highest decision-making body of the WTO is the Ministerial Conference, which usually meets every two years. It brings together all members of the WTO, all of which are countries or customs unions. The Ministerial Conference can take decisions on all matters under any of the multilateral trade agreements. The inaugural ministerial conference was held in Singapore in 1996. Disagreements between largely developed and developing economies emerged during this conference over four issues initiated by this conference, which led to them being collectively referred to as the \"Singapore issues\". The second ministerial conference was held in Geneva in Switzerland. The third conference in Seattle, Washington ended in failure, with massive demonstrations and police and National Guard crowd-control efforts drawing worldwide attention. The fourth ministerial conference was held in Doha in the Persian Gulf nation of Qatar. The Doha Development Round was launched at the conference. The conference also approved the joining of China, which became the 143rd member to join. The fifth ministerial conference was held in Cancún, Mexico, aiming at forging agreement on the Doha round. An alliance of 22 southern states, the G20 developing nations (led by India, China,[30] Brazil, ASEAN led by the Philippines), resisted demands from the North for agreements on the so-called \"Singapore issues\" and called for an end to agricultural subsidies within the EU and the US. The talks broke down without progress.\nThe sixth WTO ministerial conference was held in Hong Kong from 13–18 December 2005. It was considered vital if the four-year-old Doha Development Round negotiations were to move forward sufficiently to conclude the round in 2006. In this meeting, countries agreed to phase out all their agricultural export subsidies by the end of 2013, and terminate any cotton export subsidies by the end of 2006. Further concessions to developing countries included an agreement to introduce duty-free, tariff-free access for goods from the Least Developed Countries, following the Everything but Arms initiative of the European Union — but with up to 3% of tariff lines exempted. Other major issues were left for further negotiation to be completed by the end of 2010. The WTO General Council, on 26 May 2009, agreed to hold a seventh WTO ministerial conference session in Geneva from 30 November-3 December 2009. A statement by chairman Amb. Mario Matus acknowledged that the prime purpose was to remedy a breach of protocol requiring two-yearly \"regular\" meetings, which had lapsed with the Doha Round failure in 2005, and that the \"scaled-down\" meeting would not be a negotiating session, but \"emphasis will be on transparency and open discussion rather than on small group processes and informal negotiating structures\". The general theme for discussion was \"The WTO, the Multilateral Trading System and the Current Global Economic Environment\"[31]\nDoha Round (Doha Agenda)\nMain article: Doha Development Round\n\nThe Doha Development Round started in 2001 is at an impasse.\nThe WTO launched the current round of negotiations, the Doha Development Round, at the fourth ministerial conference in Doha, Qatar in November 2001. This was to be an ambitious effort to make globalization more inclusive and help the world's poor, particularly by slashing barriers and subsidies in farming.[32] The initial agenda comprised both further trade liberalization and new rule-making, underpinned by commitments to strengthen substantial assistance to developing countries.[33]\nThe negotiations have been highly contentious. Disagreements still continue over several key areas including agriculture subsidies, which emerged as critical in July 2006.[34] According to a European Union statement, \"The 2008 Ministerial meeting broke down over a disagreement between exporters of agricultural bulk commodities and countries with large numbers of subsistence farmers on the precise terms of a 'special safeguard measure' to protect farmers from surges in imports.\"[35] The position of the European Commission is that \"The successful conclusion of the Doha negotiations would confirm the central role of multilateral liberalisation and rule-making. It would confirm the WTO as a powerful shield against protectionist backsliding.\"[33] An impasse remains and, as of August 2013, agreement has not been reached, despite intense negotiations at several ministerial conferences and at other sessions. On 27 March 2013, the chairman of agriculture talks announced \"a proposal to loosen price support disciplines for developing countries’ public stocks and domestic food aid.\" He added: “...we are not yet close to agreement—in fact, the substantive discussion of the proposal is only beginning.”[36]\n[show]v · t · eGATT and WTO trade rounds[37]\nFunctions\nAmong the various functions of the WTO, these are regarded by analysts as the most important:\nIt oversees the implementation, administration and operation of the covered agreements.[38][39]\nIt provides a forum for negotiations and for settling disputes.[40][41]\nAdditionally, it is the WTO's duty to review and propagate the national trade policies, and to ensure the coherence and transparency of trade policies through surveillance in global economic policy-making.[39][41] Another priority of the WTO is the assistance of developing, least-developed and low-income countries in transition to adjust to WTO rules and disciplines through technical cooperation and training.[42]\n(i) The WTO shall facilitate the implementation, administration and operation and further the objec­tives of this Agreement and of the Multilateral Trade Agreements, and shall also provide the frame work for the implementation, administration and operation of the multilateral Trade Agreements.\n(ii) The WTO shall provide the forum for negotiations among its members concerning their multilateral trade relations in matters dealt with under the Agreement in the Annexes to this Agreement.\n(iii) The WTO shall administer the Understanding on Rules and Procedures Governing the Settlement of Disputes.\n(iv) The WTO shall administer Trade Policy Review Mechanism.\n(v) With a view to achieving greater coherence in global economic policy making, the WTO shall cooperate, as appropriate, with the international Monetary Fund (IMF) and with the International Bank for Reconstruction and Development (IBRD) and its affiliated agencies. [43]\nThe above five listings are the additional functions of the World Trade Organization. As globalization proceeds in today's society, the necessity of an International Organization to manage the trading systems has been of vital importance. As the trade volume increases, issues such as protectionism, trade barriers, subsidies, violation of intellectual property arise due to the differences in the trading rules of every nation. The World Trade Organization serves as the mediator between the nations when such problems arise. WTO could be referred to as the product of globalization and also as one of the most important organizations in today's globalized society.\nThe WTO is also a center of economic research and analysis: regular assessments of the global trade picture in its annual publications and research reports on specific topics are produced by the organization.[44] Finally, the WTO cooperates closely with the two other components of the Bretton Woods system, the IMF and the World Bank.[40]\nPrinciples of the trading system\nThe WTO establishes a framework for trade policies; it does not define or specify outcomes. That is, it is concerned with setting the rules of the trade policy games.[45] Five principles are of particular importance in understanding both the pre-1994 GATT and the WTO:\nNon-discrimination. It has two major components: the most favoured nation (MFN) rule, and the national treatment policy. Both are embedded in the main WTO rules on goods, services, and intellectual property, but their precise scope and nature differ across these areas. The MFN rule requires that a WTO member must apply the same conditions on all trade with other WTO members, i.e. a WTO member has to grant the most favorable conditions under which it allows trade in a certain product type to all other WTO members.[45] \"Grant someone a special favour and you have to do the same for all other WTO members.\"[29] National treatment means that imported goods should be treated no less favorably than domestically produced goods (at least after the foreign goods have entered the market) and was introduced to tackle non-tariff barriers to trade (e.g. technical standards, security standards et al. discriminating against imported goods).[45]\nReciprocity. It reflects both a desire to limit the scope of free-riding that may arise because of the MFN rule, and a desire to obtain better access to foreign markets. A related point is that for a nation to negotiate, it is necessary that the gain from doing so be greater than the gain available from unilateral liberalization; reciprocal concessions intend to ensure that such gains will materialise.[46]\nBinding and enforceable commitments. The tariff commitments made by WTO members in a multilateral trade negotiation and on accession are enumerated in a schedule (list) of concessions. These schedules establish \"ceiling bindings\": a country can change its bindings, but only after negotiating with its trading partners, which could mean compensating them for loss of trade. If satisfaction is not obtained, the complaining country may invoke the WTO dispute settlement procedures.[29][46]\nTransparency. The WTO members are required to publish their trade regulations, to maintain institutions allowing for the review of administrative decisions affecting trade, to respond to requests for information by other members, and to notify changes in trade policies to the WTO. These internal transparency requirements are supplemented and facilitated by periodic country-specific reports (trade policy reviews) through the Trade Policy Review Mechanism (TPRM).[47] The WTO system tries also to improve predictability and stability, discouraging the use of quotas and other measures used to set limits on quantities of imports.[29]\nSafety valves. In specific circumstances, governments are able to restrict trade. The WTO's agreements permit members to take measures to protect not only the environment but also public health, animal health and plant health.[48]\nThere are three types of provision in this direction:\narticles allowing for the use of trade measures to attain non-economic objectives;\narticles aimed at ensuring \"fair competition\"; members must not use environmental protection measures as a means of disguising protectionist policies.[48]\nprovisions permitting intervention in trade for economic reasons.[47]\nExceptions to the MFN principle also allow for preferential treatment of developing countries, regional free trade areas and customs unions.[6]:fol.93\nOrganizational structure\nThe General Council has the following subsidiary bodies which oversee committees in different areas:\nCouncil for Trade in Goods\nThere are 11 committees under the jurisdiction of the Goods Council each with a specific task. All members of the WTO participate in the committees. The Textiles Monitoring Body is separate from the other committees but still under the jurisdiction of Goods Council. The body has its own chairman and only 10 members. The body also has several groups relating to textiles.[49]\nCouncil for Trade-Related Aspects of Intellectual Property Rights\nInformation on intellectual property in the WTO, news and official records of the activities of the TRIPS Council, and details of the WTO's work with other international organizations in the field.[50]\nCouncil for Trade in Services\nThe Council for Trade in Services operates under the guidance of the General Council and is responsible for overseeing the functioning of the General Agreement on Trade in Services (GATS). It is open to all WTO members, and can create subsidiary bodies as required.[51]\nTrade Negotiations Committee\nThe Trade Negotiations Committee (TNC) is the committee that deals with the current trade talks round. The chair is WTO's director-general. As of June 2012 the committee was tasked with the Doha Development Round.[52]\nThe Service Council has three subsidiary bodies: financial services, domestic regulations, GATS rules and specific commitments.[49] The council has several different committees, working groups, and working parties.[53] There are committees on the following: Trade and Environment; Trade and Development (Subcommittee on Least-Developed Countries); Regional Trade Agreements; Balance of Payments Restrictions; and Budget, Finance and Administration. There are working parties on the following: Accession. There are working groups on the following: Trade, debt and finance; and Trade and technology transfer.\nDecision-making\nThe WTO describes itself as \"a rules-based, member-driven organization — all decisions are made by the member governments, and the rules are the outcome of negotiations among members\".[54] The WTO Agreement foresees votes where consensus cannot be reached, but the practice of consensus dominates the process of decision-making.[55]\nRichard Harold Steinberg (2002) argues that although the WTO's consensus governance model provides law-based initial bargaining, trading rounds close through power-based bargaining favouring Europe and the U.S., and may not lead to Pareto improvement.[56]\nDispute settlement\nMain article: Dispute settlement in the WTO\nIn 1994, the WTO members agreed on the Understanding on Rules and Procedures Governing the Settlement of Disputes (DSU) annexed to the \"Final Act\" signed in Marrakesh in 1994.[57] Dispute settlement is regarded by the WTO as the central pillar of the multilateral trading system, and as a \"unique contribution to the stability of the global economy\".[58] WTO members have agreed that, if they believe fellow-members are violating trade rules, they will use the multilateral system of settling disputes instead of taking action unilaterally.[59]\nThe operation of the WTO dispute settlement process involves the DSB panels, the Appellate Body, the WTO Secretariat, arbitrators, independent experts and several specialized institutions.[60] Bodies involved in the dispute settlement process, World Trade Organization.\nAccession and membership\nMain article: World Trade Organization accession and membership\nThe process of becoming a WTO member is unique to each applicant country, and the terms of accession are dependent upon the country's stage of economic development and current trade regime.[61] The process takes about five years, on average, but it can last more if the country is less than fully committed to the process or if political issues interfere. The shortest accession negotiation was that of the Kyrgyz Republic, while the longest was that of Russia, which, having first applied to join GATT in 1993, was approved for membership in December 2011 and became a WTO member on 22 August 2012.[62] The second longest was that of Vanuatu, whose Working Party on the Accession of Vanuatu was established on 11 July 1995. After a final meeting of the Working Party in October 2001, Vanuatu requested more time to consider its accession terms. In 2008, it indicated its interest to resume and conclude its WTO accession. The Working Party on the Accession of Vanuatu was reconvened informally on 4 April 2011 to discuss Vanuatu's future WTO membership. The re-convened Working Party completed its mandate on 2 May 2011. The General Council formally approved the Accession Package of Vanuatu on 26 October 2011. On 24 August 2012, the WTO welcomed Vanuatu as its 157th member.[63] An offer of accession is only given once consensus is reached among interested parties.[64]\nAccession process\n\nWTO accession progress:\n  Members (including dual-representation with the European Union)\n  Draft Working Party Report or Factual Summary adopted\n  Goods and/or Services offers submitted\n  Memorandum on Foreign Trade Regime (FTR) submitted\n  Observer, negotiations to start later or no Memorandum on FTR submitted\n  Frozen procedures or no negotiations in the last 3 years\n  No official interaction with the WTO\nA country wishing to accede to the WTO submits an application to the General Council, and has to describe all aspects of its trade and economic policies that have a bearing on WTO agreements.[65] The application is submitted to the WTO in a memorandum which is examined by a working party open to all interested WTO Members.[66]\nAfter all necessary background information has been acquired, the working party focuses on issues of discrepancy between the WTO rules and the applicant's international and domestic trade policies and laws. The working party determines the terms and conditions of entry into the WTO for the applicant nation, and may consider transitional periods to allow countries some leeway in complying with the WTO rules.[61]\nThe final phase of accession involves bilateral negotiations between the applicant nation and other working party members regarding the concessions and commitments on tariff levels and market access for goods and services. The new member's commitments are to apply equally to all WTO members under normal non-discrimination rules, even though they are negotiated bilaterally.[65]\nWhen the bilateral talks conclude, the working party sends to the general council or ministerial conference an accession package, which includes a summary of all the working party meetings, the Protocol of Accession (a draft membership treaty), and lists (\"schedules\") of the member-to-be's commitments. Once the general council or ministerial conference approves of the terms of accession, the applicant's parliament must ratify the Protocol of Accession before it can become a member.[67] Some countries may have faced tougher and a much longer accession process due to challenges during negotiations with other WTO members, such as Vietnam, whose negotiations took more than 11 years before it became official member in January 2007.[68]\nMembers and observers\nThe WTO has 160 members and 24 observer governments.[69] In addition to states, the European Union is a member. WTO members do not have to be full sovereign nation-members. Instead, they must be a customs territory with full autonomy in the conduct of their external commercial relations. Thus Hong Kong has been a member since 1995 (as \"Hong Kong, China\" since 1997) predating the People's Republic of China, which joined in 2001 after 15 years of negotiations. The Republic of China (Taiwan) acceded to the WTO in 2002 as \"Separate Customs Territory of Taiwan, Penghu, Kinmen and Matsu\" (Chinese Taipei) despite its disputed status.[70] The WTO Secretariat omits the official titles (such as Counselor, First Secretary, Second Secretary and Third Secretary) of the members of Chinese Taipei's Permanent Mission to the WTO, except for the titles of the Permanent Representative and the Deputy Permanent Representative.[71]\nAs of 2007, WTO member states represented 96.4% of global trade and 96.7% of global GDP.[72] Iran, followed by Algeria, are the economies with the largest GDP and trade outside the WTO, using 2005 data.[73][74] With the exception of the Holy See, observers must start accession negotiations within five years of becoming observers. A number of international intergovernmental organizations have also been granted observer status to WTO bodies.[75] 14 UN member states have no official affiliation with the WTO.\nAgreements\nFurther information: Uruguay Round\nThe WTO oversees about 60 different agreements which have the status of international legal texts. Member countries must sign and ratify all WTO agreements on accession.[76] A discussion of some of the most important agreements follows. The Agreement on Agriculture came into effect with the establishment of the WTO at the beginning of 1995. The AoA has three central concepts, or \"pillars\": domestic support, market access and export subsidies. The General Agreement on Trade in Services was created to extend the multilateral trading system to service sector, in the same way as the General Agreement on Tariffs and Trade (GATT) provided such a system for merchandise trade. The agreement entered into force in January 1995. The Agreement on Trade-Related Aspects of Intellectual Property Rights sets down minimum standards for many forms of intellectual property (IP) regulation. It was negotiated at the end of the Uruguay Round of the General Agreement on Tariffs and Trade (GATT) in 1994.[77]\nThe Agreement on the Application of Sanitary and Phytosanitary Measures—also known as the SPS Agreement—was negotiated during the Uruguay Round of GATT, and entered into force with the establishment of the WTO at the beginning of 1995. Under the SPS agreement, the WTO sets constraints on members' policies relating to food safety (bacterial contaminants, pesticides, inspection and labelling) as well as animal and plant health (imported pests and diseases). The Agreement on Technical Barriers to Trade is an international treaty of the World Trade Organization. It was negotiated during the Uruguay Round of the General Agreement on Tariffs and Trade, and entered into force with the establishment of the WTO at the end of 1994. The object ensures that technical negotiations and standards, as well as testing and certification procedures, do not create unnecessary obstacles to trade\".[78] The Agreement on Customs Valuation, formally known as the Agreement on Implementation of Article VII of GATT, prescribes methods of customs valuation that Members are to follow. Chiefly, it adopts the \"transaction value\" approach.\nIn December 2013, the biggest agreement within the WTO was signed and known as the Bali Package.[79]\nOffice of director-general\n\nThe headquarters of the World Trade Organization, in Geneva, Switzerland.\nThe procedures for the appointment of the WTO director-general were published in January 2003.[80] Additionally, there are four deputy directors-general. As of 1 October 2013, under director-general Roberto Azevêdo, the four deputy directors-general are Yi Xiaozhun of China, Karl-Ernst Brauner of Germany, Yonov Frederick Agah of Nigeria and David Shark of the United States.[81]\nList of directors-general\nSource: Official website[82]\nBrazil Roberto Azevedo, 2013–\nFrance Pascal Lamy, 2005–2013\nThailand Supachai Panitchpakdi, 2002–2005\nNew Zealand Mike Moore, 1999–2002\nItaly Renato Ruggiero, 1995–1999\nRepublic of Ireland Peter Sutherland, 1995\n(Heads of the precursor organization, GATT):\nRepublic of Ireland Peter Sutherland, 1993–1995\nSwitzerland Arthur Dunkel, 1980–1993\nSwitzerland Olivier Long, 1968–1980\nUnited Kingdom Eric Wyndham White, 1948–1968\nSee also\nAgreement on Trade Related Investment Measures (TRIMS)\nAgreement on Trade-Related Aspects of Intellectual Property Rights (TRIPS)\nAide-mémoire non-paper\nAnti-globalization movement\nCriticism of the World Trade Organization\nForeign Affiliate Trade Statistics\nGlobal administrative law\nGlobality\nInformation Technology Agreement\nInternational Trade Centre\nLabour Standards in the World Trade Organisation\nList of member states of the World Trade Organization\nNorth American Free Trade Agreement (NAFTA)\nSubsidy\nSwiss Formula\nTrade bloc\nWashington Consensus\nWorld Trade Report\nWorld Trade Organization Ministerial Conference of 1999 protest activity\nChina and the World Trade Organization\nNotes and references\nJump up ^ Members and Observers at WTO official website\nJump up ^ Languages, Documentation and Information Management Division at WTO official site\nJump up ^ \"WTO Secretariat budget for 2011\". WTO official site. Retrieved 25 August 2008.\nJump up ^ Understanding the WTO: What We Stand For_ Fact File\nJump up ^ World Trade Organization - UNDERSTANDING THE WTO: BASICS\n^ Jump up to: a b Understanding the WTO Handbook at WTO official website. (Note that the document's printed folio numbers do not match the pdf page numbers.)\nJump up ^ Malanczuk, P. (1999). \"International Organisations and Space Law: World Trade Organization\". Encyclopaedia Britannica 442. p. 305. Bibcode:1999ESASP.442..305M.\nJump up ^ Understanding the WTO: The Doha Agenda\nJump up ^ The Challenges to the World Trade Organization: It’s All About Legitimacy THE BROOKINGS INSTITUTION, Policy Paper 2011-04\nJump up ^ GROUPS IN THE WTO Updated 1 July 2013\nJump up ^ Bourcier, Nicolas (21 May 2013). \"Roberto Azevedo's WTO appointment gives Brazil a seat at the top table\". Guardian Weekly. Retrieved 2 September 2013.\nJump up ^ \"Roberto Azevêdo takes over\". WTO official website. 1 September 2013. Retrieved 2 September 2013.\nJump up ^ \"Overview of the WTO Secretariat\". WTO official website. Retrieved 2 September 2013.\nJump up ^ Ninth WTO Ministerial Conference | WTO - MC9\nJump up ^ BBC News - WTO agrees global trade deal worth $1tn\nJump up ^ A.E. Eckes Jr., US Trade History, 73\n* A. Smithies, Reflections on the Work of Keynes, 578–601\n* N. Warren, Internet and Globalization, 193\nJump up ^ P. van den Bossche, The Law and Policy of the World Trade Organization, 80\nJump up ^ Palmeter-Mavroidis, Dispute Settlement, 2\nJump up ^ Fergusson, Ian F. (9 May 2007). \"The World Trade Organization: Background and Issues\" (PDF). Congressional Research Service. p. 4. Retrieved 15 August 2008.\nJump up ^ It was contemplated that the GATT would be applied for several years until the ITO came into force. However, since the ITO was never brought into being, the GATT gradually became the focus for international governmental cooperation on trade matters with economist Nicholas Halford overseeing the implementation of GATT in members policies. (P. van den Bossche, The Law and Policy of the World Trade Organization, 81; J.H. Jackson, Managing the Trading System, 134).\n^ Jump up to: a b The GATT Years: from Havana to Marrakesh, WTO official site\nJump up ^ Footer, M. E. Analysis of the World Trade Organization, 17\nJump up ^ B.S. Klapper, With a \"Short Window\"\nJump up ^ Lula, Time to Get Serious about Agricultural Subsidies\n^ Jump up to: a b c P. Gallagher, The First Ten Years of the WTO, 4\n^ Jump up to: a b The Uruguay Round, WTO official site\nJump up ^ \"Legal texts – Marrakesh agreement\". WTO. Retrieved 30 May 2010.\nJump up ^ Overview: a Navigational Guide, WTO official site. For the complete list of \"The Uruguay Round Agreements\", see WTO legal texts, WTO official site, and Uruguay Round Agreements, Understandings, Decisions and Declarations, WorldTradeLaw.net\n^ Jump up to: a b c d Principles of the Trading System, WTO official site\nJump up ^ \"Five Years of China WTO Membership. EU and US Perspectives about China's Compliance with Transparency Commitments and the Transitional Review Mechanism\". Papers.ssrn.com. Retrieved 30 May 2010.\nJump up ^ WTO to hold 7th Ministerial Conference on 30 November-2 December 2009 WTO official website\nJump up ^ \"In the twilight of Doha\". The Economist (The Economist): 65. 27 July 2006.\n^ Jump up to: a b European Commission The Doha Round\nJump up ^ Fergusson, Ian F. (18 January 2008). \"World Trade Organization Negotiations: The Doha Development Agenda\" (PDF). Congressional Research Service. Retrieved 13 April 2012. Page 9 (folio CRS-6)\nJump up ^ WTO trade negotiations: Doha Development Agenda Europa press release, 31 October 2011\nJump up ^ \"Members start negotiating proposal on poor countries’ food stockholding\". WTO official website. 27 March 2013. Retrieved 2 September 2013.\nJump up ^ a)The GATT years: from Havana to Marrakesh, World Trade Organization\nb)Timeline: World Trade Organization – A chronology of key events, BBC News\nc)Brakman-Garretsen-Marrewijk-Witteloostuijn, Nations and Firms in the Global Economy, Chapter 10: Trade and Capital Restriction\nJump up ^ Functions of the WTO, IISD\n^ Jump up to: a b Main Functions, WTO official site\n^ Jump up to: a b A Bredimas, International Economic Law, II, 17\n^ Jump up to: a b C. Deere, Decision-making in the WTO: Medieval or Up-to-Date?\nJump up ^ WTO Assistance for Developing Countries[dead link], WTO official site\nJump up ^ Sinha, Aparijita. [1]. \"What are the functions and objectives of the WTO?\". Retrieved on 13 April, 2014.\nJump up ^ Economic research and analysis, WTO official site\n^ Jump up to: a b c B. Hoekman, The WTO: Functions and Basic Principles, 42\n^ Jump up to: a b B. Hoekman, The WTO: Functions and Basic Principles, 43\n^ Jump up to: a b B. Hoekman, The WTO: Functions and Basic Principles, 44\n^ Jump up to: a b Understanding the WTO: What we stand for\n^ Jump up to: a b \"Fourth level: down to the nitty-gritty\". WTO official site. Retrieved 18 August 2008.\nJump up ^ \"Intellectual property – overview of TRIPS Agreement\". Wto.org. 15 April 1994. Retrieved 30 May 2010.\nJump up ^ \"The Services Council, its Committees and other subsidiary bodies\". WTO official site. Retrieved 14 August 2008.\nJump up ^ \"The Trade Negotiations Committee\". WTO official site. Retrieved 14 August 2008.\nJump up ^ \"WTO organization chart\". WTO official site. Retrieved 14 August 2008.\nJump up ^ Decision-making at WTO official site\nJump up ^ Decision-Making in the World Trade Organization Abstract from Journal of International Economic Law at Oxford Journals\nJump up ^ Steinberg, Richard H. \"In the Shadow of Law or Power? Consensus-based Bargaining and Outcomes in the GATT/WTO.\" International Organization. Spring 2002. pp. 339–374.\nJump up ^ Stewart-Dawyer, The WTO Dispute Settlement System, 7\nJump up ^ S. Panitchpakdi, The WTO at ten, 8.\nJump up ^ Settling Disputes:a Unique Contribution, WTO official site\nJump up ^ \"Disputes – Dispute Settlement CBT – WTO Bodies involved in the dispute settlement process – The Dispute Settlement Body (DSB) – Page 1\". WTO. 25 July 1996. Retrieved 21 May 2011.\n^ Jump up to: a b Accessions Summary, Center for International Development\nJump up ^ Ministerial Conference approves Russia's WTO membership WTO News Item, 16 December 2011\nJump up ^ Accession status: Vanuatu. WTO. Retrieved on 12 July 2013.\nJump up ^ C. Michalopoulos, WTO Accession, 64\n^ Jump up to: a b Membership, Alliances and Bureaucracy, WTO official site\nJump up ^ C. Michalopoulos, WTO Accession, 62–63\nJump up ^ How to Become a Member of the WTO, WTO official site\nJump up ^ Napier, Nancy K.; Vuong, Quan Hoang (2013). What we see, why we worry, why we hope: Vietnam going forward. Boise, ID, USA: Boise State University CCI Press. p. 140. ISBN 978-0985530587.\nJump up ^ \"Members and Observers\". World Trade Organization. 24 August 2012.\nJump up ^ Jackson, J. H. Sovereignty, 109\nJump up ^ ROC Government Publication\nJump up ^ \"Accession in perspective\". World Trade Organization. Retrieved 22 December 2013.\nJump up ^ \"ANNEX 1. STATISTICAL SURVEY\". World Trade Organization. 2005. Retrieved 22 December 2013.\nJump up ^ Arjomandy, Danial (21 November 2013). \"Iranian Membership in the World Trade Organization: An Unclear Future\". Iranian Studies. Retrieved 22 December 2013.\nJump up ^ International intergovernmental organizations granted observer status to WTO bodies at WTO official website\nJump up ^ \"Legal texts – the WTO agreements\". WTO. Retrieved 30 May 2010.\nJump up ^ Understanding the WTO - Intellectual property: protection and enforcement. WTO. Retrieved on 29 July 2013.\nJump up ^ \"A Summary of the Final Act of the Uruguay Round\". Wto.org. Retrieved 30 May 2010.\nJump up ^ Zarocostas, John (7 December 2013). \"Global Trade Deal Reached\". WWD. Retrieved 8 December 2013.\nJump up ^ \"WT/L/509\". WTO. Retrieved 18 February 2013.\nJump up ^ \"Director-General Elect Azevêdo announces his four Deputy Directors-General\". 17 August 2013. Retrieved 2 September 2013.\nJump up ^ \"Previous GATT and WTO Directors-General\". WTO. Retrieved 21 May 2011.\nExternal links\n\tWikiquote has quotations related to: World Trade Organization\n\tWikimedia Commons has media related to World Trade Organization.\nOfficial pages\nOfficial WTO homepage\nWTO 10th Anniversary PDF (1.40 MB) — Highlights of the first decade, Annual Report 2005 pages 116–166\nGlossary of terms—a guide to 'WTO-speak'\nInternational Trade Centre — joint UN/WTO agency\nGovernment pages on the WTO\nEuropean Union position on the WTO\nMedia pages on the WTO\nWorld Trade Organization\nBBC News — Profile: WTO\nGuardian Unlimited — Special Report: The World Trade Organisation ongoing coverage\nNon-governmental organization pages on the WTO\nGatt.org — Parody of official WTO page by The Yes Men\nPublic Citizen\nTransnational Institute: Beyond the WTO\n[show] v t e\nWorld Trade Organization\n[show] v t e\nInternational trade\n[show] v t e\nInternational organizations\nAuthority control\t\nWorldCat VIAF: 149937768 LCCN: no94018277 ISNI: 0000 0001 2296 2735 GND: 2145784-0 SELIBR: 135910 ULAN: 500292980 NDL: 00577475 NKC: kn20010711437 BNE: XX4574846\nCategories: World Trade OrganizationInternational tradeInternational trade organizationsOrganisations based in GenevaOrganizations established in 1995World government\nNavigation menu\nCreate accountLog inArticleTalkReadView sourceView history\n\nMain page\nContents\nFeatured content\nCurrent events\nRandom article\nDonate to Wikipedia\nWikimedia Shop\nInteraction\nHelp\nAbout Wikipedia\nCommunity portal\nRecent changes\nContact page\nTools\nWhat links here\nRelated changes\nUpload file\nSpecial pages\nPermanent link\nPage information\nWikidata item\nCite this page\nPrint/export\nCreate a book\nDownload as PDF\nPrintable version\nLanguages\nAfrikaans\nالعربية\nAragonés\nAsturianu\nAzərbaycanca\nবাংলা\nBân-lâm-gú\nБеларуская\nБеларуская (тарашкевіца)‎\nБългарски\nBosanski\nBrezhoneg\nCatalà\nČeština\nCymraeg\nDansk\nDeutsch\nEesti\nΕλληνικά\nEspañol\nEsperanto\nEuskara\nفارسی\nFiji Hindi\nFøroyskt\nFrançais\nFrysk\nGalego\nગુજરાતી\n客家語/Hak-kâ-ngî\n한국어\nՀայերեն\nहिन्दी\nHrvatski\nIdo\nIlokano\nBahasa Indonesia\nÍslenska\nItaliano\nעברית\nBasa Jawa\nಕನ್ನಡ\nКъарачай-малкъар\nქართული\nҚазақша\nKiswahili\nLatina\nLatviešu\nLietuvių\nMagyar\nМакедонски\nമലയാളം\nमराठी\nمصرى\nBahasa Melayu\nBaso Minangkabau\nမြန်မာဘာသာ\nNederlands\nनेपाली\nनेपाल भाषा\n日本語\nНохчийн\nNorsk bokmål\nNorsk nynorsk\nOccitan\nOʻzbekcha\nਪੰਜਾਬੀ\nپنجابی\nپښتو\nភាសាខ្មែរ\nPiemontèis\nPolski\nPortuguês\nRomână\nРусиньскый\nРусский\nСаха тыла\nShqip\nසිංහල\nSimple English\nSlovenčina\nSlovenščina\nکوردی\nСрпски / srpski\nSrpskohrvatski / српскохрватски\nSuomi\nSvenska\nTagalog\nதமிழ்\nТатарча/tatarça\nతెలుగు\nไทย\nТоҷикӣ\nTürkçe\nTürkmençe\nУкраїнська\nاردو\nئۇيغۇرچە / Uyghurche\nTiếng Việt\nWinaray\nייִדיש\nYorùbá\n粵語\nŽemaitėška\n中文\nEdit links\nThis page was last modified on 22 November 2014 at 14:33.\nText is available under the Creative Commons Attribution-ShareAlike License; additional terms may apply. By using this site, you agree to the Terms of Use and Privacy Policy. Wikipedia® is a registered trademark of the Wikimedia Foundation, Inc., a non-profit organization.\nPrivacy policyAbout WikipediaDisclaimersContact WikipediaDevelopersMobile viewWikimedia Foundation Powered by MediaWiki"
  },
  {
    "path": "search/facet/facet_builder_datetime.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage facet\n\nimport (\n\t\"reflect\"\n\t\"sort\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n)\n\nvar (\n\treflectStaticSizeDateTimeFacetBuilder int\n\treflectStaticSizedateTimeRange        int\n)\n\nfunc init() {\n\tvar dtfb DateTimeFacetBuilder\n\treflectStaticSizeDateTimeFacetBuilder = int(reflect.TypeOf(dtfb).Size())\n\tvar dtr dateTimeRange\n\treflectStaticSizedateTimeRange = int(reflect.TypeOf(dtr).Size())\n}\n\ntype dateTimeRange struct {\n\tstart time.Time\n\tend   time.Time\n}\n\ntype DateTimeFacetBuilder struct {\n\tsize       int\n\tfield      string\n\ttermsCount map[string]int\n\ttotal      int\n\tmissing    int\n\tranges     map[string]*dateTimeRange\n\tsawValue   bool\n}\n\nfunc NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder {\n\treturn &DateTimeFacetBuilder{\n\t\tsize:       size,\n\t\tfield:      field,\n\t\ttermsCount: make(map[string]int),\n\t\tranges:     make(map[string]*dateTimeRange, 0),\n\t}\n}\n\nfunc (fb *DateTimeFacetBuilder) Size() int {\n\tsizeInBytes := reflectStaticSizeDateTimeFacetBuilder + size.SizeOfPtr +\n\t\tlen(fb.field)\n\n\tfor k := range fb.termsCount {\n\t\tsizeInBytes += size.SizeOfString + len(k) +\n\t\t\tsize.SizeOfInt\n\t}\n\n\tfor k := range fb.ranges {\n\t\tsizeInBytes += size.SizeOfString + len(k) +\n\t\t\tsize.SizeOfPtr + reflectStaticSizedateTimeRange\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) {\n\tr := dateTimeRange{\n\t\tstart: start,\n\t\tend:   end,\n\t}\n\tfb.ranges[name] = &r\n}\n\nfunc (fb *DateTimeFacetBuilder) Field() string {\n\treturn fb.field\n}\n\nfunc (fb *DateTimeFacetBuilder) UpdateVisitor(term []byte) {\n\tfb.sawValue = true\n\t// only consider the values which are shifted 0\n\tprefixCoded := numeric.PrefixCoded(term)\n\tshift, err := prefixCoded.Shift()\n\tif err == nil && shift == 0 {\n\t\ti64, err := prefixCoded.Int64()\n\t\tif err == nil {\n\t\t\tt := time.Unix(0, i64)\n\n\t\t\t// look at each of the ranges for a match\n\t\t\tfor rangeName, r := range fb.ranges {\n\t\t\t\tif (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) {\n\t\t\t\t\tfb.termsCount[rangeName] = fb.termsCount[rangeName] + 1\n\t\t\t\t\tfb.total++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (fb *DateTimeFacetBuilder) StartDoc() {\n\tfb.sawValue = false\n}\n\nfunc (fb *DateTimeFacetBuilder) EndDoc() {\n\tif !fb.sawValue {\n\t\tfb.missing++\n\t}\n}\n\nfunc (fb *DateTimeFacetBuilder) Result() *search.FacetResult {\n\trv := search.FacetResult{\n\t\tField:   fb.field,\n\t\tTotal:   fb.total,\n\t\tMissing: fb.missing,\n\t}\n\n\trv.DateRanges = make([]*search.DateRangeFacet, 0, len(fb.termsCount))\n\n\tfor term, count := range fb.termsCount {\n\t\tdateRange := fb.ranges[term]\n\t\ttf := &search.DateRangeFacet{\n\t\t\tName:  term,\n\t\t\tCount: count,\n\t\t}\n\t\tif !dateRange.start.IsZero() {\n\t\t\tstart := dateRange.start.Format(time.RFC3339Nano)\n\t\t\ttf.Start = &start\n\t\t}\n\t\tif !dateRange.end.IsZero() {\n\t\t\tend := dateRange.end.Format(time.RFC3339Nano)\n\t\t\ttf.End = &end\n\t\t}\n\t\trv.DateRanges = append(rv.DateRanges, tf)\n\t}\n\n\tsort.Sort(rv.DateRanges)\n\n\t// we now have the list of the top N facets\n\tif fb.size < len(rv.DateRanges) {\n\t\trv.DateRanges = rv.DateRanges[:fb.size]\n\t}\n\n\tnotOther := 0\n\tfor _, nr := range rv.DateRanges {\n\t\tnotOther += nr.Count\n\t}\n\trv.Other = fb.total - notOther\n\n\treturn &rv\n}\n"
  },
  {
    "path": "search/facet/facet_builder_numeric.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage facet\n\nimport (\n\t\"reflect\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n)\n\nvar (\n\treflectStaticSizeNumericFacetBuilder int\n\treflectStaticSizenumericRange        int\n)\n\nfunc init() {\n\tvar nfb NumericFacetBuilder\n\treflectStaticSizeNumericFacetBuilder = int(reflect.TypeOf(nfb).Size())\n\tvar nr numericRange\n\treflectStaticSizenumericRange = int(reflect.TypeOf(nr).Size())\n}\n\ntype numericRange struct {\n\tmin *float64\n\tmax *float64\n}\n\ntype NumericFacetBuilder struct {\n\tsize       int\n\tfield      string\n\ttermsCount map[string]int\n\ttotal      int\n\tmissing    int\n\tranges     map[string]*numericRange\n\tsawValue   bool\n}\n\nfunc NewNumericFacetBuilder(field string, size int) *NumericFacetBuilder {\n\treturn &NumericFacetBuilder{\n\t\tsize:       size,\n\t\tfield:      field,\n\t\ttermsCount: make(map[string]int),\n\t\tranges:     make(map[string]*numericRange, 0),\n\t}\n}\n\nfunc (fb *NumericFacetBuilder) Size() int {\n\tsizeInBytes := reflectStaticSizeNumericFacetBuilder + size.SizeOfPtr +\n\t\tlen(fb.field)\n\n\tfor k := range fb.termsCount {\n\t\tsizeInBytes += size.SizeOfString + len(k) +\n\t\t\tsize.SizeOfInt\n\t}\n\n\tfor k := range fb.ranges {\n\t\tsizeInBytes += size.SizeOfString + len(k) +\n\t\t\tsize.SizeOfPtr + reflectStaticSizenumericRange\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (fb *NumericFacetBuilder) AddRange(name string, min, max *float64) {\n\tr := numericRange{\n\t\tmin: min,\n\t\tmax: max,\n\t}\n\tfb.ranges[name] = &r\n}\n\nfunc (fb *NumericFacetBuilder) Field() string {\n\treturn fb.field\n}\n\nfunc (fb *NumericFacetBuilder) UpdateVisitor(term []byte) {\n\tfb.sawValue = true\n\t// only consider the values which are shifted 0\n\tprefixCoded := numeric.PrefixCoded(term)\n\tshift, err := prefixCoded.Shift()\n\tif err == nil && shift == 0 {\n\t\ti64, err := prefixCoded.Int64()\n\t\tif err == nil {\n\t\t\tf64 := numeric.Int64ToFloat64(i64)\n\n\t\t\t// look at each of the ranges for a match\n\t\t\tfor rangeName, r := range fb.ranges {\n\t\t\t\tif (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) {\n\t\t\t\t\tfb.termsCount[rangeName] = fb.termsCount[rangeName] + 1\n\t\t\t\t\tfb.total++\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (fb *NumericFacetBuilder) StartDoc() {\n\tfb.sawValue = false\n}\n\nfunc (fb *NumericFacetBuilder) EndDoc() {\n\tif !fb.sawValue {\n\t\tfb.missing++\n\t}\n}\n\nfunc (fb *NumericFacetBuilder) Result() *search.FacetResult {\n\trv := search.FacetResult{\n\t\tField:   fb.field,\n\t\tTotal:   fb.total,\n\t\tMissing: fb.missing,\n\t}\n\n\trv.NumericRanges = make([]*search.NumericRangeFacet, 0, len(fb.termsCount))\n\n\tfor term, count := range fb.termsCount {\n\t\tnumericRange := fb.ranges[term]\n\t\ttf := &search.NumericRangeFacet{\n\t\t\tName:  term,\n\t\t\tCount: count,\n\t\t\tMin:   numericRange.min,\n\t\t\tMax:   numericRange.max,\n\t\t}\n\n\t\trv.NumericRanges = append(rv.NumericRanges, tf)\n\t}\n\n\tsort.Sort(rv.NumericRanges)\n\n\t// we now have the list of the top N facets\n\tif fb.size < len(rv.NumericRanges) {\n\t\trv.NumericRanges = rv.NumericRanges[:fb.size]\n\t}\n\n\tnotOther := 0\n\tfor _, nr := range rv.NumericRanges {\n\t\tnotOther += nr.Count\n\t}\n\trv.Other = fb.total - notOther\n\n\treturn &rv\n}\n"
  },
  {
    "path": "search/facet/facet_builder_numeric_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage facet\n\nimport (\n\t\"strconv\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n)\n\nvar pcodedvalues []numeric.PrefixCoded\n\nfunc init() {\n\tpcodedvalues = []numeric.PrefixCoded{{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, {0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, {0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7a, 0x1d, 0xa}, {0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x16, 0x9, 0x4a, 0x7b}}\n}\n\nfunc BenchmarkNumericFacet10(b *testing.B) {\n\tnumericFacetN(b, 10)\n}\n\nfunc BenchmarkNumericFacet100(b *testing.B) {\n\tnumericFacetN(b, 100)\n}\n\nfunc BenchmarkNumericFacet1000(b *testing.B) {\n\tnumericFacetN(b, 1000)\n}\n\nfunc numericFacetN(b *testing.B, numTerms int) {\n\tfield := \"test\"\n\tnfb := NewNumericFacetBuilder(field, numTerms)\n\tmin, max := 0.0, 9999999998.0\n\n\tfor i := 0; i <= numTerms; i++ {\n\t\tmax++\n\t\tmin--\n\n\t\tnfb.AddRange(\"rangename\"+strconv.Itoa(i), &min, &max)\n\n\t\tfor _, pv := range pcodedvalues {\n\t\t\tnfb.StartDoc()\n\t\t\tnfb.UpdateVisitor(pv)\n\t\t\tnfb.EndDoc()\n\t\t}\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnfb.Result()\n\t}\n}\n"
  },
  {
    "path": "search/facet/facet_builder_terms.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage facet\n\nimport (\n\t\"bytes\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n)\n\nvar reflectStaticSizeTermsFacetBuilder int\n\nfunc init() {\n\tvar tfb TermsFacetBuilder\n\treflectStaticSizeTermsFacetBuilder = int(reflect.TypeOf(tfb).Size())\n}\n\ntype TermsFacetBuilder struct {\n\tsize        int\n\tfield       string\n\tprefixBytes []byte\n\tregex       *regexp.Regexp\n\ttermsCount  map[string]int\n\ttotal       int\n\tmissing     int\n\tsawValue    bool\n}\n\nfunc NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder {\n\treturn &TermsFacetBuilder{\n\t\tsize:       size,\n\t\tfield:      field,\n\t\ttermsCount: make(map[string]int),\n\t}\n}\n\nfunc (fb *TermsFacetBuilder) Size() int {\n\tsizeInBytes := reflectStaticSizeTermsFacetBuilder + size.SizeOfPtr +\n\t\tlen(fb.field) +\n\t\tlen(fb.prefixBytes) +\n\t\tsize.SizeOfPtr // regex pointer (does not include actual regexp.Regexp object size)\n\n\t// Estimate regex object size if present.\n\tif fb.regex != nil {\n\t\t// This is only the static size of regexp.Regexp struct, not including heap allocations.\n\t\tsizeInBytes += int(reflect.TypeOf(*fb.regex).Size())\n\t\t// NOTE: Actual memory usage of regexp.Regexp may be higher due to internal allocations.\n\t}\n\n\tfor k := range fb.termsCount {\n\t\tsizeInBytes += size.SizeOfString + len(k) +\n\t\t\tsize.SizeOfInt\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (fb *TermsFacetBuilder) Field() string {\n\treturn fb.field\n}\n\n// SetPrefixFilter sets the prefix filter for term facets.\nfunc (fb *TermsFacetBuilder) SetPrefixFilter(prefix string) {\n\tif prefix != \"\" {\n\t\tfb.prefixBytes = []byte(prefix)\n\t} else {\n\t\tfb.prefixBytes = nil\n\t}\n}\n\n// SetRegexFilter sets the compiled regex filter for term facets.\nfunc (fb *TermsFacetBuilder) SetRegexFilter(regex *regexp.Regexp) {\n\tfb.regex = regex\n}\n\nfunc (fb *TermsFacetBuilder) UpdateVisitor(term []byte) {\n\t// Total represents all terms visited, not just matching ones.\n\t// This is necessary for the \"Other\" calculation.\n\tfb.total++\n\n\t// Fast prefix check on []byte - zero allocation\n\tif len(fb.prefixBytes) > 0 && !bytes.HasPrefix(term, fb.prefixBytes) {\n\t\treturn\n\t}\n\n\t// Fast regex check on []byte - zero allocation\n\tif fb.regex != nil && !fb.regex.Match(term) {\n\t\treturn\n\t}\n\n\t// Only convert to string if term matches filters\n\ttermStr := string(term)\n\tfb.sawValue = true\n\tfb.termsCount[termStr] = fb.termsCount[termStr] + 1\n}\n\nfunc (fb *TermsFacetBuilder) StartDoc() {\n\tfb.sawValue = false\n}\n\nfunc (fb *TermsFacetBuilder) EndDoc() {\n\tif !fb.sawValue {\n\t\tfb.missing++\n\t}\n}\n\nfunc (fb *TermsFacetBuilder) Result() *search.FacetResult {\n\trv := search.FacetResult{\n\t\tField:   fb.field,\n\t\tTotal:   fb.total,\n\t\tMissing: fb.missing,\n\t}\n\n\trv.Terms = &search.TermFacets{}\n\n\tfor term, count := range fb.termsCount {\n\t\ttf := &search.TermFacet{\n\t\t\tTerm:  term,\n\t\t\tCount: count,\n\t\t}\n\n\t\trv.Terms.Add(tf)\n\t}\n\n\tsort.Sort(rv.Terms)\n\n\t// we now have the list of the top N facets\n\ttrimTopN := fb.size\n\tif trimTopN > rv.Terms.Len() {\n\t\ttrimTopN = rv.Terms.Len()\n\t}\n\trv.Terms.TrimToTopN(trimTopN)\n\n\tnotOther := 0\n\tfor _, tf := range rv.Terms.Terms() {\n\t\tnotOther += tf.Count\n\t}\n\trv.Other = fb.total - notOther\n\n\treturn &rv\n}\n"
  },
  {
    "path": "search/facet/facet_builder_terms_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage facet\n\nimport (\n\t\"os\"\n\t\"regexp\"\n\t\"testing\"\n)\n\nvar terms []string\n\nfunc init() {\n\twsRegexp := regexp.MustCompile(`\\W+`)\n\tinput, err := os.ReadFile(\"benchmark_data.txt\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tterms = wsRegexp.Split(string(input), -1)\n}\n\nfunc BenchmarkTermsFacet10(b *testing.B) {\n\ttermsFacetN(b, 10)\n}\n\nfunc BenchmarkTermsFacet100(b *testing.B) {\n\ttermsFacetN(b, 100)\n}\n\nfunc BenchmarkTermsFacet1000(b *testing.B) {\n\ttermsFacetN(b, 1000)\n}\n\nfunc BenchmarkTermsFacet10000(b *testing.B) {\n\ttermsFacetN(b, 10000)\n}\n\n// func BenchmarkTermsFacet100000(b *testing.B) {\n// \ttermsFacetN(b, 100000)\n// }\n\nfunc termsFacetN(b *testing.B, numTerms int) {\n\tfield := \"test\"\n\ttermsLen := len(terms)\n\ttfb := NewTermsFacetBuilder(field, 3)\n\ti := 0\n\tfor len(tfb.termsCount) < numTerms && i <= termsLen {\n\t\tj := i % termsLen\n\t\tterm := terms[j]\n\t\ttfb.StartDoc()\n\t\ttfb.UpdateVisitor([]byte(term))\n\t\ttfb.EndDoc()\n\t\ti++\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\ttfb.Result()\n\t}\n}\n\nfunc TestTermsFacetPrefix(t *testing.T) {\n\tfield := \"category\"\n\ttfb := NewTermsFacetBuilder(field, 10)\n\ttfb.SetPrefixFilter(\"prod-\")\n\n\t// Add terms with various prefixes\n\tterms := []string{\n\t\t\"prod-server\",\n\t\t\"prod-database\",\n\t\t\"dev-server\",\n\t\t\"dev-database\",\n\t\t\"test-server\",\n\t\t\"prod-cache\",\n\t}\n\n\tfor _, term := range terms {\n\t\ttfb.StartDoc()\n\t\ttfb.UpdateVisitor([]byte(term))\n\t\ttfb.EndDoc()\n\t}\n\n\tresult := tfb.Result()\n\n\t// Should only have terms with \"prod-\" prefix\n\tif result.Terms.Len() != 3 {\n\t\tt.Fatalf(\"expected 3 matching terms, got %d\", result.Terms.Len())\n\t}\n\n\t// Verify the terms are correct\n\texpectedTerms := map[string]bool{\n\t\t\"prod-server\":   true,\n\t\t\"prod-database\": true,\n\t\t\"prod-cache\":    true,\n\t}\n\n\tfor _, facet := range result.Terms.Terms() {\n\t\tif !expectedTerms[facet.Term] {\n\t\t\tt.Errorf(\"unexpected term in results: %s\", facet.Term)\n\t\t}\n\t\tif facet.Count != 1 {\n\t\t\tt.Errorf(\"expected count 1 for %s, got %d\", facet.Term, facet.Count)\n\t\t}\n\t}\n\n\t// Total should include all terms (matching + non-matching)\n\tif result.Total != 6 {\n\t\tt.Errorf(\"expected total 6, got %d\", result.Total)\n\t}\n\n\t// Other should be 3 (the non-matching terms)\n\tif result.Other != 3 {\n\t\tt.Errorf(\"expected other 3, got %d\", result.Other)\n\t}\n}\n\nfunc TestTermsFacetRegex(t *testing.T) {\n\tfield := \"product_code\"\n\t// Match pattern: ABC-#### (3 letters, dash, 4 digits) - pattern: ^[A-Z]{3}-\\\\d{4}$\n\ttfb := NewTermsFacetBuilder(field, 10)\n\tregex, err := regexp.Compile(\"^[A-Z]{3}-\\\\d{4}$\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttfb.SetRegexFilter(regex)\n\n\t// Add terms with various formats\n\tterms := []string{\n\t\t\"ABC-1234\",\n\t\t\"XYZ-5678\",\n\t\t\"ABC-999\",   // too few digits\n\t\t\"ABCD-1234\", // too many letters\n\t\t\"ABC-ABCD\",  // letters instead of digits\n\t\t\"DEF-0000\",\n\t}\n\n\tfor _, term := range terms {\n\t\ttfb.StartDoc()\n\t\ttfb.UpdateVisitor([]byte(term))\n\t\ttfb.EndDoc()\n\t}\n\n\tresult := tfb.Result()\n\n\t// Should only have 3 terms matching the pattern\n\tif result.Terms.Len() != 3 {\n\t\tt.Fatalf(\"expected 3 matching terms, got %d\", result.Terms.Len())\n\t}\n\n\t// Verify the terms are correct\n\texpectedTerms := map[string]bool{\n\t\t\"ABC-1234\": true,\n\t\t\"XYZ-5678\": true,\n\t\t\"DEF-0000\": true,\n\t}\n\n\tfor _, facet := range result.Terms.Terms() {\n\t\tif !expectedTerms[facet.Term] {\n\t\t\tt.Errorf(\"unexpected term in results: %s\", facet.Term)\n\t\t}\n\t\tif facet.Count != 1 {\n\t\t\tt.Errorf(\"expected count 1 for %s, got %d\", facet.Term, facet.Count)\n\t\t}\n\t}\n\n\t// Total should include all terms\n\tif result.Total != 6 {\n\t\tt.Errorf(\"expected total 6, got %d\", result.Total)\n\t}\n\n\t// Other should be 3 (the non-matching terms)\n\tif result.Other != 3 {\n\t\tt.Errorf(\"expected other 3, got %d\", result.Other)\n\t}\n}\n\nfunc TestTermsFacetPrefixAndRegex(t *testing.T) {\n\tfield := \"tag\"\n\t// Both prefix \"env:\" and regex pattern for prod/staging only\n\ttfb := NewTermsFacetBuilder(field, 10)\n\ttfb.SetPrefixFilter(\"env:\")\n\tregex, err := regexp.Compile(\"^env:(prod|staging)$\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttfb.SetRegexFilter(regex)\n\n\t// Add various terms\n\tterms := []string{\n\t\t\"env:prod\",\n\t\t\"env:staging\",\n\t\t\"env:dev\",     // has prefix but doesn't match regex\n\t\t\"env:test\",    // has prefix but doesn't match regex\n\t\t\"type:server\", // no prefix\n\t\t\"env:prod\",    // duplicate\n\t\t\"env:staging\", // duplicate\n\t}\n\n\tfor _, term := range terms {\n\t\ttfb.StartDoc()\n\t\ttfb.UpdateVisitor([]byte(term))\n\t\ttfb.EndDoc()\n\t}\n\n\tresult := tfb.Result()\n\n\t// Should only have 2 unique terms (env:prod and env:staging)\n\tif result.Terms.Len() != 2 {\n\t\tt.Fatalf(\"expected 2 matching terms, got %d\", result.Terms.Len())\n\t}\n\n\t// Verify the terms and counts\n\ttermCounts := make(map[string]int)\n\tfor _, facet := range result.Terms.Terms() {\n\t\ttermCounts[facet.Term] = facet.Count\n\t}\n\n\tif termCounts[\"env:prod\"] != 2 {\n\t\tt.Errorf(\"expected count 2 for env:prod, got %d\", termCounts[\"env:prod\"])\n\t}\n\tif termCounts[\"env:staging\"] != 2 {\n\t\tt.Errorf(\"expected count 2 for env:staging, got %d\", termCounts[\"env:staging\"])\n\t}\n\n\t// Total should be all 7 terms\n\tif result.Total != 7 {\n\t\tt.Errorf(\"expected total 7, got %d\", result.Total)\n\t}\n\n\t// Other should be 3 (env:dev, env:test, type:server)\n\tif result.Other != 3 {\n\t\tt.Errorf(\"expected other 3, got %d\", result.Other)\n\t}\n}\n\nfunc TestTermsFacetInvalidRegex(t *testing.T) {\n\t// Invalid regex pattern (unmatched bracket)\n\t_, err := regexp.Compile(\"[invalid\")\n\tif err == nil {\n\t\tt.Fatal(\"expected error for invalid regex, got nil\")\n\t}\n}\n\nfunc TestTermsFacetNoFilter(t *testing.T) {\n\tfield := \"tag\"\n\ttfb := NewTermsFacetBuilder(field, 2)\n\n\tterms := []string{\"apple\", \"banana\", \"cherry\", \"apple\"}\n\n\tfor _, term := range terms {\n\t\ttfb.StartDoc()\n\t\ttfb.UpdateVisitor([]byte(term))\n\t\ttfb.EndDoc()\n\t}\n\n\tresult := tfb.Result()\n\n\t// Should return top 2 by count\n\tif result.Terms.Len() != 2 {\n\t\tt.Fatalf(\"expected 2 terms, got %d\", result.Terms.Len())\n\t}\n\n\t// Apple should be first with count 2\n\tfacets := result.Terms.Terms()\n\tif facets[0].Term != \"apple\" || facets[0].Count != 2 {\n\t\tt.Errorf(\"expected apple with count 2, got %s with count %d\", facets[0].Term, facets[0].Count)\n\t}\n\n\t// Total should be 4\n\tif result.Total != 4 {\n\t\tt.Errorf(\"expected total 4, got %d\", result.Total)\n\t}\n\n\t// Other should be 1 (cherry was trimmed)\n\tif result.Other != 1 {\n\t\tt.Errorf(\"expected other 1, got %d\", result.Other)\n\t}\n}\n"
  },
  {
    "path": "search/facets_builder.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"reflect\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeFacetsBuilder int\nvar reflectStaticSizeFacetResult int\nvar reflectStaticSizeTermFacet int\nvar reflectStaticSizeNumericRangeFacet int\nvar reflectStaticSizeDateRangeFacet int\n\nfunc init() {\n\tvar fb FacetsBuilder\n\treflectStaticSizeFacetsBuilder = int(reflect.TypeOf(fb).Size())\n\tvar fr FacetResult\n\treflectStaticSizeFacetResult = int(reflect.TypeOf(fr).Size())\n\tvar tf TermFacet\n\treflectStaticSizeTermFacet = int(reflect.TypeOf(tf).Size())\n\tvar nrf NumericRangeFacet\n\treflectStaticSizeNumericRangeFacet = int(reflect.TypeOf(nrf).Size())\n\tvar drf DateRangeFacet\n\treflectStaticSizeDateRangeFacet = int(reflect.TypeOf(drf).Size())\n}\n\ntype FacetBuilder interface {\n\tStartDoc()\n\tUpdateVisitor(term []byte)\n\tEndDoc()\n\n\tResult() *FacetResult\n\tField() string\n\n\tSize() int\n}\n\ntype FacetsBuilder struct {\n\tindexReader   index.IndexReader\n\tfacetNames    []string\n\tfacets        []FacetBuilder\n\tfacetsByField map[string][]FacetBuilder\n\tfields        []string\n}\n\nfunc NewFacetsBuilder(indexReader index.IndexReader) *FacetsBuilder {\n\treturn &FacetsBuilder{\n\t\tindexReader: indexReader,\n\t}\n}\n\nfunc (fb *FacetsBuilder) Size() int {\n\tsizeInBytes := reflectStaticSizeFacetsBuilder + size.SizeOfPtr\n\n\tfor k, v := range fb.facets {\n\t\tsizeInBytes += size.SizeOfString + v.Size() + len(fb.facetNames[k])\n\t}\n\n\tfor _, entry := range fb.fields {\n\t\tsizeInBytes += size.SizeOfString + len(entry)\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) {\n\tif fb.facetsByField == nil {\n\t\tfb.facetsByField = map[string][]FacetBuilder{}\n\t}\n\n\tfb.facetNames = append(fb.facetNames, name)\n\tfb.facets = append(fb.facets, facetBuilder)\n\tfb.facetsByField[facetBuilder.Field()] = append(fb.facetsByField[facetBuilder.Field()], facetBuilder)\n\tfb.fields = append(fb.fields, facetBuilder.Field())\n}\n\nfunc (fb *FacetsBuilder) RequiredFields() []string {\n\treturn fb.fields\n}\n\nfunc (fb *FacetsBuilder) StartDoc() {\n\tfor _, facetBuilder := range fb.facets {\n\t\tfacetBuilder.StartDoc()\n\t}\n}\n\nfunc (fb *FacetsBuilder) EndDoc() {\n\tfor _, facetBuilder := range fb.facets {\n\t\tfacetBuilder.EndDoc()\n\t}\n}\n\nfunc (fb *FacetsBuilder) UpdateVisitor(field string, term []byte) {\n\tif facetBuilders, ok := fb.facetsByField[field]; ok {\n\t\tfor _, facetBuilder := range facetBuilders {\n\t\t\tfacetBuilder.UpdateVisitor(term)\n\t\t}\n\t}\n}\n\ntype TermFacet struct {\n\tTerm  string `json:\"term\"`\n\tCount int    `json:\"count\"`\n}\n\ntype TermFacets struct {\n\ttermFacets []*TermFacet\n\ttermLookup map[string]*TermFacet\n}\n\nfunc (tf *TermFacets) Terms() []*TermFacet {\n\tif tf == nil {\n\t\treturn []*TermFacet{}\n\t}\n\treturn tf.termFacets\n}\n\nfunc (tf *TermFacets) TrimToTopN(n int) {\n\ttf.termFacets = tf.termFacets[:n]\n}\n\nfunc (tf *TermFacets) Add(termFacets ...*TermFacet) {\n\tfor _, termFacet := range termFacets {\n\t\tif tf.termLookup == nil {\n\t\t\ttf.termLookup = map[string]*TermFacet{}\n\t\t}\n\n\t\tif term, ok := tf.termLookup[termFacet.Term]; ok {\n\t\t\tterm.Count += termFacet.Count\n\t\t\treturn\n\t\t}\n\n\t\t// if we got here it wasn't already in the existing terms\n\t\ttf.termFacets = append(tf.termFacets, termFacet)\n\t\ttf.termLookup[termFacet.Term] = termFacet\n\t}\n}\n\nfunc (tf *TermFacets) Len() int {\n\t// Handle case where *TermFacets is not fully initialized in index_impl.go.init()\n\tif tf == nil {\n\t\treturn 0\n\t}\n\n\treturn len(tf.termFacets)\n}\nfunc (tf *TermFacets) Swap(i, j int) {\n\ttf.termFacets[i], tf.termFacets[j] = tf.termFacets[j], tf.termFacets[i]\n}\nfunc (tf *TermFacets) Less(i, j int) bool {\n\tif tf.termFacets[i].Count == tf.termFacets[j].Count {\n\t\treturn tf.termFacets[i].Term < tf.termFacets[j].Term\n\t}\n\treturn tf.termFacets[i].Count > tf.termFacets[j].Count\n}\n\n// TermFacets used to be a type alias for []*TermFacet.\n// To maintain backwards compatibility, we have to implement custom\n// JSON marshalling.\nfunc (tf *TermFacets) MarshalJSON() ([]byte, error) {\n\treturn util.MarshalJSON(tf.termFacets)\n}\n\nfunc (tf *TermFacets) UnmarshalJSON(b []byte) error {\n\ttermFacets := []*TermFacet{}\n\terr := util.UnmarshalJSON(b, &termFacets)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, termFacet := range termFacets {\n\t\ttf.Add(termFacet)\n\t}\n\n\treturn nil\n}\n\ntype NumericRangeFacet struct {\n\tName  string   `json:\"name\"`\n\tMin   *float64 `json:\"min,omitempty\"`\n\tMax   *float64 `json:\"max,omitempty\"`\n\tCount int      `json:\"count\"`\n}\n\nfunc (nrf *NumericRangeFacet) Same(other *NumericRangeFacet) bool {\n\tif nrf.Min == nil && other.Min != nil {\n\t\treturn false\n\t}\n\tif nrf.Min != nil && other.Min == nil {\n\t\treturn false\n\t}\n\tif nrf.Min != nil && other.Min != nil && *nrf.Min != *other.Min {\n\t\treturn false\n\t}\n\tif nrf.Max == nil && other.Max != nil {\n\t\treturn false\n\t}\n\tif nrf.Max != nil && other.Max == nil {\n\t\treturn false\n\t}\n\tif nrf.Max != nil && other.Max != nil && *nrf.Max != *other.Max {\n\t\treturn false\n\t}\n\n\treturn true\n}\n\ntype NumericRangeFacets []*NumericRangeFacet\n\nfunc (nrf NumericRangeFacets) Add(numericRangeFacet *NumericRangeFacet) NumericRangeFacets {\n\tfor _, existingNr := range nrf {\n\t\tif numericRangeFacet.Same(existingNr) {\n\t\t\texistingNr.Count += numericRangeFacet.Count\n\t\t\treturn nrf\n\t\t}\n\t}\n\t// if we got here it wasn't already in the existing terms\n\tnrf = append(nrf, numericRangeFacet)\n\treturn nrf\n}\n\nfunc (nrf NumericRangeFacets) Len() int      { return len(nrf) }\nfunc (nrf NumericRangeFacets) Swap(i, j int) { nrf[i], nrf[j] = nrf[j], nrf[i] }\nfunc (nrf NumericRangeFacets) Less(i, j int) bool {\n\tif nrf[i].Count == nrf[j].Count {\n\t\treturn nrf[i].Name < nrf[j].Name\n\t}\n\treturn nrf[i].Count > nrf[j].Count\n}\n\ntype DateRangeFacet struct {\n\tName  string  `json:\"name\"`\n\tStart *string `json:\"start,omitempty\"`\n\tEnd   *string `json:\"end,omitempty\"`\n\tCount int     `json:\"count\"`\n}\n\nfunc (drf *DateRangeFacet) Same(other *DateRangeFacet) bool {\n\tif drf.Start == nil && other.Start != nil {\n\t\treturn false\n\t}\n\tif drf.Start != nil && other.Start == nil {\n\t\treturn false\n\t}\n\tif drf.Start != nil && other.Start != nil && *drf.Start != *other.Start {\n\t\treturn false\n\t}\n\tif drf.End == nil && other.End != nil {\n\t\treturn false\n\t}\n\tif drf.End != nil && other.End == nil {\n\t\treturn false\n\t}\n\tif drf.End != nil && other.End != nil && *drf.End != *other.End {\n\t\treturn false\n\t}\n\n\treturn true\n}\n\ntype DateRangeFacets []*DateRangeFacet\n\nfunc (drf DateRangeFacets) Add(dateRangeFacet *DateRangeFacet) DateRangeFacets {\n\tfor _, existingDr := range drf {\n\t\tif dateRangeFacet.Same(existingDr) {\n\t\t\texistingDr.Count += dateRangeFacet.Count\n\t\t\treturn drf\n\t\t}\n\t}\n\t// if we got here it wasn't already in the existing terms\n\tdrf = append(drf, dateRangeFacet)\n\treturn drf\n}\n\nfunc (drf DateRangeFacets) Len() int      { return len(drf) }\nfunc (drf DateRangeFacets) Swap(i, j int) { drf[i], drf[j] = drf[j], drf[i] }\nfunc (drf DateRangeFacets) Less(i, j int) bool {\n\tif drf[i].Count == drf[j].Count {\n\t\treturn drf[i].Name < drf[j].Name\n\t}\n\treturn drf[i].Count > drf[j].Count\n}\n\ntype FacetResult struct {\n\tField         string             `json:\"field\"`\n\tTotal         int                `json:\"total\"`\n\tMissing       int                `json:\"missing\"`\n\tOther         int                `json:\"other\"`\n\tTerms         *TermFacets        `json:\"terms,omitempty\"`\n\tNumericRanges NumericRangeFacets `json:\"numeric_ranges,omitempty\"`\n\tDateRanges    DateRangeFacets    `json:\"date_ranges,omitempty\"`\n}\n\nfunc (fr *FacetResult) Size() int {\n\treturn reflectStaticSizeFacetResult + size.SizeOfPtr +\n\t\tlen(fr.Field) +\n\t\tfr.Terms.Len()*(reflectStaticSizeTermFacet+size.SizeOfPtr) +\n\t\tlen(fr.NumericRanges)*(reflectStaticSizeNumericRangeFacet+size.SizeOfPtr) +\n\t\tlen(fr.DateRanges)*(reflectStaticSizeDateRangeFacet+size.SizeOfPtr)\n}\n\nfunc (fr *FacetResult) Merge(other *FacetResult) {\n\tfr.Total += other.Total\n\tfr.Missing += other.Missing\n\tfr.Other += other.Other\n\tif other.Terms != nil {\n\t\tif fr.Terms == nil {\n\t\t\tfr.Terms = other.Terms\n\t\t\treturn\n\t\t}\n\t\tfor _, term := range other.Terms.termFacets {\n\t\t\tfr.Terms.Add(term)\n\t\t}\n\t}\n\tif other.NumericRanges != nil {\n\t\tif fr.NumericRanges == nil {\n\t\t\tfr.NumericRanges = other.NumericRanges\n\t\t\treturn\n\t\t}\n\t\tfor _, nr := range other.NumericRanges {\n\t\t\tfr.NumericRanges = fr.NumericRanges.Add(nr)\n\t\t}\n\t}\n\tif other.DateRanges != nil {\n\t\tif fr.DateRanges == nil {\n\t\t\tfr.DateRanges = other.DateRanges\n\t\t\treturn\n\t\t}\n\t\tfor _, dr := range other.DateRanges {\n\t\t\tfr.DateRanges = fr.DateRanges.Add(dr)\n\t\t}\n\t}\n}\n\nfunc (fr *FacetResult) Fixup(size int) {\n\tif fr.Terms != nil {\n\t\tsort.Sort(fr.Terms)\n\t\tif fr.Terms.Len() > size {\n\t\t\tmoveToOther := fr.Terms.termFacets[size:]\n\t\t\tfor _, mto := range moveToOther {\n\t\t\t\tfr.Other += mto.Count\n\t\t\t}\n\t\t\tfr.Terms.termFacets = fr.Terms.termFacets[0:size]\n\t\t}\n\t} else if fr.NumericRanges != nil {\n\t\tsort.Sort(fr.NumericRanges)\n\t\tif len(fr.NumericRanges) > size {\n\t\t\tmoveToOther := fr.NumericRanges[size:]\n\t\t\tfor _, mto := range moveToOther {\n\t\t\t\tfr.Other += mto.Count\n\t\t\t}\n\t\t\tfr.NumericRanges = fr.NumericRanges[0:size]\n\t\t}\n\t} else if fr.DateRanges != nil {\n\t\tsort.Sort(fr.DateRanges)\n\t\tif len(fr.DateRanges) > size {\n\t\t\tmoveToOther := fr.DateRanges[size:]\n\t\t\tfor _, mto := range moveToOther {\n\t\t\t\tfr.Other += mto.Count\n\t\t\t}\n\t\t\tfr.DateRanges = fr.DateRanges[0:size]\n\t\t}\n\t}\n}\n\ntype FacetResults map[string]*FacetResult\n\nfunc (fr FacetResults) Merge(other FacetResults) {\n\tfor name, oFacetResult := range other {\n\t\tfacetResult, ok := fr[name]\n\t\tif ok {\n\t\t\tfacetResult.Merge(oFacetResult)\n\t\t} else {\n\t\t\tfr[name] = oFacetResult\n\t\t}\n\t}\n}\n\nfunc (fr FacetResults) Fixup(name string, size int) {\n\tfacetResult, ok := fr[name]\n\tif ok {\n\t\tfacetResult.Fixup(size)\n\t}\n}\n\nfunc (fb *FacetsBuilder) Results() FacetResults {\n\tfr := make(FacetResults)\n\tfor i, facetBuilder := range fb.facets {\n\t\tfacetResult := facetBuilder.Result()\n\t\tfr[fb.facetNames[i]] = facetResult\n\t}\n\treturn fr\n}\n"
  },
  {
    "path": "search/facets_builder_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestTermFacetResultsMerge(t *testing.T) {\n\ttype testCase struct {\n\t\t// Input\n\t\tfrs1   FacetResults   // first facet results\n\t\tfrs2   FacetResults   // second facet results (to be merged into first)\n\t\tfixups map[string]int // {facetName:size} (to be applied after merge)\n\n\t\t// Expected output\n\t\texpFrs FacetResults // facet results after merge and fixup\n\t}\n\n\ttests := []*testCase{\n\t\tfunc() *testCase {\n\t\t\trv := &testCase{}\n\n\t\t\trv.frs1 = FacetResults{\n\t\t\t\t\"types\": &FacetResult{\n\t\t\t\t\tField:   \"type\",\n\t\t\t\t\tTotal:   100,\n\t\t\t\t\tMissing: 25,\n\t\t\t\t\tOther:   25,\n\t\t\t\t\tTerms: func() *TermFacets {\n\t\t\t\t\t\ttfs := &TermFacets{}\n\t\t\t\t\t\ttfs.Add(\n\t\t\t\t\t\t\t&TermFacet{\n\t\t\t\t\t\t\t\tTerm:  \"blog\",\n\t\t\t\t\t\t\t\tCount: 25,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t&TermFacet{\n\t\t\t\t\t\t\t\tTerm:  \"comment\",\n\t\t\t\t\t\t\t\tCount: 24,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t&TermFacet{\n\t\t\t\t\t\t\t\tTerm:  \"feedback\",\n\t\t\t\t\t\t\t\tCount: 1,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t)\n\t\t\t\t\t\treturn tfs\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\t\"categories\": &FacetResult{\n\t\t\t\t\tField:   \"category\",\n\t\t\t\t\tTotal:   97,\n\t\t\t\t\tMissing: 22,\n\t\t\t\t\tOther:   15,\n\t\t\t\t\tTerms: func() *TermFacets {\n\t\t\t\t\t\ttfs := &TermFacets{}\n\t\t\t\t\t\ttfs.Add(\n\t\t\t\t\t\t\t&TermFacet{\n\t\t\t\t\t\t\t\tTerm:  \"clothing\",\n\t\t\t\t\t\t\t\tCount: 35,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t&TermFacet{\n\t\t\t\t\t\t\t\tTerm:  \"electronics\",\n\t\t\t\t\t\t\t\tCount: 25,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t)\n\t\t\t\t\t\treturn tfs\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t}\n\t\t\trv.frs2 = FacetResults{\n\t\t\t\t\"types\": &FacetResult{\n\t\t\t\t\tField:   \"type\",\n\t\t\t\t\tTotal:   100,\n\t\t\t\t\tMissing: 25,\n\t\t\t\t\tOther:   25,\n\t\t\t\t\tTerms: func() *TermFacets {\n\t\t\t\t\t\ttfs := &TermFacets{}\n\t\t\t\t\t\ttfs.Add(\n\t\t\t\t\t\t\t&TermFacet{\n\t\t\t\t\t\t\t\tTerm:  \"blog\",\n\t\t\t\t\t\t\t\tCount: 25,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t&TermFacet{\n\t\t\t\t\t\t\t\tTerm:  \"comment\",\n\t\t\t\t\t\t\t\tCount: 22,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t&TermFacet{\n\t\t\t\t\t\t\t\tTerm:  \"flag\",\n\t\t\t\t\t\t\t\tCount: 3,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t)\n\t\t\t\t\t\treturn tfs\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t}\n\t\t\trv.fixups = map[string]int{\n\t\t\t\t\"types\": 3, // we want top 3 terms based on count\n\t\t\t}\n\n\t\t\trv.expFrs = FacetResults{\n\t\t\t\t\"types\": &FacetResult{\n\t\t\t\t\tField:   \"type\",\n\t\t\t\t\tTotal:   200,\n\t\t\t\t\tMissing: 50,\n\t\t\t\t\tOther:   51,\n\t\t\t\t\tTerms: &TermFacets{\n\t\t\t\t\t\ttermFacets: []*TermFacet{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tTerm:  \"blog\",\n\t\t\t\t\t\t\t\tCount: 50,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tTerm:  \"comment\",\n\t\t\t\t\t\t\t\tCount: 46,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tTerm:  \"flag\",\n\t\t\t\t\t\t\t\tCount: 3,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"categories\": rv.frs1[\"categories\"],\n\t\t\t}\n\n\t\t\treturn rv\n\t\t}(),\n\t\tfunc() *testCase {\n\t\t\trv := &testCase{}\n\n\t\t\trv.frs1 = FacetResults{\n\t\t\t\t\"facetName\": &FacetResult{\n\t\t\t\t\tField:   \"docField\",\n\t\t\t\t\tTotal:   0,\n\t\t\t\t\tMissing: 0,\n\t\t\t\t\tOther:   0,\n\t\t\t\t\tTerms:   nil,\n\t\t\t\t},\n\t\t\t}\n\t\t\trv.frs2 = FacetResults{\n\t\t\t\t\"facetName\": &FacetResult{\n\t\t\t\t\tField:   \"docField\",\n\t\t\t\t\tTotal:   3,\n\t\t\t\t\tMissing: 0,\n\t\t\t\t\tOther:   0,\n\t\t\t\t\tTerms: &TermFacets{\n\t\t\t\t\t\ttermFacets: []*TermFacet{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tTerm:  \"firstTerm\",\n\t\t\t\t\t\t\t\tCount: 1,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tTerm:  \"secondTerm\",\n\t\t\t\t\t\t\t\tCount: 2,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t}\n\t\t\trv.fixups = map[string]int{\n\t\t\t\t\"facetName\": 1,\n\t\t\t}\n\n\t\t\trv.expFrs = FacetResults{\n\t\t\t\t\"facetName\": &FacetResult{\n\t\t\t\t\tField:   \"docField\",\n\t\t\t\t\tTotal:   3,\n\t\t\t\t\tMissing: 0,\n\t\t\t\t\tOther:   1,\n\t\t\t\t\tTerms: &TermFacets{\n\t\t\t\t\t\ttermFacets: []*TermFacet{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tTerm:  \"secondTerm\",\n\t\t\t\t\t\t\t\tCount: 2,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t}\n\t\t\treturn rv\n\t\t}(),\n\t}\n\n\tfor tcIdx, tc := range tests {\n\t\tt.Run(fmt.Sprintf(\"T#%d\", tcIdx), func(t *testing.T) {\n\t\t\ttc.frs1.Merge(tc.frs2)\n\t\t\tfor facetName, size := range tc.fixups {\n\t\t\t\ttc.frs1.Fixup(facetName, size)\n\t\t\t}\n\n\t\t\t// clear termLookup, so we can compare the facet results\n\t\t\tfor _, fr := range tc.frs1 {\n\t\t\t\tif fr.Terms != nil {\n\t\t\t\t\tfr.Terms.termLookup = nil\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(tc.frs1, tc.expFrs) {\n\t\t\t\tt.Errorf(\"expected %v, got %v\", tc.expFrs, tc.frs1)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNumericFacetResultsMerge(t *testing.T) {\n\n\tlowmed := 3.0\n\tmedhi := 6.0\n\thihigher := 9.0\n\n\t// why second copy? the pointers may be different, but values the same\n\tlowmed2 := 3.0\n\tmedhi2 := 6.0\n\thihigher2 := 9.0\n\n\tfr1 := &FacetResult{\n\t\tField:   \"rating\",\n\t\tTotal:   100,\n\t\tMissing: 25,\n\t\tOther:   25,\n\t\tNumericRanges: []*NumericRangeFacet{\n\t\t\t{\n\t\t\t\tName:  \"low\",\n\t\t\t\tMax:   &lowmed,\n\t\t\t\tCount: 25,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"med\",\n\t\t\t\tCount: 24,\n\t\t\t\tMax:   &lowmed,\n\t\t\t\tMin:   &medhi,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"hi\",\n\t\t\t\tCount: 1,\n\t\t\t\tMin:   &medhi,\n\t\t\t\tMax:   &hihigher,\n\t\t\t},\n\t\t},\n\t}\n\tfrs1 := FacetResults{\n\t\t\"ratings\": fr1,\n\t}\n\n\tfr2 := &FacetResult{\n\t\tField:   \"rating\",\n\t\tTotal:   100,\n\t\tMissing: 25,\n\t\tOther:   25,\n\t\tNumericRanges: []*NumericRangeFacet{\n\t\t\t{\n\t\t\t\tName:  \"low\",\n\t\t\t\tMax:   &lowmed2,\n\t\t\t\tCount: 25,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"med\",\n\t\t\t\tMax:   &lowmed2,\n\t\t\t\tMin:   &medhi2,\n\t\t\t\tCount: 22,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"highest\",\n\t\t\t\tMin:   &hihigher2,\n\t\t\t\tCount: 3,\n\t\t\t},\n\t\t},\n\t}\n\tfrs2 := FacetResults{\n\t\t\"ratings\": fr2,\n\t}\n\n\texpectedFr := &FacetResult{\n\t\tField:   \"rating\",\n\t\tTotal:   200,\n\t\tMissing: 50,\n\t\tOther:   51,\n\t\tNumericRanges: []*NumericRangeFacet{\n\t\t\t{\n\t\t\t\tName:  \"low\",\n\t\t\t\tCount: 50,\n\t\t\t\tMax:   &lowmed,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"med\",\n\t\t\t\tMax:   &lowmed,\n\t\t\t\tMin:   &medhi,\n\t\t\t\tCount: 46,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"highest\",\n\t\t\t\tMin:   &hihigher,\n\t\t\t\tCount: 3,\n\t\t\t},\n\t\t},\n\t}\n\texpectedFrs := FacetResults{\n\t\t\"ratings\": expectedFr,\n\t}\n\n\tfrs1.Merge(frs2)\n\tfrs1.Fixup(\"ratings\", 3)\n\tif !reflect.DeepEqual(frs1, expectedFrs) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedFrs, frs1)\n\t}\n}\n\nfunc TestDateFacetResultsMerge(t *testing.T) {\n\n\tlowmed := \"2010-01-01\"\n\tmedhi := \"2011-01-01\"\n\thihigher := \"2012-01-01\"\n\n\t// why second copy? the pointer are to strings done by date time parsing\n\t// inside the facet generation, so comparing pointers will not work\n\tlowmed2 := \"2010-01-01\"\n\tmedhi2 := \"2011-01-01\"\n\thihigher2 := \"2012-01-01\"\n\n\tfr1 := &FacetResult{\n\t\tField:   \"birthday\",\n\t\tTotal:   100,\n\t\tMissing: 25,\n\t\tOther:   25,\n\t\tDateRanges: []*DateRangeFacet{\n\t\t\t{\n\t\t\t\tName:  \"low\",\n\t\t\t\tEnd:   &lowmed,\n\t\t\t\tCount: 25,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"med\",\n\t\t\t\tCount: 24,\n\t\t\t\tStart: &lowmed,\n\t\t\t\tEnd:   &medhi,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"hi\",\n\t\t\t\tCount: 1,\n\t\t\t\tStart: &medhi,\n\t\t\t\tEnd:   &hihigher,\n\t\t\t},\n\t\t},\n\t}\n\tfrs1 := FacetResults{\n\t\t\"birthdays\": fr1,\n\t}\n\n\tfr2 := &FacetResult{\n\t\tField:   \"birthday\",\n\t\tTotal:   100,\n\t\tMissing: 25,\n\t\tOther:   25,\n\t\tDateRanges: []*DateRangeFacet{\n\t\t\t{\n\t\t\t\tName:  \"low\",\n\t\t\t\tEnd:   &lowmed2,\n\t\t\t\tCount: 25,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"med\",\n\t\t\t\tStart: &lowmed2,\n\t\t\t\tEnd:   &medhi2,\n\t\t\t\tCount: 22,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"highest\",\n\t\t\t\tStart: &hihigher2,\n\t\t\t\tCount: 3,\n\t\t\t},\n\t\t},\n\t}\n\tfrs2 := FacetResults{\n\t\t\"birthdays\": fr2,\n\t}\n\n\texpectedFr := &FacetResult{\n\t\tField:   \"birthday\",\n\t\tTotal:   200,\n\t\tMissing: 50,\n\t\tOther:   51,\n\t\tDateRanges: []*DateRangeFacet{\n\t\t\t{\n\t\t\t\tName:  \"low\",\n\t\t\t\tCount: 50,\n\t\t\t\tEnd:   &lowmed,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"med\",\n\t\t\t\tStart: &lowmed,\n\t\t\t\tEnd:   &medhi,\n\t\t\t\tCount: 46,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"highest\",\n\t\t\t\tStart: &hihigher,\n\t\t\t\tCount: 3,\n\t\t\t},\n\t\t},\n\t}\n\texpectedFrs := FacetResults{\n\t\t\"birthdays\": expectedFr,\n\t}\n\n\tfrs1.Merge(frs2)\n\tfrs1.Fixup(\"birthdays\", 3)\n\tif !reflect.DeepEqual(frs1, expectedFrs) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedFrs, frs1)\n\t}\n}\n"
  },
  {
    "path": "search/highlight/format/ansi/ansi.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ansi\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nconst Name = \"ansi\"\n\nconst DefaultAnsiHighlight = BgYellow\n\ntype FragmentFormatter struct {\n\tcolor string\n}\n\nfunc NewFragmentFormatter(color string) *FragmentFormatter {\n\treturn &FragmentFormatter{\n\t\tcolor: color,\n\t}\n}\n\nfunc (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string {\n\trv := \"\"\n\tcurr := f.Start\n\tfor _, termLocation := range orderedTermLocations {\n\t\tif termLocation == nil {\n\t\t\tcontinue\n\t\t}\n\t\t// make sure the array positions match\n\t\tif !termLocation.ArrayPositions.Equals(f.ArrayPositions) {\n\t\t\tcontinue\n\t\t}\n\t\tif termLocation.Start < curr {\n\t\t\tcontinue\n\t\t}\n\t\tif termLocation.End > f.End {\n\t\t\tbreak\n\t\t}\n\t\t// add the stuff before this location\n\t\trv += string(f.Orig[curr:termLocation.Start])\n\t\t// add the color\n\t\trv += a.color\n\t\t// add the term itself\n\t\trv += string(f.Orig[termLocation.Start:termLocation.End])\n\t\t// reset the color\n\t\trv += Reset\n\t\t// update current\n\t\tcurr = termLocation.End\n\t}\n\t// add any remaining text after the last token\n\trv += string(f.Orig[curr:f.End])\n\n\treturn rv\n}\n\n// ANSI color control escape sequences.\n// Shamelessly copied from https://github.com/sqp/godock/blob/master/libs/log/colors.go\nconst (\n\tReset      = \"\\x1b[0m\"\n\tBright     = \"\\x1b[1m\"\n\tDim        = \"\\x1b[2m\"\n\tUnderscore = \"\\x1b[4m\"\n\tBlink      = \"\\x1b[5m\"\n\tReverse    = \"\\x1b[7m\"\n\tHidden     = \"\\x1b[8m\"\n\tFgBlack    = \"\\x1b[30m\"\n\tFgRed      = \"\\x1b[31m\"\n\tFgGreen    = \"\\x1b[32m\"\n\tFgYellow   = \"\\x1b[33m\"\n\tFgBlue     = \"\\x1b[34m\"\n\tFgMagenta  = \"\\x1b[35m\"\n\tFgCyan     = \"\\x1b[36m\"\n\tFgWhite    = \"\\x1b[37m\"\n\tBgBlack    = \"\\x1b[40m\"\n\tBgRed      = \"\\x1b[41m\"\n\tBgGreen    = \"\\x1b[42m\"\n\tBgYellow   = \"\\x1b[43m\"\n\tBgBlue     = \"\\x1b[44m\"\n\tBgMagenta  = \"\\x1b[45m\"\n\tBgCyan     = \"\\x1b[46m\"\n\tBgWhite    = \"\\x1b[47m\"\n)\n\nfunc Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) {\n\tcolor := DefaultAnsiHighlight\n\tcolorVal, ok := config[\"color\"].(string)\n\tif ok {\n\t\tcolor = colorVal\n\t}\n\treturn NewFragmentFormatter(color), nil\n}\n\nfunc init() {\n\terr := registry.RegisterFragmentFormatter(Name, Constructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "search/highlight/format/html/html.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage html\n\nimport (\n\t\"html\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nconst Name = \"html\"\n\nconst defaultHTMLHighlightBefore = \"<mark>\"\nconst defaultHTMLHighlightAfter = \"</mark>\"\n\ntype FragmentFormatter struct {\n\tbefore string\n\tafter  string\n}\n\nfunc NewFragmentFormatter(before, after string) *FragmentFormatter {\n\treturn &FragmentFormatter{\n\t\tbefore: before,\n\t\tafter:  after,\n\t}\n}\n\nfunc (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string {\n\trv := \"\"\n\tcurr := f.Start\n\tfor _, termLocation := range orderedTermLocations {\n\t\tif termLocation == nil {\n\t\t\tcontinue\n\t\t}\n\t\t// make sure the array positions match\n\t\tif !termLocation.ArrayPositions.Equals(f.ArrayPositions) {\n\t\t\tcontinue\n\t\t}\n\t\tif termLocation.Start < curr {\n\t\t\tcontinue\n\t\t}\n\t\tif termLocation.End > f.End {\n\t\t\tbreak\n\t\t}\n\t\t// add the stuff before this location\n\t\trv += html.EscapeString(string(f.Orig[curr:termLocation.Start]))\n\t\t// start the <mark> tag\n\t\trv += a.before\n\t\t// add the term itself\n\t\trv += html.EscapeString(string(f.Orig[termLocation.Start:termLocation.End]))\n\t\t// end the <mark> tag\n\t\trv += a.after\n\t\t// update current\n\t\tcurr = termLocation.End\n\t}\n\t// add any remaining text after the last token\n\trv += html.EscapeString(string(f.Orig[curr:f.End]))\n\n\treturn rv\n}\n\nfunc Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) {\n\tbefore := defaultHTMLHighlightBefore\n\tbeforeVal, ok := config[\"before\"].(string)\n\tif ok {\n\t\tbefore = beforeVal\n\t}\n\tafter := defaultHTMLHighlightAfter\n\tafterVal, ok := config[\"after\"].(string)\n\tif ok {\n\t\tafter = afterVal\n\t}\n\treturn NewFragmentFormatter(before, after), nil\n}\n\nfunc init() {\n\terr := registry.RegisterFragmentFormatter(Name, Constructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "search/highlight/format/html/html_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage html\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nfunc TestHTMLFragmentFormatter(t *testing.T) {\n\ttests := []struct {\n\t\tfragment *highlight.Fragment\n\t\ttlm      search.TermLocationMap\n\t\toutput   string\n\t\tstart    string\n\t\tend      string\n\t}{\n\t\t{\n\t\t\tfragment: &highlight.Fragment{\n\t\t\t\tOrig:  []byte(\"the quick brown fox\"),\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   19,\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"quick\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   2,\n\t\t\t\t\t\tStart: 4,\n\t\t\t\t\t\tEnd:   9,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: \"the <b>quick</b> brown fox\",\n\t\t\tstart:  \"<b>\",\n\t\t\tend:    \"</b>\",\n\t\t},\n\t\t{\n\t\t\tfragment: &highlight.Fragment{\n\t\t\t\tOrig:  []byte(\"the quick brown fox\"),\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   19,\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"quick\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   2,\n\t\t\t\t\t\tStart: 4,\n\t\t\t\t\t\tEnd:   9,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: \"the <em>quick</em> brown fox\",\n\t\t\tstart:  \"<em>\",\n\t\t\tend:    \"</em>\",\n\t\t},\n\t\t// test html escaping\n\t\t{\n\t\t\tfragment: &highlight.Fragment{\n\t\t\t\tOrig:  []byte(\"<the> quick brown & fox\"),\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   23,\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"quick\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   2,\n\t\t\t\t\t\tStart: 6,\n\t\t\t\t\t\tEnd:   11,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: \"&lt;the&gt; <em>quick</em> brown &amp; fox\",\n\t\t\tstart:  \"<em>\",\n\t\t\tend:    \"</em>\",\n\t\t},\n\t\t// test html escaping inside search term\n\t\t{\n\t\t\tfragment: &highlight.Fragment{\n\t\t\t\tOrig:  []byte(\"<the> qu&ick brown & fox\"),\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   24,\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"qu&ick\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   2,\n\t\t\t\t\t\tStart: 6,\n\t\t\t\t\t\tEnd:   12,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: \"&lt;the&gt; <em>qu&amp;ick</em> brown &amp; fox\",\n\t\t\tstart:  \"<em>\",\n\t\t\tend:    \"</em>\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\temHTMLFormatter := NewFragmentFormatter(test.start, test.end)\n\t\totl := highlight.OrderTermLocations(test.tlm)\n\t\tresult := emHTMLFormatter.Format(test.fragment, otl)\n\t\tif result != test.output {\n\t\t\tt.Errorf(\"expected `%s`, got `%s`\", test.output, result)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/highlight/format/plain/plain.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plain\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nconst Name = \"plain\"\n\nconst defaultPlainHighlightBefore = \"<start>\"\nconst defaultPlainHighlightAfter = \"<end>\"\n\ntype FragmentFormatter struct {\n\tbefore string\n\tafter  string\n}\n\nfunc NewFragmentFormatter(before, after string) *FragmentFormatter {\n\treturn &FragmentFormatter{\n\t\tbefore: before,\n\t\tafter:  after,\n\t}\n}\n\nfunc (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string {\n\trv := \"\"\n\tcurr := f.Start\n\tfor _, termLocation := range orderedTermLocations {\n\t\tif termLocation == nil {\n\t\t\tcontinue\n\t\t}\n\t\t// make sure the array positions match\n\t\tif !termLocation.ArrayPositions.Equals(f.ArrayPositions) {\n\t\t\tcontinue\n\t\t}\n\t\tif termLocation.Start < curr {\n\t\t\tcontinue\n\t\t}\n\t\tif termLocation.End > f.End {\n\t\t\tbreak\n\t\t}\n\t\t// add the stuff before this location\n\t\trv += string(f.Orig[curr:termLocation.Start])\n\t\t// start the highlight tag\n\t\trv += a.before\n\t\t// add the term itself\n\t\trv += string(f.Orig[termLocation.Start:termLocation.End])\n\t\t// end the highlight tag\n\t\trv += a.after\n\t\t// update current\n\t\tcurr = termLocation.End\n\t}\n\t// add any remaining text after the last token\n\trv += string(f.Orig[curr:f.End])\n\n\treturn rv\n}\n\nfunc Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) {\n\tbefore := defaultPlainHighlightBefore\n\tbeforeVal, ok := config[\"before\"].(string)\n\tif ok {\n\t\tbefore = beforeVal\n\t}\n\tafter := defaultPlainHighlightAfter\n\tafterVal, ok := config[\"after\"].(string)\n\tif ok {\n\t\tafter = afterVal\n\t}\n\treturn NewFragmentFormatter(before, after), nil\n}\n\nfunc init() {\n\terr := registry.RegisterFragmentFormatter(Name, Constructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "search/highlight/format/plain/plain_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plain\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nfunc TestPlainFragmentFormatter(t *testing.T) {\n\ttests := []struct {\n\t\tfragment *highlight.Fragment\n\t\ttlm      search.TermLocationMap\n\t\toutput   string\n\t\tstart    string\n\t\tend      string\n\t}{\n\t\t{\n\t\t\tfragment: &highlight.Fragment{\n\t\t\t\tOrig:  []byte(\"the quick brown fox\"),\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   19,\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"quick\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   2,\n\t\t\t\t\t\tStart: 4,\n\t\t\t\t\t\tEnd:   9,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: \"the <b>quick</b> brown fox\",\n\t\t\tstart:  \"<b>\",\n\t\t\tend:    \"</b>\",\n\t\t},\n\t\t{\n\t\t\tfragment: &highlight.Fragment{\n\t\t\t\tOrig:  []byte(\"the quick brown fox\"),\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   19,\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"quick\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   2,\n\t\t\t\t\t\tStart: 4,\n\t\t\t\t\t\tEnd:   9,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: \"the <em>quick</em> brown fox\",\n\t\t\tstart:  \"<em>\",\n\t\t\tend:    \"</em>\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tplainFormatter := NewFragmentFormatter(test.start, test.end)\n\t\totl := highlight.OrderTermLocations(test.tlm)\n\t\tresult := plainFormatter.Format(test.fragment, otl)\n\t\tif result != test.output {\n\t\t\tt.Errorf(\"expected `%s`, got `%s`\", test.output, result)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/highlight/fragmenter/simple/simple.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage simple\n\nimport (\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nconst Name = \"simple\"\n\nconst defaultFragmentSize = 200\n\ntype Fragmenter struct {\n\tfragmentSize int\n}\n\nfunc NewFragmenter(fragmentSize int) *Fragmenter {\n\treturn &Fragmenter{\n\t\tfragmentSize: fragmentSize,\n\t}\n}\n\nfunc (s *Fragmenter) Fragment(orig []byte, ot highlight.TermLocations) []*highlight.Fragment {\n\tvar rv []*highlight.Fragment\n\tmaxbegin := 0\nOUTER:\n\tfor currTermIndex, termLocation := range ot {\n\t\t// start with this\n\t\t// it should be the highest scoring fragment with this term first\n\t\tstart := termLocation.Start\n\t\tend := start\n\t\tused := 0\n\t\tfor end < len(orig) && used < s.fragmentSize {\n\t\t\tr, size := utf8.DecodeRune(orig[end:])\n\t\t\tif r == utf8.RuneError {\n\t\t\t\tcontinue OUTER // bail\n\t\t\t}\n\t\t\tend += size\n\t\t\tused++\n\t\t}\n\n\t\t// if we still have more characters available to us\n\t\t// push back towards beginning\n\t\t// without cross maxbegin\n\t\tfor start > 0 && used < s.fragmentSize {\n\t\t\tif start > len(orig) {\n\t\t\t\t// bail if out of bounds, possibly due to token replacement\n\t\t\t\t// e.g with a regexp replacement\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\t\t\tr, size := utf8.DecodeLastRune(orig[0:start])\n\t\t\tif r == utf8.RuneError {\n\t\t\t\tcontinue OUTER // bail\n\t\t\t}\n\t\t\tif start-size >= maxbegin {\n\t\t\t\tstart -= size\n\t\t\t\tused++\n\t\t\t} else {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// however, we'd rather have the tokens centered more in the frag\n\t\t// lets try to do that as best we can, without affecting the score\n\t\t// find the end of the last term in this fragment\n\t\tminend := end\n\t\tfor _, innerTermLocation := range ot[currTermIndex:] {\n\t\t\tif innerTermLocation.End > end {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tminend = innerTermLocation.End\n\t\t}\n\n\t\t// find the smaller of the two rooms to move\n\t\troomToMove := utf8.RuneCount(orig[minend:end])\n\t\troomToMoveStart := 0\n\t\tif start >= maxbegin {\n\t\t\troomToMoveStart = utf8.RuneCount(orig[maxbegin:start])\n\t\t}\n\t\tif roomToMoveStart < roomToMove {\n\t\t\troomToMove = roomToMoveStart\n\t\t}\n\n\t\toffset := roomToMove / 2\n\n\t\tfor offset > 0 {\n\t\t\tr, size := utf8.DecodeLastRune(orig[0:start])\n\t\t\tif r == utf8.RuneError {\n\t\t\t\tcontinue OUTER // bail\n\t\t\t}\n\t\t\tstart -= size\n\n\t\t\tr, size = utf8.DecodeLastRune(orig[0:end])\n\t\t\tif r == utf8.RuneError {\n\t\t\t\tcontinue OUTER // bail\n\t\t\t}\n\t\t\tend -= size\n\t\t\toffset--\n\t\t}\n\n\t\trv = append(rv, &highlight.Fragment{Orig: orig, Start: start - offset, End: end - offset})\n\t\t// set maxbegin to the end of the current term location\n\t\t// so that next one won't back up to include it\n\t\tmaxbegin = termLocation.End\n\n\t}\n\tif len(ot) == 0 {\n\t\t// if there were no terms to highlight\n\t\t// produce a single fragment from the beginning\n\t\tstart := 0\n\t\tend := start\n\t\tused := 0\n\t\tfor end < len(orig) && used < s.fragmentSize {\n\t\t\tr, size := utf8.DecodeRune(orig[end:])\n\t\t\tif r == utf8.RuneError {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tend += size\n\t\t\tused++\n\t\t}\n\t\trv = append(rv, &highlight.Fragment{Orig: orig, Start: start, End: end})\n\t}\n\n\treturn rv\n}\n\nfunc Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Fragmenter, error) {\n\tsize := defaultFragmentSize\n\tsizeVal, ok := config[\"size\"].(float64)\n\tif ok {\n\t\tsize = int(sizeVal)\n\t}\n\treturn NewFragmenter(size), nil\n}\n\nfunc init() {\n\terr := registry.RegisterFragmenter(Name, Constructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "search/highlight/fragmenter/simple/simple_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage simple\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nfunc TestSimpleFragmenter(t *testing.T) {\n\n\ttests := []struct {\n\t\torig      []byte\n\t\tfragments []*highlight.Fragment\n\t\tot        highlight.TermLocations\n\t\tsize      int\n\t}{\n\t\t{\n\t\t\torig: []byte(\"this is a test\"),\n\t\t\tfragments: []*highlight.Fragment{\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"this is a test\"),\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   14,\n\t\t\t\t},\n\t\t\t},\n\t\t\tot: highlight.TermLocations{\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"test\",\n\t\t\t\t\tPos:   4,\n\t\t\t\t\tStart: 10,\n\t\t\t\t\tEnd:   14,\n\t\t\t\t},\n\t\t\t},\n\t\t\tsize: 100,\n\t\t},\n\t\t{\n\t\t\torig: []byte(\"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\"),\n\t\t\tfragments: []*highlight.Fragment{\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\"),\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   100,\n\t\t\t\t},\n\t\t\t},\n\t\t\tot: highlight.TermLocations{\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\",\n\t\t\t\t\tPos:   1,\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   100,\n\t\t\t\t},\n\t\t\t},\n\t\t\tsize: 100,\n\t\t},\n\t\t{\n\t\t\torig: []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\tfragments: []*highlight.Fragment{\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   100,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 10,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 20,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 30,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 40,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 50,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 60,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 70,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 80,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\"),\n\t\t\t\t\tStart: 90,\n\t\t\t\t\tEnd:   101,\n\t\t\t\t},\n\t\t\t},\n\t\t\tot: highlight.TermLocations{\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   1,\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   10,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   2,\n\t\t\t\t\tStart: 10,\n\t\t\t\t\tEnd:   20,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   3,\n\t\t\t\t\tStart: 20,\n\t\t\t\t\tEnd:   30,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   4,\n\t\t\t\t\tStart: 30,\n\t\t\t\t\tEnd:   40,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   5,\n\t\t\t\t\tStart: 40,\n\t\t\t\t\tEnd:   50,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   6,\n\t\t\t\t\tStart: 50,\n\t\t\t\t\tEnd:   60,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   7,\n\t\t\t\t\tStart: 60,\n\t\t\t\t\tEnd:   70,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   8,\n\t\t\t\t\tStart: 70,\n\t\t\t\t\tEnd:   80,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   9,\n\t\t\t\t\tStart: 80,\n\t\t\t\t\tEnd:   90,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"0123456789\",\n\t\t\t\t\tPos:   10,\n\t\t\t\t\tStart: 90,\n\t\t\t\t\tEnd:   100,\n\t\t\t\t},\n\t\t\t},\n\t\t\tsize: 100,\n\t\t},\n\t\t{\n\t\t\torig: []byte(\"[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}\"),\n\t\t\tfragments: []*highlight.Fragment{\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}\"),\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   411,\n\t\t\t\t},\n\t\t\t},\n\t\t\tot: highlight.TermLocations{\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"पानी\",\n\t\t\t\t\tPos:   1,\n\t\t\t\t\tStart: 2,\n\t\t\t\t\tEnd:   14,\n\t\t\t\t},\n\t\t\t},\n\t\t\tsize: 200,\n\t\t},\n\t\t{\n\t\t\torig: []byte(\"交换机\"),\n\t\t\tfragments: []*highlight.Fragment{\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"交换机\"),\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   9,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"交换机\"),\n\t\t\t\t\tStart: 3,\n\t\t\t\t\tEnd:   9,\n\t\t\t\t},\n\t\t\t},\n\t\t\tot: highlight.TermLocations{\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"交换\",\n\t\t\t\t\tPos:   1,\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   6,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"换机\",\n\t\t\t\t\tPos:   2,\n\t\t\t\t\tStart: 3,\n\t\t\t\t\tEnd:   9,\n\t\t\t\t},\n\t\t\t},\n\t\t\tsize: 200,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tfragmenter := NewFragmenter(test.size)\n\t\tfragments := fragmenter.Fragment(test.orig, test.ot)\n\t\tif !reflect.DeepEqual(fragments, test.fragments) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.fragments, fragments)\n\t\t\tfor _, fragment := range fragments {\n\t\t\t\tt.Logf(\"frag: %s\", fragment.Orig[fragment.Start:fragment.End])\n\t\t\t\tt.Logf(\"frag: %d - %d\", fragment.Start, fragment.End)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestSimpleFragmenterWithSize(t *testing.T) {\n\n\ttests := []struct {\n\t\torig      []byte\n\t\tfragments []*highlight.Fragment\n\t\tot        highlight.TermLocations\n\t}{\n\t\t{\n\t\t\torig: []byte(\"this is a test\"),\n\t\t\tfragments: []*highlight.Fragment{\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"this is a test\"),\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"this is a test\"),\n\t\t\t\t\tStart: 9,\n\t\t\t\t\tEnd:   14,\n\t\t\t\t},\n\t\t\t},\n\t\t\tot: highlight.TermLocations{\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"this\",\n\t\t\t\t\tPos:   1,\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t\t&highlight.TermLocation{\n\t\t\t\t\tTerm:  \"test\",\n\t\t\t\t\tPos:   4,\n\t\t\t\t\tStart: 10,\n\t\t\t\t\tEnd:   14,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\torig: []byte(\"避免出现 rune 越界问题\"),\n\t\t\tfragments: []*highlight.Fragment{\n\t\t\t\t{\n\t\t\t\t\tOrig:  []byte(\"避免出现 rune 越界问题\"),\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   13,\n\t\t\t\t},\n\t\t\t},\n\t\t\tot: nil,\n\t\t},\n\t}\n\n\tfragmenter := NewFragmenter(5)\n\tfor _, test := range tests {\n\t\tfragments := fragmenter.Fragment(test.orig, test.ot)\n\t\tif !reflect.DeepEqual(fragments, test.fragments) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.fragments, fragments)\n\t\t\tfor _, fragment := range fragments {\n\t\t\t\tt.Logf(\"frag: %#v\", fragment)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/highlight/highlighter/ansi/ansi.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ansi\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n\tansiFormatter \"github.com/blevesearch/bleve/v2/search/highlight/format/ansi\"\n\tsimpleFragmenter \"github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple\"\n\tsimpleHighlighter \"github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple\"\n)\n\nconst Name = \"ansi\"\n\nfunc Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) {\n\n\tfragmenter, err := cache.FragmenterNamed(simpleFragmenter.Name)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building fragmenter: %v\", err)\n\t}\n\n\tformatter, err := cache.FragmentFormatterNamed(ansiFormatter.Name)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building fragment formatter: %v\", err)\n\t}\n\n\treturn simpleHighlighter.NewHighlighter(\n\t\t\tfragmenter,\n\t\t\tformatter,\n\t\t\tsimpleHighlighter.DefaultSeparator),\n\t\tnil\n}\n\nfunc init() {\n\terr := registry.RegisterHighlighter(Name, Constructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "search/highlight/highlighter/html/html.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage html\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n\thtmlFormatter \"github.com/blevesearch/bleve/v2/search/highlight/format/html\"\n\tsimpleFragmenter \"github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple\"\n\tsimpleHighlighter \"github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple\"\n)\n\nconst Name = \"html\"\n\nfunc Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) {\n\n\tfragmenter, err := cache.FragmenterNamed(simpleFragmenter.Name)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building fragmenter: %v\", err)\n\t}\n\n\tformatter, err := cache.FragmentFormatterNamed(htmlFormatter.Name)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building fragment formatter: %v\", err)\n\t}\n\n\treturn simpleHighlighter.NewHighlighter(\n\t\t\tfragmenter,\n\t\t\tformatter,\n\t\t\tsimpleHighlighter.DefaultSeparator),\n\t\tnil\n}\n\nfunc init() {\n\terr := registry.RegisterHighlighter(Name, Constructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "search/highlight/highlighter/simple/fragment_scorer_simple.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage simple\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\n// FragmentScorer will score fragments by how many\n// unique terms occur in the fragment with no regard for\n// any boost values used in the original query\ntype FragmentScorer struct {\n\ttlm search.TermLocationMap\n}\n\nfunc NewFragmentScorer(tlm search.TermLocationMap) *FragmentScorer {\n\treturn &FragmentScorer{\n\t\ttlm: tlm,\n\t}\n}\n\nfunc (s *FragmentScorer) Score(f *highlight.Fragment) {\n\tscore := 0.0\nOUTER:\n\tfor _, locations := range s.tlm {\n\t\tfor _, location := range locations {\n\t\t\tif location.ArrayPositions.Equals(f.ArrayPositions) && int(location.Start) >= f.Start && int(location.End) <= f.End {\n\t\t\t\tscore += 1.0\n\t\t\t\t// once we find a term in the fragment\n\t\t\t\t// don't care about additional matches\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\t\t}\n\t}\n\tf.Score = score\n}\n"
  },
  {
    "path": "search/highlight/highlighter/simple/fragment_scorer_simple_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage simple\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nfunc TestSimpleFragmentScorer(t *testing.T) {\n\n\ttests := []struct {\n\t\tfragment *highlight.Fragment\n\t\ttlm      search.TermLocationMap\n\t\tscore    float64\n\t}{\n\t\t{\n\t\t\tfragment: &highlight.Fragment{\n\t\t\t\tOrig:  []byte(\"cat in the hat\"),\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   14,\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 0,\n\t\t\t\t\t\tEnd:   3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tscore: 1,\n\t\t},\n\t\t{\n\t\t\tfragment: &highlight.Fragment{\n\t\t\t\tOrig:  []byte(\"cat in the hat\"),\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   14,\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   1,\n\t\t\t\t\t\tStart: 0,\n\t\t\t\t\t\tEnd:   3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"hat\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   4,\n\t\t\t\t\t\tStart: 11,\n\t\t\t\t\t\tEnd:   14,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tscore: 2,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tscorer := NewFragmentScorer(test.tlm)\n\t\tscorer.Score(test.fragment)\n\t\tif test.fragment.Score != test.score {\n\t\t\tt.Errorf(\"expected score %f, got %f\", test.score, test.fragment.Score)\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "search/highlight/highlighter/simple/highlighter_simple.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage simple\n\nimport (\n\t\"container/heap\"\n\t\"fmt\"\n\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight\"\n)\n\nconst Name = \"simple\"\nconst DefaultSeparator = \"…\"\n\ntype Highlighter struct {\n\tfragmenter highlight.Fragmenter\n\tformatter  highlight.FragmentFormatter\n\tsep        string\n}\n\nfunc NewHighlighter(fragmenter highlight.Fragmenter, formatter highlight.FragmentFormatter, separator string) *Highlighter {\n\treturn &Highlighter{\n\t\tfragmenter: fragmenter,\n\t\tformatter:  formatter,\n\t\tsep:        separator,\n\t}\n}\n\nfunc (s *Highlighter) Fragmenter() highlight.Fragmenter {\n\treturn s.fragmenter\n}\n\nfunc (s *Highlighter) SetFragmenter(f highlight.Fragmenter) {\n\ts.fragmenter = f\n}\n\nfunc (s *Highlighter) FragmentFormatter() highlight.FragmentFormatter {\n\treturn s.formatter\n}\n\nfunc (s *Highlighter) SetFragmentFormatter(f highlight.FragmentFormatter) {\n\ts.formatter = f\n}\n\nfunc (s *Highlighter) Separator() string {\n\treturn s.sep\n}\n\nfunc (s *Highlighter) SetSeparator(sep string) {\n\ts.sep = sep\n}\n\nfunc (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc index.Document, field string) string {\n\tfragments := s.BestFragmentsInField(dm, doc, field, 1)\n\tif len(fragments) > 0 {\n\t\treturn fragments[0]\n\t}\n\treturn \"\"\n}\n\nfunc (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc index.Document, field string, num int) []string {\n\ttlm := dm.Locations[field]\n\torderedTermLocations := highlight.OrderTermLocations(tlm)\n\tscorer := NewFragmentScorer(tlm)\n\n\t// score the fragments and put them into a priority queue ordered by score\n\tfq := make(FragmentQueue, 0)\n\theap.Init(&fq)\n\tdoc.VisitFields(func(f index.Field) {\n\t\tif f.Name() == field {\n\t\t\t_, ok := f.(index.TextField)\n\t\t\tif ok {\n\t\t\t\ttermLocationsSameArrayPosition := make(highlight.TermLocations, 0)\n\t\t\t\tfor _, otl := range orderedTermLocations {\n\t\t\t\t\tif otl.ArrayPositions.Equals(f.ArrayPositions()) {\n\t\t\t\t\t\ttermLocationsSameArrayPosition = append(termLocationsSameArrayPosition, otl)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tfieldData := f.Value()\n\t\t\t\tfragments := s.fragmenter.Fragment(fieldData, termLocationsSameArrayPosition)\n\t\t\t\tfor _, fragment := range fragments {\n\t\t\t\t\tfragment.ArrayPositions = f.ArrayPositions()\n\t\t\t\t\tscorer.Score(fragment)\n\t\t\t\t\theap.Push(&fq, fragment)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t})\n\n\t// now find the N best non-overlapping fragments\n\tvar bestFragments []*highlight.Fragment\n\tif len(fq) > 0 {\n\t\tcandidate := heap.Pop(&fq)\n\tOUTER:\n\t\tfor candidate != nil && len(bestFragments) < num {\n\t\t\t// see if this overlaps with any of the best already identified\n\t\t\tif len(bestFragments) > 0 {\n\t\t\t\tfor _, frag := range bestFragments {\n\t\t\t\t\tif candidate.(*highlight.Fragment).Overlaps(frag) {\n\t\t\t\t\t\tif len(fq) < 1 {\n\t\t\t\t\t\t\tbreak OUTER\n\t\t\t\t\t\t}\n\t\t\t\t\t\tcandidate = heap.Pop(&fq)\n\t\t\t\t\t\tcontinue OUTER\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tbestFragments = append(bestFragments, candidate.(*highlight.Fragment))\n\t\t\t} else {\n\t\t\t\tbestFragments = append(bestFragments, candidate.(*highlight.Fragment))\n\t\t\t}\n\n\t\t\tif len(fq) < 1 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tcandidate = heap.Pop(&fq)\n\t\t}\n\t}\n\n\t// now that we have the best fragments, we can format them\n\torderedTermLocations.MergeOverlapping()\n\tformattedFragments := make([]string, len(bestFragments))\n\tfor i, fragment := range bestFragments {\n\t\tformattedFragments[i] = \"\"\n\t\tif fragment.Start != 0 {\n\t\t\tformattedFragments[i] += s.sep\n\t\t}\n\t\tformattedFragments[i] += s.formatter.Format(fragment, orderedTermLocations)\n\t\tif fragment.End != len(fragment.Orig) {\n\t\t\tformattedFragments[i] += s.sep\n\t\t}\n\t}\n\tif len(formattedFragments) > 0 {\n\t\tdm.AddFragments(field, formattedFragments)\n\t}\n\n\treturn formattedFragments\n}\n\n// FragmentQueue implements heap.Interface and holds Items.\ntype FragmentQueue []*highlight.Fragment\n\nfunc (fq FragmentQueue) Len() int { return len(fq) }\n\nfunc (fq FragmentQueue) Less(i, j int) bool {\n\t// We want Pop to give us the highest, not lowest, priority so we use greater-than here.\n\treturn fq[i].Score > fq[j].Score\n}\n\nfunc (fq FragmentQueue) Swap(i, j int) {\n\tfq[i], fq[j] = fq[j], fq[i]\n\tfq[i].Index = i\n\tfq[j].Index = j\n}\n\nfunc (fq *FragmentQueue) Push(x interface{}) {\n\tn := len(*fq)\n\titem := x.(*highlight.Fragment)\n\titem.Index = n\n\t*fq = append(*fq, item)\n}\n\nfunc (fq *FragmentQueue) Pop() interface{} {\n\told := *fq\n\tn := len(old)\n\titem := old[n-1]\n\titem.Index = -1 // for safety\n\t*fq = old[0 : n-1]\n\treturn item\n}\n\nfunc Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) {\n\tseparator := DefaultSeparator\n\tseparatorVal, ok := config[\"separator\"].(string)\n\tif ok {\n\t\tseparator = separatorVal\n\t}\n\n\tfragmenterName, ok := config[\"fragmenter\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify fragmenter\")\n\t}\n\tfragmenter, err := cache.FragmenterNamed(fragmenterName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building fragmenter: %v\", err)\n\t}\n\n\tformatterName, ok := config[\"formatter\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"must specify formatter\")\n\t}\n\tformatter, err := cache.FragmentFormatterNamed(formatterName)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error building fragment formatter: %v\", err)\n\t}\n\n\treturn NewHighlighter(fragmenter, formatter, separator), nil\n}\n\nfunc init() {\n\terr := registry.RegisterHighlighter(Name, Constructor)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "search/highlight/highlighter/simple/highlighter_simple_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage simple\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight/format/ansi\"\n\tsfrag \"github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple\"\n)\n\nconst (\n\treset                = \"\\x1b[0m\"\n\tDefaultAnsiHighlight = \"\\x1b[43m\"\n)\n\nfunc TestSimpleHighlighter(t *testing.T) {\n\tfragmenter := sfrag.NewFragmenter(100)\n\tformatter := ansi.NewFragmentFormatter(ansi.DefaultAnsiHighlight)\n\thighlighter := NewHighlighter(fragmenter, formatter, DefaultSeparator)\n\n\tdocMatch := search.DocumentMatch{\n\t\tID:    \"a\",\n\t\tScore: 1.0,\n\t\tLocations: search.FieldTermLocationMap{\n\t\t\t\"desc\": search.TermLocationMap{\n\t\t\t\t\"quick\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   2,\n\t\t\t\t\t\tStart: 4,\n\t\t\t\t\t\tEnd:   9,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"fox\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   4,\n\t\t\t\t\t\tStart: 16,\n\t\t\t\t\t\tEnd:   19,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\texpectedFragment := \"the \" + DefaultAnsiHighlight + \"quick\" + reset + \" brown \" + DefaultAnsiHighlight + \"fox\" + reset + \" jumps over the lazy dog\"\n\tdoc := document.NewDocument(\"a\").AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"the quick brown fox jumps over the lazy dog\")))\n\n\tfragment := highlighter.BestFragmentInField(&docMatch, doc, \"desc\")\n\tif fragment != expectedFragment {\n\t\tt.Errorf(\"expected `%s`, got `%s`\", expectedFragment, fragment)\n\t}\n}\n\nfunc TestSimpleHighlighterLonger(t *testing.T) {\n\n\tfieldBytes := []byte(`Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sed semper nulla, sed pellentesque urna. Suspendisse potenti. Aliquam dignissim pulvinar erat vel ullamcorper. Nullam sed diam at dolor dapibus varius. Vestibulum at semper nunc. Integer ullamcorper enim ut nisi condimentum lacinia. Nulla ipsum ipsum, dictum in dapibus non, bibendum eget neque. Vestibulum malesuada erat quis malesuada dictum. Mauris luctus viverra lorem, nec hendrerit lacus lacinia ut. Donec suscipit sit amet nisi et dictum. Maecenas ultrices mollis diam, vel commodo libero lobortis nec. Nunc non dignissim dolor. Nulla non tempus risus, eget porttitor lectus. Suspendisse vitae gravida magna, a sagittis urna. Curabitur nec dui volutpat, hendrerit nisi non, adipiscing erat. Maecenas aliquet sem sit amet nibh ultrices accumsan.\n\nMauris lobortis sem sed blandit bibendum. In scelerisque eros sed metus aliquet convallis ac eget metus. Donec eget feugiat sem. Quisque venenatis, augue et blandit vulputate, velit odio viverra dolor, eu iaculis eros urna ut nunc. Duis faucibus mattis enim ut ultricies. Donec scelerisque volutpat elit, vel varius ante porttitor vel. Duis neque nulla, ultrices vel est id, molestie semper odio. Maecenas condimentum felis vitae nibh venenatis, ut feugiat risus vehicula. Suspendisse non sapien neque. Etiam et lorem consequat lorem aliquam ullamcorper. Pellentesque id vestibulum neque, at aliquam turpis. Aenean ultrices nec erat sit amet aliquam. Morbi eu sem in augue cursus ullamcorper a sed dolor. Integer et lobortis nulla, sit amet laoreet elit. In elementum, nibh nec volutpat pretium, lectus est pulvinar arcu, vehicula lobortis tellus sem id mauris. Maecenas ac blandit purus, sit amet scelerisque magna.\n\nIn hac habitasse platea dictumst. In lacinia elit non risus venenatis viverra. Nulla vestibulum laoreet turpis ac accumsan. Vivamus eros felis, rhoncus vel interdum bibendum, imperdiet nec diam. Etiam sed eros sed orci pellentesque sagittis. Praesent a fermentum leo. Vivamus ipsum risus, faucibus a dignissim ut, ullamcorper nec risus. Etiam quis adipiscing velit. Nam ac cursus arcu. Sed bibendum lectus quis massa dapibus dapibus. Vestibulum fermentum eros vitae hendrerit condimentum.\n\nFusce viverra eleifend iaculis. Maecenas tempor dictum cursus. Mauris faucibus, tortor in bibendum ornare, nibh lorem sollicitudin est, sed consectetur nulla dui imperdiet urna. Fusce aliquet odio fermentum massa mollis, id feugiat lacus egestas. Integer et eleifend metus. Duis neque tellus, vulputate nec dui eu, euismod sodales orci. Vivamus turpis erat, consectetur et pulvinar nec, ornare a quam. Maecenas fermentum, ligula vitae consectetur lobortis, mi lacus fermentum ante, ut semper lacus lectus porta orci. Nulla vehicula sodales eros, in iaculis ante laoreet at. Sed venenatis interdum metus, egestas scelerisque orci laoreet ut. Donec fermentum enim eget nibh blandit laoreet. Proin lacinia adipiscing lorem vel ornare. Donec ullamcorper massa elementum urna varius viverra. Proin pharetra, erat at feugiat rhoncus, velit eros condimentum mi, ac mattis sapien dolor non elit. Aenean viverra purus id tincidunt vulputate.\n\nEtiam vel augue vel nisl commodo suscipit et ac nisl. Quisque eros diam, porttitor et aliquet sed, vulputate in odio. Aenean feugiat est quis neque vehicula, eget vulputate nunc tempor. Donec quis nulla ut quam feugiat consectetur ut et justo. Nulla congue, metus auctor facilisis scelerisque, nunc risus vulputate urna, in blandit urna nibh et neque. Etiam quis tortor ut nulla dignissim dictum non sed ligula. Vivamus accumsan ligula eget ipsum ultrices, a tincidunt urna blandit. In hac habitasse platea dictumst.`)\n\n\tdoc := document.NewDocument(\"a\").AddField(document.NewTextField(\"full\", []uint64{}, fieldBytes))\n\tdocMatch := search.DocumentMatch{\n\t\tID:    \"a\",\n\t\tScore: 1.0,\n\t\tLocations: search.FieldTermLocationMap{\n\t\t\t\"full\": search.TermLocationMap{\n\t\t\t\t\"metus\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 883,\n\t\t\t\t\t\tEnd:   888,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 915,\n\t\t\t\t\t\tEnd:   920,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 2492,\n\t\t\t\t\t\tEnd:   2497,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 2822,\n\t\t\t\t\t\tEnd:   2827,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 3417,\n\t\t\t\t\t\tEnd:   3422,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"interdum\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 1891,\n\t\t\t\t\t\tEnd:   1899,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 2813,\n\t\t\t\t\t\tEnd:   2821,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"venenatis\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 954,\n\t\t\t\t\t\tEnd:   963,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 1252,\n\t\t\t\t\t\tEnd:   1261,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 1795,\n\t\t\t\t\t\tEnd:   1804,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPos:   0,\n\t\t\t\t\t\tStart: 2803,\n\t\t\t\t\t\tEnd:   2812,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\texpectedFragments := []string{\n\t\t\"…eros, in iaculis ante laoreet at. Sed \" + DefaultAnsiHighlight + \"venenatis\" + reset + \" \" + DefaultAnsiHighlight + \"interdum\" + reset + \" \" + DefaultAnsiHighlight + \"metus\" + reset + \", egestas scelerisque orci laoreet ut.…\",\n\t\t\"… eros sed \" + DefaultAnsiHighlight + \"metus\" + reset + \" aliquet convallis ac eget \" + DefaultAnsiHighlight + \"metus\" + reset + \". Donec eget feugiat sem. Quisque \" + DefaultAnsiHighlight + \"venenatis\" + reset + \", augue et…\",\n\t\t\"… odio. Maecenas condimentum felis vitae nibh \" + DefaultAnsiHighlight + \"venenatis\" + reset + \", ut feugiat risus vehicula. Suspendisse non s…\",\n\t\t\"… id feugiat lacus egestas. Integer et eleifend \" + DefaultAnsiHighlight + \"metus\" + reset + \". Duis neque tellus, vulputate nec dui eu, euism…\",\n\t\t\"… accumsan. Vivamus eros felis, rhoncus vel \" + DefaultAnsiHighlight + \"interdum\" + reset + \" bibendum, imperdiet nec diam. Etiam sed eros sed…\",\n\t}\n\n\tfragmenter := sfrag.NewFragmenter(100)\n\tformatter := ansi.NewFragmentFormatter(ansi.DefaultAnsiHighlight)\n\thighlighter := NewHighlighter(fragmenter, formatter, DefaultSeparator)\n\tfragments := highlighter.BestFragmentsInField(&docMatch, doc, \"full\", 5)\n\n\tif !reflect.DeepEqual(fragments, expectedFragments) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expectedFragments, fragments)\n\t}\n\n}\n"
  },
  {
    "path": "search/highlight/highlighter.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage highlight\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype Fragment struct {\n\tOrig           []byte\n\tArrayPositions []uint64\n\tStart          int\n\tEnd            int\n\tScore          float64\n\tIndex          int // used by heap\n}\n\nfunc (f *Fragment) Overlaps(other *Fragment) bool {\n\tif other.Start >= f.Start && other.Start < f.End {\n\t\treturn true\n\t} else if f.Start >= other.Start && f.Start < other.End {\n\t\treturn true\n\t}\n\treturn false\n}\n\ntype Fragmenter interface {\n\tFragment([]byte, TermLocations) []*Fragment\n}\n\ntype FragmentFormatter interface {\n\tFormat(f *Fragment, orderedTermLocations TermLocations) string\n}\n\ntype FragmentScorer interface {\n\tScore(f *Fragment) float64\n}\n\ntype Highlighter interface {\n\tFragmenter() Fragmenter\n\tSetFragmenter(Fragmenter)\n\n\tFragmentFormatter() FragmentFormatter\n\tSetFragmentFormatter(FragmentFormatter)\n\n\tSeparator() string\n\tSetSeparator(string)\n\n\tBestFragmentInField(*search.DocumentMatch, index.Document, string) string\n\tBestFragmentsInField(*search.DocumentMatch, index.Document, string, int) []string\n}\n"
  },
  {
    "path": "search/highlight/term_locations.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage highlight\n\nimport (\n\t\"reflect\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\ntype TermLocation struct {\n\tTerm           string\n\tArrayPositions search.ArrayPositions\n\tPos            int\n\tStart          int\n\tEnd            int\n}\n\nfunc (tl *TermLocation) Overlaps(other *TermLocation) bool {\n\tif reflect.DeepEqual(tl.ArrayPositions, other.ArrayPositions) {\n\t\tif other.Start >= tl.Start && other.Start < tl.End {\n\t\t\treturn true\n\t\t} else if tl.Start >= other.Start && tl.Start < other.End {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\ntype TermLocations []*TermLocation\n\nfunc (t TermLocations) Len() int      { return len(t) }\nfunc (t TermLocations) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\nfunc (t TermLocations) Less(i, j int) bool {\n\n\tshortestArrayPositions := len(t[i].ArrayPositions)\n\tif len(t[j].ArrayPositions) < shortestArrayPositions {\n\t\tshortestArrayPositions = len(t[j].ArrayPositions)\n\t}\n\n\t// compare all the common array positions\n\tfor api := 0; api < shortestArrayPositions; api++ {\n\t\tif t[i].ArrayPositions[api] < t[j].ArrayPositions[api] {\n\t\t\treturn true\n\t\t}\n\t\tif t[i].ArrayPositions[api] > t[j].ArrayPositions[api] {\n\t\t\treturn false\n\t\t}\n\t}\n\t// all the common array positions are the same\n\tif len(t[i].ArrayPositions) < len(t[j].ArrayPositions) {\n\t\treturn true // j array positions, longer so greater\n\t} else if len(t[i].ArrayPositions) > len(t[j].ArrayPositions) {\n\t\treturn false // j array positions, shorter so less\n\t}\n\n\t// array positions the same, compare starts\n\treturn t[i].Start < t[j].Start\n}\n\nfunc (t TermLocations) MergeOverlapping() {\n\tvar lastTl *TermLocation\n\tfor i, tl := range t {\n\t\tif lastTl == nil && tl != nil {\n\t\t\tlastTl = tl\n\t\t} else if lastTl != nil && tl != nil {\n\t\t\tif lastTl.Overlaps(tl) {\n\t\t\t\t// ok merge this with previous\n\t\t\t\tlastTl.End = tl.End\n\t\t\t\tt[i] = nil\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc OrderTermLocations(tlm search.TermLocationMap) TermLocations {\n\trv := make(TermLocations, 0)\n\tfor term, locations := range tlm {\n\t\tfor _, location := range locations {\n\t\t\ttl := TermLocation{\n\t\t\t\tTerm:           term,\n\t\t\t\tArrayPositions: location.ArrayPositions,\n\t\t\t\tPos:            int(location.Pos),\n\t\t\t\tStart:          int(location.Start),\n\t\t\t\tEnd:            int(location.End),\n\t\t\t}\n\t\t\trv = append(rv, &tl)\n\t\t}\n\t}\n\tsort.Sort(rv)\n\treturn rv\n}\n"
  },
  {
    "path": "search/highlight/term_locations_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage highlight\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\nfunc TestTermLocationOverlaps(t *testing.T) {\n\n\ttests := []struct {\n\t\tleft     *TermLocation\n\t\tright    *TermLocation\n\t\texpected bool\n\t}{\n\t\t{\n\t\t\tleft: &TermLocation{\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   5,\n\t\t\t},\n\t\t\tright: &TermLocation{\n\t\t\t\tStart: 3,\n\t\t\t\tEnd:   7,\n\t\t\t},\n\t\t\texpected: true,\n\t\t},\n\t\t{\n\t\t\tleft: &TermLocation{\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   5,\n\t\t\t},\n\t\t\tright: &TermLocation{\n\t\t\t\tStart: 5,\n\t\t\t\tEnd:   7,\n\t\t\t},\n\t\t\texpected: false,\n\t\t},\n\t\t{\n\t\t\tleft: &TermLocation{\n\t\t\t\tStart: 0,\n\t\t\t\tEnd:   5,\n\t\t\t},\n\t\t\tright: &TermLocation{\n\t\t\t\tStart: 7,\n\t\t\t\tEnd:   11,\n\t\t\t},\n\t\t\texpected: false,\n\t\t},\n\t\t// with array positions\n\t\t{\n\t\t\tleft: &TermLocation{\n\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\tStart:          0,\n\t\t\t\tEnd:            5,\n\t\t\t},\n\t\t\tright: &TermLocation{\n\t\t\t\tArrayPositions: search.ArrayPositions{1},\n\t\t\t\tStart:          7,\n\t\t\t\tEnd:            11,\n\t\t\t},\n\t\t\texpected: false,\n\t\t},\n\t\t{\n\t\t\tleft: &TermLocation{\n\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\tStart:          0,\n\t\t\t\tEnd:            5,\n\t\t\t},\n\t\t\tright: &TermLocation{\n\t\t\t\tArrayPositions: search.ArrayPositions{1},\n\t\t\t\tStart:          3,\n\t\t\t\tEnd:            11,\n\t\t\t},\n\t\t\texpected: false,\n\t\t},\n\t\t{\n\t\t\tleft: &TermLocation{\n\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\tStart:          0,\n\t\t\t\tEnd:            5,\n\t\t\t},\n\t\t\tright: &TermLocation{\n\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\tStart:          3,\n\t\t\t\tEnd:            11,\n\t\t\t},\n\t\t\texpected: true,\n\t\t},\n\t\t{\n\t\t\tleft: &TermLocation{\n\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\tStart:          0,\n\t\t\t\tEnd:            5,\n\t\t\t},\n\t\t\tright: &TermLocation{\n\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\tStart:          7,\n\t\t\t\tEnd:            11,\n\t\t\t},\n\t\t\texpected: false,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := test.left.Overlaps(test.right)\n\t\tif actual != test.expected {\n\t\t\tt.Errorf(\"expected %t got %t for %#v\", test.expected, actual, test)\n\t\t}\n\t}\n}\n\nfunc TestTermLocationsMergeOverlapping(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  TermLocations\n\t\toutput TermLocations\n\t}{\n\t\t{\n\t\t\tinput:  TermLocations{},\n\t\t\toutput: TermLocations{},\n\t\t},\n\t\t{\n\t\t\tinput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 7,\n\t\t\t\t\tEnd:   11,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 7,\n\t\t\t\t\tEnd:   11,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 4,\n\t\t\t\t\tEnd:   11,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   11,\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 4,\n\t\t\t\t\tEnd:   11,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 9,\n\t\t\t\t\tEnd:   13,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   13,\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t\tnil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 4,\n\t\t\t\t\tEnd:   11,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 9,\n\t\t\t\t\tEnd:   13,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 15,\n\t\t\t\t\tEnd:   21,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   13,\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t\tnil,\n\t\t\t\t&TermLocation{\n\t\t\t\t\tStart: 15,\n\t\t\t\t\tEnd:   21,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// with array positions\n\t\t{\n\t\t\tinput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          0,\n\t\t\t\t\tEnd:            5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{1},\n\t\t\t\t\tStart:          7,\n\t\t\t\t\tEnd:            11,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          0,\n\t\t\t\t\tEnd:            5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{1},\n\t\t\t\t\tStart:          7,\n\t\t\t\t\tEnd:            11,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          0,\n\t\t\t\t\tEnd:            5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          7,\n\t\t\t\t\tEnd:            11,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          0,\n\t\t\t\t\tEnd:            5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          7,\n\t\t\t\t\tEnd:            11,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          0,\n\t\t\t\t\tEnd:            5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          3,\n\t\t\t\t\tEnd:            11,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          0,\n\t\t\t\t\tEnd:            11,\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          0,\n\t\t\t\t\tEnd:            5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{1},\n\t\t\t\t\tStart:          3,\n\t\t\t\t\tEnd:            11,\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tStart:          0,\n\t\t\t\t\tEnd:            5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{1},\n\t\t\t\t\tStart:          3,\n\t\t\t\t\tEnd:            11,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttest.input.MergeOverlapping()\n\t\tif !reflect.DeepEqual(test.input, test.output) {\n\t\t\tt.Errorf(\"expected: %#v got %#v\", test.output, test.input)\n\t\t}\n\t}\n}\n\nfunc TestTermLocationsOrder(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  search.TermLocationMap\n\t\toutput TermLocations\n\t}{\n\t\t{\n\t\t\tinput:  search.TermLocationMap{},\n\t\t\toutput: TermLocations{},\n\t\t},\n\t\t{\n\t\t\tinput: search.TermLocationMap{\n\t\t\t\t\"term\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tStart: 0,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStart: 5,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tTerm:  \"term\",\n\t\t\t\t\tStart: 0,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tTerm:  \"term\",\n\t\t\t\t\tStart: 5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: search.TermLocationMap{\n\t\t\t\t\"term\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tStart: 5,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStart: 0,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tTerm:  \"term\",\n\t\t\t\t\tStart: 0,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tTerm:  \"term\",\n\t\t\t\t\tStart: 5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// with array positions\n\t\t{\n\t\t\tinput: search.TermLocationMap{\n\t\t\t\t\"term\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\t\tStart:          0,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\t\tStart:          5,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tTerm:           \"term\",\n\t\t\t\t\tStart:          0,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tTerm:           \"term\",\n\t\t\t\t\tStart:          5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: search.TermLocationMap{\n\t\t\t\t\"term\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\t\tStart:          5,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\t\tStart:          0,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tTerm:           \"term\",\n\t\t\t\t\tStart:          0,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tTerm:           \"term\",\n\t\t\t\t\tStart:          5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: search.TermLocationMap{\n\t\t\t\t\"term\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\t\tStart:          5,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tArrayPositions: search.ArrayPositions{1},\n\t\t\t\t\t\tStart:          0,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tTerm:           \"term\",\n\t\t\t\t\tStart:          5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{1},\n\t\t\t\t\tTerm:           \"term\",\n\t\t\t\t\tStart:          0,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: search.TermLocationMap{\n\t\t\t\t\"term\": []*search.Location{\n\t\t\t\t\t{\n\t\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\t\tStart:          5,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tArrayPositions: search.ArrayPositions{0, 1},\n\t\t\t\t\t\tStart:          0,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\toutput: TermLocations{\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0},\n\t\t\t\t\tTerm:           \"term\",\n\t\t\t\t\tStart:          5,\n\t\t\t\t},\n\t\t\t\t&TermLocation{\n\t\t\t\t\tArrayPositions: search.ArrayPositions{0, 1},\n\t\t\t\t\tTerm:           \"term\",\n\t\t\t\t\tStart:          0,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := OrderTermLocations(test.input)\n\t\tif !reflect.DeepEqual(actual, test.output) {\n\t\t\tt.Errorf(\"expected: %#v got %#v\", test.output, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/levenshtein.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"math\"\n)\n\nfunc LevenshteinDistance(a, b string) int {\n\tla := len(a)\n\tlb := len(b)\n\td := make([]int, la+1)\n\tvar lastdiag, olddiag, temp int\n\n\tfor i := 1; i <= la; i++ {\n\t\td[i] = i\n\t}\n\tfor i := 1; i <= lb; i++ {\n\t\td[0] = i\n\t\tlastdiag = i - 1\n\t\tfor j := 1; j <= la; j++ {\n\t\t\tolddiag = d[j]\n\t\t\tmin := d[j] + 1\n\t\t\tif (d[j-1] + 1) < min {\n\t\t\t\tmin = d[j-1] + 1\n\t\t\t}\n\t\t\tif a[j-1] == b[i-1] {\n\t\t\t\ttemp = 0\n\t\t\t} else {\n\t\t\t\ttemp = 1\n\t\t\t}\n\t\t\tif (lastdiag + temp) < min {\n\t\t\t\tmin = lastdiag + temp\n\t\t\t}\n\t\t\td[j] = min\n\t\t\tlastdiag = olddiag\n\t\t}\n\t}\n\treturn d[la]\n}\n\n// LevenshteinDistanceMax same as LevenshteinDistance but\n// attempts to bail early once we know the distance\n// will be greater than max\n// in which case the first return val will be the max\n// and the second will be true, indicating max was exceeded\nfunc LevenshteinDistanceMax(a, b string, max int) (int, bool) {\n\tv, wasMax, _ := LevenshteinDistanceMaxReuseSlice(a, b, max, nil)\n\treturn v, wasMax\n}\n\nfunc LevenshteinDistanceMaxReuseSlice(a, b string, max int, d []int) (int, bool, []int) {\n\tla := len(a)\n\tlb := len(b)\n\n\tld := int(math.Abs(float64(la - lb)))\n\tif ld > max {\n\t\treturn max, true, d\n\t} else if la == 0 || lb == 0 {\n\t\t// if one string of the two strings is empty, then ld is\n\t\t// the length of the other string and as such is <= max\n\t\treturn ld, false, d\n\t}\n\n\tif cap(d) < la+1 {\n\t\td = make([]int, la+1)\n\t}\n\td = d[:la+1]\n\n\tvar lastdiag, olddiag, temp int\n\n\tfor i := 1; i <= la; i++ {\n\t\td[i] = i\n\t}\n\tfor i := 1; i <= lb; i++ {\n\t\td[0] = i\n\t\tlastdiag = i - 1\n\t\trowmin := max + 1\n\t\tfor j := 1; j <= la; j++ {\n\t\t\tolddiag = d[j]\n\t\t\tmin := d[j] + 1\n\t\t\tif (d[j-1] + 1) < min {\n\t\t\t\tmin = d[j-1] + 1\n\t\t\t}\n\t\t\tif a[j-1] == b[i-1] {\n\t\t\t\ttemp = 0\n\t\t\t} else {\n\t\t\t\ttemp = 1\n\t\t\t}\n\t\t\tif (lastdiag + temp) < min {\n\t\t\t\tmin = lastdiag + temp\n\t\t\t}\n\t\t\tif min < rowmin {\n\t\t\t\trowmin = min\n\t\t\t}\n\t\t\td[j] = min\n\n\t\t\tlastdiag = olddiag\n\t\t}\n\t\t// after each row if rowmin isn't less than max stop\n\t\tif rowmin > max {\n\t\t\treturn max, true, d\n\t\t}\n\t}\n\treturn d[la], false, d\n}\n"
  },
  {
    "path": "search/levenshtein_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"testing\"\n)\n\nfunc TestLevenshteinDistance(t *testing.T) {\n\n\ttests := []struct {\n\t\ta    string\n\t\tb    string\n\t\tdist int\n\t}{\n\t\t{\n\t\t\t\"water\",\n\t\t\t\"atec\",\n\t\t\t2,\n\t\t},\n\t\t{\n\t\t\t\"water\",\n\t\t\t\"aphex\",\n\t\t\t4,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := LevenshteinDistance(test.a, test.b)\n\t\tif actual != test.dist {\n\t\t\tt.Errorf(\"expected %d, got %d for %s and %s\", test.dist, actual, test.a, test.b)\n\t\t}\n\t}\n}\n\nfunc TestLevenshteinDistanceMax(t *testing.T) {\n\n\ttests := []struct {\n\t\ta        string\n\t\tb        string\n\t\tmax      int\n\t\tdist     int\n\t\texceeded bool\n\t}{\n\t\t{\n\t\t\ta:        \"water\",\n\t\t\tb:        \"atec\",\n\t\t\tmax:      1,\n\t\t\tdist:     1,\n\t\t\texceeded: true,\n\t\t},\n\t\t{\n\t\t\ta:        \"water\",\n\t\t\tb:        \"christmas\",\n\t\t\tmax:      3,\n\t\t\tdist:     3,\n\t\t\texceeded: true,\n\t\t},\n\t\t{\n\t\t\ta:        \"\",\n\t\t\tb:        \"water\",\n\t\t\tmax:      10,\n\t\t\tdist:     5,\n\t\t\texceeded: false,\n\t\t},\n\t\t{\n\t\t\ta:        \"water\",\n\t\t\tb:        \"\",\n\t\t\tmax:      3,\n\t\t\tdist:     3,\n\t\t\texceeded: true,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual, exceeded := LevenshteinDistanceMax(test.a, test.b, test.max)\n\t\tif actual != test.dist || exceeded != test.exceeded {\n\t\t\tt.Errorf(\"expected %d %t, got %d %t for %s and %s\", test.dist, test.exceeded, actual, exceeded, test.a, test.b)\n\t\t}\n\t}\n}\n\n// 5 terms that are less than 2\n// 5 terms that are more than 2\nvar benchmarkTerms = []string{\n\t\"watex\",\n\t\"aters\",\n\t\"wayer\",\n\t\"wbter\",\n\t\"yater\",\n\t\"christmas\",\n\t\"waterwaterwater\",\n\t\"watcatdogfish\",\n\t\"q\",\n\t\"couchbase\",\n}\n\nfunc BenchmarkLevenshteinDistance(b *testing.B) {\n\ta := \"water\"\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, t := range benchmarkTerms {\n\t\t\tLevenshteinDistance(a, t)\n\t\t}\n\t}\n}\n\nfunc BenchmarkLevenshteinDistanceMax(b *testing.B) {\n\ta := \"water\"\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, t := range benchmarkTerms {\n\t\t\tLevenshteinDistanceMax(a, t, 2)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/pool.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"reflect\"\n)\n\nvar reflectStaticSizeDocumentMatchPool int\n\nfunc init() {\n\tvar dmp DocumentMatchPool\n\treflectStaticSizeDocumentMatchPool = int(reflect.TypeOf(dmp).Size())\n}\n\n// DocumentMatchPoolTooSmall is a callback function that can be executed\n// when the DocumentMatchPool does not have sufficient capacity\n// By default we just perform just-in-time allocation, but you could log\n// a message, or panic, etc.\ntype DocumentMatchPoolTooSmall func(p *DocumentMatchPool) *DocumentMatch\n\n// DocumentMatchPool manages use/reuse of DocumentMatch instances\n// it pre-allocates space from a single large block with the expected\n// number of instances.  It is not thread-safe as currently all\n// aspects of search take place in a single goroutine.\ntype DocumentMatchPool struct {\n\tavail    DocumentMatchCollection\n\tTooSmall DocumentMatchPoolTooSmall\n}\n\nfunc defaultDocumentMatchPoolTooSmall(p *DocumentMatchPool) *DocumentMatch {\n\treturn &DocumentMatch{}\n}\n\n// NewDocumentMatchPool will build a DocumentMatchPool with memory\n// pre-allocated to accommodate the requested number of DocumentMatch\n// instances\nfunc NewDocumentMatchPool(size, sortsize int) *DocumentMatchPool {\n\tavail := make(DocumentMatchCollection, size)\n\t// pre-allocate the expected number of instances\n\tstartBlock := make([]DocumentMatch, size)\n\tstartSorts := make([]string, size*sortsize)\n\t// make these initial instances available\n\ti, j := 0, 0\n\tfor i < size {\n\t\tavail[i] = &startBlock[i]\n\t\tavail[i].Sort = startSorts[j:j]\n\t\ti += 1\n\t\tj += sortsize\n\t}\n\treturn &DocumentMatchPool{\n\t\tavail:    avail,\n\t\tTooSmall: defaultDocumentMatchPoolTooSmall,\n\t}\n}\n\n// Get returns an available DocumentMatch from the pool\n// if the pool was not allocated with sufficient size, an allocation will\n// occur to satisfy this request.  As a side-effect this will grow the size\n// of the pool.\nfunc (p *DocumentMatchPool) Get() *DocumentMatch {\n\tvar rv *DocumentMatch\n\tif len(p.avail) > 0 {\n\t\trv, p.avail = p.avail[len(p.avail)-1], p.avail[:len(p.avail)-1]\n\t} else {\n\t\trv = p.TooSmall(p)\n\t}\n\treturn rv\n}\n\n// Put returns a DocumentMatch to the pool\nfunc (p *DocumentMatchPool) Put(d *DocumentMatch) {\n\tif d == nil {\n\t\treturn\n\t}\n\t// reset DocumentMatch before returning it to available pool\n\td.Reset()\n\tp.avail = append(p.avail, d)\n}\n"
  },
  {
    "path": "search/pool_test.go",
    "content": "//  Copyright (c) 2013 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport \"testing\"\n\nfunc TestDocumentMatchPool(t *testing.T) {\n\n\ttooManyCalled := false\n\n\t// create a pool\n\tdmp := NewDocumentMatchPool(10, 0)\n\tdmp.TooSmall = func(inner *DocumentMatchPool) *DocumentMatch {\n\t\ttooManyCalled = true\n\t\treturn &DocumentMatch{}\n\t}\n\n\t// get 10 instances without returning\n\treturned := make(DocumentMatchCollection, 10)\n\n\tfor i := 0; i < 10; i++ {\n\t\treturned[i] = dmp.Get()\n\t\tif tooManyCalled {\n\t\t\tt.Fatal(\"too many function called before expected\")\n\t\t}\n\t}\n\n\t// get one more and see if too many function is called\n\textra := dmp.Get()\n\tif !tooManyCalled {\n\t\tt.Fatal(\"expected too many function to be called, but wasn't\")\n\t}\n\n\t// return the first 10\n\tfor i := 0; i < 10; i++ {\n\t\tdmp.Put(returned[i])\n\t}\n\n\t// check len and cap\n\tif len(dmp.avail) != 10 {\n\t\tt.Fatalf(\"expected 10 available, got %d\", len(dmp.avail))\n\t}\n\tif cap(dmp.avail) != 10 {\n\t\tt.Fatalf(\"expected avail cap still 10, got %d\", cap(dmp.avail))\n\t}\n\n\t// return the extra\n\tdmp.Put(extra)\n\n\t// check len and cap grown to 11\n\tif len(dmp.avail) != 11 {\n\t\tt.Fatalf(\"expected 11 available, got %d\", len(dmp.avail))\n\t}\n\t// cap grows, but not by 1 (append behavior)\n\tif cap(dmp.avail) <= 10 {\n\t\tt.Fatalf(\"expected avail cap mpore than 10, got %d\", cap(dmp.avail))\n\t}\n}\n"
  },
  {
    "path": "search/query/bool_field.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype BoolFieldQuery struct {\n\tBool     bool   `json:\"bool\"`\n\tFieldVal string `json:\"field,omitempty\"`\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\n// NewBoolFieldQuery creates a new Query for boolean fields\nfunc NewBoolFieldQuery(val bool) *BoolFieldQuery {\n\treturn &BoolFieldQuery{\n\t\tBool: val,\n\t}\n}\n\nfunc (q *BoolFieldQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *BoolFieldQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *BoolFieldQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *BoolFieldQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *BoolFieldQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\tterm := \"F\"\n\tif q.Bool {\n\t\tterm = \"T\"\n\t}\n\treturn searcher.NewTermSearcher(ctx, i, term, field, q.BoostVal.Value(), options)\n}\n"
  },
  {
    "path": "search/query/boolean.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype BooleanQuery struct {\n\tMust            Query  `json:\"must,omitempty\"`\n\tShould          Query  `json:\"should,omitempty\"`\n\tMustNot         Query  `json:\"must_not,omitempty\"`\n\tFilter          Query  `json:\"filter,omitempty\"`\n\tBoostVal        *Boost `json:\"boost,omitempty\"`\n\tqueryStringMode bool\n}\n\n// NewBooleanQuery creates a compound Query composed\n// of several other Query objects.\n// Result documents must satisfy ALL of the\n// must Queries.\n// Result documents must satisfy NONE of the must not\n// Queries.\n// Result documents that ALSO satisfy any of the should\n// Queries will score higher.\nfunc NewBooleanQuery(must []Query, should []Query, mustNot []Query) *BooleanQuery {\n\n\trv := BooleanQuery{}\n\tif len(must) > 0 {\n\t\trv.Must = NewConjunctionQuery(must)\n\t}\n\tif len(should) > 0 {\n\t\trv.Should = NewDisjunctionQuery(should)\n\t}\n\tif len(mustNot) > 0 {\n\t\trv.MustNot = NewDisjunctionQuery(mustNot)\n\t}\n\n\treturn &rv\n}\n\nfunc NewBooleanQueryForQueryString(must []Query, should []Query, mustNot []Query) *BooleanQuery {\n\trv := NewBooleanQuery(nil, nil, nil)\n\trv.queryStringMode = true\n\trv.AddMust(must...)\n\trv.AddShould(should...)\n\trv.AddMustNot(mustNot...)\n\treturn rv\n}\n\n// SetMinShould requires that at least minShould of the\n// should Queries must be satisfied.\nfunc (q *BooleanQuery) SetMinShould(minShould float64) {\n\tq.Should.(*DisjunctionQuery).SetMin(minShould)\n}\n\nfunc (q *BooleanQuery) AddMust(m ...Query) {\n\tif m == nil {\n\t\treturn\n\t}\n\tif q.Must == nil {\n\t\ttmp := NewConjunctionQuery([]Query{})\n\t\ttmp.queryStringMode = q.queryStringMode\n\t\tq.Must = tmp\n\t}\n\tfor _, mq := range m {\n\t\tq.Must.(*ConjunctionQuery).AddQuery(mq)\n\t}\n}\n\nfunc (q *BooleanQuery) AddShould(m ...Query) {\n\tif m == nil {\n\t\treturn\n\t}\n\tif q.Should == nil {\n\t\ttmp := NewDisjunctionQuery([]Query{})\n\t\ttmp.queryStringMode = q.queryStringMode\n\t\tq.Should = tmp\n\t}\n\tfor _, mq := range m {\n\t\tq.Should.(*DisjunctionQuery).AddQuery(mq)\n\t}\n}\n\nfunc (q *BooleanQuery) AddMustNot(m ...Query) {\n\tif m == nil {\n\t\treturn\n\t}\n\tif q.MustNot == nil {\n\t\ttmp := NewDisjunctionQuery([]Query{})\n\t\ttmp.queryStringMode = q.queryStringMode\n\t\tq.MustNot = tmp\n\t}\n\tfor _, mq := range m {\n\t\tq.MustNot.(*DisjunctionQuery).AddQuery(mq)\n\t}\n}\n\nfunc (q *BooleanQuery) AddFilter(m Query) {\n\tif m == nil {\n\t\treturn\n\t}\n\tq.Filter = m\n}\n\nfunc (q *BooleanQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *BooleanQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *BooleanQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tvar err error\n\tvar mustNotSearcher search.Searcher\n\tif q.MustNot != nil {\n\t\tmustNotSearcher, err = q.MustNot.Searcher(ctx, i, m, options)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// if must not is MatchNone, reset it to nil\n\t\tif _, ok := mustNotSearcher.(*searcher.MatchNoneSearcher); ok {\n\t\t\tmustNotSearcher = nil\n\t\t}\n\t}\n\n\tvar mustSearcher search.Searcher\n\tif q.Must != nil {\n\t\tmustSearcher, err = q.Must.Searcher(ctx, i, m, options)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// if must searcher is MatchNone, reset it to nil\n\t\tif _, ok := mustSearcher.(*searcher.MatchNoneSearcher); ok {\n\t\t\tmustSearcher = nil\n\t\t}\n\t}\n\n\tvar shouldSearcher search.Searcher\n\tif q.Should != nil {\n\t\tshouldSearcher, err = q.Should.Searcher(ctx, i, m, options)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// if should searcher is MatchNone, reset it to nil\n\t\tif _, ok := shouldSearcher.(*searcher.MatchNoneSearcher); ok {\n\t\t\tshouldSearcher = nil\n\t\t}\n\t}\n\n\tvar filterFunc searcher.FilterFunc\n\tif q.Filter != nil {\n\t\t// create a new searcher options with disabled scoring, since filter should not affect scoring\n\t\t// and we don't want to pay the cost of scoring if we don't need it, also disable term vectors\n\t\t// and explain, since we don't need them for filters\n\t\tfilterOptions := search.SearcherOptions{\n\t\t\tExplain:            false,\n\t\t\tIncludeTermVectors: false,\n\t\t\tScore:              \"none\",\n\t\t}\n\t\tfilterSearcher, err := q.Filter.Searcher(ctx, i, m, filterOptions)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tvar init bool\n\t\tvar refDoc *search.DocumentMatch\n\t\tfilterFunc = func(sctx *search.SearchContext, d *search.DocumentMatch) bool {\n\t\t\t// Initialize the reference document to point\n\t\t\t// to the first document in the filterSearcher\n\t\t\tvar err error\n\t\t\tif !init {\n\t\t\t\trefDoc, err = filterSearcher.Next(sctx)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\tinit = true\n\t\t\t}\n\t\t\tif refDoc == nil {\n\t\t\t\t// filterSearcher is exhausted, d is not in filter\n\t\t\t\treturn false\n\t\t\t}\n\t\t\t// Compare document IDs\n\t\t\tcmp := refDoc.IndexInternalID.Compare(d.IndexInternalID)\n\t\t\tif cmp < 0 {\n\t\t\t\t// recycle refDoc now that we do not need it\n\t\t\t\tsctx.DocumentMatchPool.Put(refDoc)\n\t\t\t\t// filterSearcher is behind the current document, Advance() it\n\t\t\t\trefDoc, err = filterSearcher.Advance(sctx, d.IndexInternalID)\n\t\t\t\tif err != nil || refDoc == nil {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\t// After advance, check if they're now equal\n\t\t\t\tcmp = refDoc.IndexInternalID.Compare(d.IndexInternalID)\n\t\t\t}\n\t\t\t// cmp >= 0: either equal (match) or filterSearcher is ahead (no match)\n\t\t\treturn cmp == 0\n\t\t}\n\t}\n\n\t// if all 4 are nil, return MatchNone\n\tif mustSearcher == nil && shouldSearcher == nil && mustNotSearcher == nil && filterFunc == nil {\n\t\treturn searcher.NewMatchNoneSearcher(i)\n\t}\n\n\t// optimization, if only must searcher, just return it instead\n\tif mustSearcher != nil && shouldSearcher == nil && mustNotSearcher == nil && filterFunc == nil {\n\t\treturn mustSearcher, nil\n\t}\n\n\t// optimization, if only should searcher, just return it instead\n\tif mustSearcher == nil && shouldSearcher != nil && mustNotSearcher == nil && filterFunc == nil {\n\t\treturn shouldSearcher, nil\n\t}\n\n\t// optimization, if only filter searcher, wrap around a MatchAllSearcher\n\tif mustSearcher == nil && shouldSearcher == nil && mustNotSearcher == nil && filterFunc != nil {\n\t\tmustSearcher, err = searcher.NewMatchAllSearcher(ctx, i, 1.0, options)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn searcher.NewFilteringSearcher(ctx,\n\t\t\tmustSearcher,\n\t\t\tfilterFunc,\n\t\t), nil\n\t}\n\n\t// if only mustNotSearcher, start with MatchAll\n\tif mustSearcher == nil && shouldSearcher == nil && mustNotSearcher != nil {\n\t\tmustSearcher, err = searcher.NewMatchAllSearcher(ctx, i, 1.0, options)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tbs, err := searcher.NewBooleanSearcher(ctx, i, mustSearcher, shouldSearcher, mustNotSearcher, options)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif filterFunc != nil {\n\t\treturn searcher.NewFilteringSearcher(ctx, bs, filterFunc), nil\n\t}\n\treturn bs, nil\n}\n\nfunc (q *BooleanQuery) Validate() error {\n\tif qm, ok := q.Must.(ValidatableQuery); ok {\n\t\terr := qm.Validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif qs, ok := q.Should.(ValidatableQuery); ok {\n\t\terr := qs.Validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif qmn, ok := q.MustNot.(ValidatableQuery); ok {\n\t\terr := qmn.Validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif qf, ok := q.Filter.(ValidatableQuery); ok {\n\t\terr := qf.Validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif q.Must == nil && q.Should == nil && q.MustNot == nil && q.Filter == nil {\n\t\treturn fmt.Errorf(\"boolean query must contain at least one must or should or not must or filter clause\")\n\t}\n\treturn nil\n}\n\nfunc (q *BooleanQuery) UnmarshalJSON(data []byte) error {\n\ttmp := struct {\n\t\tMust    json.RawMessage `json:\"must,omitempty\"`\n\t\tShould  json.RawMessage `json:\"should,omitempty\"`\n\t\tMustNot json.RawMessage `json:\"must_not,omitempty\"`\n\t\tFilter  json.RawMessage `json:\"filter,omitempty\"`\n\t\tBoost   *Boost          `json:\"boost,omitempty\"`\n\t}{}\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tmp.Must != nil {\n\t\tq.Must, err = ParseQuery(tmp.Must)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, isConjunctionQuery := q.Must.(*ConjunctionQuery)\n\t\tif !isConjunctionQuery {\n\t\t\treturn fmt.Errorf(\"must clause must be conjunction\")\n\t\t}\n\t}\n\n\tif tmp.Should != nil {\n\t\tq.Should, err = ParseQuery(tmp.Should)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, isDisjunctionQuery := q.Should.(*DisjunctionQuery)\n\t\tif !isDisjunctionQuery {\n\t\t\treturn fmt.Errorf(\"should clause must be disjunction\")\n\t\t}\n\t}\n\n\tif tmp.MustNot != nil {\n\t\tq.MustNot, err = ParseQuery(tmp.MustNot)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, isDisjunctionQuery := q.MustNot.(*DisjunctionQuery)\n\t\tif !isDisjunctionQuery {\n\t\t\treturn fmt.Errorf(\"must not clause must be disjunction\")\n\t\t}\n\t}\n\n\tif tmp.Filter != nil {\n\t\tq.Filter, err = ParseQuery(tmp.Filter)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tq.BoostVal = tmp.Boost\n\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/boost.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport \"fmt\"\n\ntype Boost float64\n\nfunc (b *Boost) Value() float64 {\n\tif b == nil {\n\t\treturn 1.0\n\t}\n\treturn float64(*b)\n}\n\nfunc (b *Boost) GoString() string {\n\tif b == nil {\n\t\treturn \"boost unspecified\"\n\t}\n\treturn fmt.Sprintf(\"%f\", *b)\n}\n"
  },
  {
    "path": "search/query/conjunction.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype ConjunctionQuery struct {\n\tConjuncts       []Query `json:\"conjuncts\"`\n\tBoostVal        *Boost  `json:\"boost,omitempty\"`\n\tqueryStringMode bool\n}\n\n// NewConjunctionQuery creates a new compound Query.\n// Result documents must satisfy all of the queries.\nfunc NewConjunctionQuery(conjuncts []Query) *ConjunctionQuery {\n\treturn &ConjunctionQuery{\n\t\tConjuncts: conjuncts,\n\t}\n}\n\nfunc (q *ConjunctionQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *ConjunctionQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *ConjunctionQuery) AddQuery(aq ...Query) {\n\tq.Conjuncts = append(q.Conjuncts, aq...)\n}\n\nfunc (q *ConjunctionQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tss := make([]search.Searcher, 0, len(q.Conjuncts))\n\tcleanup := func() {\n\t\tfor _, searcher := range ss {\n\t\t\tif searcher != nil {\n\t\t\t\t_ = searcher.Close()\n\t\t\t}\n\t\t}\n\t}\n\tnestedMode, _ := ctx.Value(search.NestedSearchKey).(bool)\n\tvar nm mapping.NestedMapping\n\tif nestedMode {\n\t\tvar ok bool\n\t\t// get the nested mapping\n\t\tif nm, ok = m.(mapping.NestedMapping); !ok {\n\t\t\t// shouldn't be in nested mode if no nested mapping\n\t\t\tnestedMode = false\n\t\t}\n\t}\n\t// set of fields used in this query\n\tvar qfs search.FieldSet\n\tvar err error\n\n\tfor _, conjunct := range q.Conjuncts {\n\t\t// Gather fields when nested mode is enabled\n\t\tif nestedMode {\n\t\t\tqfs, err = ExtractFields(conjunct, m, qfs)\n\t\t\tif err != nil {\n\t\t\t\tcleanup()\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tsr, err := conjunct.Searcher(ctx, i, m, options)\n\t\tif err != nil {\n\t\t\tcleanup()\n\t\t\treturn nil, err\n\t\t}\n\t\tif _, ok := sr.(*searcher.MatchNoneSearcher); ok && q.queryStringMode {\n\t\t\t// in query string mode, skip match none\n\t\t\tcontinue\n\t\t}\n\t\tss = append(ss, sr)\n\t}\n\n\tif len(ss) < 1 {\n\t\treturn searcher.NewMatchNoneSearcher(i)\n\t}\n\n\tif nestedMode {\n\t\t// first determine the nested depth info for the query fields\n\t\tcommonDepth, maxDepth := nm.NestedDepth(qfs)\n\t\t// if we have common depth == max depth then we can just use\n\t\t// the normal conjunction searcher, as all fields share the same\n\t\t// nested context, otherwise we need to use the nested conjunction searcher\n\t\t// also, if we are querying the _all or _id fields, we need to use\n\t\t// the nested conjunction searcher as well, with common depth 0\n\t\t// indicating matches happen only at the root level\n\t\tif qfs.HasAll() || qfs.HasID() {\n\t\t\tcommonDepth = 0\n\t\t}\n\t\tif commonDepth < maxDepth {\n\t\t\treturn searcher.NewNestedConjunctionSearcher(ctx, i, ss, commonDepth, options)\n\t\t}\n\t}\n\n\treturn searcher.NewConjunctionSearcher(ctx, i, ss, options)\n}\n\nfunc (q *ConjunctionQuery) Validate() error {\n\tfor _, q := range q.Conjuncts {\n\t\tif q, ok := q.(ValidatableQuery); ok {\n\t\t\terr := q.Validate()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (q *ConjunctionQuery) UnmarshalJSON(data []byte) error {\n\ttmp := struct {\n\t\tConjuncts []json.RawMessage `json:\"conjuncts\"`\n\t\tBoost     *Boost            `json:\"boost,omitempty\"`\n\t}{}\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\tq.Conjuncts = make([]Query, len(tmp.Conjuncts))\n\tfor i, term := range tmp.Conjuncts {\n\t\tquery, err := ParseQuery(term)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tq.Conjuncts[i] = query\n\t}\n\tq.BoostVal = tmp.Boost\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/date_range.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/optional\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\n// QueryDateTimeParser controls the default query date time parser.\nvar QueryDateTimeParser = optional.Name\n\n// QueryDateTimeFormat controls the format when Marshaling to JSON.\nvar QueryDateTimeFormat = time.RFC3339\n\nvar cache = registry.NewCache()\n\ntype BleveQueryTime struct {\n\ttime.Time\n}\n\nvar MinRFC3339CompatibleTime time.Time\nvar MaxRFC3339CompatibleTime time.Time\n\nfunc init() {\n\tMinRFC3339CompatibleTime, _ = time.Parse(time.RFC3339, \"1677-12-01T00:00:00Z\")\n\tMaxRFC3339CompatibleTime, _ = time.Parse(time.RFC3339, \"2262-04-11T11:59:59Z\")\n}\n\nfunc queryTimeFromString(t string) (time.Time, error) {\n\tdateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser)\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\trv, _, err := dateTimeParser.ParseDateTime(t)\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\treturn rv, nil\n}\n\nfunc (t *BleveQueryTime) MarshalJSON() ([]byte, error) {\n\ttt := time.Time(t.Time)\n\treturn []byte(\"\\\"\" + tt.Format(QueryDateTimeFormat) + \"\\\"\"), nil\n}\n\nfunc (t *BleveQueryTime) UnmarshalJSON(data []byte) error {\n\tvar timeString string\n\terr := util.UnmarshalJSON(data, &timeString)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser)\n\tif err != nil {\n\t\treturn err\n\t}\n\tt.Time, _, err = dateTimeParser.ParseDateTime(timeString)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\ntype DateRangeQuery struct {\n\tStart          BleveQueryTime `json:\"start,omitempty\"`\n\tEnd            BleveQueryTime `json:\"end,omitempty\"`\n\tInclusiveStart *bool          `json:\"inclusive_start,omitempty\"`\n\tInclusiveEnd   *bool          `json:\"inclusive_end,omitempty\"`\n\tFieldVal       string         `json:\"field,omitempty\"`\n\tBoostVal       *Boost         `json:\"boost,omitempty\"`\n}\n\n// NewDateRangeQuery creates a new Query for ranges\n// of date values.\n// Date strings are parsed using the DateTimeParser configured in the\n// top-level config.QueryDateTimeParser\n// Either, but not both endpoints can be nil.\nfunc NewDateRangeQuery(start, end time.Time) *DateRangeQuery {\n\treturn NewDateRangeInclusiveQuery(start, end, nil, nil)\n}\n\n// NewDateRangeInclusiveQuery creates a new Query for ranges\n// of date values.\n// Date strings are parsed using the DateTimeParser configured in the\n// top-level config.QueryDateTimeParser\n// Either, but not both endpoints can be nil.\n// startInclusive and endInclusive control inclusion of the endpoints.\nfunc NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *DateRangeQuery {\n\treturn &DateRangeQuery{\n\t\tStart:          BleveQueryTime{start},\n\t\tEnd:            BleveQueryTime{end},\n\t\tInclusiveStart: startInclusive,\n\t\tInclusiveEnd:   endInclusive,\n\t}\n}\n\nfunc (q *DateRangeQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *DateRangeQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *DateRangeQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *DateRangeQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *DateRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tmin, max, err := q.parseEndpoints()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\treturn searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *DateRangeQuery) parseEndpoints() (*float64, *float64, error) {\n\tmin := math.Inf(-1)\n\tmax := math.Inf(1)\n\tif !q.Start.IsZero() {\n\t\tif !isDatetimeCompatible(q.Start) {\n\t\t\t// overflow\n\t\t\treturn nil, nil, fmt.Errorf(\"invalid/unsupported date range, start: %v\", q.Start)\n\t\t}\n\t\tstartInt64 := q.Start.UnixNano()\n\t\tmin = numeric.Int64ToFloat64(startInt64)\n\t}\n\tif !q.End.IsZero() {\n\t\tif !isDatetimeCompatible(q.End) {\n\t\t\t// overflow\n\t\t\treturn nil, nil, fmt.Errorf(\"invalid/unsupported date range, end: %v\", q.End)\n\t\t}\n\t\tendInt64 := q.End.UnixNano()\n\t\tmax = numeric.Int64ToFloat64(endInt64)\n\t}\n\n\treturn &min, &max, nil\n}\n\nfunc (q *DateRangeQuery) Validate() error {\n\tif q.Start.IsZero() && q.End.IsZero() {\n\t\treturn fmt.Errorf(\"must specify start or end\")\n\t}\n\t_, _, err := q.parseEndpoints()\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc isDatetimeCompatible(t BleveQueryTime) bool {\n\tif QueryDateTimeFormat == time.RFC3339 &&\n\t\t(t.Before(MinRFC3339CompatibleTime) || t.After(MaxRFC3339CompatibleTime)) {\n\t\treturn false\n\t}\n\n\treturn true\n}\n"
  },
  {
    "path": "search/query/date_range_string.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\n// DateRangeStringQuery represents a query for a range of date values.\n// Start and End are the range endpoints, as strings.\n// Start and End are parsed using DateTimeParser, which is a custom date time parser\n// defined in the index mapping. If DateTimeParser is not specified, then the\n// top-level config.QueryDateTimeParser is used.\ntype DateRangeStringQuery struct {\n\tStart          string `json:\"start,omitempty\"`\n\tEnd            string `json:\"end,omitempty\"`\n\tInclusiveStart *bool  `json:\"inclusive_start,omitempty\"`\n\tInclusiveEnd   *bool  `json:\"inclusive_end,omitempty\"`\n\tFieldVal       string `json:\"field,omitempty\"`\n\tBoostVal       *Boost `json:\"boost,omitempty\"`\n\tDateTimeParser string `json:\"datetime_parser,omitempty\"`\n}\n\n// NewDateRangeStringQuery creates a new Query for ranges\n// of date values.\n// Date strings are parsed using the DateTimeParser field of the query struct,\n// which is a custom date time parser defined in the index mapping.\n// if DateTimeParser is not specified, then the\n// top-level config.QueryDateTimeParser is used.\n// Either, but not both endpoints can be nil.\nfunc NewDateRangeStringQuery(start, end string) *DateRangeStringQuery {\n\treturn NewDateRangeStringInclusiveQuery(start, end, nil, nil)\n}\n\n// NewDateRangeStringInclusiveQuery creates a new Query for ranges\n// of date values.\n// Date strings are parsed using the DateTimeParser field of the query struct,\n// which is a custom date time parser defined in the index mapping.\n// if DateTimeParser is not specified, then the\n// top-level config.QueryDateTimeParser is used.\n// Either, but not both endpoints can be nil.\n// startInclusive and endInclusive control inclusion of the endpoints.\nfunc NewDateRangeStringInclusiveQuery(start, end string, startInclusive, endInclusive *bool) *DateRangeStringQuery {\n\treturn &DateRangeStringQuery{\n\t\tStart:          start,\n\t\tEnd:            end,\n\t\tInclusiveStart: startInclusive,\n\t\tInclusiveEnd:   endInclusive,\n\t}\n}\n\nfunc (q *DateRangeStringQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *DateRangeStringQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *DateRangeStringQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *DateRangeStringQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *DateRangeStringQuery) SetDateTimeParser(d string) {\n\tq.DateTimeParser = d\n}\n\nfunc (q *DateRangeStringQuery) DateTimeParserName() string {\n\treturn q.DateTimeParser\n}\n\nfunc (q *DateRangeStringQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\tdateTimeParserName := QueryDateTimeParser\n\tif q.DateTimeParser != \"\" {\n\t\tdateTimeParserName = q.DateTimeParser\n\t}\n\tdateTimeParser := m.DateTimeParserNamed(dateTimeParserName)\n\tif dateTimeParser == nil {\n\t\treturn nil, fmt.Errorf(\"no dateTimeParser named '%s' registered\", dateTimeParserName)\n\t}\n\n\tvar startTime, endTime time.Time\n\tvar err error\n\tif q.Start != \"\" {\n\t\tstartTime, _, err = dateTimeParser.ParseDateTime(q.Start)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"%v, date time parser name: %s\", err, dateTimeParserName)\n\t\t}\n\t}\n\tif q.End != \"\" {\n\t\tendTime, _, err = dateTimeParser.ParseDateTime(q.End)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"%v, date time parser name: %s\", err, dateTimeParserName)\n\t\t}\n\t}\n\n\tmin, max, err := q.parseEndpoints(startTime, endTime)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *DateRangeStringQuery) parseEndpoints(startTime, endTime time.Time) (*float64, *float64, error) {\n\tmin := math.Inf(-1)\n\tmax := math.Inf(1)\n\n\tif startTime.IsZero() && endTime.IsZero() {\n\t\treturn nil, nil, fmt.Errorf(\"date range query must specify at least one of start/end\")\n\t}\n\n\tif !startTime.IsZero() {\n\t\tif !isDateTimeWithinRange(startTime) {\n\t\t\t// overflow\n\t\t\treturn nil, nil, fmt.Errorf(\"invalid/unsupported date range, start: %v\", q.Start)\n\t\t}\n\t\tstartInt64 := startTime.UnixNano()\n\t\tmin = numeric.Int64ToFloat64(startInt64)\n\t}\n\tif !endTime.IsZero() {\n\t\tif !isDateTimeWithinRange(endTime) {\n\t\t\t// overflow\n\t\t\treturn nil, nil, fmt.Errorf(\"invalid/unsupported date range, end: %v\", q.End)\n\t\t}\n\t\tendInt64 := endTime.UnixNano()\n\t\tmax = numeric.Int64ToFloat64(endInt64)\n\t}\n\n\treturn &min, &max, nil\n}\n\nfunc (q *DateRangeStringQuery) Validate() error {\n\t// either start or end must be specified\n\tif q.Start == \"\" && q.End == \"\" {\n\t\treturn fmt.Errorf(\"date range query must specify at least one of start/end\")\n\t}\n\treturn nil\n}\n\nfunc isDateTimeWithinRange(t time.Time) bool {\n\tif t.Before(MinRFC3339CompatibleTime) || t.After(MaxRFC3339CompatibleTime) {\n\t\treturn false\n\t}\n\treturn true\n}\n"
  },
  {
    "path": "search/query/date_range_test.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestBleveQueryTime(t *testing.T) {\n\ttestTimes := []time.Time{\n\t\ttime.Now(),\n\t\t{},\n\t}\n\n\tfor i, testTime := range testTimes {\n\t\tbqt := &BleveQueryTime{testTime}\n\n\t\tbuf, err := json.Marshal(bqt)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"expected no err\")\n\t\t}\n\n\t\tvar bqt2 BleveQueryTime\n\t\terr = json.Unmarshal(buf, &bqt2)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"expected no unmarshal err, got: %v\", err)\n\t\t}\n\n\t\tif bqt.Time.Format(time.RFC3339) != bqt2.Time.Format(time.RFC3339) {\n\t\t\tt.Errorf(\"test %d - expected same time, %#v != %#v\", i, bqt.Time, bqt2.Time)\n\t\t}\n\n\t\tif testTime.Format(time.RFC3339) != bqt2.Time.Format(time.RFC3339) {\n\t\t\tt.Errorf(\"test %d - expected orig time, %#v != %#v\", i, testTime, bqt2.Time)\n\t\t}\n\t}\n}\n\nfunc TestValidateDatetimeRanges(t *testing.T) {\n\ttests := []struct {\n\t\tstart  string\n\t\tend    string\n\t\texpect bool\n\t}{\n\t\t{\n\t\t\tstart:  \"2019-03-22T13:25:00Z\",\n\t\t\tend:    \"2019-03-22T18:25:00Z\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tstart:  \"2019-03-22T13:25:00Z\",\n\t\t\tend:    \"9999-03-22T13:25:00Z\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tstart:  \"2019-03-22T13:25:00Z\",\n\t\t\tend:    \"2262-04-11T11:59:59Z\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tstart:  \"2019-03-22T13:25:00Z\",\n\t\t\tend:    \"2262-04-12T00:00:00Z\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tstart:  \"1950-03-22T12:23:23Z\",\n\t\t\tend:    \"1960-02-21T15:23:34Z\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tstart:  \"0001-01-01T00:00:00Z\",\n\t\t\tend:    \"0001-01-01T00:00:00Z\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tstart:  \"0001-01-01T00:00:00Z\",\n\t\t\tend:    \"2000-01-01T00:00:00Z\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tstart:  \"1677-11-30T11:59:59Z\",\n\t\t\tend:    \"2262-04-11T11:59:59Z\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tstart:  \"2262-04-12T00:00:00Z\",\n\t\t\tend:    \"2262-04-11T11:59:59Z\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tstart:  \"1677-12-01T00:00:00Z\",\n\t\t\tend:    \"2262-04-12T00:00:00Z\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tstart:  \"1677-12-01T00:00:00Z\",\n\t\t\tend:    \"1677-11-30T11:59:59Z\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tstart:  \"1677-12-01T00:00:00Z\",\n\t\t\tend:    \"2262-04-11T11:59:59Z\",\n\t\t\texpect: true,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tstartTime, _ := time.Parse(time.RFC3339, test.start)\n\t\tendTime, _ := time.Parse(time.RFC3339, test.end)\n\n\t\tdateRangeQuery := NewDateRangeQuery(startTime, endTime)\n\t\tif (dateRangeQuery.Validate() == nil) != test.expect {\n\t\t\tt.Errorf(\"unexpected results while validating date range query with\"+\n\t\t\t\t\" {start: %v, end: %v}, expected: %v\",\n\t\t\t\ttest.start, test.end, test.expect)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/query/disjunction.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype DisjunctionQuery struct {\n\tDisjuncts              []Query `json:\"disjuncts\"`\n\tBoostVal               *Boost  `json:\"boost,omitempty\"`\n\tMin                    float64 `json:\"min\"`\n\tretrieveScoreBreakdown bool\n\tqueryStringMode        bool\n}\n\nfunc (q *DisjunctionQuery) RetrieveScoreBreakdown(b bool) {\n\tq.retrieveScoreBreakdown = b\n}\n\n// NewDisjunctionQuery creates a new compound Query.\n// Result documents satisfy at least one Query.\nfunc NewDisjunctionQuery(disjuncts []Query) *DisjunctionQuery {\n\treturn &DisjunctionQuery{\n\t\tDisjuncts: disjuncts,\n\t}\n}\n\nfunc (q *DisjunctionQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *DisjunctionQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *DisjunctionQuery) AddQuery(aq ...Query) {\n\tq.Disjuncts = append(q.Disjuncts, aq...)\n}\n\nfunc (q *DisjunctionQuery) SetMin(m float64) {\n\tq.Min = m\n}\n\nfunc (q *DisjunctionQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping,\n\toptions search.SearcherOptions,\n) (search.Searcher, error) {\n\tss := make([]search.Searcher, 0, len(q.Disjuncts))\n\tfor _, disjunct := range q.Disjuncts {\n\t\tsr, err := disjunct.Searcher(ctx, i, m, options)\n\t\tif err != nil {\n\t\t\tfor _, searcher := range ss {\n\t\t\t\tif searcher != nil {\n\t\t\t\t\t_ = searcher.Close()\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil, err\n\t\t}\n\t\tif sr != nil {\n\t\t\tif _, ok := sr.(*searcher.MatchNoneSearcher); ok && q.queryStringMode {\n\t\t\t\t// in query string mode, skip match none\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tss = append(ss, sr)\n\t\t}\n\t}\n\n\tif len(ss) < 1 {\n\t\treturn searcher.NewMatchNoneSearcher(i)\n\t}\n\n\tnctx := context.WithValue(ctx, search.IncludeScoreBreakdownKey, q.retrieveScoreBreakdown)\n\n\treturn searcher.NewDisjunctionSearcher(nctx, i, ss, q.Min, options)\n}\n\nfunc (q *DisjunctionQuery) Validate() error {\n\tif int(q.Min) > len(q.Disjuncts) {\n\t\treturn fmt.Errorf(\"disjunction query has fewer than the minimum number of clauses to satisfy\")\n\t}\n\tfor _, q := range q.Disjuncts {\n\t\tif q, ok := q.(ValidatableQuery); ok {\n\t\t\terr := q.Validate()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (q *DisjunctionQuery) UnmarshalJSON(data []byte) error {\n\ttmp := struct {\n\t\tDisjuncts []json.RawMessage `json:\"disjuncts\"`\n\t\tBoost     *Boost            `json:\"boost,omitempty\"`\n\t\tMin       float64           `json:\"min\"`\n\t}{}\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\tq.Disjuncts = make([]Query, len(tmp.Disjuncts))\n\tfor i, term := range tmp.Disjuncts {\n\t\tquery, err := ParseQuery(term)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tq.Disjuncts[i] = query\n\t}\n\tq.BoostVal = tmp.Boost\n\tq.Min = tmp.Min\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/docid.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype DocIDQuery struct {\n\tIDs      []string `json:\"ids\"`\n\tBoostVal *Boost   `json:\"boost,omitempty\"`\n}\n\n// NewDocIDQuery creates a new Query object returning indexed documents among\n// the specified set. Combine it with ConjunctionQuery to restrict the scope of\n// other queries output.\nfunc NewDocIDQuery(ids []string) *DocIDQuery {\n\treturn &DocIDQuery{\n\t\tIDs: ids,\n\t}\n}\n\nfunc (q *DocIDQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *DocIDQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *DocIDQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\treturn searcher.NewDocIDSearcher(ctx, i, q.IDs, q.BoostVal.Value(), options)\n}\n"
  },
  {
    "path": "search/query/fuzzy.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype FuzzyQuery struct {\n\tTerm      string `json:\"term\"`\n\tPrefix    int    `json:\"prefix_length\"`\n\tFuzziness int    `json:\"fuzziness\"`\n\tFieldVal  string `json:\"field,omitempty\"`\n\tBoostVal  *Boost `json:\"boost,omitempty\"`\n\tautoFuzzy bool\n}\n\n// NewFuzzyQuery creates a new Query which finds\n// documents containing terms within a specific\n// fuzziness of the specified term.\n// The default fuzziness is 1.\n//\n// The current implementation uses Levenshtein edit\n// distance as the fuzziness metric.\nfunc NewFuzzyQuery(term string) *FuzzyQuery {\n\treturn &FuzzyQuery{\n\t\tTerm:      term,\n\t\tFuzziness: 1,\n\t}\n}\n\nfunc (q *FuzzyQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *FuzzyQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *FuzzyQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *FuzzyQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *FuzzyQuery) SetFuzziness(f int) {\n\tq.Fuzziness = f\n}\n\nfunc (q *FuzzyQuery) SetAutoFuzziness(a bool) {\n\tq.autoFuzzy = a\n}\n\nfunc (q *FuzzyQuery) SetPrefix(p int) {\n\tq.Prefix = p\n}\n\nfunc (q *FuzzyQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\tif q.autoFuzzy {\n\t\treturn searcher.NewAutoFuzzySearcher(ctx, i, q.Term, q.Prefix, field, q.BoostVal.Value(), options)\n\t}\n\treturn searcher.NewFuzzySearcher(ctx, i, q.Term, q.Prefix, q.Fuzziness, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *FuzzyQuery) UnmarshalJSON(data []byte) error {\n\ttype Alias FuzzyQuery\n\taux := &struct {\n\t\tFuzziness interface{} `json:\"fuzziness\"`\n\t\t*Alias\n\t}{\n\t\tAlias: (*Alias)(q),\n\t}\n\tif err := util.UnmarshalJSON(data, &aux); err != nil {\n\t\treturn err\n\t}\n\tswitch v := aux.Fuzziness.(type) {\n\tcase float64:\n\t\tq.Fuzziness = int(v)\n\tcase string:\n\t\tif v == \"auto\" {\n\t\t\tq.autoFuzzy = true\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (f *FuzzyQuery) MarshalJSON() ([]byte, error) {\n\tvar fuzzyValue interface{}\n\tif f.autoFuzzy {\n\t\tfuzzyValue = \"auto\"\n\t} else {\n\t\tfuzzyValue = f.Fuzziness\n\t}\n\ttype fuzzyQuery struct {\n\t\tTerm      string      `json:\"term\"`\n\t\tPrefix    int         `json:\"prefix_length\"`\n\t\tFuzziness interface{} `json:\"fuzziness\"`\n\t\tFieldVal  string      `json:\"field,omitempty\"`\n\t\tBoostVal  *Boost      `json:\"boost,omitempty\"`\n\t}\n\taux := fuzzyQuery{\n\t\tTerm:      f.Term,\n\t\tPrefix:    f.Prefix,\n\t\tFuzziness: fuzzyValue,\n\t\tFieldVal:  f.FieldVal,\n\t\tBoostVal:  f.BoostVal,\n\t}\n\treturn util.MarshalJSON(aux)\n}\n"
  },
  {
    "path": "search/query/geo_boundingbox.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype GeoBoundingBoxQuery struct {\n\tTopLeft     []float64 `json:\"top_left,omitempty\"`\n\tBottomRight []float64 `json:\"bottom_right,omitempty\"`\n\tFieldVal    string    `json:\"field,omitempty\"`\n\tBoostVal    *Boost    `json:\"boost,omitempty\"`\n}\n\nfunc NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *GeoBoundingBoxQuery {\n\treturn &GeoBoundingBoxQuery{\n\t\tTopLeft:     []float64{topLeftLon, topLeftLat},\n\t\tBottomRight: []float64{bottomRightLon, bottomRightLat},\n\t}\n}\n\nfunc (q *GeoBoundingBoxQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *GeoBoundingBoxQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *GeoBoundingBoxQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *GeoBoundingBoxQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *GeoBoundingBoxQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\tctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo)\n\n\tif q.BottomRight[0] < q.TopLeft[0] {\n\t\t// cross date line, rewrite as two parts\n\n\t\tleftSearcher, err := searcher.NewGeoBoundingBoxSearcher(ctx, i, -180, q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trightSearcher, err := searcher.NewGeoBoundingBoxSearcher(ctx, i, q.TopLeft[0], q.BottomRight[1], 180, q.TopLeft[1], field, q.BoostVal.Value(), options, true)\n\t\tif err != nil {\n\t\t\t_ = leftSearcher.Close()\n\t\t\treturn nil, err\n\t\t}\n\n\t\treturn searcher.NewDisjunctionSearcher(ctx, i, []search.Searcher{leftSearcher, rightSearcher}, 0, options)\n\t}\n\n\treturn searcher.NewGeoBoundingBoxSearcher(ctx, i, q.TopLeft[0], q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true)\n}\n\nfunc (q *GeoBoundingBoxQuery) Validate() error {\n\tif q.TopLeft[1] < q.BottomRight[1] {\n\t\treturn fmt.Errorf(\"geo bounding box top left should be higher than bottom right\")\n\t}\n\treturn nil\n}\n\nfunc (q *GeoBoundingBoxQuery) UnmarshalJSON(data []byte) error {\n\ttmp := struct {\n\t\tTopLeft     interface{} `json:\"top_left,omitempty\"`\n\t\tBottomRight interface{} `json:\"bottom_right,omitempty\"`\n\t\tFieldVal    string      `json:\"field,omitempty\"`\n\t\tBoostVal    *Boost      `json:\"boost,omitempty\"`\n\t}{}\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// now use our generic point parsing code from the geo package\n\tlon, lat, found := geo.ExtractGeoPoint(tmp.TopLeft)\n\tif !found {\n\t\treturn fmt.Errorf(\"geo location top_left not in a valid format\")\n\t}\n\tq.TopLeft = []float64{lon, lat}\n\tlon, lat, found = geo.ExtractGeoPoint(tmp.BottomRight)\n\tif !found {\n\t\treturn fmt.Errorf(\"geo location bottom_right not in a valid format\")\n\t}\n\tq.BottomRight = []float64{lon, lat}\n\tq.FieldVal = tmp.FieldVal\n\tq.BoostVal = tmp.BoostVal\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/geo_boundingpolygon.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype GeoBoundingPolygonQuery struct {\n\tPoints   []geo.Point `json:\"polygon_points\"`\n\tFieldVal string      `json:\"field,omitempty\"`\n\tBoostVal *Boost      `json:\"boost,omitempty\"`\n}\n\nfunc NewGeoBoundingPolygonQuery(points []geo.Point) *GeoBoundingPolygonQuery {\n\treturn &GeoBoundingPolygonQuery{\n\t\tPoints: points}\n}\n\nfunc (q *GeoBoundingPolygonQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *GeoBoundingPolygonQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *GeoBoundingPolygonQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *GeoBoundingPolygonQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *GeoBoundingPolygonQuery) Searcher(ctx context.Context, i index.IndexReader,\n\tm mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\tctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo)\n\n\treturn searcher.NewGeoBoundedPolygonSearcher(ctx, i, q.Points, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *GeoBoundingPolygonQuery) Validate() error {\n\treturn nil\n}\n\nfunc (q *GeoBoundingPolygonQuery) UnmarshalJSON(data []byte) error {\n\ttmp := struct {\n\t\tPoints   []interface{} `json:\"polygon_points\"`\n\t\tFieldVal string        `json:\"field,omitempty\"`\n\t\tBoostVal *Boost        `json:\"boost,omitempty\"`\n\t}{}\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tq.Points = make([]geo.Point, 0, len(tmp.Points))\n\tfor _, i := range tmp.Points {\n\t\t// now use our generic point parsing code from the geo package\n\t\tlon, lat, found := geo.ExtractGeoPoint(i)\n\t\tif !found {\n\t\t\treturn fmt.Errorf(\"geo polygon point: %v is not in a valid format\", i)\n\t\t}\n\t\tq.Points = append(q.Points, geo.Point{Lon: lon, Lat: lat})\n\t}\n\n\tq.FieldVal = tmp.FieldVal\n\tq.BoostVal = tmp.BoostVal\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/geo_distance.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype GeoDistanceQuery struct {\n\tLocation []float64 `json:\"location,omitempty\"`\n\tDistance string    `json:\"distance,omitempty\"`\n\tFieldVal string    `json:\"field,omitempty\"`\n\tBoostVal *Boost    `json:\"boost,omitempty\"`\n}\n\nfunc NewGeoDistanceQuery(lon, lat float64, distance string) *GeoDistanceQuery {\n\treturn &GeoDistanceQuery{\n\t\tLocation: []float64{lon, lat},\n\t\tDistance: distance,\n\t}\n}\n\nfunc (q *GeoDistanceQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *GeoDistanceQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *GeoDistanceQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *GeoDistanceQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *GeoDistanceQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping,\n\toptions search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\tctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo)\n\n\tdist, err := geo.ParseDistance(q.Distance)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn searcher.NewGeoPointDistanceSearcher(ctx, i, q.Location[0], q.Location[1],\n\t\tdist, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *GeoDistanceQuery) Validate() error {\n\treturn nil\n}\n\nfunc (q *GeoDistanceQuery) UnmarshalJSON(data []byte) error {\n\ttmp := struct {\n\t\tLocation interface{} `json:\"location,omitempty\"`\n\t\tDistance string      `json:\"distance,omitempty\"`\n\t\tFieldVal string      `json:\"field,omitempty\"`\n\t\tBoostVal *Boost      `json:\"boost,omitempty\"`\n\t}{}\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// now use our generic point parsing code from the geo package\n\tlon, lat, found := geo.ExtractGeoPoint(tmp.Location)\n\tif !found {\n\t\treturn fmt.Errorf(\"geo location not in a valid format\")\n\t}\n\tq.Location = []float64{lon, lat}\n\tq.Distance = tmp.Distance\n\tq.FieldVal = tmp.FieldVal\n\tq.BoostVal = tmp.BoostVal\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/geo_shape.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype Geometry struct {\n\tShape    index.GeoJSON `json:\"shape\"`\n\tRelation string        `json:\"relation\"`\n}\n\ntype GeoShapeQuery struct {\n\tGeometry Geometry `json:\"geometry\"`\n\tFieldVal string   `json:\"field,omitempty\"`\n\tBoostVal *Boost   `json:\"boost,omitempty\"`\n}\n\n// NewGeoShapeQuery creates a geoshape query for the\n// given shape type. This method can be used for\n// creating geoshape queries for shape types like: point,\n// linestring, polygon, multipoint, multilinestring,\n// multipolygon and envelope.\nfunc NewGeoShapeQuery(coordinates [][][][]float64, typ,\n\trelation string) (*GeoShapeQuery, error) {\n\ts, _, err := geo.NewGeoJsonShape(coordinates, typ)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &GeoShapeQuery{Geometry: Geometry{Shape: s,\n\t\tRelation: relation}}, nil\n}\n\n// NewGeoShapeCircleQuery creates a geoshape query for the\n// given center point and the radius. Radius formats supported:\n// \"5in\" \"5inch\" \"7yd\" \"7yards\" \"9ft\" \"9feet\" \"11km\" \"11kilometers\"\n// \"3nm\" \"3nauticalmiles\" \"13mm\" \"13millimeters\" \"15cm\" \"15centimeters\"\n// \"17mi\" \"17miles\" \"19m\" \"19meters\" If the unit cannot be determined,\n// the entire string is parsed and the unit of meters is assumed.\nfunc NewGeoShapeCircleQuery(coordinates []float64, radius,\n\trelation string) (*GeoShapeQuery, error) {\n\n\ts, _, err := geo.NewGeoCircleShape(coordinates, radius)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &GeoShapeQuery{Geometry: Geometry{Shape: s,\n\t\tRelation: relation}}, nil\n}\n\n// NewGeometryCollectionQuery creates a geoshape query for the\n// given geometrycollection coordinates and types.\nfunc NewGeometryCollectionQuery(coordinates [][][][][]float64, types []string,\n\trelation string) (*GeoShapeQuery, error) {\n\ts, _, err := geo.NewGeometryCollection(coordinates, types)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &GeoShapeQuery{Geometry: Geometry{Shape: s,\n\t\tRelation: relation}}, nil\n}\n\nfunc (q *GeoShapeQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *GeoShapeQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *GeoShapeQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *GeoShapeQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *GeoShapeQuery) Searcher(ctx context.Context, i index.IndexReader,\n\tm mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\tctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo)\n\n\treturn searcher.NewGeoShapeSearcher(ctx, i, q.Geometry.Shape, q.Geometry.Relation, field,\n\t\tq.BoostVal.Value(), options)\n}\n\nfunc (q *GeoShapeQuery) Validate() error {\n\treturn nil\n}\n\nfunc (q *Geometry) UnmarshalJSON(data []byte) error {\n\ttmp := struct {\n\t\tShape    json.RawMessage `json:\"shape\"`\n\t\tRelation string          `json:\"relation\"`\n\t}{}\n\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tq.Shape, err = geo.ParseGeoJSONShape(tmp.Shape)\n\tif err != nil {\n\t\treturn err\n\t}\n\tq.Relation = tmp.Relation\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/ip_range.go",
    "content": "//  Copyright (c) 2021 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype IPRangeQuery struct {\n\tCIDR     string `json:\"cidr,omitempty\"`\n\tFieldVal string `json:\"field,omitempty\"`\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\nfunc NewIPRangeQuery(cidr string) *IPRangeQuery {\n\treturn &IPRangeQuery{\n\t\tCIDR: cidr,\n\t}\n}\n\nfunc (q *IPRangeQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *IPRangeQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *IPRangeQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *IPRangeQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *IPRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\t_, ipNet, err := net.ParseCIDR(q.CIDR)\n\tif err != nil {\n\t\tip := net.ParseIP(q.CIDR)\n\t\tif ip == nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// If we are searching for a specific ip rather than members of a network, just use a term search.\n\t\treturn searcher.NewTermSearcherBytes(ctx, i, ip.To16(), field, q.BoostVal.Value(), options)\n\t}\n\treturn searcher.NewIPRangeSearcher(ctx, i, ipNet, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *IPRangeQuery) Validate() error {\n\t_, _, err := net.ParseCIDR(q.CIDR)\n\tif err == nil {\n\t\treturn nil\n\t}\n\t// We also allow search for a specific IP.\n\tip := net.ParseIP(q.CIDR)\n\tif ip != nil {\n\t\treturn nil // we have a valid ip\n\t}\n\treturn fmt.Errorf(\"IPRangeQuery must be for a network or ip address, %q\", q.CIDR)\n}\n"
  },
  {
    "path": "search/query/knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage query\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype KNNQuery struct {\n\tVectorField string    `json:\"field\"`\n\tVector      []float32 `json:\"vector\"`\n\tK           int64     `json:\"k\"`\n\tBoostVal    *Boost    `json:\"boost,omitempty\"`\n\n\t// see KNNRequest.Params for description\n\tParams json.RawMessage `json:\"params\"`\n\t// elegibleSelector is used to filter out documents that are\n\t// eligible for the KNN search from a pre-filter query.\n\telegibleSelector index.EligibleDocumentSelector\n}\n\nfunc NewKNNQuery(vector []float32) *KNNQuery {\n\treturn &KNNQuery{Vector: vector}\n}\n\nfunc (q *KNNQuery) Field() string {\n\treturn q.VectorField\n}\n\nfunc (q *KNNQuery) SetK(k int64) {\n\tq.K = k\n}\n\nfunc (q *KNNQuery) SetField(field string) {\n\tq.VectorField = field\n}\n\nfunc (q *KNNQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *KNNQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *KNNQuery) SetParams(params json.RawMessage) {\n\tq.Params = params\n}\n\nfunc (q *KNNQuery) SetEligibleSelector(eligibleSelector index.EligibleDocumentSelector) {\n\tq.elegibleSelector = eligibleSelector\n}\n\nfunc (q *KNNQuery) Searcher(ctx context.Context, i index.IndexReader,\n\tm mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfieldMapping := m.FieldMappingForPath(q.VectorField)\n\tsimilarityMetric := fieldMapping.Similarity\n\tif similarityMetric == \"\" {\n\t\tsimilarityMetric = index.DefaultVectorSimilarityMetric\n\t}\n\tif q.K <= 0 || len(q.Vector) == 0 {\n\t\treturn nil, fmt.Errorf(\"k must be greater than 0 and vector must be non-empty\")\n\t}\n\t// bivf-sq8 indexes only supports hamming distance for the primary\n\t// binary index. Similarity here is used for the backing flat index,\n\t// which is set to cosine similarity for recall reasons\n\tif index.OptimizationRequiresBinaryIndex(fieldMapping.VectorIndexOptimizedFor) {\n\t\tsimilarityMetric = index.CosineSimilarity\n\t}\n\tif similarityMetric == index.CosineSimilarity {\n\t\t// normalize the vector\n\t\tq.Vector = mapping.NormalizeVector(q.Vector)\n\t}\n\n\treturn searcher.NewKNNSearcher(ctx, i, m, options, q.VectorField,\n\t\tq.Vector, q.K, q.BoostVal.Value(), similarityMetric, q.Params,\n\t\tq.elegibleSelector)\n}\n"
  },
  {
    "path": "search/query/match.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype MatchQuery struct {\n\tMatch     string             `json:\"match\"`\n\tFieldVal  string             `json:\"field,omitempty\"`\n\tAnalyzer  string             `json:\"analyzer,omitempty\"`\n\tBoostVal  *Boost             `json:\"boost,omitempty\"`\n\tPrefix    int                `json:\"prefix_length\"`\n\tFuzziness int                `json:\"fuzziness\"`\n\tOperator  MatchQueryOperator `json:\"operator,omitempty\"`\n\tautoFuzzy bool\n}\n\ntype MatchQueryOperator int\n\nconst (\n\t// Document must satisfy AT LEAST ONE of term searches.\n\tMatchQueryOperatorOr = MatchQueryOperator(0)\n\t// Document must satisfy ALL of term searches.\n\tMatchQueryOperatorAnd = MatchQueryOperator(1)\n)\n\nfunc (o MatchQueryOperator) MarshalJSON() ([]byte, error) {\n\tswitch o {\n\tcase MatchQueryOperatorOr:\n\t\treturn util.MarshalJSON(\"or\")\n\tcase MatchQueryOperatorAnd:\n\t\treturn util.MarshalJSON(\"and\")\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"cannot marshal match operator %d to JSON\", o)\n\t}\n}\n\nfunc (o *MatchQueryOperator) UnmarshalJSON(data []byte) error {\n\tvar operatorString string\n\terr := util.UnmarshalJSON(data, &operatorString)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tswitch operatorString {\n\tcase \"or\":\n\t\t*o = MatchQueryOperatorOr\n\t\treturn nil\n\tcase \"and\":\n\t\t*o = MatchQueryOperatorAnd\n\t\treturn nil\n\tdefault:\n\t\treturn fmt.Errorf(\"cannot unmarshal match operator '%v' from JSON\", o)\n\t}\n}\n\n// NewMatchQuery creates a Query for matching text.\n// An Analyzer is chosen based on the field.\n// Input text is analyzed using this analyzer.\n// Token terms resulting from this analysis are\n// used to perform term searches.  Result documents\n// must satisfy at least one of these term searches.\nfunc NewMatchQuery(match string) *MatchQuery {\n\treturn &MatchQuery{\n\t\tMatch:    match,\n\t\tOperator: MatchQueryOperatorOr,\n\t}\n}\n\nfunc (q *MatchQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *MatchQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *MatchQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *MatchQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *MatchQuery) SetFuzziness(f int) {\n\tq.Fuzziness = f\n}\n\nfunc (q *MatchQuery) SetAutoFuzziness(auto bool) {\n\tq.autoFuzzy = auto\n}\n\nfunc (q *MatchQuery) SetPrefix(p int) {\n\tq.Prefix = p\n}\n\nfunc (q *MatchQuery) SetOperator(operator MatchQueryOperator) {\n\tq.Operator = operator\n}\n\nfunc (q *MatchQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\tanalyzerName := \"\"\n\tif q.Analyzer != \"\" {\n\t\tanalyzerName = q.Analyzer\n\t} else {\n\t\tanalyzerName = m.AnalyzerNameForPath(field)\n\t}\n\tanalyzer := m.AnalyzerNamed(analyzerName)\n\n\tif analyzer == nil {\n\t\treturn nil, fmt.Errorf(\"no analyzer named '%s' registered\", q.Analyzer)\n\t}\n\n\ttokens := analyzer.Analyze([]byte(q.Match))\n\tif len(tokens) > 0 {\n\n\t\ttqs := make([]Query, len(tokens))\n\t\tif q.Fuzziness != 0 || q.autoFuzzy {\n\t\t\tfor i, token := range tokens {\n\t\t\t\tquery := NewFuzzyQuery(string(token.Term))\n\t\t\t\tif q.autoFuzzy {\n\t\t\t\t\tquery.SetAutoFuzziness(true)\n\t\t\t\t} else {\n\t\t\t\t\tquery.SetFuzziness(q.Fuzziness)\n\t\t\t\t}\n\t\t\t\tquery.SetPrefix(q.Prefix)\n\t\t\t\tquery.SetField(field)\n\t\t\t\tquery.SetBoost(q.BoostVal.Value())\n\t\t\t\ttqs[i] = query\n\t\t\t}\n\t\t} else {\n\t\t\tfor i, token := range tokens {\n\t\t\t\ttq := NewTermQuery(string(token.Term))\n\t\t\t\ttq.SetField(field)\n\t\t\t\ttq.SetBoost(q.BoostVal.Value())\n\t\t\t\ttqs[i] = tq\n\t\t\t}\n\t\t}\n\n\t\tswitch q.Operator {\n\t\tcase MatchQueryOperatorOr:\n\t\t\tshouldQuery := NewDisjunctionQuery(tqs)\n\t\t\tshouldQuery.SetMin(1)\n\t\t\tshouldQuery.SetBoost(q.BoostVal.Value())\n\t\t\treturn shouldQuery.Searcher(ctx, i, m, options)\n\n\t\tcase MatchQueryOperatorAnd:\n\t\t\tmustQuery := NewConjunctionQuery(tqs)\n\t\t\tmustQuery.SetBoost(q.BoostVal.Value())\n\t\t\treturn mustQuery.Searcher(ctx, i, m, options)\n\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"unhandled operator %d\", q.Operator)\n\t\t}\n\t}\n\tnoneQuery := NewMatchNoneQuery()\n\treturn noneQuery.Searcher(ctx, i, m, options)\n}\n\nfunc (q *MatchQuery) UnmarshalJSON(data []byte) error {\n\ttype Alias MatchQuery\n\taux := &struct {\n\t\tFuzziness interface{} `json:\"fuzziness\"`\n\t\t*Alias\n\t}{\n\t\tAlias: (*Alias)(q),\n\t}\n\tif err := util.UnmarshalJSON(data, &aux); err != nil {\n\t\treturn err\n\t}\n\tswitch v := aux.Fuzziness.(type) {\n\tcase float64:\n\t\tq.Fuzziness = int(v)\n\tcase string:\n\t\tif v == \"auto\" {\n\t\t\tq.autoFuzzy = true\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (f *MatchQuery) MarshalJSON() ([]byte, error) {\n\tvar fuzzyValue interface{}\n\tif f.autoFuzzy {\n\t\tfuzzyValue = \"auto\"\n\t} else {\n\t\tfuzzyValue = f.Fuzziness\n\t}\n\ttype match struct {\n\t\tMatch     string             `json:\"match\"`\n\t\tFieldVal  string             `json:\"field,omitempty\"`\n\t\tAnalyzer  string             `json:\"analyzer,omitempty\"`\n\t\tBoostVal  *Boost             `json:\"boost,omitempty\"`\n\t\tPrefix    int                `json:\"prefix_length\"`\n\t\tFuzziness interface{}        `json:\"fuzziness\"`\n\t\tOperator  MatchQueryOperator `json:\"operator,omitempty\"`\n\t}\n\taux := match{\n\t\tMatch:     f.Match,\n\t\tFieldVal:  f.FieldVal,\n\t\tAnalyzer:  f.Analyzer,\n\t\tBoostVal:  f.BoostVal,\n\t\tPrefix:    f.Prefix,\n\t\tFuzziness: fuzzyValue,\n\t\tOperator:  f.Operator,\n\t}\n\treturn util.MarshalJSON(aux)\n}\n"
  },
  {
    "path": "search/query/match_all.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype MatchAllQuery struct {\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\n// NewMatchAllQuery creates a Query which will\n// match all documents in the index.\nfunc NewMatchAllQuery() *MatchAllQuery {\n\treturn &MatchAllQuery{}\n}\n\nfunc (q *MatchAllQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *MatchAllQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *MatchAllQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\treturn searcher.NewMatchAllSearcher(ctx, i, q.BoostVal.Value(), options)\n}\n\nfunc (q *MatchAllQuery) MarshalJSON() ([]byte, error) {\n\ttmp := map[string]interface{}{\n\t\t\"boost\":     q.BoostVal,\n\t\t\"match_all\": map[string]interface{}{},\n\t}\n\treturn json.Marshal(tmp)\n}\n"
  },
  {
    "path": "search/query/match_none.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype MatchNoneQuery struct {\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\n// NewMatchNoneQuery creates a Query which will not\n// match any documents in the index.\nfunc NewMatchNoneQuery() *MatchNoneQuery {\n\treturn &MatchNoneQuery{}\n}\n\nfunc (q *MatchNoneQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *MatchNoneQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *MatchNoneQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\treturn searcher.NewMatchNoneSearcher(i)\n}\n\nfunc (q *MatchNoneQuery) MarshalJSON() ([]byte, error) {\n\ttmp := map[string]interface{}{\n\t\t\"boost\":      q.BoostVal,\n\t\t\"match_none\": map[string]interface{}{},\n\t}\n\treturn json.Marshal(tmp)\n}\n"
  },
  {
    "path": "search/query/match_phrase.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype MatchPhraseQuery struct {\n\tMatchPhrase string `json:\"match_phrase\"`\n\tFieldVal    string `json:\"field,omitempty\"`\n\tAnalyzer    string `json:\"analyzer,omitempty\"`\n\tBoostVal    *Boost `json:\"boost,omitempty\"`\n\tFuzziness   int    `json:\"fuzziness\"`\n\tautoFuzzy   bool\n}\n\n// NewMatchPhraseQuery creates a new Query object\n// for matching phrases in the index.\n// An Analyzer is chosen based on the field.\n// Input text is analyzed using this analyzer.\n// Token terms resulting from this analysis are\n// used to build a search phrase.  Result documents\n// must match this phrase. Queried field must have been indexed with\n// IncludeTermVectors set to true.\nfunc NewMatchPhraseQuery(matchPhrase string) *MatchPhraseQuery {\n\treturn &MatchPhraseQuery{\n\t\tMatchPhrase: matchPhrase,\n\t}\n}\n\nfunc (q *MatchPhraseQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *MatchPhraseQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *MatchPhraseQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *MatchPhraseQuery) SetFuzziness(f int) {\n\tq.Fuzziness = f\n}\n\nfunc (q *MatchPhraseQuery) SetAutoFuzziness(auto bool) {\n\tq.autoFuzzy = auto\n}\n\nfunc (q *MatchPhraseQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *MatchPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\tanalyzerName := \"\"\n\tif q.Analyzer != \"\" {\n\t\tanalyzerName = q.Analyzer\n\t} else {\n\t\tanalyzerName = m.AnalyzerNameForPath(field)\n\t}\n\tanalyzer := m.AnalyzerNamed(analyzerName)\n\tif analyzer == nil {\n\t\treturn nil, fmt.Errorf(\"no analyzer named '%s' registered\", q.Analyzer)\n\t}\n\n\ttokens := analyzer.Analyze([]byte(q.MatchPhrase))\n\tif len(tokens) > 0 {\n\t\tphrase := tokenStreamToPhrase(tokens)\n\t\tphraseQuery := NewMultiPhraseQuery(phrase, field)\n\t\tphraseQuery.SetBoost(q.BoostVal.Value())\n\t\tif q.autoFuzzy {\n\t\t\tphraseQuery.SetAutoFuzziness(true)\n\t\t} else {\n\t\t\tphraseQuery.SetFuzziness(q.Fuzziness)\n\t\t}\n\t\treturn phraseQuery.Searcher(ctx, i, m, options)\n\t}\n\tnoneQuery := NewMatchNoneQuery()\n\treturn noneQuery.Searcher(ctx, i, m, options)\n}\n\nfunc tokenStreamToPhrase(tokens analysis.TokenStream) [][]string {\n\tfirstPosition := int(^uint(0) >> 1)\n\tlastPosition := 0\n\tfor _, token := range tokens {\n\t\tif token.Position < firstPosition {\n\t\t\tfirstPosition = token.Position\n\t\t}\n\t\tif token.Position > lastPosition {\n\t\t\tlastPosition = token.Position\n\t\t}\n\t}\n\tphraseLen := lastPosition - firstPosition + 1\n\tif phraseLen > 0 {\n\t\trv := make([][]string, phraseLen)\n\t\tfor _, token := range tokens {\n\t\t\tpos := token.Position - firstPosition\n\t\t\trv[pos] = append(rv[pos], string(token.Term))\n\t\t}\n\t\treturn rv\n\t}\n\treturn nil\n}\n\nfunc (q *MatchPhraseQuery) UnmarshalJSON(data []byte) error {\n\ttype Alias MatchPhraseQuery\n\taux := &struct {\n\t\tFuzziness interface{} `json:\"fuzziness\"`\n\t\t*Alias\n\t}{\n\t\tAlias: (*Alias)(q),\n\t}\n\tif err := util.UnmarshalJSON(data, &aux); err != nil {\n\t\treturn err\n\t}\n\tswitch v := aux.Fuzziness.(type) {\n\tcase float64:\n\t\tq.Fuzziness = int(v)\n\tcase string:\n\t\tif v == \"auto\" {\n\t\t\tq.autoFuzzy = true\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (f *MatchPhraseQuery) MarshalJSON() ([]byte, error) {\n\tvar fuzzyValue interface{}\n\tif f.autoFuzzy {\n\t\tfuzzyValue = \"auto\"\n\t} else {\n\t\tfuzzyValue = f.Fuzziness\n\t}\n\ttype matchPhrase struct {\n\t\tMatchPhrase string      `json:\"match_phrase\"`\n\t\tFieldVal    string      `json:\"field,omitempty\"`\n\t\tAnalyzer    string      `json:\"analyzer,omitempty\"`\n\t\tBoostVal    *Boost      `json:\"boost,omitempty\"`\n\t\tFuzziness   interface{} `json:\"fuzziness\"`\n\t}\n\taux := matchPhrase{\n\t\tMatchPhrase: f.MatchPhrase,\n\t\tFieldVal:    f.FieldVal,\n\t\tAnalyzer:    f.Analyzer,\n\t\tBoostVal:    f.BoostVal,\n\t\tFuzziness:   fuzzyValue,\n\t}\n\treturn util.MarshalJSON(aux)\n}\n"
  },
  {
    "path": "search/query/match_phrase_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n)\n\nfunc TestTokenStreamToPhrase(t *testing.T) {\n\n\ttests := []struct {\n\t\ttokens analysis.TokenStream\n\t\tresult [][]string\n\t}{\n\t\t// empty token stream returns nil\n\t\t{\n\t\t\ttokens: analysis.TokenStream{},\n\t\t\tresult: nil,\n\t\t},\n\t\t// typical token\n\t\t{\n\t\t\ttokens: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"one\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"two\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: [][]string{{\"one\"}, {\"two\"}},\n\t\t},\n\t\t// token stream containing a gap (usually from stop words)\n\t\t{\n\t\t\ttokens: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"wag\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"dog\"),\n\t\t\t\t\tPosition: 3,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: [][]string{{\"wag\"}, nil, {\"dog\"}},\n\t\t},\n\t\t// token stream containing multiple tokens at the same position\n\t\t{\n\t\t\ttokens: analysis.TokenStream{\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"nia\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"onia\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"donia\"),\n\t\t\t\t\tPosition: 1,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"imo\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"nimo\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t\t&analysis.Token{\n\t\t\t\t\tTerm:     []byte(\"ónimo\"),\n\t\t\t\t\tPosition: 2,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: [][]string{{\"nia\", \"onia\", \"donia\"}, {\"imo\", \"nimo\", \"ónimo\"}},\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tactual := tokenStreamToPhrase(test.tokens)\n\t\tif !reflect.DeepEqual(actual, test.result) {\n\t\t\tt.Fatalf(\"expected %#v got %#v for test %d\", test.result, actual, i)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/query/multi_phrase.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype MultiPhraseQuery struct {\n\tTerms     [][]string `json:\"terms\"`\n\tFieldVal  string     `json:\"field,omitempty\"`\n\tBoostVal  *Boost     `json:\"boost,omitempty\"`\n\tFuzziness int        `json:\"fuzziness\"`\n\tautoFuzzy bool\n}\n\n// NewMultiPhraseQuery creates a new Query for finding\n// term phrases in the index.\n// It is like PhraseQuery, but each position in the\n// phrase may be satisfied by a list of terms\n// as opposed to just one.\n// At least one of the terms must exist in the correct\n// order, at the correct index offsets, in the\n// specified field. Queried field must have been indexed with\n// IncludeTermVectors set to true.\nfunc NewMultiPhraseQuery(terms [][]string, field string) *MultiPhraseQuery {\n\treturn &MultiPhraseQuery{\n\t\tTerms:    terms,\n\t\tFieldVal: field,\n\t}\n}\n\nfunc (q *MultiPhraseQuery) SetFuzziness(f int) {\n\tq.Fuzziness = f\n}\n\nfunc (q *MultiPhraseQuery) SetAutoFuzziness(auto bool) {\n\tq.autoFuzzy = auto\n}\n\nfunc (q *MultiPhraseQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *MultiPhraseQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *MultiPhraseQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *MultiPhraseQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *MultiPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\treturn searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, q.Fuzziness, q.autoFuzzy, q.FieldVal, q.BoostVal.Value(), options)\n}\n\nfunc (q *MultiPhraseQuery) Validate() error {\n\tif len(q.Terms) < 1 {\n\t\treturn fmt.Errorf(\"phrase query must contain at least one term\")\n\t}\n\treturn nil\n}\n\nfunc (q *MultiPhraseQuery) UnmarshalJSON(data []byte) error {\n\ttype Alias MultiPhraseQuery\n\taux := &struct {\n\t\tFuzziness interface{} `json:\"fuzziness\"`\n\t\t*Alias\n\t}{\n\t\tAlias: (*Alias)(q),\n\t}\n\tif err := util.UnmarshalJSON(data, &aux); err != nil {\n\t\treturn err\n\t}\n\tswitch v := aux.Fuzziness.(type) {\n\tcase float64:\n\t\tq.Fuzziness = int(v)\n\tcase string:\n\t\tif v == \"auto\" {\n\t\t\tq.autoFuzzy = true\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (f *MultiPhraseQuery) MarshalJSON() ([]byte, error) {\n\tvar fuzzyValue interface{}\n\tif f.autoFuzzy {\n\t\tfuzzyValue = \"auto\"\n\t} else {\n\t\tfuzzyValue = f.Fuzziness\n\t}\n\ttype multiPhraseQuery struct {\n\t\tTerms     [][]string  `json:\"terms\"`\n\t\tFieldVal  string      `json:\"field,omitempty\"`\n\t\tBoostVal  *Boost      `json:\"boost,omitempty\"`\n\t\tFuzziness interface{} `json:\"fuzziness\"`\n\t}\n\taux := multiPhraseQuery{\n\t\tTerms:     f.Terms,\n\t\tFieldVal:  f.FieldVal,\n\t\tBoostVal:  f.BoostVal,\n\t\tFuzziness: fuzzyValue,\n\t}\n\treturn util.MarshalJSON(aux)\n}\n"
  },
  {
    "path": "search/query/numeric_range.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype NumericRangeQuery struct {\n\tMin          *float64 `json:\"min,omitempty\"`\n\tMax          *float64 `json:\"max,omitempty\"`\n\tInclusiveMin *bool    `json:\"inclusive_min,omitempty\"`\n\tInclusiveMax *bool    `json:\"inclusive_max,omitempty\"`\n\tFieldVal     string   `json:\"field,omitempty\"`\n\tBoostVal     *Boost   `json:\"boost,omitempty\"`\n}\n\n// NewNumericRangeQuery creates a new Query for ranges\n// of numeric values.\n// Either, but not both endpoints can be nil.\n// The minimum value is inclusive.\n// The maximum value is exclusive.\nfunc NewNumericRangeQuery(min, max *float64) *NumericRangeQuery {\n\treturn NewNumericRangeInclusiveQuery(min, max, nil, nil)\n}\n\n// NewNumericRangeInclusiveQuery creates a new Query for ranges\n// of numeric values.\n// Either, but not both endpoints can be nil.\n// Control endpoint inclusion with inclusiveMin, inclusiveMax.\nfunc NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *NumericRangeQuery {\n\treturn &NumericRangeQuery{\n\t\tMin:          min,\n\t\tMax:          max,\n\t\tInclusiveMin: minInclusive,\n\t\tInclusiveMax: maxInclusive,\n\t}\n}\n\nfunc (q *NumericRangeQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *NumericRangeQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *NumericRangeQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *NumericRangeQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *NumericRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\tctx = context.WithValue(ctx, search.QueryTypeKey, search.Numeric)\n\treturn searcher.NewNumericRangeSearcher(ctx, i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *NumericRangeQuery) Validate() error {\n\tif q.Min == nil && q.Min == q.Max {\n\t\treturn fmt.Errorf(\"numeric range query must specify min or max\")\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/phrase.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype PhraseQuery struct {\n\tTerms     []string `json:\"terms\"`\n\tFieldVal  string   `json:\"field,omitempty\"`\n\tBoostVal  *Boost   `json:\"boost,omitempty\"`\n\tFuzziness int      `json:\"fuzziness\"`\n\tautoFuzzy bool\n}\n\n// NewPhraseQuery creates a new Query for finding\n// exact term phrases in the index.\n// The provided terms must exist in the correct\n// order, at the correct index offsets, in the\n// specified field. Queried field must have been indexed with\n// IncludeTermVectors set to true.\nfunc NewPhraseQuery(terms []string, field string) *PhraseQuery {\n\treturn &PhraseQuery{\n\t\tTerms:    terms,\n\t\tFieldVal: field,\n\t}\n}\n\nfunc (q *PhraseQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *PhraseQuery) SetFuzziness(f int) {\n\tq.Fuzziness = f\n}\n\nfunc (q *PhraseQuery) SetAutoFuzziness(auto bool) {\n\tq.autoFuzzy = auto\n}\n\nfunc (q *PhraseQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *PhraseQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *PhraseQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *PhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\treturn searcher.NewPhraseSearcher(ctx, i, q.Terms, q.Fuzziness, q.autoFuzzy, q.FieldVal, q.BoostVal.Value(), options)\n}\n\nfunc (q *PhraseQuery) Validate() error {\n\tif len(q.Terms) < 1 {\n\t\treturn fmt.Errorf(\"phrase query must contain at least one term\")\n\t}\n\treturn nil\n}\n\nfunc (q *PhraseQuery) UnmarshalJSON(data []byte) error {\n\ttype Alias PhraseQuery\n\taux := &struct {\n\t\tFuzziness interface{} `json:\"fuzziness\"`\n\t\t*Alias\n\t}{\n\t\tAlias: (*Alias)(q),\n\t}\n\tif err := util.UnmarshalJSON(data, &aux); err != nil {\n\t\treturn err\n\t}\n\tswitch v := aux.Fuzziness.(type) {\n\tcase float64:\n\t\tq.Fuzziness = int(v)\n\tcase string:\n\t\tif v == \"auto\" {\n\t\t\tq.autoFuzzy = true\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (f *PhraseQuery) MarshalJSON() ([]byte, error) {\n\tvar fuzzyValue interface{}\n\tif f.autoFuzzy {\n\t\tfuzzyValue = \"auto\"\n\t} else {\n\t\tfuzzyValue = f.Fuzziness\n\t}\n\ttype phraseQuery struct {\n\t\tTerms     []string    `json:\"terms\"`\n\t\tFieldVal  string      `json:\"field,omitempty\"`\n\t\tBoostVal  *Boost      `json:\"boost,omitempty\"`\n\t\tFuzziness interface{} `json:\"fuzziness\"`\n\t}\n\taux := phraseQuery{\n\t\tTerms:     f.Terms,\n\t\tFieldVal:  f.FieldVal,\n\t\tBoostVal:  f.BoostVal,\n\t\tFuzziness: fuzzyValue,\n\t}\n\treturn util.MarshalJSON(aux)\n}\n"
  },
  {
    "path": "search/query/prefix.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype PrefixQuery struct {\n\tPrefix   string `json:\"prefix\"`\n\tFieldVal string `json:\"field,omitempty\"`\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\n// NewPrefixQuery creates a new Query which finds\n// documents containing terms that start with the\n// specified prefix.\nfunc NewPrefixQuery(prefix string) *PrefixQuery {\n\treturn &PrefixQuery{\n\t\tPrefix: prefix,\n\t}\n}\n\nfunc (q *PrefixQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *PrefixQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *PrefixQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *PrefixQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *PrefixQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\treturn searcher.NewTermPrefixSearcher(ctx, i, q.Prefix, field, q.BoostVal.Value(), options)\n}\n"
  },
  {
    "path": "search/query/query.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar logger = log.New(io.Discard, \"bleve mapping \", log.LstdFlags)\n\n// SetLog sets the logger used for logging\n// by default log messages are sent to io.Discard\nfunc SetLog(l *log.Logger) {\n\tlogger = l\n}\n\n// A Query represents a description of the type\n// and parameters for a query into the index.\ntype Query interface {\n\tSearcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping,\n\t\toptions search.SearcherOptions) (search.Searcher, error)\n}\n\n// A BoostableQuery represents a Query which can be boosted\n// relative to other queries.\ntype BoostableQuery interface {\n\tQuery\n\tSetBoost(b float64)\n\tBoost() float64\n}\n\n// A FieldableQuery represents a Query which can be restricted\n// to a single field.\ntype FieldableQuery interface {\n\tQuery\n\tSetField(f string)\n\tField() string\n}\n\n// A ValidatableQuery represents a Query which can be validated\n// prior to execution.\ntype ValidatableQuery interface {\n\tQuery\n\tValidate() error\n}\n\n// ParsePreSearchData deserializes a JSON representation of\n// a PreSearchData object.\nfunc ParsePreSearchData(input []byte) (map[string]interface{}, error) {\n\tvar rv map[string]interface{}\n\n\tvar tmp map[string]json.RawMessage\n\terr := util.UnmarshalJSON(input, &tmp)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor k, v := range tmp {\n\t\tswitch k {\n\t\tcase search.KnnPreSearchDataKey:\n\t\t\tvar value []*search.DocumentMatch\n\t\t\tif v != nil {\n\t\t\t\terr := util.UnmarshalJSON(v, &value)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif rv == nil {\n\t\t\t\trv = make(map[string]interface{})\n\t\t\t}\n\t\t\trv[search.KnnPreSearchDataKey] = value\n\t\tcase search.SynonymPreSearchDataKey:\n\t\t\tvar value search.FieldTermSynonymMap\n\t\t\tif v != nil {\n\t\t\t\terr := util.UnmarshalJSON(v, &value)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif rv == nil {\n\t\t\t\trv = make(map[string]interface{})\n\t\t\t}\n\t\t\trv[search.SynonymPreSearchDataKey] = value\n\t\tcase search.BM25PreSearchDataKey:\n\t\t\tvar value *search.BM25Stats\n\t\t\tif v != nil {\n\t\t\t\terr := util.UnmarshalJSON(v, &value)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif rv == nil {\n\t\t\t\trv = make(map[string]interface{})\n\t\t\t}\n\t\t\trv[search.BM25PreSearchDataKey] = value\n\n\t\t}\n\t}\n\treturn rv, nil\n}\n\n// ParseQuery deserializes a JSON representation of\n// a Query object.\nfunc ParseQuery(input []byte) (Query, error) {\n\tif len(input) == 0 {\n\t\t// interpret as a match_none query\n\t\treturn NewMatchNoneQuery(), nil\n\t}\n\n\tvar tmp map[string]interface{}\n\terr := util.UnmarshalJSON(input, &tmp)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(tmp) == 0 {\n\t\t// interpret as a match_none query\n\t\treturn NewMatchNoneQuery(), nil\n\t}\n\n\t_, hasFuzziness := tmp[\"fuzziness\"]\n\t_, isMatchQuery := tmp[\"match\"]\n\t_, isMatchPhraseQuery := tmp[\"match_phrase\"]\n\t_, hasTerms := tmp[\"terms\"]\n\tif hasFuzziness && !isMatchQuery && !isMatchPhraseQuery && !hasTerms {\n\t\tvar rv FuzzyQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\tif isMatchQuery {\n\t\tvar rv MatchQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\tif isMatchPhraseQuery {\n\t\tvar rv MatchPhraseQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\tif hasTerms {\n\t\tvar rv PhraseQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\t// now try multi-phrase\n\t\t\tvar rv2 MultiPhraseQuery\n\t\t\terr = util.UnmarshalJSON(input, &rv2)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn &rv2, nil\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, isTermQuery := tmp[\"term\"]\n\tif isTermQuery {\n\t\tvar rv TermQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasMust := tmp[\"must\"]\n\t_, hasShould := tmp[\"should\"]\n\t_, hasMustNot := tmp[\"must_not\"]\n\t_, hasFilter := tmp[\"filter\"]\n\tif hasMust || hasShould || hasMustNot || hasFilter {\n\t\tvar rv BooleanQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasConjuncts := tmp[\"conjuncts\"]\n\tif hasConjuncts {\n\t\tvar rv ConjunctionQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasDisjuncts := tmp[\"disjuncts\"]\n\tif hasDisjuncts {\n\t\tvar rv DisjunctionQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\n\t_, hasSyntaxQuery := tmp[\"query\"]\n\tif hasSyntaxQuery {\n\t\tvar rv QueryStringQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasMin := tmp[\"min\"].(float64)\n\t_, hasMax := tmp[\"max\"].(float64)\n\tif hasMin || hasMax {\n\t\tvar rv NumericRangeQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasMinStr := tmp[\"min\"].(string)\n\t_, hasMaxStr := tmp[\"max\"].(string)\n\tif hasMinStr || hasMaxStr {\n\t\tvar rv TermRangeQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasStart := tmp[\"start\"]\n\t_, hasEnd := tmp[\"end\"]\n\tif hasStart || hasEnd {\n\t\tvar rv DateRangeStringQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasPrefix := tmp[\"prefix\"]\n\tif hasPrefix {\n\t\tvar rv PrefixQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasRegexp := tmp[\"regexp\"]\n\tif hasRegexp {\n\t\tvar rv RegexpQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasWildcard := tmp[\"wildcard\"]\n\tif hasWildcard {\n\t\tvar rv WildcardQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasMatchAll := tmp[\"match_all\"]\n\tif hasMatchAll {\n\t\tvar rv MatchAllQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasMatchNone := tmp[\"match_none\"]\n\tif hasMatchNone {\n\t\tvar rv MatchNoneQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasDocIds := tmp[\"ids\"]\n\tif hasDocIds {\n\t\tvar rv DocIDQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasBool := tmp[\"bool\"]\n\tif hasBool {\n\t\tvar rv BoolFieldQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasTopLeft := tmp[\"top_left\"]\n\t_, hasBottomRight := tmp[\"bottom_right\"]\n\tif hasTopLeft && hasBottomRight {\n\t\tvar rv GeoBoundingBoxQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasDistance := tmp[\"distance\"]\n\tif hasDistance {\n\t\tvar rv GeoDistanceQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\t_, hasPoints := tmp[\"polygon_points\"]\n\tif hasPoints {\n\t\tvar rv GeoBoundingPolygonQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\n\t_, hasGeo := tmp[\"geometry\"]\n\tif hasGeo {\n\t\tvar rv GeoShapeQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\n\t_, hasCIDR := tmp[\"cidr\"]\n\tif hasCIDR {\n\t\tvar rv IPRangeQuery\n\t\terr := util.UnmarshalJSON(input, &rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &rv, nil\n\t}\n\n\treturn nil, fmt.Errorf(\"unknown query type\")\n}\n\n// expandQuery traverses the input query tree and returns a new tree where\n// query string queries have been expanded into base queries. Returned tree may\n// reference queries from the input tree or new queries.\nfunc expandQuery(m mapping.IndexMapping, query Query) (Query, error) {\n\tvar expand func(query Query) (Query, error)\n\tvar expandSlice func(queries []Query) ([]Query, error) = func(queries []Query) ([]Query, error) {\n\t\texpanded := []Query{}\n\t\tfor _, q := range queries {\n\t\t\texp, err := expand(q)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\texpanded = append(expanded, exp)\n\t\t}\n\t\treturn expanded, nil\n\t}\n\n\texpand = func(query Query) (Query, error) {\n\t\tswitch q := query.(type) {\n\t\tcase *QueryStringQuery:\n\t\t\tparsed, err := parseQuerySyntax(q.Query)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"could not parse '%s': %s\", q.Query, err)\n\t\t\t}\n\t\t\treturn expand(parsed)\n\t\tcase *ConjunctionQuery:\n\t\t\tchildren, err := expandSlice(q.Conjuncts)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tq.Conjuncts = children\n\t\t\treturn q, nil\n\t\tcase *DisjunctionQuery:\n\t\t\tchildren, err := expandSlice(q.Disjuncts)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tq.Disjuncts = children\n\t\t\treturn q, nil\n\t\tcase *BooleanQuery:\n\t\t\tvar err error\n\t\t\tq.Must, err = expand(q.Must)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tq.Should, err = expand(q.Should)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tq.MustNot, err = expand(q.MustNot)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tq.Filter, err = expand(q.Filter)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn q, nil\n\t\tdefault:\n\t\t\treturn query, nil\n\t\t}\n\t}\n\treturn expand(query)\n}\n\n// DumpQuery returns a string representation of the query tree, where query\n// string queries have been expanded into base queries. The output format is\n// meant for debugging purpose and may change in the future.\nfunc DumpQuery(m mapping.IndexMapping, query Query) (string, error) {\n\tq, err := expandQuery(m, query)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdata, err := json.MarshalIndent(q, \"\", \"  \")\n\treturn string(data), err\n}\n\n// ExtractFields returns a set of fields referenced by the query.\n// The returned set may be nil if the query does not explicitly reference any field\n// and the DefaultSearchField is unset in the index mapping.\nfunc ExtractFields(q Query, m mapping.IndexMapping, fs search.FieldSet) (search.FieldSet, error) {\n\tif q == nil || m == nil {\n\t\treturn fs, nil\n\t}\n\tvar err error\n\tswitch q := q.(type) {\n\tcase FieldableQuery:\n\t\tf := q.Field()\n\t\tif f == \"\" {\n\t\t\tf = m.DefaultSearchField()\n\t\t}\n\t\tif f != \"\" {\n\t\t\tif fs == nil {\n\t\t\t\tfs = search.NewFieldSet()\n\t\t\t}\n\t\t\tfs.AddField(f)\n\t\t}\n\tcase *QueryStringQuery:\n\t\tvar expandedQuery Query\n\t\texpandedQuery, err = expandQuery(m, q)\n\t\tif err == nil {\n\t\t\tfs, err = ExtractFields(expandedQuery, m, fs)\n\t\t}\n\tcase *BooleanQuery:\n\t\tfor _, subq := range []Query{q.Must, q.Should, q.MustNot, q.Filter} {\n\t\t\tfs, err = ExtractFields(subq, m, fs)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\tcase *ConjunctionQuery:\n\t\tfor _, subq := range q.Conjuncts {\n\t\t\tfs, err = ExtractFields(subq, m, fs)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\tcase *DisjunctionQuery:\n\t\tfor _, subq := range q.Disjuncts {\n\t\t\tfs, err = ExtractFields(subq, m, fs)\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\tcase *DocIDQuery, *MatchAllQuery:\n\t\tif fs == nil {\n\t\t\tfs = search.NewFieldSet()\n\t\t}\n\t\tfs.AddField(\"_id\")\n\t}\n\treturn fs, err\n}\n\nconst (\n\tFuzzyMatchType = iota\n\tRegexpMatchType\n\tPrefixMatchType\n)\n\n// ExtractSynonyms extracts synonyms from the query tree and returns a map of\n// field-term pairs to their synonyms. The input query tree is traversed and\n// for each term query, the synonyms are extracted from the synonym source\n// associated with the field. The synonyms are then added to the provided map.\n// The map is returned and may be nil if no synonyms were found.\nfunc ExtractSynonyms(ctx context.Context, m mapping.SynonymMapping, r index.ThesaurusReader,\n\tquery Query, rv search.FieldTermSynonymMap,\n) (search.FieldTermSynonymMap, error) {\n\tif r == nil || m == nil || query == nil {\n\t\treturn rv, nil\n\t}\n\tvar err error\n\tresolveFieldAndSource := func(field string) (string, string) {\n\t\tif field == \"\" {\n\t\t\tfield = m.DefaultSearchField()\n\t\t}\n\t\treturn field, m.SynonymSourceForPath(field)\n\t}\n\thandleAnalyzer := func(analyzerName, field string) (analysis.Analyzer, error) {\n\t\tif analyzerName == \"\" {\n\t\t\tanalyzerName = m.AnalyzerNameForPath(field)\n\t\t}\n\t\tanalyzer := m.AnalyzerNamed(analyzerName)\n\t\tif analyzer == nil {\n\t\t\treturn nil, fmt.Errorf(\"no analyzer named '%s' registered\", analyzerName)\n\t\t}\n\t\treturn analyzer, nil\n\t}\n\tswitch q := query.(type) {\n\tcase *BooleanQuery:\n\t\trv, err = ExtractSynonyms(ctx, m, r, q.Must, rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trv, err = ExtractSynonyms(ctx, m, r, q.Should, rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trv, err = ExtractSynonyms(ctx, m, r, q.MustNot, rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trv, err = ExtractSynonyms(ctx, m, r, q.Filter, rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase *ConjunctionQuery:\n\t\tfor _, child := range q.Conjuncts {\n\t\t\trv, err = ExtractSynonyms(ctx, m, r, child, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\tcase *DisjunctionQuery:\n\t\tfor _, child := range q.Disjuncts {\n\t\t\trv, err = ExtractSynonyms(ctx, m, r, child, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\tcase *FuzzyQuery:\n\t\tfield, source := resolveFieldAndSource(q.FieldVal)\n\t\tif source != \"\" {\n\t\t\tfuzziness := q.Fuzziness\n\t\t\tif q.autoFuzzy {\n\t\t\t\tfuzziness = searcher.GetAutoFuzziness(q.Term)\n\t\t\t}\n\t\t\trv, err = addSynonymsForTermWithMatchType(ctx, FuzzyMatchType, source, field, q.Term, fuzziness, q.Prefix, r, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\tcase *MatchQuery, *MatchPhraseQuery:\n\t\tvar analyzerName, matchString, fieldVal string\n\t\tvar fuzziness, prefix int\n\t\tvar autoFuzzy bool\n\t\tif mq, ok := q.(*MatchQuery); ok {\n\t\t\tanalyzerName, fieldVal, matchString, fuzziness, prefix, autoFuzzy = mq.Analyzer, mq.FieldVal, mq.Match, mq.Fuzziness, mq.Prefix, mq.autoFuzzy\n\t\t} else if mpq, ok := q.(*MatchPhraseQuery); ok {\n\t\t\tanalyzerName, fieldVal, matchString, fuzziness, autoFuzzy = mpq.Analyzer, mpq.FieldVal, mpq.MatchPhrase, mpq.Fuzziness, mpq.autoFuzzy\n\t\t}\n\t\tfield, source := resolveFieldAndSource(fieldVal)\n\t\tif source != \"\" {\n\t\t\tanalyzer, err := handleAnalyzer(analyzerName, field)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\ttokens := analyzer.Analyze([]byte(matchString))\n\t\t\tfor _, token := range tokens {\n\t\t\t\tif autoFuzzy {\n\t\t\t\t\tfuzziness = searcher.GetAutoFuzziness(string(token.Term))\n\t\t\t\t}\n\t\t\t\trv, err = addSynonymsForTermWithMatchType(ctx, FuzzyMatchType, source, field, string(token.Term), fuzziness, prefix, r, rv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\tcase *MultiPhraseQuery, *PhraseQuery:\n\t\tvar fieldVal string\n\t\tvar fuzziness int\n\t\tvar autoFuzzy bool\n\t\tif mpq, ok := q.(*MultiPhraseQuery); ok {\n\t\t\tfieldVal, fuzziness, autoFuzzy = mpq.FieldVal, mpq.Fuzziness, mpq.autoFuzzy\n\t\t} else if pq, ok := q.(*PhraseQuery); ok {\n\t\t\tfieldVal, fuzziness, autoFuzzy = pq.FieldVal, pq.Fuzziness, pq.autoFuzzy\n\t\t}\n\t\tfield, source := resolveFieldAndSource(fieldVal)\n\t\tif source != \"\" {\n\t\t\tvar terms []string\n\t\t\tif mpq, ok := q.(*MultiPhraseQuery); ok {\n\t\t\t\tfor _, termGroup := range mpq.Terms {\n\t\t\t\t\tterms = append(terms, termGroup...)\n\t\t\t\t}\n\t\t\t} else if pq, ok := q.(*PhraseQuery); ok {\n\t\t\t\tterms = pq.Terms\n\t\t\t}\n\t\t\tfor _, term := range terms {\n\t\t\t\tif autoFuzzy {\n\t\t\t\t\tfuzziness = searcher.GetAutoFuzziness(term)\n\t\t\t\t}\n\t\t\t\trv, err = addSynonymsForTermWithMatchType(ctx, FuzzyMatchType, source, field, term, fuzziness, 0, r, rv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\tcase *PrefixQuery:\n\t\tfield, source := resolveFieldAndSource(q.FieldVal)\n\t\tif source != \"\" {\n\t\t\trv, err = addSynonymsForTermWithMatchType(ctx, PrefixMatchType, source, field, q.Prefix, 0, 0, r, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\tcase *QueryStringQuery:\n\t\texpanded, err := expandQuery(m, q)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trv, err = ExtractSynonyms(ctx, m, r, expanded, rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase *TermQuery:\n\t\tfield, source := resolveFieldAndSource(q.FieldVal)\n\t\tif source != \"\" {\n\t\t\trv, err = addSynonymsForTerm(ctx, source, field, q.Term, r, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\tcase *RegexpQuery:\n\t\tfield, source := resolveFieldAndSource(q.FieldVal)\n\t\tif source != \"\" {\n\t\t\trv, err = addSynonymsForTermWithMatchType(ctx, RegexpMatchType, source, field, strings.TrimPrefix(q.Regexp, \"^\"), 0, 0, r, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\tcase *WildcardQuery:\n\t\tfield, source := resolveFieldAndSource(q.FieldVal)\n\t\tif source != \"\" {\n\t\t\trv, err = addSynonymsForTermWithMatchType(ctx, RegexpMatchType, source, field, wildcardRegexpReplacer.Replace(q.Wildcard), 0, 0, r, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\treturn rv, nil\n}\n\n// addFuzzySynonymsForTerm finds all terms that match the given term with the\n// given fuzziness and adds their synonyms to the provided map.\nfunc addSynonymsForTermWithMatchType(ctx context.Context, matchType int, src, field, term string, fuzziness, prefix int,\n\tr index.ThesaurusReader, rv search.FieldTermSynonymMap,\n) (search.FieldTermSynonymMap, error) {\n\t// Determine the terms based on the match type (fuzzy, prefix, or regexp)\n\tvar thesKeys index.ThesaurusKeys\n\tvar err error\n\tvar terms []string\n\tswitch matchType {\n\tcase FuzzyMatchType:\n\t\t// Ensure valid fuzziness\n\t\tif fuzziness == 0 {\n\t\t\trv, err = addSynonymsForTerm(ctx, src, field, term, r, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn rv, nil\n\t\t}\n\t\tif fuzziness > searcher.MaxFuzziness {\n\t\t\treturn nil, fmt.Errorf(\"fuzziness exceeds max (%d)\", searcher.MaxFuzziness)\n\t\t}\n\t\tif fuzziness < 0 {\n\t\t\treturn nil, fmt.Errorf(\"invalid fuzziness, negative\")\n\t\t}\n\t\t// Handle fuzzy match\n\t\tprefixTerm := \"\"\n\t\tfor i, r := range term {\n\t\t\tif i < prefix {\n\t\t\t\tprefixTerm += string(r)\n\t\t\t} else {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tthesKeys, err = r.ThesaurusKeysFuzzy(src, term, fuzziness, prefixTerm)\n\tcase RegexpMatchType:\n\t\t// Handle regexp match\n\t\tthesKeys, err = r.ThesaurusKeysRegexp(src, term)\n\tcase PrefixMatchType:\n\t\t// Handle prefix match\n\t\tthesKeys, err = r.ThesaurusKeysPrefix(src, []byte(term))\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"invalid match type: %d\", matchType)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := thesKeys.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\t// Collect the matching terms\n\tterms = []string{}\n\ttfd, err := thesKeys.Next()\n\tfor err == nil && tfd != nil {\n\t\tterms = append(terms, tfd.Term)\n\t\ttfd, err = thesKeys.Next()\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, synTerm := range terms {\n\t\trv, err = addSynonymsForTerm(ctx, src, field, synTerm, r, rv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn rv, nil\n}\n\nfunc addSynonymsForTerm(ctx context.Context, src, field, term string,\n\tr index.ThesaurusReader, rv search.FieldTermSynonymMap,\n) (search.FieldTermSynonymMap, error) {\n\ttermReader, err := r.ThesaurusTermReader(ctx, src, []byte(term))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := termReader.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\tvar synonyms []string\n\tsynonym, err := termReader.Next()\n\tfor err == nil && synonym != \"\" {\n\t\tsynonyms = append(synonyms, synonym)\n\t\tsynonym, err = termReader.Next()\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(synonyms) > 0 {\n\t\tif rv == nil {\n\t\t\trv = make(search.FieldTermSynonymMap)\n\t\t}\n\t\tif _, exists := rv[field]; !exists {\n\t\t\trv[field] = make(map[string][]string)\n\t\t}\n\t\trv[field][term] = synonyms\n\t}\n\treturn rv, nil\n}\n"
  },
  {
    "path": "search/query/query_string.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype QueryStringQuery struct {\n\tQuery    string `json:\"query\"`\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\n// NewQueryStringQuery creates a new Query used for\n// finding documents that satisfy a query string.  The\n// query string is a small query language for humans.\nfunc NewQueryStringQuery(query string) *QueryStringQuery {\n\treturn &QueryStringQuery{\n\t\tQuery: query,\n\t}\n}\n\nfunc (q *QueryStringQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *QueryStringQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *QueryStringQuery) Parse() (Query, error) {\n\treturn parseQuerySyntax(q.Query)\n}\n\nfunc (q *QueryStringQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tnewQuery, err := parseQuerySyntax(q.Query)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn newQuery.Searcher(ctx, i, m, options)\n}\n\nfunc (q *QueryStringQuery) Validate() error {\n\tnewQuery, err := parseQuerySyntax(q.Query)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif newQuery, ok := newQuery.(ValidatableQuery); ok {\n\t\treturn newQuery.Validate()\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/query_string.y",
    "content": "%{\npackage query\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\nfunc logDebugGrammar(format string, v ...interface{}) {\n\tif debugParser {\n    \tlogger.Printf(format, v...)\n    }\n}\n%}\n\n%union {\ns string\nn int\nf float64\nq Query\npf *float64}\n\n%token tSTRING tPHRASE tPLUS tMINUS tCOLON tBOOST tNUMBER tSTRING tGREATER tLESS\ntEQUAL tTILDE\n\n%type <s>                tSTRING\n%type <s>                tPHRASE\n%type <s>                tNUMBER\n%type <s>                posOrNegNumber\n%type <s>                fieldName\n%type <s>                tTILDE\n%type <s>                tBOOST\n%type <q>                searchBase\n%type <pf>               searchSuffix\n%type <n>                searchPrefix\n\n%%\n\ninput:\nsearchParts {\n\tlogDebugGrammar(\"INPUT\")\n};\n\nsearchParts:\nsearchPart searchParts {\n\tlogDebugGrammar(\"SEARCH PARTS\")\n}\n|\nsearchPart {\n\tlogDebugGrammar(\"SEARCH PART\")\n};\n\nsearchPart:\nsearchPrefix searchBase searchSuffix {\n\tquery := $2\n\tif $3 != nil {\n\t\tif query, ok := query.(BoostableQuery); ok {\n\t\t\tquery.SetBoost(*$3)\n\t\t\t}\n\t}\n\tswitch($1) {\n\t\tcase queryShould:\n\t\t\tyylex.(*lexerWrapper).query.AddShould(query)\n\t\tcase queryMust:\n\t\t\tyylex.(*lexerWrapper).query.AddMust(query)\n\t\tcase queryMustNot:\n\t\t\tyylex.(*lexerWrapper).query.AddMustNot(query)\n\t}\n};\n\n\nsearchPrefix:\n/* empty */ {\n\t$$ = queryShould\n}\n|\ntPLUS {\n\tlogDebugGrammar(\"PLUS\")\n\t$$ = queryMust\n}\n|\ntMINUS {\n\tlogDebugGrammar(\"MINUS\")\n\t$$ = queryMustNot\n};\n\nsearchBase:\ntSTRING {\n\tstr := $1\n\tlogDebugGrammar(\"STRING - %s\", str)\n\tvar q FieldableQuery\n\tif strings.HasPrefix(str, \"/\") && strings.HasSuffix(str, \"/\") {\n\t  q = NewRegexpQuery(str[1:len(str)-1])\n\t} else if strings.ContainsAny(str, \"*?\"){\n\t  q = NewWildcardQuery(str)\n\t} else {\n\t  q = NewMatchQuery(str)\n\t}\n\t$$ = q\n}\n|\ntSTRING tTILDE {\n\tstr := $1\n\tfuzziness, err := strconv.ParseFloat($2, 64)\n\tif err != nil {\n\t  yylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid fuzziness value: %v\", err))\n\t}\n\tlogDebugGrammar(\"FUZZY STRING - %s %f\", str, fuzziness)\n\tq := NewMatchQuery(str)\n\tq.SetFuzziness(int(fuzziness))\n\t$$ = q\n}\n|\nfieldName tCOLON tSTRING tTILDE {\n\tfield := $1\n\tstr := $3\n\tfuzziness, err := strconv.ParseFloat($4, 64)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid fuzziness value: %v\", err))\n\t}\n\tlogDebugGrammar(\"FIELD - %s FUZZY STRING - %s %f\", field, str, fuzziness)\n\tq := NewMatchQuery(str)\n\tq.SetFuzziness(int(fuzziness))\n\tq.SetField(field)\n\t$$ = q\n}\n|\ntNUMBER {\n\tstr := $1\n\tlogDebugGrammar(\"STRING - %s\", str)\n\tq1 := NewMatchQuery(str)\n\tval, err := strconv.ParseFloat($1, 64)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t}\n\tinclusive := true\n\tq2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive)\n\tq := NewDisjunctionQuery([]Query{q1,q2})\n\tq.queryStringMode = true\n\t$$ = q\n}\n|\ntPHRASE {\n\tphrase := $1\n\tlogDebugGrammar(\"PHRASE - %s\", phrase)\n\tq := NewMatchPhraseQuery(phrase)\n\t$$ = q\n}\n|\nfieldName tCOLON tSTRING {\n\tfield := $1\n\tstr := $3\n\tlogDebugGrammar(\"FIELD - %s STRING - %s\", field, str)\n\tvar q FieldableQuery\n\tif strings.HasPrefix(str, \"/\") && strings.HasSuffix(str, \"/\") {\n\t\tq = NewRegexpQuery(str[1:len(str)-1])\n\t} else if strings.ContainsAny(str, \"*?\"){\n\t  q = NewWildcardQuery(str)\n\t}  else {\n\t\tq = NewMatchQuery(str)\n\t}\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON posOrNegNumber {\n\tfield := $1\n\tstr := $3\n\tlogDebugGrammar(\"FIELD - %s STRING - %s\", field, str)\n\tq1 := NewMatchQuery(str)\n\tq1.SetField(field)\n\tval, err := strconv.ParseFloat($3, 64)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t}\n\tinclusive := true\n\tq2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive)\n\tq2.SetField(field)\n\tq := NewDisjunctionQuery([]Query{q1,q2})\n\tq.queryStringMode = true\n\t$$ = q\n}\n|\nfieldName tCOLON tPHRASE {\n\tfield := $1\n\tphrase := $3\n\tlogDebugGrammar(\"FIELD - %s PHRASE - %s\", field, phrase)\n\tq := NewMatchPhraseQuery(phrase)\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON tGREATER posOrNegNumber {\n\tfield := $1\n\tmin, err := strconv.ParseFloat($4, 64)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t}\n\tminInclusive := false\n\tlogDebugGrammar(\"FIELD - GREATER THAN %f\", min)\n\tq := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil)\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON tGREATER tEQUAL posOrNegNumber {\n\tfield := $1\n\tmin, err := strconv.ParseFloat($5, 64)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t}\n\tminInclusive := true\n\tlogDebugGrammar(\"FIELD - GREATER THAN OR EQUAL %f\", min)\n\tq := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil)\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON tLESS posOrNegNumber {\n\tfield := $1\n\tmax, err := strconv.ParseFloat($4, 64)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t}\n\tmaxInclusive := false\n\tlogDebugGrammar(\"FIELD - LESS THAN %f\", max)\n\tq := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive)\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON tLESS tEQUAL posOrNegNumber {\n\tfield := $1\n\tmax, err := strconv.ParseFloat($5, 64)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t}\n\tmaxInclusive := true\n\tlogDebugGrammar(\"FIELD - LESS THAN OR EQUAL %f\", max)\n\tq := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive)\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON tGREATER tPHRASE {\n\tfield := $1\n\tminInclusive := false\n\tphrase := $4\n\n\tlogDebugGrammar(\"FIELD - GREATER THAN DATE %s\", phrase)\n\tminTime, err := queryTimeFromString(phrase)\n\tif err != nil {\n\t  yylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid time: %v\", err))\n\t}\n\tq := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil)\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON tGREATER tEQUAL tPHRASE {\n\tfield := $1\n\tminInclusive := true\n\tphrase := $5\n\n\tlogDebugGrammar(\"FIELD - GREATER THAN OR EQUAL DATE %s\", phrase)\n\tminTime, err := queryTimeFromString(phrase)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid time: %v\", err))\n\t}\n\tq := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil)\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON tLESS tPHRASE {\n\tfield := $1\n\tmaxInclusive := false\n\tphrase := $4\n\n\tlogDebugGrammar(\"FIELD - LESS THAN DATE %s\", phrase)\n\tmaxTime, err := queryTimeFromString(phrase)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid time: %v\", err))\n\t}\n\tq := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive)\n\tq.SetField(field)\n\t$$ = q\n}\n|\nfieldName tCOLON tLESS tEQUAL tPHRASE {\n\tfield := $1\n\tmaxInclusive := true\n\tphrase := $5\n\n\tlogDebugGrammar(\"FIELD - LESS THAN OR EQUAL DATE %s\", phrase)\n\tmaxTime, err := queryTimeFromString(phrase)\n\tif err != nil {\n\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid time: %v\", err))\n\t}\n\tq := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive)\n\tq.SetField(field)\n\t$$ = q\n};\n\nsearchSuffix:\n/* empty */ {\n\t$$ = nil\n}\n|\ntBOOST {\n  $$ = nil\n\tboost, err := strconv.ParseFloat($1, 64)\n\tif err != nil {\n\t  yylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid boost value: %v\", err))\n\t} else {\n\t\t$$ = &boost\n\t}\n\tlogDebugGrammar(\"BOOST %f\", boost)\n};\n\nposOrNegNumber:\ntNUMBER {\n\t$$ = $1\n}\n|\ntMINUS tNUMBER {\n\t$$ = \"-\" + $2\n};\n\nfieldName:\ntPHRASE {\n    $$ = $1\n}\n|\ntSTRING {\n    $$ = $1\n};\n"
  },
  {
    "path": "search/query/query_string.y.go",
    "content": "// Code generated by goyacc -o query_string.y.go query_string.y. DO NOT EDIT.\n\n//line query_string.y:2\npackage query\n\nimport __yyfmt__ \"fmt\"\n\n//line query_string.y:2\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\nfunc logDebugGrammar(format string, v ...interface{}) {\n\tif debugParser {\n\t\tlogger.Printf(format, v...)\n\t}\n}\n\n//line query_string.y:17\ntype yySymType struct {\n\tyys int\n\ts   string\n\tn   int\n\tf   float64\n\tq   Query\n\tpf  *float64\n}\n\nconst tSTRING = 57346\nconst tPHRASE = 57347\nconst tPLUS = 57348\nconst tMINUS = 57349\nconst tCOLON = 57350\nconst tBOOST = 57351\nconst tNUMBER = 57352\nconst tGREATER = 57353\nconst tLESS = 57354\nconst tEQUAL = 57355\nconst tTILDE = 57356\n\nvar yyToknames = [...]string{\n\t\"$end\",\n\t\"error\",\n\t\"$unk\",\n\t\"tSTRING\",\n\t\"tPHRASE\",\n\t\"tPLUS\",\n\t\"tMINUS\",\n\t\"tCOLON\",\n\t\"tBOOST\",\n\t\"tNUMBER\",\n\t\"tGREATER\",\n\t\"tLESS\",\n\t\"tEQUAL\",\n\t\"tTILDE\",\n}\n\nvar yyStatenames = [...]string{}\n\nconst yyEofCode = 1\nconst yyErrCode = 2\nconst yyInitialStackSize = 16\n\n//line yacctab:1\nvar yyExca = [...]int{\n\t-1, 1,\n\t1, -1,\n\t-2, 0,\n\t-1, 3,\n\t1, 3,\n\t-2, 5,\n\t-1, 9,\n\t8, 29,\n\t-2, 8,\n\t-1, 12,\n\t8, 28,\n\t-2, 12,\n}\n\nconst yyPrivate = 57344\n\nconst yyLast = 43\n\nvar yyAct = [...]int{\n\t18, 17, 19, 24, 23, 15, 31, 22, 20, 21,\n\t30, 27, 23, 23, 3, 22, 22, 14, 29, 26,\n\t16, 25, 28, 35, 33, 23, 23, 32, 22, 22,\n\t34, 9, 12, 1, 5, 6, 2, 11, 4, 13,\n\t7, 8, 10,\n}\n\nvar yyPact = [...]int{\n\t28, -1000, -1000, 28, 27, -1000, -1000, -1000, 8, -9,\n\t12, -1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000,\n\t6, 5, -1000, -4, -1000, -1000, 19, -1000, -1000, 18,\n\t-1000, -1000, -1000, -1000, -1000, -1000,\n}\n\nvar yyPgo = [...]int{\n\t0, 0, 42, 41, 39, 38, 33, 36, 14,\n}\n\nvar yyR1 = [...]int{\n\t0, 6, 7, 7, 8, 5, 5, 5, 3, 3,\n\t3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n\t3, 3, 3, 3, 4, 4, 1, 1, 2, 2,\n}\n\nvar yyR2 = [...]int{\n\t0, 1, 2, 1, 3, 0, 1, 1, 1, 2,\n\t4, 1, 1, 3, 3, 3, 4, 5, 4, 5,\n\t4, 5, 4, 5, 0, 1, 1, 2, 1, 1,\n}\n\nvar yyChk = [...]int{\n\t-1000, -6, -7, -8, -5, 6, 7, -7, -3, 4,\n\t-2, 10, 5, -4, 9, 14, 8, 4, -1, 5,\n\t11, 12, 10, 7, 14, -1, 13, 5, -1, 13,\n\t5, 10, -1, 5, -1, 5,\n}\n\nvar yyDef = [...]int{\n\t5, -2, 1, -2, 0, 6, 7, 2, 24, -2,\n\t0, 11, -2, 4, 25, 9, 0, 13, 14, 15,\n\t0, 0, 26, 0, 10, 16, 0, 20, 18, 0,\n\t22, 27, 17, 21, 19, 23,\n}\n\nvar yyTok1 = [...]int{\n\t1,\n}\n\nvar yyTok2 = [...]int{\n\t2, 3, 4, 5, 6, 7, 8, 9, 10, 11,\n\t12, 13, 14,\n}\n\nvar yyTok3 = [...]int{\n\t0,\n}\n\nvar yyErrorMessages = [...]struct {\n\tstate int\n\ttoken int\n\tmsg   string\n}{}\n\n//line yaccpar:1\n\n/*\tparser for yacc output\t*/\n\nvar (\n\tyyDebug        = 0\n\tyyErrorVerbose = false\n)\n\ntype yyLexer interface {\n\tLex(lval *yySymType) int\n\tError(s string)\n}\n\ntype yyParser interface {\n\tParse(yyLexer) int\n\tLookahead() int\n}\n\ntype yyParserImpl struct {\n\tlval  yySymType\n\tstack [yyInitialStackSize]yySymType\n\tchar  int\n}\n\nfunc (p *yyParserImpl) Lookahead() int {\n\treturn p.char\n}\n\nfunc yyNewParser() yyParser {\n\treturn &yyParserImpl{}\n}\n\nconst yyFlag = -1000\n\nfunc yyTokname(c int) string {\n\tif c >= 1 && c-1 < len(yyToknames) {\n\t\tif yyToknames[c-1] != \"\" {\n\t\t\treturn yyToknames[c-1]\n\t\t}\n\t}\n\treturn __yyfmt__.Sprintf(\"tok-%v\", c)\n}\n\nfunc yyStatname(s int) string {\n\tif s >= 0 && s < len(yyStatenames) {\n\t\tif yyStatenames[s] != \"\" {\n\t\t\treturn yyStatenames[s]\n\t\t}\n\t}\n\treturn __yyfmt__.Sprintf(\"state-%v\", s)\n}\n\nfunc yyErrorMessage(state, lookAhead int) string {\n\tconst TOKSTART = 4\n\n\tif !yyErrorVerbose {\n\t\treturn \"syntax error\"\n\t}\n\n\tfor _, e := range yyErrorMessages {\n\t\tif e.state == state && e.token == lookAhead {\n\t\t\treturn \"syntax error: \" + e.msg\n\t\t}\n\t}\n\n\tres := \"syntax error: unexpected \" + yyTokname(lookAhead)\n\n\t// To match Bison, suggest at most four expected tokens.\n\texpected := make([]int, 0, 4)\n\n\t// Look for shiftable tokens.\n\tbase := yyPact[state]\n\tfor tok := TOKSTART; tok-1 < len(yyToknames); tok++ {\n\t\tif n := base + tok; n >= 0 && n < yyLast && yyChk[yyAct[n]] == tok {\n\t\t\tif len(expected) == cap(expected) {\n\t\t\t\treturn res\n\t\t\t}\n\t\t\texpected = append(expected, tok)\n\t\t}\n\t}\n\n\tif yyDef[state] == -2 {\n\t\ti := 0\n\t\tfor yyExca[i] != -1 || yyExca[i+1] != state {\n\t\t\ti += 2\n\t\t}\n\n\t\t// Look for tokens that we accept or reduce.\n\t\tfor i += 2; yyExca[i] >= 0; i += 2 {\n\t\t\ttok := yyExca[i]\n\t\t\tif tok < TOKSTART || yyExca[i+1] == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(expected) == cap(expected) {\n\t\t\t\treturn res\n\t\t\t}\n\t\t\texpected = append(expected, tok)\n\t\t}\n\n\t\t// If the default action is to accept or reduce, give up.\n\t\tif yyExca[i+1] != 0 {\n\t\t\treturn res\n\t\t}\n\t}\n\n\tfor i, tok := range expected {\n\t\tif i == 0 {\n\t\t\tres += \", expecting \"\n\t\t} else {\n\t\t\tres += \" or \"\n\t\t}\n\t\tres += yyTokname(tok)\n\t}\n\treturn res\n}\n\nfunc yylex1(lex yyLexer, lval *yySymType) (char, token int) {\n\ttoken = 0\n\tchar = lex.Lex(lval)\n\tif char <= 0 {\n\t\ttoken = yyTok1[0]\n\t\tgoto out\n\t}\n\tif char < len(yyTok1) {\n\t\ttoken = yyTok1[char]\n\t\tgoto out\n\t}\n\tif char >= yyPrivate {\n\t\tif char < yyPrivate+len(yyTok2) {\n\t\t\ttoken = yyTok2[char-yyPrivate]\n\t\t\tgoto out\n\t\t}\n\t}\n\tfor i := 0; i < len(yyTok3); i += 2 {\n\t\ttoken = yyTok3[i+0]\n\t\tif token == char {\n\t\t\ttoken = yyTok3[i+1]\n\t\t\tgoto out\n\t\t}\n\t}\n\nout:\n\tif token == 0 {\n\t\ttoken = yyTok2[1] /* unknown char */\n\t}\n\tif yyDebug >= 3 {\n\t\t__yyfmt__.Printf(\"lex %s(%d)\\n\", yyTokname(token), uint(char))\n\t}\n\treturn char, token\n}\n\nfunc yyParse(yylex yyLexer) int {\n\treturn yyNewParser().Parse(yylex)\n}\n\nfunc (yyrcvr *yyParserImpl) Parse(yylex yyLexer) int {\n\tvar yyn int\n\tvar yyVAL yySymType\n\tvar yyDollar []yySymType\n\t_ = yyDollar // silence set and not used\n\tyyS := yyrcvr.stack[:]\n\n\tNerrs := 0   /* number of errors */\n\tErrflag := 0 /* error recovery flag */\n\tyystate := 0\n\tyyrcvr.char = -1\n\tyytoken := -1 // yyrcvr.char translated into internal numbering\n\tdefer func() {\n\t\t// Make sure we report no lookahead when not parsing.\n\t\tyystate = -1\n\t\tyyrcvr.char = -1\n\t\tyytoken = -1\n\t}()\n\tyyp := -1\n\tgoto yystack\n\nret0:\n\treturn 0\n\nret1:\n\treturn 1\n\nyystack:\n\t/* put a state and value onto the stack */\n\tif yyDebug >= 4 {\n\t\t__yyfmt__.Printf(\"char %v in %v\\n\", yyTokname(yytoken), yyStatname(yystate))\n\t}\n\n\tyyp++\n\tif yyp >= len(yyS) {\n\t\tnyys := make([]yySymType, len(yyS)*2)\n\t\tcopy(nyys, yyS)\n\t\tyyS = nyys\n\t}\n\tyyS[yyp] = yyVAL\n\tyyS[yyp].yys = yystate\n\nyynewstate:\n\tyyn = yyPact[yystate]\n\tif yyn <= yyFlag {\n\t\tgoto yydefault /* simple state */\n\t}\n\tif yyrcvr.char < 0 {\n\t\tyyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval)\n\t}\n\tyyn += yytoken\n\tif yyn < 0 || yyn >= yyLast {\n\t\tgoto yydefault\n\t}\n\tyyn = yyAct[yyn]\n\tif yyChk[yyn] == yytoken { /* valid shift */\n\t\tyyrcvr.char = -1\n\t\tyytoken = -1\n\t\tyyVAL = yyrcvr.lval\n\t\tyystate = yyn\n\t\tif Errflag > 0 {\n\t\t\tErrflag--\n\t\t}\n\t\tgoto yystack\n\t}\n\nyydefault:\n\t/* default state action */\n\tyyn = yyDef[yystate]\n\tif yyn == -2 {\n\t\tif yyrcvr.char < 0 {\n\t\t\tyyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval)\n\t\t}\n\n\t\t/* look through exception table */\n\t\txi := 0\n\t\tfor {\n\t\t\tif yyExca[xi+0] == -1 && yyExca[xi+1] == yystate {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\txi += 2\n\t\t}\n\t\tfor xi += 2; ; xi += 2 {\n\t\t\tyyn = yyExca[xi+0]\n\t\t\tif yyn < 0 || yyn == yytoken {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tyyn = yyExca[xi+1]\n\t\tif yyn < 0 {\n\t\t\tgoto ret0\n\t\t}\n\t}\n\tif yyn == 0 {\n\t\t/* error ... attempt to resume parsing */\n\t\tswitch Errflag {\n\t\tcase 0: /* brand new error */\n\t\t\tyylex.Error(yyErrorMessage(yystate, yytoken))\n\t\t\tNerrs++\n\t\t\tif yyDebug >= 1 {\n\t\t\t\t__yyfmt__.Printf(\"%s\", yyStatname(yystate))\n\t\t\t\t__yyfmt__.Printf(\" saw %s\\n\", yyTokname(yytoken))\n\t\t\t}\n\t\t\tfallthrough\n\n\t\tcase 1, 2: /* incompletely recovered error ... try again */\n\t\t\tErrflag = 3\n\n\t\t\t/* find a state where \"error\" is a legal shift action */\n\t\t\tfor yyp >= 0 {\n\t\t\t\tyyn = yyPact[yyS[yyp].yys] + yyErrCode\n\t\t\t\tif yyn >= 0 && yyn < yyLast {\n\t\t\t\t\tyystate = yyAct[yyn] /* simulate a shift of \"error\" */\n\t\t\t\t\tif yyChk[yystate] == yyErrCode {\n\t\t\t\t\t\tgoto yystack\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t/* the current p has no shift on \"error\", pop stack */\n\t\t\t\tif yyDebug >= 2 {\n\t\t\t\t\t__yyfmt__.Printf(\"error recovery pops state %d\\n\", yyS[yyp].yys)\n\t\t\t\t}\n\t\t\t\tyyp--\n\t\t\t}\n\t\t\t/* there is no state on the stack with an error shift ... abort */\n\t\t\tgoto ret1\n\n\t\tcase 3: /* no shift yet; clobber input char */\n\t\t\tif yyDebug >= 2 {\n\t\t\t\t__yyfmt__.Printf(\"error recovery discards %s\\n\", yyTokname(yytoken))\n\t\t\t}\n\t\t\tif yytoken == yyEofCode {\n\t\t\t\tgoto ret1\n\t\t\t}\n\t\t\tyyrcvr.char = -1\n\t\t\tyytoken = -1\n\t\t\tgoto yynewstate /* try again in the same state */\n\t\t}\n\t}\n\n\t/* reduction by production yyn */\n\tif yyDebug >= 2 {\n\t\t__yyfmt__.Printf(\"reduce %v in:\\n\\t%v\\n\", yyn, yyStatname(yystate))\n\t}\n\n\tyynt := yyn\n\tyypt := yyp\n\t_ = yypt // guard against \"declared and not used\"\n\n\tyyp -= yyR2[yyn]\n\t// yyp is now the index of $0. Perform the default action. Iff the\n\t// reduced production is ε, $1 is possibly out of range.\n\tif yyp+1 >= len(yyS) {\n\t\tnyys := make([]yySymType, len(yyS)*2)\n\t\tcopy(nyys, yyS)\n\t\tyyS = nyys\n\t}\n\tyyVAL = yyS[yyp+1]\n\n\t/* consult goto table to find next state */\n\tyyn = yyR1[yyn]\n\tyyg := yyPgo[yyn]\n\tyyj := yyg + yyS[yyp].yys + 1\n\n\tif yyj >= yyLast {\n\t\tyystate = yyAct[yyg]\n\t} else {\n\t\tyystate = yyAct[yyj]\n\t\tif yyChk[yystate] != -yyn {\n\t\t\tyystate = yyAct[yyg]\n\t\t}\n\t}\n\t// dummy call; replaced with literal code\n\tswitch yynt {\n\n\tcase 1:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:41\n\t\t{\n\t\t\tlogDebugGrammar(\"INPUT\")\n\t\t}\n\tcase 2:\n\t\tyyDollar = yyS[yypt-2 : yypt+1]\n//line query_string.y:46\n\t\t{\n\t\t\tlogDebugGrammar(\"SEARCH PARTS\")\n\t\t}\n\tcase 3:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:50\n\t\t{\n\t\t\tlogDebugGrammar(\"SEARCH PART\")\n\t\t}\n\tcase 4:\n\t\tyyDollar = yyS[yypt-3 : yypt+1]\n//line query_string.y:55\n\t\t{\n\t\t\tquery := yyDollar[2].q\n\t\t\tif yyDollar[3].pf != nil {\n\t\t\t\tif query, ok := query.(BoostableQuery); ok {\n\t\t\t\t\tquery.SetBoost(*yyDollar[3].pf)\n\t\t\t\t}\n\t\t\t}\n\t\t\tswitch yyDollar[1].n {\n\t\t\tcase queryShould:\n\t\t\t\tyylex.(*lexerWrapper).query.AddShould(query)\n\t\t\tcase queryMust:\n\t\t\t\tyylex.(*lexerWrapper).query.AddMust(query)\n\t\t\tcase queryMustNot:\n\t\t\t\tyylex.(*lexerWrapper).query.AddMustNot(query)\n\t\t\t}\n\t\t}\n\tcase 5:\n\t\tyyDollar = yyS[yypt-0 : yypt+1]\n//line query_string.y:74\n\t\t{\n\t\t\tyyVAL.n = queryShould\n\t\t}\n\tcase 6:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:78\n\t\t{\n\t\t\tlogDebugGrammar(\"PLUS\")\n\t\t\tyyVAL.n = queryMust\n\t\t}\n\tcase 7:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:83\n\t\t{\n\t\t\tlogDebugGrammar(\"MINUS\")\n\t\t\tyyVAL.n = queryMustNot\n\t\t}\n\tcase 8:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:89\n\t\t{\n\t\t\tstr := yyDollar[1].s\n\t\t\tlogDebugGrammar(\"STRING - %s\", str)\n\t\t\tvar q FieldableQuery\n\t\t\tif strings.HasPrefix(str, \"/\") && strings.HasSuffix(str, \"/\") {\n\t\t\t\tq = NewRegexpQuery(str[1 : len(str)-1])\n\t\t\t} else if strings.ContainsAny(str, \"*?\") {\n\t\t\t\tq = NewWildcardQuery(str)\n\t\t\t} else {\n\t\t\t\tq = NewMatchQuery(str)\n\t\t\t}\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 9:\n\t\tyyDollar = yyS[yypt-2 : yypt+1]\n//line query_string.y:103\n\t\t{\n\t\t\tstr := yyDollar[1].s\n\t\t\tfuzziness, err := strconv.ParseFloat(yyDollar[2].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid fuzziness value: %v\", err))\n\t\t\t}\n\t\t\tlogDebugGrammar(\"FUZZY STRING - %s %f\", str, fuzziness)\n\t\t\tq := NewMatchQuery(str)\n\t\t\tq.SetFuzziness(int(fuzziness))\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 10:\n\t\tyyDollar = yyS[yypt-4 : yypt+1]\n//line query_string.y:115\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tstr := yyDollar[3].s\n\t\t\tfuzziness, err := strconv.ParseFloat(yyDollar[4].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid fuzziness value: %v\", err))\n\t\t\t}\n\t\t\tlogDebugGrammar(\"FIELD - %s FUZZY STRING - %s %f\", field, str, fuzziness)\n\t\t\tq := NewMatchQuery(str)\n\t\t\tq.SetFuzziness(int(fuzziness))\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 11:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:129\n\t\t{\n\t\t\tstr := yyDollar[1].s\n\t\t\tlogDebugGrammar(\"STRING - %s\", str)\n\t\t\tq1 := NewMatchQuery(str)\n\t\t\tval, err := strconv.ParseFloat(yyDollar[1].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t\t\t}\n\t\t\tinclusive := true\n\t\t\tq2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive)\n\t\t\tq := NewDisjunctionQuery([]Query{q1, q2})\n\t\t\tq.queryStringMode = true\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 12:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:144\n\t\t{\n\t\t\tphrase := yyDollar[1].s\n\t\t\tlogDebugGrammar(\"PHRASE - %s\", phrase)\n\t\t\tq := NewMatchPhraseQuery(phrase)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 13:\n\t\tyyDollar = yyS[yypt-3 : yypt+1]\n//line query_string.y:151\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tstr := yyDollar[3].s\n\t\t\tlogDebugGrammar(\"FIELD - %s STRING - %s\", field, str)\n\t\t\tvar q FieldableQuery\n\t\t\tif strings.HasPrefix(str, \"/\") && strings.HasSuffix(str, \"/\") {\n\t\t\t\tq = NewRegexpQuery(str[1 : len(str)-1])\n\t\t\t} else if strings.ContainsAny(str, \"*?\") {\n\t\t\t\tq = NewWildcardQuery(str)\n\t\t\t} else {\n\t\t\t\tq = NewMatchQuery(str)\n\t\t\t}\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 14:\n\t\tyyDollar = yyS[yypt-3 : yypt+1]\n//line query_string.y:167\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tstr := yyDollar[3].s\n\t\t\tlogDebugGrammar(\"FIELD - %s STRING - %s\", field, str)\n\t\t\tq1 := NewMatchQuery(str)\n\t\t\tq1.SetField(field)\n\t\t\tval, err := strconv.ParseFloat(yyDollar[3].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t\t\t}\n\t\t\tinclusive := true\n\t\t\tq2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive)\n\t\t\tq2.SetField(field)\n\t\t\tq := NewDisjunctionQuery([]Query{q1, q2})\n\t\t\tq.queryStringMode = true\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 15:\n\t\tyyDollar = yyS[yypt-3 : yypt+1]\n//line query_string.y:185\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tphrase := yyDollar[3].s\n\t\t\tlogDebugGrammar(\"FIELD - %s PHRASE - %s\", field, phrase)\n\t\t\tq := NewMatchPhraseQuery(phrase)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 16:\n\t\tyyDollar = yyS[yypt-4 : yypt+1]\n//line query_string.y:194\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tmin, err := strconv.ParseFloat(yyDollar[4].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t\t\t}\n\t\t\tminInclusive := false\n\t\t\tlogDebugGrammar(\"FIELD - GREATER THAN %f\", min)\n\t\t\tq := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 17:\n\t\tyyDollar = yyS[yypt-5 : yypt+1]\n//line query_string.y:207\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tmin, err := strconv.ParseFloat(yyDollar[5].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t\t\t}\n\t\t\tminInclusive := true\n\t\t\tlogDebugGrammar(\"FIELD - GREATER THAN OR EQUAL %f\", min)\n\t\t\tq := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 18:\n\t\tyyDollar = yyS[yypt-4 : yypt+1]\n//line query_string.y:220\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tmax, err := strconv.ParseFloat(yyDollar[4].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t\t\t}\n\t\t\tmaxInclusive := false\n\t\t\tlogDebugGrammar(\"FIELD - LESS THAN %f\", max)\n\t\t\tq := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 19:\n\t\tyyDollar = yyS[yypt-5 : yypt+1]\n//line query_string.y:233\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tmax, err := strconv.ParseFloat(yyDollar[5].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"error parsing number: %v\", err))\n\t\t\t}\n\t\t\tmaxInclusive := true\n\t\t\tlogDebugGrammar(\"FIELD - LESS THAN OR EQUAL %f\", max)\n\t\t\tq := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 20:\n\t\tyyDollar = yyS[yypt-4 : yypt+1]\n//line query_string.y:246\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tminInclusive := false\n\t\t\tphrase := yyDollar[4].s\n\n\t\t\tlogDebugGrammar(\"FIELD - GREATER THAN DATE %s\", phrase)\n\t\t\tminTime, err := queryTimeFromString(phrase)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid time: %v\", err))\n\t\t\t}\n\t\t\tq := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 21:\n\t\tyyDollar = yyS[yypt-5 : yypt+1]\n//line query_string.y:261\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tminInclusive := true\n\t\t\tphrase := yyDollar[5].s\n\n\t\t\tlogDebugGrammar(\"FIELD - GREATER THAN OR EQUAL DATE %s\", phrase)\n\t\t\tminTime, err := queryTimeFromString(phrase)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid time: %v\", err))\n\t\t\t}\n\t\t\tq := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 22:\n\t\tyyDollar = yyS[yypt-4 : yypt+1]\n//line query_string.y:276\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tmaxInclusive := false\n\t\t\tphrase := yyDollar[4].s\n\n\t\t\tlogDebugGrammar(\"FIELD - LESS THAN DATE %s\", phrase)\n\t\t\tmaxTime, err := queryTimeFromString(phrase)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid time: %v\", err))\n\t\t\t}\n\t\t\tq := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 23:\n\t\tyyDollar = yyS[yypt-5 : yypt+1]\n//line query_string.y:291\n\t\t{\n\t\t\tfield := yyDollar[1].s\n\t\t\tmaxInclusive := true\n\t\t\tphrase := yyDollar[5].s\n\n\t\t\tlogDebugGrammar(\"FIELD - LESS THAN OR EQUAL DATE %s\", phrase)\n\t\t\tmaxTime, err := queryTimeFromString(phrase)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid time: %v\", err))\n\t\t\t}\n\t\t\tq := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive)\n\t\t\tq.SetField(field)\n\t\t\tyyVAL.q = q\n\t\t}\n\tcase 24:\n\t\tyyDollar = yyS[yypt-0 : yypt+1]\n//line query_string.y:307\n\t\t{\n\t\t\tyyVAL.pf = nil\n\t\t}\n\tcase 25:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:311\n\t\t{\n\t\t\tyyVAL.pf = nil\n\t\t\tboost, err := strconv.ParseFloat(yyDollar[1].s, 64)\n\t\t\tif err != nil {\n\t\t\t\tyylex.(*lexerWrapper).lex.Error(fmt.Sprintf(\"invalid boost value: %v\", err))\n\t\t\t} else {\n\t\t\t\tyyVAL.pf = &boost\n\t\t\t}\n\t\t\tlogDebugGrammar(\"BOOST %f\", boost)\n\t\t}\n\tcase 26:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:323\n\t\t{\n\t\t\tyyVAL.s = yyDollar[1].s\n\t\t}\n\tcase 27:\n\t\tyyDollar = yyS[yypt-2 : yypt+1]\n//line query_string.y:327\n\t\t{\n\t\t\tyyVAL.s = \"-\" + yyDollar[2].s\n\t\t}\n\tcase 28:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:332\n\t\t{\n\t\t\tyyVAL.s = yyDollar[1].s\n\t\t}\n\tcase 29:\n\t\tyyDollar = yyS[yypt-1 : yypt+1]\n//line query_string.y:336\n\t\t{\n\t\t\tyyVAL.s = yyDollar[1].s\n\t\t}\n\t}\n\tgoto yystack /* stack new state and value */\n}\n"
  },
  {
    "path": "search/query/query_string_lex.go",
    "content": "//  Copyright (c) 2016 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"bufio\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n)\n\nconst reservedChars = \"+-=&|><!(){}[]^\\\"~*?:\\\\/ \"\n\nfunc unescape(escaped string) string {\n\t// see if this character can be escaped\n\tif strings.ContainsAny(escaped, reservedChars) {\n\t\treturn escaped\n\t}\n\t// otherwise return it with the \\ intact\n\treturn \"\\\\\" + escaped\n}\n\ntype queryStringLex struct {\n\tin            *bufio.Reader\n\tbuf           string\n\tcurrState     lexState\n\tcurrConsumed  bool\n\tinEscape      bool\n\tnextToken     *yySymType\n\tnextTokenType int\n\tseenDot       bool\n\tnextRune      rune\n\tnextRuneSize  int\n\tatEOF         bool\n}\n\nfunc (l *queryStringLex) reset() {\n\tl.buf = \"\"\n\tl.inEscape = false\n\tl.seenDot = false\n}\n\nfunc (l *queryStringLex) Error(msg string) {\n\tpanic(msg)\n}\n\nfunc (l *queryStringLex) Lex(lval *yySymType) int {\n\tvar err error\n\n\tfor l.nextToken == nil {\n\t\tif l.currConsumed {\n\t\t\tl.nextRune, l.nextRuneSize, err = l.in.ReadRune()\n\t\t\tif err != nil && err == io.EOF {\n\t\t\t\tl.nextRune = 0\n\t\t\t\tl.atEOF = true\n\t\t\t} else if err != nil {\n\t\t\t\treturn 0\n\t\t\t}\n\t\t}\n\t\tl.currState, l.currConsumed = l.currState(l, l.nextRune, l.atEOF)\n\t\tif l.currState == nil {\n\t\t\treturn 0\n\t\t}\n\t}\n\n\t*lval = *l.nextToken\n\trv := l.nextTokenType\n\tl.nextToken = nil\n\tl.nextTokenType = 0\n\treturn rv\n}\n\nfunc newQueryStringLex(in io.Reader) *queryStringLex {\n\treturn &queryStringLex{\n\t\tin:           bufio.NewReader(in),\n\t\tcurrState:    startState,\n\t\tcurrConsumed: true,\n\t}\n}\n\ntype lexState func(l *queryStringLex, next rune, eof bool) (lexState, bool)\n\nfunc startState(l *queryStringLex, next rune, eof bool) (lexState, bool) {\n\tif eof {\n\t\treturn nil, false\n\t}\n\n\t// handle inside escape case up front\n\tif l.inEscape {\n\t\tl.inEscape = false\n\t\tl.buf += unescape(string(next))\n\t\treturn inStrState, true\n\t}\n\n\tswitch next {\n\tcase '\"':\n\t\treturn inPhraseState, true\n\tcase '+', '-', ':', '>', '<', '=':\n\t\tl.buf += string(next)\n\t\treturn singleCharOpState, true\n\tcase '^':\n\t\treturn inBoostState, true\n\tcase '~':\n\t\treturn inTildeState, true\n\t}\n\n\tswitch {\n\tcase !l.inEscape && next == '\\\\':\n\t\tl.inEscape = true\n\t\treturn startState, true\n\tcase unicode.IsDigit(next):\n\t\tl.buf += string(next)\n\t\treturn inNumOrStrState, true\n\tcase !unicode.IsSpace(next):\n\t\tl.buf += string(next)\n\t\treturn inStrState, true\n\t}\n\n\t// doesn't look like anything, just eat it and stay here\n\tl.reset()\n\treturn startState, true\n}\n\nfunc inPhraseState(l *queryStringLex, next rune, eof bool) (lexState, bool) {\n\t// unterminated phrase eats the phrase\n\tif eof {\n\t\tl.Error(\"unterminated quote\")\n\t\treturn nil, false\n\t}\n\n\t// only a non-escaped \" ends the phrase\n\tif !l.inEscape && next == '\"' {\n\t\t// end phrase\n\t\tl.nextTokenType = tPHRASE\n\t\tl.nextToken = &yySymType{\n\t\t\ts: l.buf,\n\t\t}\n\t\tlogDebugTokens(\"PHRASE - '%s'\", l.nextToken.s)\n\t\tl.reset()\n\t\treturn startState, true\n\t} else if !l.inEscape && next == '\\\\' {\n\t\tl.inEscape = true\n\t} else if l.inEscape {\n\t\t// if in escape, end it\n\t\tl.inEscape = false\n\t\tl.buf += unescape(string(next))\n\t} else {\n\t\tl.buf += string(next)\n\t}\n\n\treturn inPhraseState, true\n}\n\nfunc singleCharOpState(l *queryStringLex, next rune, eof bool) (lexState, bool) {\n\tl.nextToken = &yySymType{}\n\n\tswitch l.buf {\n\tcase \"+\":\n\t\tl.nextTokenType = tPLUS\n\t\tlogDebugTokens(\"PLUS\")\n\tcase \"-\":\n\t\tl.nextTokenType = tMINUS\n\t\tlogDebugTokens(\"MINUS\")\n\tcase \":\":\n\t\tl.nextTokenType = tCOLON\n\t\tlogDebugTokens(\"COLON\")\n\tcase \">\":\n\t\tl.nextTokenType = tGREATER\n\t\tlogDebugTokens(\"GREATER\")\n\tcase \"<\":\n\t\tl.nextTokenType = tLESS\n\t\tlogDebugTokens(\"LESS\")\n\tcase \"=\":\n\t\tl.nextTokenType = tEQUAL\n\t\tlogDebugTokens(\"EQUAL\")\n\t}\n\n\tl.reset()\n\treturn startState, false\n}\n\nfunc inBoostState(l *queryStringLex, next rune, eof bool) (lexState, bool) {\n\n\t// only a non-escaped space ends the boost (or eof)\n\tif eof || (!l.inEscape && next == ' ') {\n\t\t// end boost\n\t\tl.nextTokenType = tBOOST\n\t\tif l.buf == \"\" {\n\t\t\tl.buf = \"1\"\n\t\t}\n\t\tl.nextToken = &yySymType{\n\t\t\ts: l.buf,\n\t\t}\n\t\tlogDebugTokens(\"BOOST - '%s'\", l.nextToken.s)\n\t\tl.reset()\n\t\treturn startState, true\n\t} else if !l.inEscape && next == '\\\\' {\n\t\tl.inEscape = true\n\t} else if l.inEscape {\n\t\t// if in escape, end it\n\t\tl.inEscape = false\n\t\tl.buf += unescape(string(next))\n\t} else {\n\t\tl.buf += string(next)\n\t}\n\n\treturn inBoostState, true\n}\n\nfunc inTildeState(l *queryStringLex, next rune, eof bool) (lexState, bool) {\n\n\t// only a non-escaped space ends the tilde (or eof)\n\tif eof || (!l.inEscape && next == ' ') {\n\t\t// end tilde\n\t\tl.nextTokenType = tTILDE\n\t\tif l.buf == \"\" {\n\t\t\tl.buf = \"1\"\n\t\t}\n\t\tl.nextToken = &yySymType{\n\t\t\ts: l.buf,\n\t\t}\n\t\tlogDebugTokens(\"TILDE - '%s'\", l.nextToken.s)\n\t\tl.reset()\n\t\treturn startState, true\n\t} else if !l.inEscape && next == '\\\\' {\n\t\tl.inEscape = true\n\t} else if l.inEscape {\n\t\t// if in escape, end it\n\t\tl.inEscape = false\n\t\tl.buf += unescape(string(next))\n\t} else {\n\t\tl.buf += string(next)\n\t}\n\n\treturn inTildeState, true\n}\n\nfunc inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) {\n\t// end on non-escaped space, colon, tilde, boost (or eof)\n\tif eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) {\n\t\t// end number\n\t\tl.nextTokenType = tNUMBER\n\t\tl.nextToken = &yySymType{\n\t\t\ts: l.buf,\n\t\t}\n\t\tlogDebugTokens(\"NUMBER - '%s'\", l.nextToken.s)\n\t\tl.reset()\n\n\t\tconsumed := true\n\t\tif !eof && (next == ':' || next == '^' || next == '~') {\n\t\t\tconsumed = false\n\t\t}\n\n\t\treturn startState, consumed\n\t} else if !l.inEscape && next == '\\\\' {\n\t\tl.inEscape = true\n\t\treturn inNumOrStrState, true\n\t} else if l.inEscape {\n\t\t// if in escape, end it\n\t\tl.inEscape = false\n\t\tl.buf += unescape(string(next))\n\t\t// go directly to string, no successfully or unsuccessfully\n\t\t// escaped string results in a valid number\n\t\treturn inStrState, true\n\t}\n\n\t// see where to go\n\tif !l.seenDot && next == '.' {\n\t\t// stay in this state\n\t\tl.seenDot = true\n\t\tl.buf += string(next)\n\t\treturn inNumOrStrState, true\n\t} else if unicode.IsDigit(next) {\n\t\tl.buf += string(next)\n\t\treturn inNumOrStrState, true\n\t}\n\n\t// doesn't look like an number, transition\n\tl.buf += string(next)\n\treturn inStrState, true\n}\n\nfunc inStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) {\n\t// end on non-escaped space, colon, tilde, boost (or eof)\n\tif eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) {\n\t\t// end string\n\t\tl.nextTokenType = tSTRING\n\t\tl.nextToken = &yySymType{\n\t\t\ts: l.buf,\n\t\t}\n\t\tlogDebugTokens(\"STRING - '%s'\", l.nextToken.s)\n\t\tl.reset()\n\n\t\tconsumed := true\n\t\tif !eof && (next == ':' || next == '^' || next == '~') {\n\t\t\tconsumed = false\n\t\t}\n\n\t\treturn startState, consumed\n\t} else if !l.inEscape && next == '\\\\' {\n\t\tl.inEscape = true\n\t} else if l.inEscape {\n\t\t// if in escape, end it\n\t\tl.inEscape = false\n\t\tl.buf += unescape(string(next))\n\t} else {\n\t\tl.buf += string(next)\n\t}\n\n\treturn inStrState, true\n}\n\nfunc logDebugTokens(format string, v ...interface{}) {\n\tif debugLexer {\n\t\tlogger.Printf(format, v...)\n\t}\n}\n"
  },
  {
    "path": "search/query/query_string_lex_test.go",
    "content": "package query\n\nimport (\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestLexer(t *testing.T) {\n\n\ttests := []struct {\n\t\tinput  string\n\t\ttokens []token\n\t}{\n\t\t{\n\t\t\tinput: \"test\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"127.0.0.1\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"127.0.0.1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `\"test phrase 1\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test phrase 1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:test\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:t-est\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"t-est\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:t+est\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"t+est\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:t>est\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"t>est\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:t<est\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp:  tCOLON,\n\t\t\t\t\tlval: yySymType{},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"t<est\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tinput: \"field:t=est\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"t=est\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"+field1:test1\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tPLUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"-field2:test2\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tMINUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field3:\"test phrase 2\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field3\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test phrase 2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `+field4:\"test phrase 1\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tPLUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field4\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test phrase 1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `-field5:\"test phrase 2\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tMINUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test phrase 2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `+field6:test3 -field7:test4 field8:test5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tPLUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field6\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test3\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tMINUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field7\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test4\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field8\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"test^3\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tBOOST,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"3\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"test^3 other^6\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"test\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tBOOST,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"3\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"other\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tBOOST,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"6\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"33\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"33\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:33\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"33\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"cat-dog\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"cat-dog\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"watex~\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"watex\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tTILDE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"watex~2\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"watex\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tTILDE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"watex~ 2\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"watex\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tTILDE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:watex~\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"watex\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tTILDE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:watex~2\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"watex\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tTILDE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:555c3bb06f7a127cda000005`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"555c3bb06f7a127cda000005\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:>5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tGREATER,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:>=5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tGREATER,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tEQUAL,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:<5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tLESS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:<=5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tLESS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tEQUAL,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: \"field:-5\",\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tMINUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:>-5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tGREATER,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tMINUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:>=-5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tGREATER,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tEQUAL,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tMINUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:<-5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tLESS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tMINUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:<=-5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tLESS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tEQUAL,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tMINUS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:>\"2006-01-02T15:04:05Z\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tGREATER,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"2006-01-02T15:04:05Z\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:>=\"2006-01-02T15:04:05Z\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tGREATER,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tEQUAL,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"2006-01-02T15:04:05Z\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:<\"2006-01-02T15:04:05Z\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tLESS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"2006-01-02T15:04:05Z\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `field:<=\"2006-01-02T15:04:05Z\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"field\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tLESS,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tEQUAL,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"2006-01-02T15:04:05Z\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `/mar.*ty/`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `/mar.*ty/`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `name:/mar.*ty/`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"name\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `/mar.*ty/`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `mart*`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `mart*`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `name:mart*`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"name\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `mart*`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `name\\:marty`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `name:marty`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `name:marty\\:couchbase`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"name\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `marty:couchbase`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `marty\\ couchbase`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `marty couchbase`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `\\+marty`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `+marty`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `\\-marty`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `-marty`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `\"what does \\\"quote\\\" mean\"`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tPHRASE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `what does \"quote\" mean`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `can\\ i\\ escap\\e`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `can i escap\\e`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `   what`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `what`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `term^`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `term`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tBOOST,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `3.0\\:`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `3.0:`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `3.0\\a`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: `3.0\\a`,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `age:65^10`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"age\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"65\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tBOOST,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"10\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `age:65^10 age:18^5`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"age\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"65\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tBOOST,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"10\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"age\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"18\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tBOOST,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"5\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `age:65~2`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"age\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"65\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tTILDE,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: `65:cat`,\n\t\t\ttokens: []token{\n\t\t\t\t{\n\t\t\t\t\ttyp: tNUMBER,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"65\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tCOLON,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\ttyp: tSTRING,\n\t\t\t\t\tlval: yySymType{\n\t\t\t\t\t\ts: \"cat\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttest := test\n\t\tt.Run(test.input, func(t *testing.T) {\n\n\t\t\tr := strings.NewReader(test.input)\n\t\t\tl := newQueryStringLex(r)\n\t\t\tvar tokens []token\n\t\t\tvar lval yySymType\n\t\t\trv := l.Lex(&lval)\n\t\t\tfor rv > 0 {\n\t\t\t\t//tokenTypes = append(tokenTypes, rv)\n\t\t\t\ttokens = append(tokens, token{typ: rv, lval: lval})\n\t\t\t\tlval.s = \"\"\n\t\t\t\tlval.n = 0\n\t\t\t\trv = l.Lex(&lval)\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(tokens, test.tokens) {\n\t\t\t\tt.Fatalf(\"\\nexpected: %#v\\n     got: %#v\\n\", test.tokens, tokens)\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype token struct {\n\ttyp  int\n\tlval yySymType\n}\n"
  },
  {
    "path": "search/query/query_string_parser.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// as of Go 1.8 this requires the goyacc external tool\n// available from golang.org/x/tools/cmd/goyacc\n\n//go:generate goyacc -o query_string.y.go query_string.y\n//go:generate sed -i.tmp -e 1d query_string.y.go\n//go:generate rm query_string.y.go.tmp\n\n// note: OSX sed and gnu sed handle the -i (in-place) option differently.\n// using -i.tmp works on both, at the expense of having to remove\n// the unsightly .tmp files\n\npackage query\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\nvar debugParser bool\nvar debugLexer bool\n\nfunc parseQuerySyntax(query string) (rq Query, err error) {\n\tif query == \"\" {\n\t\treturn NewMatchNoneQuery(), nil\n\t}\n\tlex := newLexerWrapper(newQueryStringLex(strings.NewReader(query)))\n\tdoParse(lex)\n\n\tif len(lex.errs) > 0 {\n\t\treturn nil, fmt.Errorf(\"%s\", strings.Join(lex.errs, \"\\n\"))\n\t}\n\treturn lex.query, nil\n}\n\nfunc doParse(lex *lexerWrapper) {\n\tdefer func() {\n\t\tr := recover()\n\t\tif r != nil {\n\t\t\tlex.errs = append(lex.errs, fmt.Sprintf(\"parse error: %v\", r))\n\t\t}\n\t}()\n\n\tyyParse(lex)\n}\n\nconst (\n\tqueryShould = iota\n\tqueryMust\n\tqueryMustNot\n)\n\ntype lexerWrapper struct {\n\tlex   yyLexer\n\terrs  []string\n\tquery *BooleanQuery\n}\n\nfunc newLexerWrapper(lex yyLexer) *lexerWrapper {\n\treturn &lexerWrapper{\n\t\tlex:   lex,\n\t\tquery: NewBooleanQueryForQueryString(nil, nil, nil),\n\t}\n}\n\nfunc (l *lexerWrapper) Lex(lval *yySymType) int {\n\treturn l.lex.Lex(lval)\n}\n\nfunc (l *lexerWrapper) Error(s string) {\n\tl.errs = append(l.errs, s)\n}\n"
  },
  {
    "path": "search/query/query_string_parser_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n)\n\nfunc TestQuerySyntaxParserValid(t *testing.T) {\n\tthirtyThreePointOh := 33.0\n\ttwoPointOh := 2.0\n\tfivePointOh := 5.0\n\tminusFivePointOh := -5.0\n\ttheTruth := true\n\ttheFalsehood := false\n\ttheDate, err := time.Parse(time.RFC3339, \"2006-01-02T15:04:05Z\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttests := []struct {\n\t\tinput   string\n\t\tresult  Query\n\t\tmapping mapping.IndexMapping\n\t}{\n\t\t{\n\t\t\tinput:   \"test\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(\"test\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"127.0.0.1\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(\"127.0.0.1\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `\"test phrase 1\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchPhraseQuery(\"test phrase 1\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"field:test\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test\")\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// - is allowed inside a term, just not the start\n\t\t{\n\t\t\tinput:   \"field:t-est\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"t-est\")\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// + is allowed inside a term, just not the start\n\t\t{\n\t\t\tinput:   \"field:t+est\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"t+est\")\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// > is allowed inside a term, just not the start\n\t\t{\n\t\t\tinput:   \"field:t>est\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"t>est\")\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// < is allowed inside a term, just not the start\n\t\t{\n\t\t\tinput:   \"field:t<est\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"t<est\")\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// = is allowed inside a term, just not the start\n\t\t{\n\t\t\tinput:   \"field:t=est\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"t=est\")\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"+field1:test1\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test1\")\n\t\t\t\t\t\tq.SetField(\"field1\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"-field2:test2\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test2\")\n\t\t\t\t\t\tq.SetField(\"field2\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t}),\n\t\t},\n\t\t{\n\t\t\tinput:   `field3:\"test phrase 2\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchPhraseQuery(\"test phrase 2\")\n\t\t\t\t\t\tq.SetField(\"field3\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `+field4:\"test phrase 1\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchPhraseQuery(\"test phrase 1\")\n\t\t\t\t\t\tq.SetField(\"field4\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `-field5:\"test phrase 2\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchPhraseQuery(\"test phrase 2\")\n\t\t\t\t\t\tq.SetField(\"field5\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t}),\n\t\t},\n\t\t{\n\t\t\tinput:   `+field6:test3 -field7:test4 field8:test5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test3\")\n\t\t\t\t\t\tq.SetField(\"field6\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test5\")\n\t\t\t\t\t\tq.SetField(\"field8\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test4\")\n\t\t\t\t\t\tq.SetField(\"field7\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t}),\n\t\t},\n\t\t{\n\t\t\tinput:   \"test^3\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test\")\n\t\t\t\t\t\tq.SetBoost(3.0)\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"test^3 other^6\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test\")\n\t\t\t\t\t\tq.SetBoost(3.0)\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"other\")\n\t\t\t\t\t\tq.SetBoost(6.0)\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"33\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tqo := NewDisjunctionQuery(\n\t\t\t\t\t\t\t[]Query{\n\t\t\t\t\t\t\t\tNewMatchQuery(\"33\"),\n\t\t\t\t\t\t\t\tNewNumericRangeInclusiveQuery(&thirtyThreePointOh, &thirtyThreePointOh, &theTruth, &theTruth),\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\tqo.queryStringMode = true\n\t\t\t\t\t\treturn qo\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"field:33\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tqo := NewDisjunctionQuery(\n\t\t\t\t\t\t\t[]Query{\n\t\t\t\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\t\t\t\tq := NewMatchQuery(\"33\")\n\t\t\t\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\t\t\t\treturn q\n\t\t\t\t\t\t\t\t}(),\n\t\t\t\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(&thirtyThreePointOh, &thirtyThreePointOh, &theTruth, &theTruth)\n\t\t\t\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\t\t\t\treturn q\n\t\t\t\t\t\t\t\t}(),\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\tqo.queryStringMode = true\n\t\t\t\t\t\treturn qo\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"cat-dog\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(\"cat-dog\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"watex~\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"watex\")\n\t\t\t\t\t\tq.SetFuzziness(1)\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"watex~2\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"watex\")\n\t\t\t\t\t\tq.SetFuzziness(2)\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"watex~ 2\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"watex\")\n\t\t\t\t\t\tq.SetFuzziness(1)\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tqo := NewDisjunctionQuery(\n\t\t\t\t\t\t\t[]Query{\n\t\t\t\t\t\t\t\tNewMatchQuery(\"2\"),\n\t\t\t\t\t\t\t\tNewNumericRangeInclusiveQuery(&twoPointOh, &twoPointOh, &theTruth, &theTruth),\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\tqo.queryStringMode = true\n\t\t\t\t\t\treturn qo\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"field:watex~\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"watex\")\n\t\t\t\t\t\tq.SetFuzziness(1)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   \"field:watex~2\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"watex\")\n\t\t\t\t\t\tq.SetFuzziness(2)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:555c3bb06f7a127cda000005`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"555c3bb06f7a127cda000005\")\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:>5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(&fivePointOh, nil, &theFalsehood, nil)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:>=5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(&fivePointOh, nil, &theTruth, nil)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:<5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(nil, &fivePointOh, nil, &theFalsehood)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:<=5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(nil, &fivePointOh, nil, &theTruth)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// new range tests with negative number\n\t\t{\n\t\t\tinput:   \"field:-5\",\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tqo := NewDisjunctionQuery(\n\t\t\t\t\t\t\t[]Query{\n\t\t\t\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\t\t\t\tq := NewMatchQuery(\"-5\")\n\t\t\t\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\t\t\t\treturn q\n\t\t\t\t\t\t\t\t}(),\n\t\t\t\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(&minusFivePointOh, &minusFivePointOh, &theTruth, &theTruth)\n\t\t\t\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\t\t\t\treturn q\n\t\t\t\t\t\t\t\t}(),\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\tqo.queryStringMode = true\n\t\t\t\t\t\treturn qo\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:>-5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(&minusFivePointOh, nil, &theFalsehood, nil)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:>=-5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(&minusFivePointOh, nil, &theTruth, nil)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:<-5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(nil, &minusFivePointOh, nil, &theFalsehood)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:<=-5`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewNumericRangeInclusiveQuery(nil, &minusFivePointOh, nil, &theTruth)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:>\"2006-01-02T15:04:05Z\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewDateRangeInclusiveQuery(theDate, time.Time{}, &theFalsehood, nil)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:>=\"2006-01-02T15:04:05Z\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewDateRangeInclusiveQuery(theDate, time.Time{}, &theTruth, nil)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:<\"2006-01-02T15:04:05Z\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewDateRangeInclusiveQuery(time.Time{}, theDate, nil, &theFalsehood)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `field:<=\"2006-01-02T15:04:05Z\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewDateRangeInclusiveQuery(time.Time{}, theDate, nil, &theTruth)\n\t\t\t\t\t\tq.SetField(\"field\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `/mar.*ty/`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewRegexpQuery(\"mar.*ty\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `name:/mar.*ty/`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewRegexpQuery(\"mar.*ty\")\n\t\t\t\t\t\tq.SetField(\"name\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `mart*`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewWildcardQuery(\"mart*\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `name:mart*`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewWildcardQuery(\"mart*\")\n\t\t\t\t\t\tq.SetField(\"name\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\n\t\t// tests for escaping\n\n\t\t// escape : as field delimiter\n\t\t{\n\t\t\tinput:   `name\\:marty`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(\"name:marty\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// first colon delimiter, second escaped\n\t\t{\n\t\t\tinput:   `name:marty\\:couchbase`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"marty:couchbase\")\n\t\t\t\t\t\tq.SetField(\"name\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// escape space, single argument to match query\n\t\t{\n\t\t\tinput:   `marty\\ couchbase`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(\"marty couchbase\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// escape leading plus, not a must clause\n\t\t{\n\t\t\tinput:   `\\+marty`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(\"+marty\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// escape leading minus, not a must not clause\n\t\t{\n\t\t\tinput:   `\\-marty`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(\"-marty\"),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// escape quote inside of phrase\n\t\t{\n\t\t\tinput:   `\"what does \\\"quote\\\" mean\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchPhraseQuery(`what does \"quote\" mean`),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// escaping an unsupported character retains backslash\n\t\t{\n\t\t\tinput:   `can\\ i\\ escap\\e`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(`can i escap\\e`),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// leading spaces\n\t\t{\n\t\t\tinput:   `   what`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(`what`),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// no boost value defaults to 1\n\t\t{\n\t\t\tinput:   `term^`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(`term`)\n\t\t\t\t\t\tq.SetBoost(1.0)\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// weird lexer cases, something that starts like a number\n\t\t// but contains escape and ends up as string\n\t\t{\n\t\t\tinput:   `3.0\\:`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(`3.0:`),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `3.0\\a`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tNewMatchQuery(`3.0\\a`),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\n\t\t// field names as phrases\n\t\t{\n\t\t\tinput:   `\"fie ld\":test`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"test\")\n\t\t\t\t\t\tq.SetField(\"fie ld\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t{\n\t\t\tinput:   `\"fie ld\":\"test\"`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewMatchPhraseQuery(\"test\")\n\t\t\t\t\t\tq.SetField(\"fie ld\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t\t// exact match number with boost\n\t\t{\n\t\t\tinput:   `age:65^10`,\n\t\t\tmapping: mapping.NewIndexMapping(),\n\t\t\tresult: NewBooleanQueryForQueryString(\n\t\t\t\tnil,\n\t\t\t\t[]Query{\n\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\tq := NewDisjunctionQuery([]Query{\n\t\t\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\t\t\tmq := NewMatchQuery(\"65\")\n\t\t\t\t\t\t\t\tmq.SetField(\"age\")\n\t\t\t\t\t\t\t\treturn mq\n\t\t\t\t\t\t\t}(),\n\t\t\t\t\t\t\tfunc() Query {\n\t\t\t\t\t\t\t\tval := float64(65)\n\t\t\t\t\t\t\t\tinclusive := true\n\t\t\t\t\t\t\t\tnq := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive)\n\t\t\t\t\t\t\t\tnq.SetField(\"age\")\n\t\t\t\t\t\t\t\treturn nq\n\t\t\t\t\t\t\t}(),\n\t\t\t\t\t\t})\n\t\t\t\t\t\tq.SetBoost(10)\n\t\t\t\t\t\tq.queryStringMode = true\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}(),\n\t\t\t\t},\n\t\t\t\tnil),\n\t\t},\n\t}\n\n\t// turn on lexer debugging\n\t// debugLexer = true\n\t// debugParser = true\n\t// logger = log.New(os.Stderr, \"bleve \", log.LstdFlags)\n\n\tfor _, test := range tests {\n\n\t\tq, err := parseQuerySyntax(test.input)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(q, test.result) {\n\t\t\tt.Errorf(\"Expected %#v, got %#v: for %s\", test.result, q, test.input)\n\t\t}\n\t}\n}\n\nfunc TestQuerySyntaxParserInvalid(t *testing.T) {\n\ttests := []struct {\n\t\tinput string\n\t}{\n\t\t{\"^\"},\n\t\t{\"^5\"},\n\t\t{\"field:-text\"},\n\t\t{\"field:+text\"},\n\t\t{\"field:>text\"},\n\t\t{\"field:>=text\"},\n\t\t{\"field:<text\"},\n\t\t{\"field:<=text\"},\n\t\t{\"field:~text\"},\n\t\t{\"field:^text\"},\n\t\t{\"field::text\"},\n\t\t{`\"this is the time`},\n\t\t{`cat^3\\:`},\n\t\t{`cat^3\\0`},\n\t\t{`cat~3\\:`},\n\t\t{`cat~3\\0`},\n\t\t{`99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`},\n\t\t{`field:99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`},\n\t\t{`field:>99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`},\n\t\t{`field:>=99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`},\n\t\t{`field:<99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`},\n\t\t{`field:<=99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`},\n\t}\n\n\t// turn on lexer debugging\n\t// debugLexer = true\n\t// logger = log.New(os.Stderr, \"bleve\", log.LstdFlags)\n\n\tfor _, test := range tests {\n\t\t_, err := parseQuerySyntax(test.input)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"expected error, got nil for `%s`\", test.input)\n\t\t}\n\t}\n}\n\nfunc BenchmarkLexer(b *testing.B) {\n\tfor n := 0; n < b.N; n++ {\n\t\tvar tokenTypes []int\n\t\tvar tokens []yySymType\n\t\tr := strings.NewReader(`+field4:\"test phrase 1\"`)\n\t\tl := newQueryStringLex(r)\n\t\tvar lval yySymType\n\t\trv := l.Lex(&lval)\n\n\t\tfor rv > 0 {\n\t\t\ttokenTypes = append(tokenTypes, rv)\n\t\t\ttokens = append(tokens, lval)\n\n\t\t\t// use the slice to silence the compiler warning\n\t\t\t_ = tokenTypes\n\t\t\t_ = tokens\n\n\t\t\tlval.s = \"\"\n\t\t\tlval.n = 0\n\t\t\trv = l.Lex(&lval)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/query/query_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"reflect\"\n\t\"sort\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n)\n\nvar minNum = 5.1\nvar maxNum = 7.1\nvar minTerm = \"bob\"\nvar maxTerm = \"cat\"\nvar startDateStr = \"2011-01-01T00:00:00Z\"\nvar endDateStr = \"2012-01-01T00:00:00Z\"\nvar startDate time.Time\nvar endDate time.Time\n\nfunc init() {\n\tvar err error\n\tstartDate, err = time.Parse(time.RFC3339, startDateStr)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tendDate, err = time.Parse(time.RFC3339, endDateStr)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc TestParseQuery(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput Query\n\t\terr    bool\n\t}{\n\t\t{\n\t\t\tinput: []byte(`{\"term\":\"water\",\"field\":\"desc\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewTermQuery(\"water\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"match\":\"beer\",\"field\":\"desc\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"match\":\"beer\",\"field\":\"desc\",\"operator\":\"or\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"match\":\"beer\",\"field\":\"desc\",\"operator\":\"and\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\tq.SetOperator(MatchQueryOperatorAnd)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"match\":\"beer\",\"field\":\"desc\",\"operator\":\"and\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\toperator := MatchQueryOperatorAnd\n\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\tq.SetOperator(operator)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"match\":\"beer\",\"field\":\"desc\",\"operator\":\"or\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\tq.SetOperator(MatchQueryOperatorOr)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"match\":\"beer\",\"field\":\"desc\",\"operator\":\"or\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\toperator := MatchQueryOperatorOr\n\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\tq.SetOperator(operator)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput:  []byte(`{\"match\":\"beer\",\"field\":\"desc\",\"operator\":\"does not exist\"}`),\n\t\t\toutput: nil,\n\t\t\terr:    true,\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"match_phrase\":\"light beer\",\"field\":\"desc\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewMatchPhraseQuery(\"light beer\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"must\":{\"conjuncts\": [{\"match\":\"beer\",\"field\":\"desc\"}]},\"should\":{\"disjuncts\": [{\"match\":\"water\",\"field\":\"desc\"}],\"min\":1.0},\"must_not\":{\"disjuncts\": [{\"match\":\"devon\",\"field\":\"desc\"}]}}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewBooleanQuery(\n\t\t\t\t\t[]Query{func() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}()},\n\t\t\t\t\t[]Query{func() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"water\")\n\t\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}()},\n\t\t\t\t\t[]Query{func() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"devon\")\n\t\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}()})\n\t\t\t\tq.SetMinShould(1)\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput:  []byte(`{\"terms\":[\"watered\",\"down\"],\"field\":\"desc\"}`),\n\t\t\toutput: NewPhraseQuery([]string{\"watered\", \"down\"}, \"desc\"),\n\t\t},\n\t\t{\n\t\t\tinput:  []byte(`{\"query\":\"+beer \\\"light beer\\\" -devon\"}`),\n\t\t\toutput: NewQueryStringQuery(`+beer \"light beer\" -devon`),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"min\":5.1,\"max\":7.1,\"field\":\"desc\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewNumericRangeQuery(&minNum, &maxNum)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"min\":\"bob\",\"max\":\"cat\",\"field\":\"desc\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewTermRangeQuery(minTerm, maxTerm)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"start\":\"` + startDateStr + `\",\"end\":\"` + endDateStr + `\",\"field\":\"desc\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewDateRangeStringQuery(startDateStr, endDateStr)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"prefix\":\"budwei\",\"field\":\"desc\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewPrefixQuery(\"budwei\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput:  []byte(`{\"match_all\":{}}`),\n\t\t\toutput: NewMatchAllQuery(),\n\t\t},\n\t\t{\n\t\t\tinput:  []byte(`{\"match_none\":{}}`),\n\t\t\toutput: NewMatchNoneQuery(),\n\t\t},\n\t\t{\n\t\t\tinput:  []byte(`{\"ids\":[\"a\",\"b\",\"c\"]}`),\n\t\t\toutput: NewDocIDQuery([]string{\"a\", \"b\", \"c\"}),\n\t\t},\n\t\t{\n\t\t\tinput:  []byte(`{\"bool\": true}`),\n\t\t\toutput: NewBoolFieldQuery(true),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\"field\": \"x\", \"cidr\": \"1.2.3.0/4\"}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq := NewIPRangeQuery(\"1.2.3.0/4\")\n\t\t\t\tq.SetField(\"x\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput:  []byte(`{\"madeitup\":\"queryhere\"}`),\n\t\t\toutput: nil,\n\t\t\terr:    true,\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tactual, err := ParseQuery(test.input)\n\t\tif err != nil && test.err == false {\n\t\t\tt.Errorf(\"error %v for %d\", err, i)\n\t\t}\n\n\t\tif !reflect.DeepEqual(test.output, actual) {\n\t\t\tt.Errorf(\"expected: %#v, got: %#v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n\nfunc TestQueryValidate(t *testing.T) {\n\ttests := []struct {\n\t\tquery Query\n\t\terr   bool\n\t}{\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewTermQuery(\"water\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewMatchPhraseQuery(\"light beer\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewNumericRangeQuery(&minNum, &maxNum)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewNumericRangeQuery(nil, nil)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t\terr: true,\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewDateRangeQuery(startDate, endDate)\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewPrefixQuery(\"budwei\")\n\t\t\t\tq.SetField(\"desc\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tquery: NewQueryStringQuery(`+beer \"light beer\" -devon`),\n\t\t},\n\t\t{\n\t\t\tquery: NewPhraseQuery([]string{\"watered\", \"down\"}, \"desc\"),\n\t\t},\n\t\t{\n\t\t\tquery: NewPhraseQuery([]string{}, \"field\"),\n\t\t\terr:   true,\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewMatchNoneQuery()\n\t\t\t\tq.SetBoost(25)\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewMatchAllQuery()\n\t\t\t\tq.SetBoost(25)\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tquery: NewBooleanQuery(\n\t\t\t\t[]Query{func() Query {\n\t\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\treturn q\n\t\t\t\t}()},\n\t\t\t\t[]Query{func() Query {\n\t\t\t\t\tq := NewMatchQuery(\"water\")\n\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\treturn q\n\t\t\t\t}()},\n\t\t\t\t[]Query{func() Query {\n\t\t\t\t\tq := NewMatchQuery(\"devon\")\n\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\treturn q\n\t\t\t\t}()}),\n\t\t},\n\t\t{\n\t\t\tquery: NewBooleanQuery(\n\t\t\t\tnil,\n\t\t\t\tnil,\n\t\t\t\t[]Query{func() Query {\n\t\t\t\t\tq := NewMatchQuery(\"devon\")\n\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\treturn q\n\t\t\t\t}()}),\n\t\t},\n\t\t{\n\t\t\tquery: NewBooleanQuery(\n\t\t\t\t[]Query{},\n\t\t\t\t[]Query{},\n\t\t\t\t[]Query{func() Query {\n\t\t\t\t\tq := NewMatchQuery(\"devon\")\n\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\treturn q\n\t\t\t\t}()}),\n\t\t},\n\t\t{\n\t\t\tquery: NewBooleanQuery(\n\t\t\t\tnil,\n\t\t\t\tnil,\n\t\t\t\tnil),\n\t\t\terr: true,\n\t\t},\n\t\t{\n\t\t\tquery: NewBooleanQuery(\n\t\t\t\t[]Query{},\n\t\t\t\t[]Query{},\n\t\t\t\t[]Query{}),\n\t\t\terr: true,\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewBooleanQuery(\n\t\t\t\t\t[]Query{func() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"beer\")\n\t\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}()},\n\t\t\t\t\t[]Query{func() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"water\")\n\t\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}()},\n\t\t\t\t\t[]Query{func() Query {\n\t\t\t\t\t\tq := NewMatchQuery(\"devon\")\n\t\t\t\t\t\tq.SetField(\"desc\")\n\t\t\t\t\t\treturn q\n\t\t\t\t\t}()})\n\t\t\t\tq.SetMinShould(2)\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t\terr: true,\n\t\t},\n\t\t{\n\t\t\tquery: func() Query {\n\t\t\t\tq := NewDocIDQuery(nil)\n\t\t\t\tq.SetBoost(25)\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tif vq, ok := test.query.(ValidatableQuery); ok {\n\t\t\tactual := vq.Validate()\n\t\t\tif actual != nil && !test.err {\n\t\t\t\tt.Errorf(\"expected no error: %#v got %#v\", test.err, actual)\n\t\t\t} else if actual == nil && test.err {\n\t\t\t\tt.Errorf(\"expected error: %#v got %#v\", test.err, actual)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDumpQuery(t *testing.T) {\n\tmapping := mapping.NewIndexMapping()\n\tq := NewQueryStringQuery(\"+water -light beer\")\n\ts, err := DumpQuery(mapping, q)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ts = strings.TrimSpace(s)\n\twanted := strings.TrimSpace(`{\n  \"must\": {\n    \"conjuncts\": [\n      {\n        \"match\": \"water\",\n        \"prefix_length\": 0,\n        \"fuzziness\": 0\n      }\n    ]\n  },\n  \"should\": {\n    \"disjuncts\": [\n      {\n        \"match\": \"beer\",\n        \"prefix_length\": 0,\n        \"fuzziness\": 0\n      }\n    ],\n    \"min\": 0\n  },\n  \"must_not\": {\n    \"disjuncts\": [\n      {\n        \"match\": \"light\",\n        \"prefix_length\": 0,\n        \"fuzziness\": 0\n      }\n    ],\n    \"min\": 0\n  }\n}`)\n\tif wanted != s {\n\t\tt.Fatalf(\"query:\\n%s\\ndiffers from expected:\\n%s\", s, wanted)\n\t}\n}\n\nfunc TestGeoShapeQuery(t *testing.T) {\n\ttests := []struct {\n\t\tinput  []byte\n\t\toutput Query\n\t\terr    bool\n\t}{\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t \"geometry\": {\n\t\t\t\t\t \"shape\": {\n\t\t\t\t\t\t \"type\": \"polygon\",\n\t\t\t\t\t\t\"coordinates\": [[\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t74.1357421875,\n\t\t\t\t\t\t\t\t30.600093873550072\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t67.0166015625,\n\t\t\t\t\t\t\t\t21.57571893245848\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t68.8623046875,\n\t\t\t\t\t\t\t\t9.145486056167277\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t83.1884765625,\n\t\t\t\t\t\t\t\t4.083452772038619\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t88.9892578125,\n\t\t\t\t\t\t\t\t22.67484735118852\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t74.1357421875,\n\t\t\t\t\t\t\t\t30.600093873550072\n\t\t\t\t\t\t\t]]]\n\t\t\t\t\t\t},\n\t\t\t\t\t  \"relation\": \"intersects\"\n\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeoShapeQuery([][][][]float64{{{{74.1357421875, 30.600093873550072},\n\t\t\t\t\t{67.0166015625, 21.57571893245848}, {68.8623046875, 9.145486056167277},\n\t\t\t\t\t{83.1884765625, 4.083452772038619}, {88.9892578125, 22.67484735118852},\n\t\t\t\t\t{74.1357421875, 30.600093873550072}}}}, geo.PolygonType, \"intersects\")\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t \"geometry\": {\n\t\t\t\t\t \"shape\": {\n\t\t\t\t\t\t \"type\": \"multipolygon\",\n\t\t\t\t\t\t \"coordinates\": [\n\t\t\t\t\t\t\t [[\n\t\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t\t77.58268117904663,\n\t\t\t\t\t\t\t\t\t12.980513152175025\n\t\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t\t77.58147954940794,\n\t\t\t\t\t\t\t\t\t12.977983107483992\n\t\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t\t77.58708000183104,\n\t\t\t\t\t\t\t\t\t12.97886130773254\n\t\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t\t77.58268117904663,\n\t\t\t\t\t\t\t\t\t12.980513152175025\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t]],\n\t\t\t\t\t\t\t[[\n\t\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t\t77.5864577293396,\n\t\t\t\t\t\t\t\t\t12.97762764459667\n\t\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t\t77.58879661560059,\n\t\t\t\t\t\t\t\t\t12.975076660730531\n\t\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t\t77.59115695953369,\n\t\t\t\t\t\t\t\t\t12.979216768855913\n\t\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t\t\t77.5864577293396,\n\t\t\t\t\t\t\t\t\t12.97762764459667\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t]]\n\t\t\t\t\t\t]\n\t\t\t\t\t},\n\t\t\t\t\t\"relation\": \"contains\"\n\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeoShapeQuery([][][][]float64{\n\t\t\t\t\t{{{77.58268117904663, 12.980513152175025},\n\t\t\t\t\t\t{77.58147954940794, 12.977983107483992}, {77.58708000183104, 12.97886130773254},\n\t\t\t\t\t\t{77.58268117904663, 12.980513152175025}}},\n\t\t\t\t\t{{{77.5864577293396, 12.97762764459667}, {77.58879661560059, 12.975076660730531},\n\t\t\t\t\t\t{77.59115695953369, 12.979216768855913}, {77.5864577293396, 12.97762764459667}}}},\n\t\t\t\t\tgeo.MultiPolygonType, \"contains\")\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t \"geometry\": {\n\t\t\t\t\t \"shape\": {\n\t\t\t\t\t\t \"type\": \"point\",\n\t\t\t\t\t\t \"coordinates\": [77.58268117904663, 12.980513152175025]\n\t\t\t\t\t},\n\t\t\t\t\t\"relation\": \"contains\"\n\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeoShapeQuery([][][][]float64{\n\t\t\t\t\t{{{77.58268117904663, 12.980513152175025}}}},\n\t\t\t\t\tgeo.PointType, \"contains\")\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t \"geometry\": {\n\t\t\t\t\t \"shape\": {\n\t\t\t\t\t\t \"type\": \"multipoint\",\n\t\t\t\t\t\t \"coordinates\": [[77.58268117904663, 12.980513152175025],\n\t\t\t\t\t\t [77.5864577293396, 12.97762764459667]]\n\t\t\t\t\t},\n\t\t\t\t\t\"relation\": \"intersects\"\n\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeoShapeQuery([][][][]float64{\n\t\t\t\t\t{{{77.58268117904663, 12.980513152175025},\n\t\t\t\t\t\t{77.5864577293396, 12.97762764459667}}}},\n\t\t\t\t\tgeo.MultiPointType, \"intersects\")\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t \"geometry\": {\n\t\t\t\t\t \"shape\": {\n\t\t\t\t\t\t \"type\": \"linestring\",\n\t\t\t\t\t\t \"coordinates\": [[77.58268117904663, 12.980513152175025],\n\t\t\t\t\t\t [77.5864577293396, 12.97762764459667]]\n\t\t\t\t\t},\n\t\t\t\t\t\"relation\": \"intersects\"\n\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeoShapeQuery([][][][]float64{\n\t\t\t\t\t{{{77.58268117904663, 12.980513152175025},\n\t\t\t\t\t\t{77.5864577293396, 12.97762764459667}}}},\n\t\t\t\t\tgeo.LineStringType, \"intersects\")\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t \"geometry\": {\n\t\t\t\t\t \"shape\": {\n\t\t\t\t\t\t \"type\": \"multilinestring\",\n\t\t\t\t\t\t \"coordinates\": [\n\t\t\t\t\t\t[[77.58268117904663, 12.980513152175025],\n\t\t\t\t\t\t [77.5864577293396, 12.97762764459667]],\n\t\t\t\t\t\t [[77.5864577293396,12.97762764459667],\n\t\t\t\t\t\t [77.58879661560059, 12.975076660730531]]]\n\t\t\t\t\t},\n\t\t\t\t\t\"relation\": \"intersects\"\n\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeoShapeQuery([][][][]float64{{\n\t\t\t\t\t{{77.58268117904663, 12.980513152175025},\n\t\t\t\t\t\t{77.5864577293396, 12.97762764459667}},\n\t\t\t\t\t{{77.5864577293396, 12.97762764459667},\n\t\t\t\t\t\t{77.58879661560059, 12.975076660730531}}}},\n\t\t\t\t\tgeo.MultiLineStringType, \"intersects\")\n\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t \"geometry\": {\n\t\t\t\t\t \"shape\": {\n\t\t\t\t\t\t \"type\": \"envelope\",\n\t\t\t\t\t\t \"coordinates\": [[77.58268117904663, 12.980513152175025],\n\t\t\t\t\t\t [77.5864577293396, 12.97762764459667]]\n\t\t\t\t\t},\n\t\t\t\t\t\"relation\": \"within\"\n\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeoShapeQuery([][][][]float64{{\n\t\t\t\t\t{{77.58268117904663, 12.980513152175025},\n\t\t\t\t\t\t{77.5864577293396, 12.97762764459667}}}},\n\t\t\t\t\tgeo.EnvelopeType, \"within\")\n\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t \"geometry\": {\n\t\t\t\t\t \"shape\": {\n\t\t\t\t\t\t \"type\": \"circle\",\n\t\t\t\t\t\t \"coordinates\": [77.58268117904663, 12.980513152175025],\n\t\t\t\t\t\t \"radius\": \"100m\"\n\t\t\t\t\t},\n\t\t\t\t\t\"relation\": \"within\"\n\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeoShapeCircleQuery([]float64{\n\t\t\t\t\t77.58268117904663, 12.980513152175025},\n\t\t\t\t\t\"100m\", \"within\")\n\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t\t{\n\t\t\tinput: []byte(`{\n\t\t\t\t\"field\" : \"region\",\n\t\t\t\t\"geometry\": {\n\t\t\t\t\t\"shape\": {\n\t\t\t\t\t  \"type\": \"geometrycollection\",\n\t\t\t\t\t  \"geometries\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t  \"type\": \"point\",\n\t\t\t\t\t\t  \"coordinates\": [\n\t\t\t\t\t\t\t77.59158611297607,\n\t\t\t\t\t\t\t12.972002899506203\n\t\t\t\t\t\t  ]\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t  \"type\": \"linestring\",\n\t\t\t\t\t\t  \"coordinates\": [\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t  77.58851766586304,\n\t\t\t\t\t\t\t  12.973152950670608\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t  77.58937597274779,\n\t\t\t\t\t\t\t  12.972212000113458\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t  ]\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t  \"type\": \"polygon\",\n\t\t\t\t\t\t  \"coordinates\": [\n\t\t\t\t\t\t\t[\n\t\t\t\t\t\t\t  [\n\t\t\t\t\t\t\t\t77.59055614471436,\n\t\t\t\t\t\t\t\t12.974721193688106\n\t\t\t\t\t\t\t  ],\n\t\t\t\t\t\t\t  [\n\t\t\t\t\t\t\t\t77.58954763412476,\n\t\t\t\t\t\t\t\t12.97350841995465\n\t\t\t\t\t\t\t  ],\n\t\t\t\t\t\t\t  [\n\t\t\t\t\t\t\t\t77.59141445159912,\n\t\t\t\t\t\t\t\t12.973382960265356\n\t\t\t\t\t\t\t  ],\n\t\t\t\t\t\t\t  [\n\t\t\t\t\t\t\t\t77.59055614471436,\n\t\t\t\t\t\t\t\t12.974721193688106\n\t\t\t\t\t\t\t  ]\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t  ]\n\t\t\t\t\t\t}\n\t\t\t\t\t  ]\n\t\t\t\t\t},\n\t\t\t\t\t\"relation\": \"contains\"\n\t\t\t\t  }}`),\n\t\t\toutput: func() Query {\n\t\t\t\tq, _ := NewGeometryCollectionQuery([][][][][]float64{\n\t\t\t\t\t{{{{77.59158611297607, 12.972002899506203}}}},\n\t\t\t\t\t{{{{77.58851766586304, 12.973152950670608}, {77.58937597274779, 12.972212000113458}}}},\n\t\t\t\t\t{{{{77.59055614471436, 12.974721193688106}, {77.58954763412476, 12.97350841995465},\n\t\t\t\t\t\t{77.59141445159912, 12.973382960265356}, {77.59055614471436, 12.974721193688106}}}},\n\t\t\t\t},\n\t\t\t\t\t[]string{\"point\", \"linestring\", \"polygon\"}, \"contains\")\n\t\t\t\tq.SetField(\"region\")\n\t\t\t\treturn q\n\t\t\t}(),\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tactual, err := ParseQuery(test.input)\n\t\tif err != nil && test.err == false {\n\t\t\tt.Errorf(\"error %v for %d\", err, i)\n\t\t}\n\n\t\tif !reflect.DeepEqual(test.output, actual) {\n\t\t\tt.Errorf(\"expected: %#v, got: %#v for %s\", test.output, actual, string(test.input))\n\t\t}\n\t}\n}\n\nfunc TestParseEmptyQuery(t *testing.T) {\n\tvar qBytes []byte\n\trv, err := ParseQuery(qBytes)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texpect := NewMatchNoneQuery()\n\tif !reflect.DeepEqual(rv, expect) {\n\t\tt.Errorf(\"[1] Expected %#v, got %#v\", expect, rv)\n\t}\n\n\tqBytes = []byte(`{}`)\n\trv, err = ParseQuery(qBytes)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texpect = NewMatchNoneQuery()\n\tif !reflect.DeepEqual(rv, expect) {\n\t\tt.Errorf(\"[2] Expected %#v, got %#v\", expect, rv)\n\t}\n}\n\nfunc TestExtractFields(t *testing.T) {\n\ttestQueries := []struct {\n\t\tquery     string\n\t\texpFields []string\n\t}{\n\t\t{\n\t\t\tquery:     `{\"term\":\"water\",\"field\":\"desc\"}`,\n\t\t\texpFields: []string{\"desc\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\t\t\"must\": {\n\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\"match\": \"water\",\n\t\t\t\t\t\t\t\t\t\"prefix_length\": 0,\n\t\t\t\t\t\t\t\t\t\"fuzziness\": 0\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t},\n\t\t\t\t\t\t\"should\": {\n\t\t\t\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\"match\": \"beer\",\n\t\t\t\t\t\t\t\t\t\"prefix_length\": 0,\n\t\t\t\t\t\t\t\t\t\"fuzziness\": 0\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"min\": 0\n\t\t\t\t\t\t},\n\t\t\t\t\t\t\"must_not\": {\n\t\t\t\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\"match\": \"light\",\n\t\t\t\t\t\t\t\t\t\"prefix_length\": 0,\n\t\t\t\t\t\t\t\t\t\"fuzziness\": 0\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"min\": 0\n\t\t\t\t\t\t}\n\t\t\t\t\t}`,\n\t\t\texpFields: []string{\"_all\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\t\t\"must\": {\n\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\"match\": \"water\",\n\t\t\t\t\t\t\t\t\t\"prefix_length\": 0,\n\t\t\t\t\t\t\t\t\t\"field\": \"desc\",\n\t\t\t\t\t\t\t\t\t\"fuzziness\": 0\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t},\n\t\t\t\t\t\t\"should\": {\n\t\t\t\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\"match\": \"beer\",\n\t\t\t\t\t\t\t\t\t\"prefix_length\": 0,\n\t\t\t\t\t\t\t\t\t\"field\": \"desc\",\n\t\t\t\t\t\t\t\t\t\"fuzziness\": 0\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"min\": 0\n\t\t\t\t\t\t},\n\t\t\t\t\t\t\"must_not\": {\n\t\t\t\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\"match\": \"light\",\n\t\t\t\t\t\t\t\t\t\"prefix_length\": 0,\n\t\t\t\t\t\t\t\t\t\"field\": \"genre\",\n\t\t\t\t\t\t\t\t\t\"fuzziness\": 0\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"min\": 0\n\t\t\t\t\t\t}\n\t\t\t\t\t}`,\n\t\t\texpFields: []string{\"desc\", \"genre\"},\n\t\t},\n\t\t{\n\t\t\tquery: `\n\t\t\t\t\t{\n\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"date\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"start\": \"2002-09-05T08:09:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"end\": \"2007-03-01T03:52:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_start\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_end\": true\n\t\t\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"number\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"min\": 1260295,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"max\": 3917314,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_min\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_max\": true\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"date2\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"start\": \"2004-08-21T18:30:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"end\": \"2006-03-24T08:08:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_start\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_end\": true\n\t\t\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"number\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"min\": 165449,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"max\": 3847517,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_min\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_max\": true\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"date\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"start\": \"2004-09-02T22:15:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"end\": \"2008-06-22T15:06:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_start\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_end\": true\n\t\t\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"number2\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"min\": 876843,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"max\": 3363351,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_min\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_max\": true\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"date\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"start\": \"2000-12-03T21:35:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"end\": \"2008-02-07T05:00:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_start\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_end\": true\n\t\t\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"number\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"min\": 2021479,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"max\": 4763404,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_min\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_max\": true\n\t\t\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"date3\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"start\": \"2000-03-13T07:13:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"end\": \"2005-09-19T09:33:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_start\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_end\": true\n\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"number\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"min\": 883125,\n\t\t\t\t\t\t\t\t\t\t\t\t\"max\": 4817433,\n\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_min\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_max\": true\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"date\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"start\": \"2002-08-10T22:42:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"end\": \"2008-02-10T23:19:00Z\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_start\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_end\": true\n\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\"field\": \"number\",\n\t\t\t\t\t\t\t\t\t\t\t\t\"min\": 896115,\n\t\t\t\t\t\t\t\t\t\t\t\t\"max\": 3897074,\n\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_min\": true,\n\t\t\t\t\t\t\t\t\t\t\t\t\"inclusive_max\": true\n\t\t\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t]\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t]\n\t\t\t\t\t}`,\n\t\t\texpFields: []string{\"date\", \"number\", \"date2\", \"number2\", \"date3\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\t\t\"query\" : \"hardworking people\"\n\t\t\t\t\t}`,\n\t\t\texpFields: []string{\"_all\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\t\t\"query\" : \"text:hardworking people\"\n\t\t\t\t\t}`,\n\t\t\texpFields: []string{\"text\", \"_all\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\t\t\"query\" : \"text:\\\"hardworking people\\\"\"\n\t\t\t\t\t}`,\n\t\t\texpFields: []string{\"text\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\t\t\"match_all\": {}\n\t\t\t\t\t}`,\n\t\t\texpFields: []string{\"_id\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\t\t\"ids\": [\"a\", \"b\", \"c\"]\n\t\t\t\t\t}`,\n\t\t\texpFields: []string{\"_id\"},\n\t\t},\n\t}\n\n\tm := mapping.NewIndexMapping()\n\tfor i, test := range testQueries {\n\t\tq, err := ParseQuery([]byte(test.query))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tfields, err := ExtractFields(q, m, nil)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tvar fieldsSlice []string\n\t\tfor k := range fields {\n\t\t\tfieldsSlice = append(fieldsSlice, k)\n\t\t}\n\t\tsort.Strings(test.expFields)\n\t\tsort.Strings(fieldsSlice)\n\t\tif !reflect.DeepEqual(fieldsSlice, test.expFields) {\n\t\t\tt.Errorf(\"Test %d: expected %v, got %v\", i, test.expFields, fieldsSlice)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/query/regexp.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype RegexpQuery struct {\n\tRegexp   string `json:\"regexp\"`\n\tFieldVal string `json:\"field,omitempty\"`\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\n// NewRegexpQuery creates a new Query which finds\n// documents containing terms that match the\n// specified regular expression.  The regexp pattern\n// SHOULD NOT include ^ or $ modifiers, the search\n// will only match entire terms even without them.\nfunc NewRegexpQuery(regexp string) *RegexpQuery {\n\treturn &RegexpQuery{\n\t\tRegexp: regexp,\n\t}\n}\n\nfunc (q *RegexpQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *RegexpQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *RegexpQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *RegexpQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *RegexpQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\t// require that pattern NOT be anchored to start and end of term.\n\t// do not attempt to remove trailing $, its presence is not\n\t// known to interfere with LiteralPrefix() the way ^ does\n\t// and removing $ introduces possible ambiguities with escaped \\$, \\\\$, etc\n\tactualRegexp := q.Regexp\n\tactualRegexp = strings.TrimPrefix(actualRegexp, \"^\") // remove leading ^ if it exists\n\n\treturn searcher.NewRegexpStringSearcher(ctx, i, actualRegexp, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *RegexpQuery) Validate() error {\n\treturn nil // real validation delayed until searcher constructor\n}\n"
  },
  {
    "path": "search/query/term.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype TermQuery struct {\n\tTerm     string `json:\"term\"`\n\tFieldVal string `json:\"field,omitempty\"`\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\n// NewTermQuery creates a new Query for finding an\n// exact term match in the index.\nfunc NewTermQuery(term string) *TermQuery {\n\treturn &TermQuery{\n\t\tTerm: term,\n\t}\n}\n\nfunc (q *TermQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *TermQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *TermQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *TermQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *TermQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\treturn searcher.NewTermSearcher(ctx, i, q.Term, field, q.BoostVal.Value(), options)\n}\n"
  },
  {
    "path": "search/query/term_range.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype TermRangeQuery struct {\n\tMin          string `json:\"min,omitempty\"`\n\tMax          string `json:\"max,omitempty\"`\n\tInclusiveMin *bool  `json:\"inclusive_min,omitempty\"`\n\tInclusiveMax *bool  `json:\"inclusive_max,omitempty\"`\n\tFieldVal     string `json:\"field,omitempty\"`\n\tBoostVal     *Boost `json:\"boost,omitempty\"`\n}\n\n// NewTermRangeQuery creates a new Query for ranges\n// of text term values.\n// Either, but not both endpoints can be nil.\n// The minimum value is inclusive.\n// The maximum value is exclusive.\nfunc NewTermRangeQuery(min, max string) *TermRangeQuery {\n\treturn NewTermRangeInclusiveQuery(min, max, nil, nil)\n}\n\n// NewTermRangeInclusiveQuery creates a new Query for ranges\n// of numeric values.\n// Either, but not both endpoints can be nil.\n// Control endpoint inclusion with inclusiveMin, inclusiveMax.\nfunc NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *TermRangeQuery {\n\treturn &TermRangeQuery{\n\t\tMin:          min,\n\t\tMax:          max,\n\t\tInclusiveMin: minInclusive,\n\t\tInclusiveMax: maxInclusive,\n\t}\n}\n\nfunc (q *TermRangeQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *TermRangeQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *TermRangeQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *TermRangeQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *TermRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\tvar minTerm []byte\n\tif q.Min != \"\" {\n\t\tminTerm = []byte(q.Min)\n\t}\n\tvar maxTerm []byte\n\tif q.Max != \"\" {\n\t\tmaxTerm = []byte(q.Max)\n\t}\n\treturn searcher.NewTermRangeSearcher(ctx, i, minTerm, maxTerm, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options)\n}\n\nfunc (q *TermRangeQuery) Validate() error {\n\tif q.Min == \"\" && q.Min == q.Max {\n\t\treturn fmt.Errorf(\"term range query must specify min or max\")\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "search/query/wildcard.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage query\n\nimport (\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/searcher\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar wildcardRegexpReplacer = strings.NewReplacer(\n\t// characters in the wildcard that must\n\t// be escaped in the regexp\n\t\"+\", `\\+`,\n\t\"(\", `\\(`,\n\t\")\", `\\)`,\n\t\"^\", `\\^`,\n\t\"$\", `\\$`,\n\t\".\", `\\.`,\n\t\"{\", `\\{`,\n\t\"}\", `\\}`,\n\t\"[\", `\\[`,\n\t\"]\", `\\]`,\n\t`|`, `\\|`,\n\t`\\`, `\\\\`,\n\t// wildcard characters\n\t\"*\", \".*\",\n\t\"?\", \".\")\n\ntype WildcardQuery struct {\n\tWildcard string `json:\"wildcard\"`\n\tFieldVal string `json:\"field,omitempty\"`\n\tBoostVal *Boost `json:\"boost,omitempty\"`\n}\n\n// NewWildcardQuery creates a new Query which finds\n// documents containing terms that match the\n// specified wildcard.  In the wildcard pattern '*'\n// will match any sequence of 0 or more characters,\n// and '?' will match any single character.\nfunc NewWildcardQuery(wildcard string) *WildcardQuery {\n\treturn &WildcardQuery{\n\t\tWildcard: wildcard,\n\t}\n}\n\nfunc (q *WildcardQuery) SetBoost(b float64) {\n\tboost := Boost(b)\n\tq.BoostVal = &boost\n}\n\nfunc (q *WildcardQuery) Boost() float64 {\n\treturn q.BoostVal.Value()\n}\n\nfunc (q *WildcardQuery) SetField(f string) {\n\tq.FieldVal = f\n}\n\nfunc (q *WildcardQuery) Field() string {\n\treturn q.FieldVal\n}\n\nfunc (q *WildcardQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {\n\tfield := q.FieldVal\n\tif q.FieldVal == \"\" {\n\t\tfield = m.DefaultSearchField()\n\t}\n\n\tregexpString := wildcardRegexpReplacer.Replace(q.Wildcard)\n\n\treturn searcher.NewRegexpStringSearcher(ctx, i, regexpString, field,\n\t\tq.BoostVal.Value(), options)\n}\n\nfunc (q *WildcardQuery) Validate() error {\n\treturn nil // real validation delayed until searcher constructor\n}\n"
  },
  {
    "path": "search/scorer/scorer_conjunction.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorer\n\nimport (\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n)\n\nvar reflectStaticSizeConjunctionQueryScorer int\n\nfunc init() {\n\tvar cqs ConjunctionQueryScorer\n\treflectStaticSizeConjunctionQueryScorer = int(reflect.TypeOf(cqs).Size())\n}\n\ntype ConjunctionQueryScorer struct {\n\toptions search.SearcherOptions\n}\n\nfunc (s *ConjunctionQueryScorer) Size() int {\n\treturn reflectStaticSizeConjunctionQueryScorer + size.SizeOfPtr\n}\n\nfunc NewConjunctionQueryScorer(options search.SearcherOptions) *ConjunctionQueryScorer {\n\treturn &ConjunctionQueryScorer{\n\t\toptions: options,\n\t}\n}\n\nfunc (s *ConjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch) *search.DocumentMatch {\n\tvar sum float64\n\tvar childrenExplanations []*search.Explanation\n\tif s.options.Explain {\n\t\tchildrenExplanations = make([]*search.Explanation, len(constituents))\n\t}\n\n\tfor i, docMatch := range constituents {\n\t\tsum += docMatch.Score\n\t\tif s.options.Explain {\n\t\t\tchildrenExplanations[i] = docMatch.Expl\n\t\t}\n\t}\n\tnewScore := sum\n\tvar newExpl *search.Explanation\n\tif s.options.Explain {\n\t\tnewExpl = &search.Explanation{Value: sum, Message: \"sum of:\", Children: childrenExplanations}\n\t}\n\n\t// reuse constituents[0] as the return value\n\trv := constituents[0]\n\trv.Score = newScore\n\trv.Expl = newExpl\n\trv.FieldTermLocations = search.MergeFieldTermLocations(\n\t\trv.FieldTermLocations, constituents[1:])\n\n\treturn rv\n}\n"
  },
  {
    "path": "search/scorer/scorer_constant.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorer\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeConstantScorer int\n\nfunc init() {\n\tvar cs ConstantScorer\n\treflectStaticSizeConstantScorer = int(reflect.TypeOf(cs).Size())\n}\n\ntype ConstantScorer struct {\n\tconstant               float64\n\tboost                  float64\n\toptions                search.SearcherOptions\n\tqueryNorm              float64\n\tqueryWeight            float64\n\tqueryWeightExplanation *search.Explanation\n\tincludeScore           bool\n}\n\nfunc (s *ConstantScorer) Size() int {\n\tsizeInBytes := reflectStaticSizeConstantScorer + size.SizeOfPtr\n\n\tif s.queryWeightExplanation != nil {\n\t\tsizeInBytes += s.queryWeightExplanation.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc NewConstantScorer(constant float64, boost float64, options search.SearcherOptions) *ConstantScorer {\n\trv := ConstantScorer{\n\t\toptions:      options,\n\t\tqueryWeight:  1.0,\n\t\tconstant:     constant,\n\t\tboost:        boost,\n\t\tincludeScore: options.Score != \"none\",\n\t}\n\n\treturn &rv\n}\n\nfunc (s *ConstantScorer) Weight() float64 {\n\tsum := s.boost\n\treturn sum * sum\n}\n\nfunc (s *ConstantScorer) SetQueryNorm(qnorm float64) {\n\ts.queryNorm = qnorm\n\n\t// update the query weight\n\ts.queryWeight = s.boost * s.queryNorm\n\n\tif s.options.Explain {\n\t\tchildrenExplanations := make([]*search.Explanation, 2)\n\t\tchildrenExplanations[0] = &search.Explanation{\n\t\t\tValue:   s.boost,\n\t\t\tMessage: \"boost\",\n\t\t}\n\t\tchildrenExplanations[1] = &search.Explanation{\n\t\t\tValue:   s.queryNorm,\n\t\t\tMessage: \"queryNorm\",\n\t\t}\n\t\ts.queryWeightExplanation = &search.Explanation{\n\t\t\tValue:    s.queryWeight,\n\t\t\tMessage:  fmt.Sprintf(\"ConstantScore()^%f, product of:\", s.boost),\n\t\t\tChildren: childrenExplanations,\n\t\t}\n\t}\n}\n\nfunc (s *ConstantScorer) Score(ctx *search.SearchContext, id index.IndexInternalID) *search.DocumentMatch {\n\tvar scoreExplanation *search.Explanation\n\n\trv := ctx.DocumentMatchPool.Get()\n\trv.IndexInternalID = id\n\n\tif s.includeScore {\n\t\tscore := s.constant\n\n\t\tif s.options.Explain {\n\t\t\tscoreExplanation = &search.Explanation{\n\t\t\t\tValue:   score,\n\t\t\t\tMessage: \"ConstantScore()\",\n\t\t\t}\n\t\t}\n\n\t\t// if the query weight isn't 1, multiply\n\t\tif s.queryWeight != 1.0 {\n\t\t\tscore = score * s.queryWeight\n\t\t\tif s.options.Explain {\n\t\t\t\tchildExplanations := make([]*search.Explanation, 2)\n\t\t\t\tchildExplanations[0] = s.queryWeightExplanation\n\t\t\t\tchildExplanations[1] = scoreExplanation\n\t\t\t\tscoreExplanation = &search.Explanation{\n\t\t\t\t\tValue:    score,\n\t\t\t\t\tMessage:  fmt.Sprintf(\"weight(^%f), product of:\", s.boost),\n\t\t\t\t\tChildren: childExplanations,\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\trv.Score = score\n\t\tif s.options.Explain {\n\t\t\trv.Expl = scoreExplanation\n\t\t}\n\t}\n\n\treturn rv\n}\n"
  },
  {
    "path": "search/scorer/scorer_constant_test.go",
    "content": "//  Copyright (c) 2013 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorer\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestConstantScorer(t *testing.T) {\n\n\tscorer := NewConstantScorer(1, 1, search.SearcherOptions{Explain: true})\n\n\ttests := []struct {\n\t\ttermMatch *index.TermFieldDoc\n\t\tresult    *search.DocumentMatch\n\t}{\n\t\t// test some simple math\n\t\t{\n\t\t\ttermMatch: &index.TermFieldDoc{\n\t\t\t\tID:   index.IndexInternalID(\"one\"),\n\t\t\t\tFreq: 1,\n\t\t\t\tNorm: 1.0,\n\t\t\t\tVectors: []*index.TermFieldVector{\n\t\t\t\t\t{\n\t\t\t\t\t\tField: \"desc\",\n\t\t\t\t\t\tPos:   1,\n\t\t\t\t\t\tStart: 0,\n\t\t\t\t\t\tEnd:   4,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           1.0,\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   1.0,\n\t\t\t\t\tMessage: \"ConstantScore()\",\n\t\t\t\t},\n\t\t\t\tSort: []string{},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(1, 0),\n\t\t}\n\t\tactual := scorer.Score(ctx, test.termMatch.ID)\n\n\t\tif !reflect.DeepEqual(actual, test.result) {\n\t\t\tt.Errorf(\"expected %#v got %#v for %#v\", test.result, actual, test.termMatch)\n\t\t}\n\t}\n\n}\n\nfunc TestConstantScorerWithQueryNorm(t *testing.T) {\n\n\tscorer := NewConstantScorer(1, 1, search.SearcherOptions{Explain: true})\n\tscorer.SetQueryNorm(2.0)\n\n\ttests := []struct {\n\t\ttermMatch *index.TermFieldDoc\n\t\tresult    *search.DocumentMatch\n\t}{\n\t\t{\n\t\t\ttermMatch: &index.TermFieldDoc{\n\t\t\t\tID:   index.IndexInternalID(\"one\"),\n\t\t\t\tFreq: 1,\n\t\t\t\tNorm: 1.0,\n\t\t\t},\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           2.0,\n\t\t\t\tSort:            []string{},\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   2.0,\n\t\t\t\t\tMessage: \"weight(^1.000000), product of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   2.0,\n\t\t\t\t\t\t\tMessage: \"ConstantScore()^1.000000, product of:\",\n\t\t\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\t\t\tMessage: \"boost\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   2,\n\t\t\t\t\t\t\t\t\tMessage: \"queryNorm\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   1.0,\n\t\t\t\t\t\t\tMessage: \"ConstantScore()\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(1, 0),\n\t\t}\n\t\tactual := scorer.Score(ctx, test.termMatch.ID)\n\n\t\tif !reflect.DeepEqual(actual, test.result) {\n\t\t\tt.Errorf(\"expected %#v got %#v for %#v\", test.result, actual, test.termMatch)\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "search/scorer/scorer_disjunction.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorer\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n)\n\nvar reflectStaticSizeDisjunctionQueryScorer int\n\nfunc init() {\n\tvar dqs DisjunctionQueryScorer\n\treflectStaticSizeDisjunctionQueryScorer = int(reflect.TypeOf(dqs).Size())\n}\n\ntype DisjunctionQueryScorer struct {\n\toptions search.SearcherOptions\n}\n\nfunc (s *DisjunctionQueryScorer) Size() int {\n\treturn reflectStaticSizeDisjunctionQueryScorer + size.SizeOfPtr\n}\n\nfunc NewDisjunctionQueryScorer(options search.SearcherOptions) *DisjunctionQueryScorer {\n\treturn &DisjunctionQueryScorer{\n\t\toptions: options,\n\t}\n}\n\nfunc (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch {\n\tvar sum float64\n\tvar childrenExplanations []*search.Explanation\n\tif s.options.Explain {\n\t\tchildrenExplanations = make([]*search.Explanation, len(constituents))\n\t}\n\n\tfor i, docMatch := range constituents {\n\t\tsum += docMatch.Score\n\t\tif s.options.Explain {\n\t\t\tchildrenExplanations[i] = docMatch.Expl\n\t\t}\n\t}\n\n\tvar rawExpl *search.Explanation\n\tif s.options.Explain {\n\t\trawExpl = &search.Explanation{Value: sum, Message: \"sum of:\", Children: childrenExplanations}\n\t}\n\n\tcoord := float64(countMatch) / float64(countTotal)\n\tnewScore := sum * coord\n\tvar newExpl *search.Explanation\n\tif s.options.Explain {\n\t\tce := make([]*search.Explanation, 2)\n\t\tce[0] = rawExpl\n\t\tce[1] = &search.Explanation{Value: coord, Message: fmt.Sprintf(\"coord(%d/%d)\", countMatch, countTotal)}\n\t\tnewExpl = &search.Explanation{Value: newScore, Message: \"product of:\", Children: ce, PartialMatch: countMatch != countTotal}\n\t}\n\n\t// reuse constituents[0] as the return value\n\trv := constituents[0]\n\trv.Score = newScore\n\trv.Expl = newExpl\n\trv.FieldTermLocations = search.MergeFieldTermLocations(\n\t\trv.FieldTermLocations, constituents[1:])\n\n\treturn rv\n}\n\n// This method is used only when disjunction searcher is used over multiple\n// KNN searchers, where only the score breakdown and the optional explanation breakdown\n// is required. The final score and explanation is set when we finalize the KNN hits.\nfunc (s *DisjunctionQueryScorer) ScoreAndExplBreakdown(ctx *search.SearchContext, constituents []*search.DocumentMatch,\n\tmatchingIdxs []int, originalPositions []int, countTotal int) *search.DocumentMatch {\n\n\trv := constituents[0]\n\tif rv.ScoreBreakdown == nil {\n\t\trv.ScoreBreakdown = make(map[int]float64, len(constituents))\n\t}\n\tvar childrenExplanations []*search.Explanation\n\tif s.options.Explain {\n\t\t// since we want to notify which expl belongs to which matched searcher within the disjunction searcher\n\t\tchildrenExplanations = make([]*search.Explanation, countTotal)\n\t}\n\n\tfor i, docMatch := range constituents {\n\t\tvar index int\n\t\tif originalPositions != nil {\n\t\t\t// scorer used in disjunction slice searcher\n\t\t\tindex = originalPositions[matchingIdxs[i]]\n\t\t} else {\n\t\t\t// scorer used in disjunction heap searcher\n\t\t\tindex = matchingIdxs[i]\n\t\t}\n\t\trv.ScoreBreakdown[index] = docMatch.Score\n\t\tif s.options.Explain {\n\t\t\tchildrenExplanations[index] = docMatch.Expl\n\t\t}\n\t}\n\tvar explBreakdown *search.Explanation\n\tif s.options.Explain {\n\t\texplBreakdown = &search.Explanation{Children: childrenExplanations}\n\t}\n\trv.Expl = explBreakdown\n\trv.FieldTermLocations = search.MergeFieldTermLocations(\n\t\trv.FieldTermLocations, constituents[1:])\n\treturn rv\n}\n"
  },
  {
    "path": "search/scorer/scorer_knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage scorer\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeKNNQueryScorer int\n\nfunc init() {\n\tvar sqs KNNQueryScorer\n\treflectStaticSizeKNNQueryScorer = int(reflect.TypeOf(sqs).Size())\n}\n\ntype KNNQueryScorer struct {\n\tqueryVector            []float32\n\tqueryField             string\n\tqueryWeight            float64\n\tqueryBoost             float64\n\tqueryNorm              float64\n\toptions                search.SearcherOptions\n\tsimilarityMetric       string\n\tqueryWeightExplanation *search.Explanation\n}\n\nfunc (s *KNNQueryScorer) Size() int {\n\tsizeInBytes := reflectStaticSizeKNNQueryScorer + size.SizeOfPtr +\n\t\t(len(s.queryVector) * size.SizeOfFloat32) + len(s.queryField) +\n\t\tlen(s.similarityMetric)\n\n\tif s.queryWeightExplanation != nil {\n\t\tsizeInBytes += s.queryWeightExplanation.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc NewKNNQueryScorer(queryVector []float32, queryField string, queryBoost float64,\n\toptions search.SearcherOptions,\n\tsimilarityMetric string) *KNNQueryScorer {\n\treturn &KNNQueryScorer{\n\t\tqueryVector:      queryVector,\n\t\tqueryField:       queryField,\n\t\tqueryBoost:       queryBoost,\n\t\tqueryWeight:      1.0,\n\t\toptions:          options,\n\t\tsimilarityMetric: similarityMetric,\n\t}\n}\n\n// Score used when the knnMatch.Score = 0 ->\n// the query and indexed vector are exactly the same.\nconst maxKNNScore = math.MaxFloat32\n\nfunc (sqs *KNNQueryScorer) Score(ctx *search.SearchContext,\n\tknnMatch *index.VectorDoc) *search.DocumentMatch {\n\trv := ctx.DocumentMatchPool.Get()\n\tvar scoreExplanation *search.Explanation\n\tscore := knnMatch.Score\n\tif sqs.similarityMetric == index.EuclideanDistance {\n\t\t// in case of euclidean distance being the distance metric,\n\t\t// an exact vector (perfect match), would return distance = 0\n\t\tif score == 0 {\n\t\t\tscore = maxKNNScore\n\t\t} else {\n\t\t\t// euclidean distances need to be inverted to work with\n\t\t\t// tf-idf scoring\n\t\t\tscore = 1.0 / score\n\t\t}\n\t}\n\tif sqs.options.Explain {\n\t\tscoreExplanation = &search.Explanation{\n\t\t\tValue: score,\n\t\t\tMessage: fmt.Sprintf(\"fieldWeight(%s in doc %s), score of:\",\n\t\t\t\tsqs.queryField, knnMatch.ID),\n\t\t\tChildren: []*search.Explanation{\n\t\t\t\t{\n\t\t\t\t\tValue: score,\n\t\t\t\t\tMessage: fmt.Sprintf(\"vector(field(%s:%s) with similarity_metric(%s)=%e\",\n\t\t\t\t\t\tsqs.queryField, knnMatch.ID, sqs.similarityMetric, score),\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t}\n\t// if the query weight isn't 1, multiply\n\tif sqs.queryWeight != 1.0 && score != maxKNNScore {\n\t\tscore = score * sqs.queryWeight\n\t\tif sqs.options.Explain {\n\t\t\tscoreExplanation = &search.Explanation{\n\t\t\t\tValue: score,\n\t\t\t\t// Product of score * weight\n\t\t\t\t// Avoid adding the query vector to the explanation since vectors\n\t\t\t\t// can get quite large.\n\t\t\t\tMessage: fmt.Sprintf(\"weight(%s:query Vector^%f in %s), product of:\",\n\t\t\t\t\tsqs.queryField, sqs.queryBoost, knnMatch.ID),\n\t\t\t\tChildren: []*search.Explanation{sqs.queryWeightExplanation, scoreExplanation},\n\t\t\t}\n\t\t}\n\t}\n\trv.Score = score\n\tif sqs.options.Explain {\n\t\trv.Expl = scoreExplanation\n\t}\n\trv.IndexInternalID = index.NewIndexInternalIDFrom(rv.IndexInternalID, knnMatch.ID)\n\treturn rv\n}\n\nfunc (sqs *KNNQueryScorer) Weight() float64 {\n\treturn 1.0\n}\n\nfunc (sqs *KNNQueryScorer) SetQueryNorm(qnorm float64) {\n\tsqs.queryNorm = qnorm\n\n\t// update the query weight\n\tsqs.queryWeight = sqs.queryBoost * sqs.queryNorm\n\n\tif sqs.options.Explain {\n\t\tchildrenExplanations := make([]*search.Explanation, 2)\n\t\tchildrenExplanations[0] = &search.Explanation{\n\t\t\tValue:   sqs.queryBoost,\n\t\t\tMessage: \"boost\",\n\t\t}\n\t\tchildrenExplanations[1] = &search.Explanation{\n\t\t\tValue:   sqs.queryNorm,\n\t\t\tMessage: \"queryNorm\",\n\t\t}\n\t\tsqs.queryWeightExplanation = &search.Explanation{\n\t\t\tValue: sqs.queryWeight,\n\t\t\tMessage: fmt.Sprintf(\"queryWeight(%s:query Vector^%f), product of:\",\n\t\t\t\tsqs.queryField, sqs.queryBoost),\n\t\t\tChildren: childrenExplanations,\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/scorer/scorer_knn_test.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage scorer\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestKNNScorerExplanation(t *testing.T) {\n\tvar queryVector []float32\n\t// arbitrary vector of dims: 64\n\tfor i := 0; i < 64; i++ {\n\t\tqueryVector = append(queryVector, float32(i))\n\t}\n\n\tvar resVector []float32\n\t// arbitrary res vector.\n\tfor i := 0; i < 64; i++ {\n\t\tresVector = append(resVector, float32(i))\n\t}\n\n\ttests := []struct {\n\t\tvectorMatch *index.VectorDoc\n\t\tscorer      *KNNQueryScorer\n\t\tnorm        float64\n\t\tresult      *search.DocumentMatch\n\t}{\n\t\t{\n\t\t\tvectorMatch: &index.VectorDoc{\n\t\t\t\tID:     index.IndexInternalID(\"one\"),\n\t\t\t\tScore:  0.5,\n\t\t\t\tVector: resVector,\n\t\t\t},\n\t\t\tnorm: 1.0,\n\t\t\tscorer: NewKNNQueryScorer(queryVector, \"desc\", 1.0,\n\t\t\t\tsearch.SearcherOptions{Explain: true}, index.EuclideanDistance),\n\t\t\t// Specifically testing EuclideanDistance since that involves score inversion.\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           0.5,\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   1 / 0.5,\n\t\t\t\t\tMessage: \"fieldWeight(desc in doc one), score of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   1 / 0.5,\n\t\t\t\t\t\t\tMessage: \"vector(field(desc:one) with similarity_metric(l2_norm)=2.000000e+00\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tvectorMatch: &index.VectorDoc{\n\t\t\t\tID:    index.IndexInternalID(\"one\"),\n\t\t\t\tScore: 0.0,\n\t\t\t\t// Result vector is an exact match of an existing vector.\n\t\t\t\tVector: queryVector,\n\t\t\t},\n\t\t\tnorm: 1.0,\n\t\t\tscorer: NewKNNQueryScorer(queryVector, \"desc\", 1.0,\n\t\t\t\tsearch.SearcherOptions{Explain: true}, index.EuclideanDistance),\n\t\t\t// Specifically testing EuclideanDistance with 0 score.\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           0.0,\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   maxKNNScore,\n\t\t\t\t\tMessage: \"fieldWeight(desc in doc one), score of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   maxKNNScore,\n\t\t\t\t\t\t\tMessage: \"vector(field(desc:one) with similarity_metric(l2_norm)=3.402823e+38\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tvectorMatch: &index.VectorDoc{\n\t\t\t\tID:     index.IndexInternalID(\"one\"),\n\t\t\t\tScore:  0.5,\n\t\t\t\tVector: resVector,\n\t\t\t},\n\t\t\tnorm: 1.0,\n\t\t\tscorer: NewKNNQueryScorer(queryVector, \"desc\", 1.0,\n\t\t\t\tsearch.SearcherOptions{Explain: true}, index.InnerProduct),\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           0.5,\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   0.5,\n\t\t\t\t\tMessage: \"fieldWeight(desc in doc one), score of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   0.5,\n\t\t\t\t\t\t\tMessage: \"vector(field(desc:one) with similarity_metric(dot_product)=5.000000e-01\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tvectorMatch: &index.VectorDoc{\n\t\t\t\tID:     index.IndexInternalID(\"one\"),\n\t\t\t\tScore:  0.25,\n\t\t\t\tVector: resVector,\n\t\t\t},\n\t\t\tnorm: 0.5,\n\t\t\tscorer: NewKNNQueryScorer(queryVector, \"desc\", 1.0,\n\t\t\t\tsearch.SearcherOptions{Explain: true}, index.InnerProduct),\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           0.25,\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   0.125,\n\t\t\t\t\tMessage: \"weight(desc:query Vector^1.000000 in one), product of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   0.5,\n\t\t\t\t\t\t\tMessage: \"queryWeight(desc:query Vector^1.000000), product of:\",\n\t\t\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\t\t\tMessage: \"boost\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   0.5,\n\t\t\t\t\t\t\t\t\tMessage: \"queryNorm\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   0.25,\n\t\t\t\t\t\t\tMessage: \"fieldWeight(desc in doc one), score of:\",\n\t\t\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   0.25,\n\t\t\t\t\t\t\t\t\tMessage: \"vector(field(desc:one) with similarity_metric(dot_product)=2.500000e-01\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(1, 0),\n\t\t}\n\t\ttest.scorer.SetQueryNorm(test.norm)\n\t\tactual := test.scorer.Score(ctx, test.vectorMatch)\n\t\tactual.Complete(nil)\n\n\t\tif !reflect.DeepEqual(actual.Expl, test.result.Expl) {\n\t\t\tt.Errorf(\"expected %#v got %#v for %#v\", test.result.Expl,\n\t\t\t\tactual.Expl, test.vectorMatch)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/scorer/scorer_term.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorer\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeTermQueryScorer int\n\nfunc init() {\n\tvar tqs TermQueryScorer\n\treflectStaticSizeTermQueryScorer = int(reflect.TypeOf(tqs).Size())\n}\n\ntype TermQueryScorer struct {\n\tqueryTerm              string\n\tqueryField             string\n\tqueryBoost             float64\n\tdocTerm                uint64 // number of documents containing the term\n\tdocTotal               uint64 // total number of documents in the index\n\tavgDocLength           float64\n\tidf                    float64\n\toptions                search.SearcherOptions\n\tidfExplanation         *search.Explanation\n\tincludeScore           bool\n\tqueryNorm              float64\n\tqueryWeight            float64\n\tqueryWeightExplanation *search.Explanation\n}\n\nfunc (s *TermQueryScorer) Size() int {\n\tsizeInBytes := reflectStaticSizeTermQueryScorer + size.SizeOfPtr +\n\t\tlen(s.queryTerm) + len(s.queryField)\n\n\tif s.idfExplanation != nil {\n\t\tsizeInBytes += s.idfExplanation.Size()\n\t}\n\n\tif s.queryWeightExplanation != nil {\n\t\tsizeInBytes += s.queryWeightExplanation.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (s *TermQueryScorer) computeIDF(avgDocLength float64, docTotal, docTerm uint64) float64 {\n\tvar rv float64\n\tif avgDocLength > 0 {\n\t\t// avgDocLength is set only for bm25 scoring\n\t\trv = math.Log(1 + (float64(docTotal)-float64(docTerm)+0.5)/\n\t\t\t(float64(docTerm)+0.5))\n\t} else {\n\t\trv = 1.0 + math.Log(float64(docTotal)/\n\t\t\tfloat64(docTerm+1.0))\n\t}\n\n\treturn rv\n}\n\n// queryTerm - the specific term being scored by this scorer object\n// queryField - the field in which the term is being searched\n// queryBoost - the boost value for the query term\n// docTotal - total number of documents in the index\n// docTerm - number of documents containing the term\n// avgDocLength - average document length in the index\n// options - search options such as explain scoring, include the location of the term etc.\nfunc NewTermQueryScorer(queryTerm []byte, queryField string, queryBoost float64, docTotal,\n\tdocTerm uint64, avgDocLength float64, options search.SearcherOptions) *TermQueryScorer {\n\n\trv := TermQueryScorer{\n\t\tqueryTerm:    string(queryTerm),\n\t\tqueryField:   queryField,\n\t\tqueryBoost:   queryBoost,\n\t\tdocTerm:      docTerm,\n\t\tdocTotal:     docTotal,\n\t\tavgDocLength: avgDocLength,\n\t\toptions:      options,\n\t\tqueryWeight:  1.0,\n\t\tincludeScore: options.Score != \"none\",\n\t}\n\n\trv.idf = rv.computeIDF(avgDocLength, docTotal, docTerm)\n\tif options.Explain {\n\t\trv.idfExplanation = &search.Explanation{\n\t\t\tValue:   rv.idf,\n\t\t\tMessage: fmt.Sprintf(\"idf(docFreq=%d, maxDocs=%d)\", docTerm, docTotal),\n\t\t}\n\t}\n\n\treturn &rv\n}\n\nfunc (s *TermQueryScorer) Weight() float64 {\n\tsum := s.queryBoost * s.idf\n\treturn sum * sum\n}\n\nfunc (s *TermQueryScorer) SetQueryNorm(qnorm float64) {\n\ts.queryNorm = qnorm\n\n\t// update the query weight\n\ts.queryWeight = s.queryBoost * s.idf * s.queryNorm\n\n\tif s.options.Explain {\n\t\tchildrenExplanations := make([]*search.Explanation, 3)\n\t\tchildrenExplanations[0] = &search.Explanation{\n\t\t\tValue:   s.queryBoost,\n\t\t\tMessage: \"boost\",\n\t\t}\n\t\tchildrenExplanations[1] = s.idfExplanation\n\t\tchildrenExplanations[2] = &search.Explanation{\n\t\t\tValue:   s.queryNorm,\n\t\t\tMessage: \"queryNorm\",\n\t\t}\n\t\ts.queryWeightExplanation = &search.Explanation{\n\t\t\tValue:    s.queryWeight,\n\t\t\tMessage:  fmt.Sprintf(\"queryWeight(%s:%s^%f), product of:\", s.queryField, s.queryTerm, s.queryBoost),\n\t\t\tChildren: childrenExplanations,\n\t\t}\n\t}\n}\n\nfunc (s *TermQueryScorer) docScore(tf, norm float64) (score float64, model string) {\n\tif s.avgDocLength > 0 {\n\t\t// bm25 scoring\n\t\t// using the posting's norm value to recompute the field length for the doc num\n\t\tfieldLength := 1 / (norm * norm)\n\n\t\tscore = s.idf * (tf * search.BM25_k1) /\n\t\t\t(tf + search.BM25_k1*(1-search.BM25_b+(search.BM25_b*fieldLength/s.avgDocLength)))\n\t\tmodel = index.BM25Scoring\n\t} else {\n\t\t// tf-idf scoring by default\n\t\tscore = tf * norm * s.idf\n\t\tmodel = index.DefaultScoringModel\n\t}\n\treturn score, model\n}\n\nfunc (s *TermQueryScorer) scoreExplanation(tf float64, termMatch *index.TermFieldDoc) []*search.Explanation {\n\tvar rv []*search.Explanation\n\tif s.avgDocLength > 0 {\n\t\tfieldLength := 1 / (termMatch.Norm * termMatch.Norm)\n\t\tfieldNormVal := 1 - search.BM25_b + (search.BM25_b * fieldLength / s.avgDocLength)\n\t\tfieldNormalizeExplanation := &search.Explanation{\n\t\t\tValue: fieldNormVal,\n\t\t\tMessage: fmt.Sprintf(\"fieldNorm(field=%s), b=%f, fieldLength=%f, avgFieldLength=%f)\",\n\t\t\t\ts.queryField, search.BM25_b, fieldLength, s.avgDocLength),\n\t\t}\n\n\t\tsaturationExplanation := &search.Explanation{\n\t\t\tValue: search.BM25_k1 / (tf + search.BM25_k1*fieldNormVal),\n\t\t\tMessage: fmt.Sprintf(\"saturation(term:%s), k1=%f/(tf=%f + k1*fieldNorm=%f))\",\n\t\t\t\ttermMatch.Term, search.BM25_k1, tf, fieldNormVal),\n\t\t\tChildren: []*search.Explanation{fieldNormalizeExplanation},\n\t\t}\n\n\t\trv = make([]*search.Explanation, 3)\n\t\trv[0] = &search.Explanation{\n\t\t\tValue:   tf,\n\t\t\tMessage: fmt.Sprintf(\"tf(termFreq(%s:%s)=%d\", s.queryField, s.queryTerm, termMatch.Freq),\n\t\t}\n\t\trv[1] = saturationExplanation\n\t\trv[2] = s.idfExplanation\n\t} else {\n\t\trv = make([]*search.Explanation, 3)\n\t\trv[0] = &search.Explanation{\n\t\t\tValue:   tf,\n\t\t\tMessage: fmt.Sprintf(\"tf(termFreq(%s:%s)=%d\", s.queryField, s.queryTerm, termMatch.Freq),\n\t\t}\n\t\trv[1] = &search.Explanation{\n\t\t\tValue:   termMatch.Norm,\n\t\t\tMessage: fmt.Sprintf(\"fieldNorm(field=%s, doc=%s)\", s.queryField, termMatch.ID),\n\t\t}\n\t\trv[2] = s.idfExplanation\n\t}\n\treturn rv\n}\n\nfunc (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.TermFieldDoc) *search.DocumentMatch {\n\trv := ctx.DocumentMatchPool.Get()\n\t// perform any score computations only when needed\n\tif s.includeScore || s.options.Explain {\n\t\tvar scoreExplanation *search.Explanation\n\t\tvar tf float64\n\t\tif termMatch.Freq < MaxSqrtCache {\n\t\t\ttf = SqrtCache[int(termMatch.Freq)]\n\t\t} else {\n\t\t\ttf = math.Sqrt(float64(termMatch.Freq))\n\t\t}\n\n\t\tscore, scoringModel := s.docScore(tf, termMatch.Norm)\n\t\tif s.options.Explain {\n\t\t\tchildrenExplanations := s.scoreExplanation(tf, termMatch)\n\t\t\tscoreExplanation = &search.Explanation{\n\t\t\t\tValue: score,\n\t\t\t\tMessage: fmt.Sprintf(\"fieldWeight(%s:%s in %s), as per %s model, \"+\n\t\t\t\t\t\"product of:\", s.queryField, s.queryTerm, termMatch.ID, scoringModel),\n\t\t\t\tChildren: childrenExplanations,\n\t\t\t}\n\t\t}\n\n\t\t// if the query weight isn't 1, multiply\n\t\tif s.queryWeight != 1.0 {\n\t\t\tscore = score * s.queryWeight\n\t\t\tif s.options.Explain {\n\t\t\t\tchildExplanations := make([]*search.Explanation, 2)\n\t\t\t\tchildExplanations[0] = s.queryWeightExplanation\n\t\t\t\tchildExplanations[1] = scoreExplanation\n\t\t\t\tscoreExplanation = &search.Explanation{\n\t\t\t\t\tValue:    score,\n\t\t\t\t\tMessage:  fmt.Sprintf(\"weight(%s:%s^%f in %s), product of:\", s.queryField, s.queryTerm, s.queryBoost, termMatch.ID),\n\t\t\t\t\tChildren: childExplanations,\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif s.includeScore {\n\t\t\trv.Score = score\n\t\t}\n\n\t\tif s.options.Explain {\n\t\t\trv.Expl = scoreExplanation\n\t\t}\n\t}\n\n\trv.IndexInternalID = index.NewIndexInternalIDFrom(rv.IndexInternalID, termMatch.ID)\n\n\tif len(termMatch.Vectors) > 0 {\n\t\tif cap(rv.FieldTermLocations) < len(termMatch.Vectors) {\n\t\t\trv.FieldTermLocations = make([]search.FieldTermLocation, 0, len(termMatch.Vectors))\n\t\t}\n\n\t\tfor _, v := range termMatch.Vectors {\n\t\t\tvar ap search.ArrayPositions\n\t\t\tif len(v.ArrayPositions) > 0 {\n\t\t\t\tn := len(rv.FieldTermLocations)\n\t\t\t\tif n < cap(rv.FieldTermLocations) { // reuse ap slice if available\n\t\t\t\t\tap = rv.FieldTermLocations[:n+1][n].Location.ArrayPositions[:0]\n\t\t\t\t}\n\t\t\t\tap = append(ap, v.ArrayPositions...)\n\t\t\t}\n\t\t\trv.FieldTermLocations =\n\t\t\t\tappend(rv.FieldTermLocations, search.FieldTermLocation{\n\t\t\t\t\tField: v.Field,\n\t\t\t\t\tTerm:  s.queryTerm,\n\t\t\t\t\tLocation: search.Location{\n\t\t\t\t\t\tPos:            v.Pos,\n\t\t\t\t\t\tStart:          v.Start,\n\t\t\t\t\t\tEnd:            v.End,\n\t\t\t\t\t\tArrayPositions: ap,\n\t\t\t\t\t},\n\t\t\t\t})\n\t\t}\n\t}\n\treturn rv\n}\n"
  },
  {
    "path": "search/scorer/scorer_term_test.go",
    "content": "//  Copyright (c) 2013 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorer\n\nimport (\n\t\"math\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestTermScorer(t *testing.T) {\n\n\tvar docTotal uint64 = 100\n\tvar docTerm uint64 = 9\n\tvar queryTerm = []byte(\"beer\")\n\tvar queryField = \"desc\"\n\tvar queryBoost = 1.0\n\tscorer := NewTermQueryScorer(queryTerm, queryField, queryBoost, docTotal, docTerm, 0, search.SearcherOptions{Explain: true})\n\tidf := 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0))\n\n\ttests := []struct {\n\t\ttermMatch *index.TermFieldDoc\n\t\tresult    *search.DocumentMatch\n\t}{\n\t\t// test some simple math\n\t\t{\n\t\t\ttermMatch: &index.TermFieldDoc{\n\t\t\t\tID:   index.IndexInternalID(\"one\"),\n\t\t\t\tFreq: 1,\n\t\t\t\tNorm: 1.0,\n\t\t\t\tVectors: []*index.TermFieldVector{\n\t\t\t\t\t{\n\t\t\t\t\t\tField: \"desc\",\n\t\t\t\t\t\tPos:   1,\n\t\t\t\t\t\tStart: 0,\n\t\t\t\t\t\tEnd:   4,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           math.Sqrt(1.0) * idf,\n\t\t\t\tSort:            []string{},\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   math.Sqrt(1.0) * idf,\n\t\t\t\t\tMessage: \"fieldWeight(desc:beer in one), as per tf-idf model, product of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\tMessage: \"tf(termFreq(desc:beer)=1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\tMessage: \"fieldNorm(field=desc, doc=one)\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   idf,\n\t\t\t\t\t\t\tMessage: \"idf(docFreq=9, maxDocs=100)\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tLocations: search.FieldTermLocationMap{\n\t\t\t\t\t\"desc\": search.TermLocationMap{\n\t\t\t\t\t\t\"beer\": []*search.Location{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tPos:   1,\n\t\t\t\t\t\t\t\tStart: 0,\n\t\t\t\t\t\t\t\tEnd:   4,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test the same thing again (score should be cached this time)\n\t\t{\n\t\t\ttermMatch: &index.TermFieldDoc{\n\t\t\t\tID:   index.IndexInternalID(\"one\"),\n\t\t\t\tFreq: 1,\n\t\t\t\tNorm: 1.0,\n\t\t\t},\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           math.Sqrt(1.0) * idf,\n\t\t\t\tSort:            []string{},\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   math.Sqrt(1.0) * idf,\n\t\t\t\t\tMessage: \"fieldWeight(desc:beer in one), as per tf-idf model, product of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\tMessage: \"tf(termFreq(desc:beer)=1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\tMessage: \"fieldNorm(field=desc, doc=one)\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   idf,\n\t\t\t\t\t\t\tMessage: \"idf(docFreq=9, maxDocs=100)\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test a case where the sqrt isn't precalculated\n\t\t{\n\t\t\ttermMatch: &index.TermFieldDoc{\n\t\t\t\tID:   index.IndexInternalID(\"one\"),\n\t\t\t\tFreq: 65,\n\t\t\t\tNorm: 1.0,\n\t\t\t},\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           math.Sqrt(65) * idf,\n\t\t\t\tSort:            []string{},\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   math.Sqrt(65) * idf,\n\t\t\t\t\tMessage: \"fieldWeight(desc:beer in one), as per tf-idf model, product of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   math.Sqrt(65),\n\t\t\t\t\t\t\tMessage: \"tf(termFreq(desc:beer)=65\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\tMessage: \"fieldNorm(field=desc, doc=one)\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   idf,\n\t\t\t\t\t\t\tMessage: \"idf(docFreq=9, maxDocs=100)\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(1, 0),\n\t\t}\n\t\tactual := scorer.Score(ctx, test.termMatch)\n\t\tactual.Complete(nil)\n\t\tif len(actual.FieldTermLocations) == 0 {\n\t\t\tactual.FieldTermLocations = nil\n\t\t}\n\n\t\tif !reflect.DeepEqual(actual, test.result) {\n\t\t\tt.Errorf(\"expected %#v got %#v for %#v\", test.result, actual, test.termMatch)\n\t\t}\n\t}\n\n}\n\nfunc TestTermScorerWithQueryNorm(t *testing.T) {\n\n\tvar docTotal uint64 = 100\n\tvar docTerm uint64 = 9\n\tvar queryTerm = []byte(\"beer\")\n\tvar queryField = \"desc\"\n\tvar queryBoost = 3.0\n\tscorer := NewTermQueryScorer(queryTerm, queryField, queryBoost, docTotal, docTerm, 0, search.SearcherOptions{Explain: true})\n\tidf := 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0))\n\n\tscorer.SetQueryNorm(2.0)\n\n\texpectedQueryWeight := 3 * idf * 3 * idf\n\tactualQueryWeight := scorer.Weight()\n\tif expectedQueryWeight != actualQueryWeight {\n\t\tt.Errorf(\"expected query weight %f, got %f\", expectedQueryWeight, actualQueryWeight)\n\t}\n\n\ttests := []struct {\n\t\ttermMatch *index.TermFieldDoc\n\t\tresult    *search.DocumentMatch\n\t}{\n\t\t{\n\t\t\ttermMatch: &index.TermFieldDoc{\n\t\t\t\tID:   index.IndexInternalID(\"one\"),\n\t\t\t\tFreq: 1,\n\t\t\t\tNorm: 1.0,\n\t\t\t},\n\t\t\tresult: &search.DocumentMatch{\n\t\t\t\tIndexInternalID: index.IndexInternalID(\"one\"),\n\t\t\t\tScore:           math.Sqrt(1.0) * idf * 3.0 * idf * 2.0,\n\t\t\t\tSort:            []string{},\n\t\t\t\tExpl: &search.Explanation{\n\t\t\t\t\tValue:   math.Sqrt(1.0) * idf * 3.0 * idf * 2.0,\n\t\t\t\t\tMessage: \"weight(desc:beer^3.000000 in one), product of:\",\n\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   2.0 * idf * 3.0,\n\t\t\t\t\t\t\tMessage: \"queryWeight(desc:beer^3.000000), product of:\",\n\t\t\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   3,\n\t\t\t\t\t\t\t\t\tMessage: \"boost\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   idf,\n\t\t\t\t\t\t\t\t\tMessage: \"idf(docFreq=9, maxDocs=100)\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   2,\n\t\t\t\t\t\t\t\t\tMessage: \"queryNorm\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tValue:   math.Sqrt(1.0) * idf,\n\t\t\t\t\t\t\tMessage: \"fieldWeight(desc:beer in one), as per tf-idf model, product of:\",\n\t\t\t\t\t\t\tChildren: []*search.Explanation{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\t\t\tMessage: \"tf(termFreq(desc:beer)=1\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   1,\n\t\t\t\t\t\t\t\t\tMessage: \"fieldNorm(field=desc, doc=one)\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tValue:   idf,\n\t\t\t\t\t\t\t\t\tMessage: \"idf(docFreq=9, maxDocs=100)\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(1, 0),\n\t\t}\n\t\tactual := scorer.Score(ctx, test.termMatch)\n\n\t\tif !reflect.DeepEqual(actual, test.result) {\n\t\t\tt.Errorf(\"expected %#v got %#v for %#v\", test.result, actual, test.termMatch)\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "search/scorer/sqrt_cache.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage scorer\n\nimport (\n\t\"math\"\n)\n\nvar SqrtCache []float64\n\nconst MaxSqrtCache = 64\n\nfunc init() {\n\tSqrtCache = make([]float64, MaxSqrtCache)\n\tfor i := 0; i < MaxSqrtCache; i++ {\n\t\tSqrtCache[i] = math.Sqrt(float64(i))\n\t}\n}\n"
  },
  {
    "path": "search/search.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"slices\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar (\n\treflectStaticSizeDocumentMatch int\n\treflectStaticSizeSearchContext int\n\treflectStaticSizeLocation      int\n)\n\nfunc init() {\n\tvar dm DocumentMatch\n\treflectStaticSizeDocumentMatch = int(reflect.TypeOf(dm).Size())\n\tvar sc SearchContext\n\treflectStaticSizeSearchContext = int(reflect.TypeOf(sc).Size())\n\tvar l Location\n\treflectStaticSizeLocation = int(reflect.TypeOf(l).Size())\n}\n\ntype ArrayPositions []uint64\n\nfunc (ap ArrayPositions) Equals(other ArrayPositions) bool {\n\treturn slices.Equal(ap, other)\n}\n\nfunc (ap ArrayPositions) Compare(other ArrayPositions) int {\n\tfor i, p := range ap {\n\t\tif i >= len(other) {\n\t\t\treturn 1\n\t\t}\n\t\tif p < other[i] {\n\t\t\treturn -1\n\t\t}\n\t\tif p > other[i] {\n\t\t\treturn 1\n\t\t}\n\t}\n\tif len(ap) < len(other) {\n\t\treturn -1\n\t}\n\treturn 0\n}\n\ntype Location struct {\n\t// Pos is the position of the term within the field, starting at 1\n\tPos uint64 `json:\"pos\"`\n\n\t// Start and End are the byte offsets of the term in the field\n\tStart uint64 `json:\"start\"`\n\tEnd   uint64 `json:\"end\"`\n\n\t// ArrayPositions contains the positions of the term within any elements.\n\tArrayPositions ArrayPositions `json:\"array_positions\"`\n}\n\nfunc (l *Location) Size() int {\n\treturn reflectStaticSizeLocation + size.SizeOfPtr +\n\t\tlen(l.ArrayPositions)*size.SizeOfUint64\n}\n\ntype Locations []*Location\n\nfunc (p Locations) Len() int      { return len(p) }\nfunc (p Locations) Swap(i, j int) { p[i], p[j] = p[j], p[i] }\n\nfunc (p Locations) Less(i, j int) bool {\n\tc := p[i].ArrayPositions.Compare(p[j].ArrayPositions)\n\tif c < 0 {\n\t\treturn true\n\t}\n\tif c > 0 {\n\t\treturn false\n\t}\n\treturn p[i].Pos < p[j].Pos\n}\n\nfunc (p Locations) Dedupe() Locations { // destructive!\n\tif len(p) <= 1 {\n\t\treturn p\n\t}\n\n\tsort.Sort(p)\n\n\tslow := 0\n\n\tfor _, pfast := range p {\n\t\tpslow := p[slow]\n\t\tif pslow.Pos == pfast.Pos &&\n\t\t\tpslow.Start == pfast.Start &&\n\t\t\tpslow.End == pfast.End &&\n\t\t\tpslow.ArrayPositions.Equals(pfast.ArrayPositions) {\n\t\t\tcontinue // duplicate, so only move fast ahead\n\t\t}\n\n\t\tslow++\n\n\t\tp[slow] = pfast\n\t}\n\n\treturn p[:slow+1]\n}\n\ntype TermLocationMap map[string]Locations\n\nfunc (t TermLocationMap) AddLocation(term string, location *Location) {\n\tt[term] = append(t[term], location)\n}\n\ntype FieldTermLocationMap map[string]TermLocationMap\n\ntype FieldTermLocation struct {\n\tField    string\n\tTerm     string\n\tLocation Location\n}\n\ntype FieldFragmentMap map[string][]string\n\ntype DocumentMatch struct {\n\tIndex           string                `json:\"index,omitempty\"`\n\tID              string                `json:\"id\"`\n\tIndexInternalID index.IndexInternalID `json:\"-\"`\n\tScore           float64               `json:\"score\"`\n\tExpl            *Explanation          `json:\"explanation,omitempty\"`\n\tLocations       FieldTermLocationMap  `json:\"locations,omitempty\"`\n\tFragments       FieldFragmentMap      `json:\"fragments,omitempty\"`\n\tSort            []string              `json:\"sort,omitempty\"`\n\tDecodedSort     []string              `json:\"decoded_sort,omitempty\"`\n\n\t// Fields contains the values for document fields listed in\n\t// SearchRequest.Fields. Text fields are returned as strings, numeric\n\t// fields as float64s and date fields as strings.\n\tFields map[string]interface{} `json:\"fields,omitempty\"`\n\n\t// used to maintain natural index order\n\tHitNumber uint64 `json:\"-\"`\n\n\t// used to temporarily hold field term location information during\n\t// search processing in an efficient, recycle-friendly manner, to\n\t// be later incorporated into the Locations map when search\n\t// results are completed\n\tFieldTermLocations []FieldTermLocation `json:\"-\"`\n\n\t// used to indicate the sub-scores that combined to form the\n\t// final score for this document match.  This is only populated\n\t// when the search request's query is a DisjunctionQuery.\n\t// The map key is the index of the sub-query\n\t// in the DisjunctionQuery. The map value is the\n\t// sub-score for that sub-query.\n\tScoreBreakdown map[int]float64 `json:\"score_breakdown,omitempty\"`\n\n\t// internal variable used in PreSearch phase of search in alias\n\t// to indicate the name of the index that this match came from.\n\t// used in knn search.\n\t// it is a stack of index names, the top of the stack is the name\n\t// of the index that this match came from\n\t// of the current alias view, used in alias of aliases scenario\n\tIndexNames []string `json:\"index_names,omitempty\"`\n\n\t// Descendants holds the IDs of any child/descendant document that contributed\n\t// to this root DocumentMatch.\n\tDescendants []index.IndexInternalID `json:\"-\"`\n}\n\nfunc (dm *DocumentMatch) AddFieldValue(name string, value interface{}) {\n\tif dm.Fields == nil {\n\t\tdm.Fields = make(map[string]interface{})\n\t}\n\texistingVal, ok := dm.Fields[name]\n\tif !ok {\n\t\tdm.Fields[name] = value\n\t\treturn\n\t}\n\n\tvalSlice, ok := existingVal.([]interface{})\n\tif ok {\n\t\t// already a slice, append to it\n\t\tvalSlice = append(valSlice, value)\n\t} else {\n\t\t// create a slice\n\t\tvalSlice = []interface{}{existingVal, value}\n\t}\n\tdm.Fields[name] = valSlice\n}\n\nfunc (dm *DocumentMatch) AddFragments(field string, fragments []string) {\n\tif dm.Fragments == nil {\n\t\tdm.Fragments = make(FieldFragmentMap)\n\t}\nOUTER:\n\tfor _, newFrag := range fragments {\n\t\tfor _, existingFrag := range dm.Fragments[field] {\n\t\t\tif existingFrag == newFrag {\n\t\t\t\tcontinue OUTER // no duplicates allowed\n\t\t\t}\n\t\t}\n\t\tdm.Fragments[field] = append(dm.Fragments[field], newFrag)\n\t}\n}\n\n// Reset allows an already allocated DocumentMatch to be reused\nfunc (dm *DocumentMatch) Reset() *DocumentMatch {\n\t// remember the []byte used for the IndexInternalID\n\tindexInternalID := dm.IndexInternalID\n\t// remember the []interface{} used for sort\n\tsort := dm.Sort\n\t// remember the []string used for decoded sort\n\tdecodedSort := dm.DecodedSort\n\t// remember the FieldTermLocations backing array\n\tftls := dm.FieldTermLocations\n\tfor i := range ftls { // recycle the ArrayPositions of each location\n\t\tftls[i].Location.ArrayPositions = ftls[i].Location.ArrayPositions[:0]\n\t}\n\t// remember the score breakdown map\n\tscoreBreakdown := dm.ScoreBreakdown\n\t// clear out the score breakdown map\n\tclear(scoreBreakdown)\n\t// remember the Descendants backing array\n\tdescendants := dm.Descendants\n\tfor i := range descendants { // recycle each IndexInternalID\n\t\tdescendants[i] = descendants[i][:0]\n\t}\n\t// idiom to copy over from empty DocumentMatch (0 allocations)\n\t*dm = DocumentMatch{}\n\t// reuse the []byte already allocated (and reset len to 0)\n\tdm.IndexInternalID = indexInternalID[:0]\n\t// reuse the []interface{} already allocated (and reset len to 0)\n\tdm.Sort = sort[:0]\n\t// reuse the []string already allocated (and reset len to 0)\n\tdm.DecodedSort = decodedSort[:0]\n\t// reuse the FieldTermLocations already allocated (and reset len to 0)\n\tdm.FieldTermLocations = ftls[:0]\n\t// reuse the Descendants already allocated (and reset len to 0)\n\tdm.Descendants = descendants[:0]\n\t// reuse the score breakdown map already allocated (after clearing it)\n\tdm.ScoreBreakdown = scoreBreakdown\n\treturn dm\n}\n\nfunc (dm *DocumentMatch) Size() int {\n\tsizeInBytes := reflectStaticSizeDocumentMatch + size.SizeOfPtr +\n\t\tlen(dm.Index) +\n\t\tlen(dm.ID) +\n\t\tlen(dm.IndexInternalID)\n\n\tif dm.Expl != nil {\n\t\tsizeInBytes += dm.Expl.Size()\n\t}\n\n\tfor k, v := range dm.Locations {\n\t\tsizeInBytes += size.SizeOfString + len(k)\n\t\tfor k1, v1 := range v {\n\t\t\tsizeInBytes += size.SizeOfString + len(k1) +\n\t\t\t\tsize.SizeOfSlice\n\t\t\tfor _, entry := range v1 {\n\t\t\t\tsizeInBytes += entry.Size()\n\t\t\t}\n\t\t}\n\t}\n\n\tfor k, v := range dm.Fragments {\n\t\tsizeInBytes += size.SizeOfString + len(k) +\n\t\t\tsize.SizeOfSlice\n\n\t\tfor _, entry := range v {\n\t\t\tsizeInBytes += size.SizeOfString + len(entry)\n\t\t}\n\t}\n\n\tfor _, entry := range dm.Sort {\n\t\tsizeInBytes += size.SizeOfString + len(entry)\n\t}\n\n\tfor _, entry := range dm.DecodedSort {\n\t\tsizeInBytes += size.SizeOfString + len(entry)\n\t}\n\n\tfor k := range dm.Fields {\n\t\tsizeInBytes += size.SizeOfString + len(k) +\n\t\t\tsize.SizeOfPtr\n\t}\n\n\treturn sizeInBytes\n}\n\n// Complete performs final preparation & transformation of the\n// DocumentMatch at the end of search processing, also allowing the\n// caller to provide an optional preallocated locations slice\nfunc (dm *DocumentMatch) Complete(prealloc []Location) []Location {\n\t// transform the FieldTermLocations slice into the Locations map\n\tnlocs := len(dm.FieldTermLocations)\n\tif nlocs > 0 {\n\t\tif cap(prealloc) < nlocs {\n\t\t\tprealloc = make([]Location, nlocs)\n\t\t}\n\t\tprealloc = prealloc[:nlocs]\n\n\t\tvar lastField string\n\t\tvar tlm TermLocationMap\n\t\tvar needsDedupe bool\n\n\t\tfor i, ftl := range dm.FieldTermLocations {\n\t\t\tif i == 0 || lastField != ftl.Field {\n\t\t\t\tlastField = ftl.Field\n\n\t\t\t\tif dm.Locations == nil {\n\t\t\t\t\tdm.Locations = make(FieldTermLocationMap)\n\t\t\t\t}\n\n\t\t\t\ttlm = dm.Locations[ftl.Field]\n\t\t\t\tif tlm == nil {\n\t\t\t\t\ttlm = make(TermLocationMap)\n\t\t\t\t\tdm.Locations[ftl.Field] = tlm\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tloc := &prealloc[i]\n\t\t\t*loc = ftl.Location\n\n\t\t\tif len(loc.ArrayPositions) > 0 { // copy\n\t\t\t\tloc.ArrayPositions = append(ArrayPositions(nil), loc.ArrayPositions...)\n\t\t\t}\n\n\t\t\tlocs := tlm[ftl.Term]\n\n\t\t\t// if the loc is before or at the last location, then there\n\t\t\t// might be duplicates that need to be deduplicated\n\t\t\tif !needsDedupe && len(locs) > 0 {\n\t\t\t\tlast := locs[len(locs)-1]\n\t\t\t\tcmp := loc.ArrayPositions.Compare(last.ArrayPositions)\n\t\t\t\tif cmp < 0 || (cmp == 0 && loc.Pos <= last.Pos) {\n\t\t\t\t\tneedsDedupe = true\n\t\t\t\t}\n\t\t\t}\n\n\t\t\ttlm[ftl.Term] = append(locs, loc)\n\n\t\t\tdm.FieldTermLocations[i] = FieldTermLocation{ // recycle\n\t\t\t\tLocation: Location{\n\t\t\t\t\tArrayPositions: ftl.Location.ArrayPositions[:0],\n\t\t\t\t},\n\t\t\t}\n\t\t}\n\n\t\tif needsDedupe {\n\t\t\tfor _, tlm := range dm.Locations {\n\t\t\t\tfor term, locs := range tlm {\n\t\t\t\t\ttlm[term] = locs.Dedupe()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tdm.FieldTermLocations = dm.FieldTermLocations[:0] // recycle\n\n\treturn prealloc\n}\n\nfunc (dm *DocumentMatch) String() string {\n\treturn fmt.Sprintf(\"[%s-%f]\", dm.ID, dm.Score)\n}\n\ntype DocumentMatchCollection []*DocumentMatch\n\nfunc (c DocumentMatchCollection) Len() int           { return len(c) }\nfunc (c DocumentMatchCollection) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }\nfunc (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score }\n\ntype Searcher interface {\n\tNext(ctx *SearchContext) (*DocumentMatch, error)\n\tAdvance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error)\n\tClose() error\n\tWeight() float64\n\tSetQueryNorm(float64)\n\tCount() uint64\n\tMin() int\n\tSize() int\n\n\tDocumentMatchPoolSize() int\n}\n\ntype SearcherOptions struct {\n\tExplain            bool\n\tIncludeTermVectors bool\n\tScore              string\n}\n\n// SearchContext represents the context around a single search\ntype SearchContext struct {\n\tDocumentMatchPool *DocumentMatchPool\n\tCollector         Collector\n\tIndexReader       index.IndexReader\n}\n\nfunc (sc *SearchContext) Size() int {\n\tsizeInBytes := reflectStaticSizeSearchContext + size.SizeOfPtr +\n\t\treflectStaticSizeDocumentMatchPool + size.SizeOfPtr\n\n\tif sc.DocumentMatchPool != nil {\n\t\tfor _, entry := range sc.DocumentMatchPool.avail {\n\t\t\tif entry != nil {\n\t\t\t\tsizeInBytes += entry.Size()\n\t\t\t}\n\t\t}\n\t}\n\n\treturn sizeInBytes\n}\n\n// A NestedDocumentMatch is like a DocumentMatch but used for nested documents\n// and does not have score or locations, or a score and is mainly used to\n// hold field values and fragments, to be embedded in the parent DocumentMatch\ntype NestedDocumentMatch struct {\n\tFields    map[string]interface{} `json:\"fields,omitempty\"`\n\tFragments FieldFragmentMap       `json:\"fragments,omitempty\"`\n}\n\n// NewNestedDocumentMatch creates a new NestedDocumentMatch instance\n// with the given fields and fragments\nfunc NewNestedDocumentMatch(fields map[string]interface{}, fragments FieldFragmentMap) *NestedDocumentMatch {\n\treturn &NestedDocumentMatch{\n\t\tFields:    fields,\n\t\tFragments: fragments,\n\t}\n}\n"
  },
  {
    "path": "search/search_test.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"encoding/json\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestArrayPositionsCompare(t *testing.T) {\n\ttests := []struct {\n\t\ta      []uint64\n\t\tb      []uint64\n\t\texpect int\n\t}{\n\t\t{nil, nil, 0},\n\t\t{[]uint64{}, []uint64{}, 0},\n\t\t{[]uint64{1}, []uint64{}, 1},\n\t\t{[]uint64{1}, []uint64{1}, 0},\n\t\t{[]uint64{}, []uint64{1}, -1},\n\t\t{[]uint64{0}, []uint64{1}, -1},\n\t\t{[]uint64{1}, []uint64{0}, 1},\n\t\t{[]uint64{1}, []uint64{1, 2}, -1},\n\t\t{[]uint64{1, 2}, []uint64{1}, 1},\n\t\t{[]uint64{1, 2}, []uint64{1, 2}, 0},\n\t\t{[]uint64{1, 2}, []uint64{1, 200}, -1},\n\t\t{[]uint64{1, 2}, []uint64{100, 2}, -1},\n\t\t{[]uint64{1, 2}, []uint64{1, 2, 3}, -1},\n\t}\n\n\tfor _, test := range tests {\n\t\tres := ArrayPositions(test.a).Compare(test.b)\n\t\tif res != test.expect {\n\t\t\tt.Errorf(\"test: %+v, res: %v\", test, res)\n\t\t}\n\t}\n}\n\nfunc TestLocationsDedupe(t *testing.T) {\n\ta := &Location{}\n\tb := &Location{Pos: 1}\n\tc := &Location{Pos: 2}\n\n\ttests := []struct {\n\t\tinput  Locations\n\t\texpect Locations\n\t}{\n\t\t{Locations{}, Locations{}},\n\t\t{Locations{a}, Locations{a}},\n\t\t{Locations{a, b, c}, Locations{a, b, c}},\n\t\t{Locations{a, a}, Locations{a}},\n\t\t{Locations{a, a, a}, Locations{a}},\n\t\t{Locations{a, b}, Locations{a, b}},\n\t\t{Locations{b, a}, Locations{a, b}},\n\t\t{Locations{c, b, a, c, b, a, c, b, a}, Locations{a, b, c}},\n\t}\n\n\tfor testi, test := range tests {\n\t\tres := test.input.Dedupe()\n\t\tif !reflect.DeepEqual(res, test.expect) {\n\t\t\tt.Errorf(\"testi: %d, test: %+v, res: %+v\", testi, test, res)\n\t\t}\n\t}\n}\n\nfunc TestMarshallingHighTerm(t *testing.T) {\n\thighTermBytes, err := json.Marshal(HighTerm)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar unmarshalledHighTerm string\n\terr = json.Unmarshal(highTermBytes, &unmarshalledHighTerm)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif unmarshalledHighTerm != HighTerm {\n\t\tt.Fatalf(\"unexpected %x != %x\", unmarshalledHighTerm, HighTerm)\n\t}\n}\n"
  },
  {
    "path": "search/searcher/base_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"math\"\n\t\"regexp\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\tregexpTokenizer \"github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar twoDocIndex index.Index\n\nfunc init() {\n\ttwoDocIndex = initTwoDocUpsideDown()\n}\n\nfunc initTwoDocUpsideDown() index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ttwoDocIndex, err := upsidedown.NewUpsideDownCouch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": \"\",\n\t\t}, analysisQueue)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tinitTwoDocs(twoDocIndex)\n\treturn twoDocIndex\n}\n\nfunc initTwoDocScorch(dir string) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ttwoDocIndex, err := scorch.NewScorch(\n\t\tscorch.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": dir,\n\t\t}, analysisQueue)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tinitTwoDocs(twoDocIndex)\n\treturn twoDocIndex\n}\n\nfunc initTwoDocs(twoDocIndex index.Index) {\n\terr := twoDocIndex.Open()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tbatch := index.NewBatch()\n\tfor _, doc := range twoDocIndexDocs {\n\t\tbatch.Update(doc)\n\t}\n\terr = twoDocIndex.Batch(batch)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\n// create a simpler analyzer which will support these tests\nvar testAnalyzer = &analysis.DefaultAnalyzer{\n\tTokenizer: regexpTokenizer.NewRegexpTokenizer(regexp.MustCompile(`\\w+`)),\n}\n\n// sets up some mock data used in many tests in this package\nvar twoDocIndexDescIndexingOptions = document.DefaultTextIndexingOptions | index.IncludeTermVectors\n\nvar twoDocIndexDocs = []*document.Document{\n\t// must have 4/4 beer\n\tdocument.NewDocument(\"1\").\n\t\tAddField(document.NewTextField(\"name\", []uint64{}, []byte(\"marty\"))).\n\t\tAddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"beer beer beer beer\"), twoDocIndexDescIndexingOptions, testAnalyzer)).\n\t\tAddField(document.NewTextFieldWithAnalyzer(\"street\", []uint64{}, []byte(\"couchbase way\"), testAnalyzer)),\n\t// must have 1/4 beer\n\tdocument.NewDocument(\"2\").\n\t\tAddField(document.NewTextField(\"name\", []uint64{}, []byte(\"steve\"))).\n\t\tAddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"angst beer couch database\"), twoDocIndexDescIndexingOptions, testAnalyzer)).\n\t\tAddField(document.NewTextFieldWithAnalyzer(\"street\", []uint64{}, []byte(\"couchbase way\"), testAnalyzer)).\n\t\tAddField(document.NewTextFieldWithAnalyzer(\"title\", []uint64{}, []byte(\"mister\"), testAnalyzer)),\n\t// must have 1/4 beer\n\tdocument.NewDocument(\"3\").\n\t\tAddField(document.NewTextField(\"name\", []uint64{}, []byte(\"dustin\"))).\n\t\tAddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"apple beer column dank\"), twoDocIndexDescIndexingOptions, testAnalyzer)).\n\t\tAddField(document.NewTextFieldWithAnalyzer(\"title\", []uint64{}, []byte(\"mister\"), testAnalyzer)),\n\t// must have 65/65 beer\n\tdocument.NewDocument(\"4\").\n\t\tAddField(document.NewTextField(\"name\", []uint64{}, []byte(\"ravi\"))).\n\t\tAddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer\"), twoDocIndexDescIndexingOptions, testAnalyzer)),\n\t// must have 0/x beer\n\tdocument.NewDocument(\"5\").\n\t\tAddField(document.NewTextField(\"name\", []uint64{}, []byte(\"bobert\"))).\n\t\tAddField(document.NewTextFieldCustom(\"desc\", []uint64{}, []byte(\"water\"), twoDocIndexDescIndexingOptions, testAnalyzer)).\n\t\tAddField(document.NewTextFieldWithAnalyzer(\"title\", []uint64{}, []byte(\"mister\"), testAnalyzer)),\n}\n\nfunc scoresCloseEnough(a, b float64) bool {\n\treturn math.Abs(a-b) < 0.001\n}\n"
  },
  {
    "path": "search/searcher/geoshape_contains_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar (\n\tleftRectEdgeMultiPoint [][]float64   = [][]float64{{-1, 0.2}, {-0.9, 0.1}}\n\tleftRectWithHole       [][][]float64 = [][][]float64{\n\t\t{{-1, 0}, {0, 0}, {0, 1}, {-1, 1}, {-1, 0}},\n\t\t{{-0.75, 0.25}, {-0.75, -0.75}, {-0.25, 0.75}, {-0.25, 0.25}, {-0.74, 0.25}},\n\t}\n\tleftRectEdgePoint  []float64   = []float64{-1, 0.2}\n\tleftRectMultiPoint [][]float64 = [][]float64{{0.5, 0.5}, {-0.9, 0.1}}\n)\n\nfunc testCaseSetup(t *testing.T, docShapeName, docShapeType string, docShapeVertices [][][][]float64,\n\ti index.Index,\n) (index.IndexReader, func() error, error) {\n\tdoc := document.NewDocument(docShapeName)\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tdocShapeVertices, docShapeType, document.DefaultGeoShapeIndexingOptions))\n\terr := i.Update(doc)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcloseFn := func() error {\n\t\terr = i.Delete(doc.ID())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\n\treturn indexReader, closeFn, nil\n}\n\nfunc TestPointPolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       rightRectPoint,\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tDesc:             \"point inside polygon\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       leftRectPoint,\n\t\t\tDocShapeVertices: nil,\n\t\t\tDocShapeName:     \"\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"empty polygon\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\", [][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType, false, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\", test.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestLinestringPolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1, 2}, {3, 5}},\n\t\t\tDocShapeVertices: [][][]float64{{{1, 2}, {3, 5}, {2, 7}, {1, 2}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"linestring coinciding with edge of the polygon\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1, 0}, {0, 1}},\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"diagonal of a square\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0.2, 0.2}, {0.8, 0.8}},\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"linestring within polygon\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeLinestringQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for linestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopePointContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: []float64{0.5, 0.5},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point completely within bounded rectangle\",\n\t\t\tExpected:         nil, // will always be nil since point can't contain envelope\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopeLinestringContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][]float64{{0.5, 0.5}, {10, 10}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring partially within bounded rectangle\",\n\t\t\tExpected:         nil, // will always be nil since linestring can't contain envelope\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopePolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.5, 0.5}, {1, 0.5}, {1, 1}, {0.5, 1}, {0.5, 0.5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon completely within bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][][]float64{{{10.5, 10.5}, {11.5, 10.5}, {11.5, 11.5}, {10.5, 11.5}, {10.5, 10.5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon completely outside bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon coincident with bounded rectangle\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonPointContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       rightRect,\n\t\t\tDocShapeVertices: rightRectPoint,\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil, // nil since point is a non-closed shape\n\t\t\tDesc:             \"point inside polygon\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       leftRect,\n\t\t\tDocShapeVertices: leftRectEdgePoint,\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point on edge of polygon\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       leftRectWithHole,\n\t\t\tDocShapeVertices: leftRectPoint,\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point in polygon's hole\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonLinestringContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       rightRect,\n\t\t\tDocShapeVertices: [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil, // nil since linestring is a non-closed shape\n\t\t\tDesc:             \"diagonal of a square\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonEnvelopeContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0.5, 0.5}, {1, 0.5}, {1, 1}, {0.5, 1}, {0.5, 0.5}}},\n\t\t\tDocShapeVertices: [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeName:     \"envelope1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"polygon contained inside envelope with edge overlaps\", // this fails since\n\t\t\t// contains doesn't include edges or vertices\n\t\t\tQueryType: \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0.25, 0.25}, {0.5, 0.25}, {0.5, 0.5}, {0.25, 0.25}, {0.25, 0.25}}},\n\t\t\tDocShapeVertices: [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeName:     \"envelope1\",\n\t\t\tExpected:         []string{\"envelope1\"},\n\t\t\tDesc:             \"polygon contained completely inside envelope\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"envelope\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPointPolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       leftRectEdgeMultiPoint,\n\t\t\tDocShapeVertices: leftRectWithHole,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tDesc:             \"multi point inside polygon with hole\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1, 0.5}},\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multi point on polygon edge\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][]float64{{0.3, 0.3}, {0.5, 0.5}},\n\t\t\tDocShapeVertices: [][][]float64{\n\t\t\t\t{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}},\n\t\t\t\t{{0.2, 0.2}, {0.4, 0.2}, {0.4, 0.4}, {0.2, 0.4}, {0.2, 0.2}},\n\t\t\t},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tExpected:     nil, // returns nil since one of the points is within the hole\n\t\t\tDesc:         \"multi point inside polygon and hole\",\n\t\t\tQueryType:    \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType,\n\t\t\t\ttrue, indexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipoint: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPointLinestringContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       leftRectEdgeMultiPoint,\n\t\t\tDocShapeVertices: [][]float64{{-1, 0.2}, {-0.9, 0.1}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t\tDesc:             \"multi point overlaps with all linestring end points\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-1, 0.2}, {-0.9, 0.1}, {0.5, 0.5}},\n\t\t\tDocShapeVertices: [][]float64{{-1, 0.2}, {-0.9, 0.1}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multi point overlaps with some linestring end points\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType,\n\t\t\t\ttrue, indexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipoint: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPointContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       leftRectEdgeMultiPoint,\n\t\t\tDocShapeVertices: [][]float64{{-1, 0.2}, {-0.9, 0.1}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         []string{\"multipoint1\"},\n\t\t\tDesc:             \"multi point overlaps with all multi points\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-1, 0.2}, {-0.9, 0.1}, {0.5, 0.5}},\n\t\t\tDocShapeVertices: [][]float64{{-1, 0.2}, {-0.9, 0.1}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multi point overlaps with some multi points\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipoint\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType,\n\t\t\t\ttrue, indexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipoint: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       leftRect,\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"polygons sharing an edge\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       rightRect,\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tDesc:             \"coincident polygons\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonMultiPointContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       leftRect,\n\t\t\tDocShapeVertices: leftRectEdgeMultiPoint,\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         nil, // nil since multipoint is a non-closed shape\n\t\t\tDesc:             \"multiple points on polygon edge\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       leftRect,\n\t\t\tDocShapeVertices: leftRectMultiPoint,\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multiple points, both outside and inside polygon\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       leftRectWithHole,\n\t\t\tDocShapeVertices: leftRectMultiPoint,\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multiple points in polygon hole\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipoint\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPolygonPolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][]float64{leftRect},\n\t\t\tDocShapeVertices: leftRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tDesc:             \"coincident polygons\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][][]float64{{{{2, 2}, {-2, 2}, {-2, -2}, {2, -2}}}},\n\t\t\tDocShapeVertices: leftRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"polygon larger than polygons in query shape\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiPolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiLinestringMultiPolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0.2, 1}, {0.8, 1}}, {{1, 0.2}, {1, 0.8}}},\n\t\t\tDocShapeVertices: [][][][]float64{rightRect},\n\t\t\tDocShapeName:     \"multipolygon1\",\n\t\t\tExpected:         nil, // contains doesn't include edges or vertices\n\t\t\tDesc:             \"linestrings on edge of polygon\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0.2, 0.2}, {0.8, 0.8}}, {{0.8, 0.2}, {0.2, 0.8}}},\n\t\t\tDocShapeVertices: [][][][]float64{{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}},\n\t\t\tDocShapeName:     \"multipolygon1\",\n\t\t\tExpected:         []string{\"multipolygon1\"},\n\t\t\tDesc:             \"linestrings within polygon\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipolygon\", test.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiLinestringQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multilinestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestGeometryCollectionPolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][][]float64\n\t\tQueryShapeTypes  []string\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{0, 1}, {1, 0}}}}},\n\t\t\tQueryShapeTypes:  []string{\"linestring\"},\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tDesc:             \"linestring on edge of polygon\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, test.QueryShapeTypes, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestGeometryCollectionMultiPolygonContains(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][][]float64\n\t\tQueryShapeTypes  []string\n\t\tDocShapeVertices [][][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{1, 1}}}}},\n\t\t\tQueryShapeTypes:  []string{\"point\"},\n\t\t\tDocShapeVertices: [][][][]float64{rightRect, leftRect},\n\t\t\tDocShapeName:     \"multipolygon1\",\n\t\t\tExpected:         []string{\"multipolygon1\"},\n\t\t\tDesc:             \"point on vertex of one of the polygons\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t\t{\n\t\t\t// WIP - Adding a point (-0.5,-0.5)\n\t\t\tQueryShape:       [][][][][]float64{{{{{0.2, 0.4}, {0.2, 0.2}, {0.4, 0.2}, {0.4, 0.4}}}}},\n\t\t\tQueryShapeTypes:  []string{\"polygon\"},\n\t\t\tDocShapeVertices: [][][][]float64{rightRect, leftRect},\n\t\t\tDocShapeName:     \"multipolygon1\",\n\t\t\tExpected:         []string{\"multipolygon1\"},\n\t\t\tDesc:             \"polygon contained completely within multipolygons\",\n\t\t\tQueryType:        \"contains\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipolygon\", test.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, test.QueryShapeTypes, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipolygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/geoshape_intersects_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc setupIndex(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": \"\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n\nfunc TestPointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{2.0, 2.0},\n\t\t\tDocShapeVertices: []float64{2.0, 2.0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"coincident points\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{2.0, 2.0},\n\t\t\tDocShapeVertices: []float64{2.0, 2.1},\n\t\t\tDocShapeName:     \"point2\",\n\t\t\tDesc:             \"non coincident points\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\t// indexing and searching independently for each case.\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPointMultiPointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{2.0, 2.0},\n\t\t\tDocShapeVertices: [][]float64{{2.0, 2.0}, {3.0, 2.0}},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point coincides with one point in multipoint\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{2.0, 2.0},\n\t\t\tDocShapeVertices: [][]float64{{2.0, 2.1}, {3.0, 3.1}},\n\t\t\tDocShapeName:     \"point2\",\n\t\t\tDesc:             \"non coincident points\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\t// indexing and searching independently for each case.\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPointLinestringIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{4.0, 4.0},\n\t\t\tDocShapeVertices: [][]float64{{2.0, 2.0}, {3.0, 3.0}, {4.0, 4.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"point at the vertex of linestring\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.5, 1.5001714},\n\t\t\tDocShapeVertices: [][]float64{{0.0, 0.0}, {1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"point along linestring\",\n\t\t\tExpected:         nil, // nil since point is said to intersect only when it matches any\n\t\t\t// of the endpoints of the linestring\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.5, 1.6001714},\n\t\t\tDocShapeVertices: [][]float64{{0.0, 0.0}, {1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"point outside linestring\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPointMultiLinestringIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{3.0, 3.0},\n\t\t\tDocShapeVertices: [][][]float64{{{2.0, 2.0}, {3.0, 3.0}, {4.0, 4.0}}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"point at the vertex of linestring\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.5, 1.5001714},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"point along a linestring\",\n\t\t\tExpected:         nil, // nil since point is said to intersect only when it matches any\n\t\t\t// of the endpoints of any of the linestrings\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.5, 1.6001714},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, {{1, 1.1}, {2, 2.1}, {3, 3.4}}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"point outside all linestrings\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multilinestring\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPointPolygonIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{3.0, 3.0},\n\t\t\tDocShapeVertices: [][][]float64{{{2.0, 2.0}, {3.0, 3.0}, {1.0, 3.0}, {2.0, 2.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"point on polygon vertex\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.5, 1.500714},\n\t\t\tDocShapeVertices: [][][]float64{{{1.0, 1.0}, {2.0, 2.0}, {0.0, 2.0}, {1.0, 1.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"point on polygon edge\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.5, 1.9},\n\t\t\tDocShapeVertices: [][][]float64{{{1.0, 1.0}, {2.0, 2.0}, {0.0, 2.0}, {1.0, 1.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"point inside polygon\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: []float64{0.3, 0.3},\n\t\t\tDocShapeVertices: [][][]float64{\n\t\t\t\t{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}},\n\t\t\t\t{{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}},\n\t\t\t},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"point inside hole inside polygon\",\n\t\t\tExpected:     nil,\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPointMultiPolygonIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{3.0, 3.0},\n\t\t\tDocShapeVertices: [][][][]float64{{{{2.0, 2.0}, {3.0, 3.0}, {1.0, 3.0}, {2.0, 2.0}}}},\n\t\t\tDocShapeName:     \"multipolygon1\",\n\t\t\tDesc:             \"point on a polygon vertex\",\n\t\t\tExpected:         []string{\"multipolygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.5, 1.500714},\n\t\t\tDocShapeVertices: [][][][]float64{{{{1.0, 1.0}, {2.0, 2.0}, {0.0, 2.0}, {1.0, 1.0}}}},\n\t\t\tDocShapeName:     \"multipolygon1\",\n\t\t\tDesc:             \"point on polygon edge\",\n\t\t\tExpected:         []string{\"multipolygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: []float64{1.5, 1.9},\n\t\t\tDocShapeVertices: [][][][]float64{\n\t\t\t\t{{{1.0, 1.0}, {2.0, 2.0}, {0.0, 2.0}, {1.0, 1.0}}},\n\t\t\t\t{{{1.5, 1.9}, {2.5, 2.9}, {0.5, 2.9}, {1.5, 1.9}}},\n\t\t\t},\n\t\t\tDocShapeName: \"multipolygon1\",\n\t\t\tDesc:         \"point inside a polygon and on vertex of another polygon\",\n\t\t\tExpected:     []string{\"multipolygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: []float64{0.3, 0.3},\n\t\t\tDocShapeVertices: [][][][]float64{{\n\t\t\t\t{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}},\n\t\t\t\t{{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}},\n\t\t\t}},\n\t\t\tDocShapeName: \"multipolygon1\",\n\t\t\tDesc:         \"point inside hole inside one of the polygons\",\n\t\t\tExpected:     nil,\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipolygon\", test.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopePointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: rightRectPoint,\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point on vertex of bounded rectangle\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tQueryType:        \"intersects\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: []float64{10, 10},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point outside bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"intersects\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopeLinestringIntersect(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][]float64{{0.25, 0.25}, {0.5, 0.5}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring completely in bounded rectangle\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t\tQueryType:        \"intersects\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][]float64{{2.5, 2.5}, {4.5, 4.5}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring outside bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"intersects\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][]float64{{0.25, 0.25}, {4.5, 4.5}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring partially in bounded rectangle\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t\tQueryType:        \"intersects\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopePolygonIntersect(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.5, 0.5}, {1.5, 0.5}, {1.5, 1.5}, {0.5, 1.5}, {0.5, 0.5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon intersects bounded rectangle\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tQueryType:        \"intersects\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][][]float64{{{10.5, 10.5}, {11.5, 10.5}, {11.5, 11.5}, {10.5, 11.5}, {10.5, 10.5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon completely outside bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"intersects\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{3.0, 3.0}, {4.0, 4.0}},\n\t\t\tDocShapeVertices: [][]float64{{4.0, 4.0}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tDesc:             \"single coincident multipoint\",\n\t\t\tExpected:         []string{\"multipoint1\"},\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipoint\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\ttrue, indexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipoint: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestLinestringIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{3.0, 2.0}, {4.0, 2.0}},\n\t\t\tDocShapeVertices: [][]float64{{3.0, 2.0}, {4.0, 2.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"coincident linestrings\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {1.5, 1.5}, {2.0, 2.0}},\n\t\t\tDocShapeVertices: [][]float64{{2.0, 2.0}, {4.0, 3.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestrings intersecting at the ends\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {3.0, 3.0}},\n\t\t\tDocShapeVertices: [][]float64{{1.5499860, 1.5501575}, {4.0, 6.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"subline not at vertex\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {2.0, 2.0}},\n\t\t\tDocShapeVertices: [][]float64{{1.5499860, 1.5501575}, {1.5, 1.5001714}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"subline inside linestring\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {1.5, 1.5}, {2.0, 2.0}},\n\t\t\tDocShapeVertices: [][]float64{{1.0, 2.0}, {2.0, 1.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestrings intersecting at some edge\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {1.5, 1.5}, {2.0, 2.0}},\n\t\t\tDocShapeVertices: [][]float64{{1.0, 2.0}, {1.0, 4.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"non intersecting linestrings\",\n\t\t\tExpected:         nil,\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{59.32, 0.52}, {68.99, -7.36}, {75.49, -12.21}},\n\t\t\tDocShapeVertices: [][]float64{{71.98, 0}, {67.58, -6.57}, {63.19, -12.72}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestrings with more than 2 points intersecting at some edges\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeLinestringQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestLinestringPolygonIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {1.5, 1.5}, {2.0, 2.0}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"linestring intersects polygon at a vertex\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0.2, 0.2}, {0.4, 0.4}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"linestring within polygon\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-0.5, 0.5}, {0.5, 0.5}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"linestring intersects polygon at an edge\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-0.5, 0.5}, {1.5, 0.5}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"linestring intersects polygon as a whole\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-0.5, 0.5}, {-1.5, -1.5}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"linestring does not intersect polygon\",\n\t\t\tExpected:         nil,\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][]float64{{0.3, 0.3}, {0.35, 0.35}},\n\t\t\tDocShapeVertices: [][][]float64{\n\t\t\t\t{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}},\n\t\t\t\t{{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}},\n\t\t\t},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"linestring does not intersect polygon when contained in the hole\",\n\t\t\tExpected:     nil,\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][]float64{{0.3, 0.3}, {0.5, 0.5}},\n\t\t\tDocShapeVertices: [][][]float64{\n\t\t\t\t{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}},\n\t\t\t\t{{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}},\n\t\t\t},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"linestring intersects polygon in the hole\",\n\t\t\tExpected:     []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][]float64{{0.4, 0.3}, {0.6, 0.3}},\n\t\t\tDocShapeVertices: [][][]float64{\n\t\t\t\t{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}},\n\t\t\t\t{{0.3, 0.3}, {0.4, 0.2}, {0.5, 0.3}, {0.4, 0.4}, {0.3, 0.3}},\n\t\t\t\t{{0.5, 0.3}, {0.6, 0.2}, {0.7, 0.3}, {0.6, 0.4}, {0.5, 0.3}},\n\t\t\t},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"linestring intersects polygon through touching holes\",\n\t\t\tExpected:     []string{\"polygon1\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeLinestringQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for linestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestLinestringPointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{179, 0}, {-179, 0}},\n\t\t\tDocShapeVertices: []float64{179.1, 0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point across longitudinal boundary of linestring\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-179, 0}, {179, 0}},\n\t\t\tDocShapeVertices: []float64{179.1, 0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point across longitudinal boundary of reversed linestring\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{179, 0}, {-179, 0}},\n\t\t\tDocShapeVertices: []float64{170, 0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point does not intersect linestring\",\n\t\t\tExpected:         nil,\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-179, 0}, {179, 0}},\n\t\t\tDocShapeVertices: []float64{170, 0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point does not intersect reversed linestring\",\n\t\t\tExpected:         nil,\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-179, 0}, {179, 0}, {178, 0}},\n\t\t\tDocShapeVertices: []float64{178, 0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point intersects linestring at end vertex\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{-179, 0}, {179, 0}, {178, 0}, {180, 0}},\n\t\t\tDocShapeVertices: []float64{178, 0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point intersects linestring with more than two points\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeLinestringQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for linestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiLinestringIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{1.0, 1.0}, {1.1, 1.1}, {2.0, 2.0}, {2.1, 2.1}}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.5132}, {-1.1, -1.1}, {1.5, 1.512}, {2.1, 2.1}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"intersecting multilinestrings\",\n\t\t\tExpected:         []string{\"multilinestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{1.0, 1.0}, {1.1, 1.1}, {2.0, 2.0}, {2.1, 2.1}}},\n\t\t\tDocShapeVertices: [][][]float64{{{10.1, 100.5}, {11.5, 102.5}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"non-intersecting multilinestrings\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multilinestring\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiLinestringQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multilinestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiLinestringMultiPointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{2.0, 2.0}, {2.1, 2.1}}, {{3.0, 3.0}, {3.1, 3.1}}},\n\t\t\tDocShapeVertices: [][]float64{{5.0, 6.0}, {67, 67}, {3.1, 3.1}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tDesc:             \"multilinestring intersects one of the multipoints\",\n\t\t\tExpected:         []string{\"multipoint1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{2.0, 2.0}, {2.1, 2.1}}, {{3.0, 3.0}, {3.1, 3.1}}},\n\t\t\tDocShapeVertices: [][]float64{{56.0, 56.0}, {66, 66}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tDesc:             \"multilinestring does not intersect any of the multipoints\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipoint\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiLinestringQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape: [][][]float64{{\n\t\t\t\t{1.0, 1.0},\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{2.0, 2.0},\n\t\t\t\t{1.0, 2.0},\n\t\t\t\t{1.0, 1.0},\n\t\t\t}},\n\t\t\tDocShapeVertices: [][][]float64{{\n\t\t\t\t{1.0, 1.0},\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{2.0, 2.0},\n\t\t\t\t{1.0, 2.0},\n\t\t\t\t{1.0, 1.0},\n\t\t\t}},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"coincident polygons\",\n\t\t\tExpected:     []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][]float64{{\n\t\t\t\t{1.0, 1.0},\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{2.0, 2.0},\n\t\t\t\t{1.0, 2.0},\n\t\t\t\t{1.0, 1.0},\n\t\t\t}},\n\t\t\tDocShapeVertices: [][][]float64{{\n\t\t\t\t{1.2, 1.2},\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{2.0, 2.0},\n\t\t\t\t{1.0, 2.0},\n\t\t\t\t{1.2, 1.2},\n\t\t\t}},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"polygon and a window polygon\",\n\t\t\tExpected:     []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][]float64{{\n\t\t\t\t{1.0, 1.0},\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{2.0, 2.0},\n\t\t\t\t{1.0, 2.0},\n\t\t\t\t{1.0, 1.0},\n\t\t\t}},\n\t\t\tDocShapeVertices: [][][]float64{{\n\t\t\t\t{1.1, 1.1},\n\t\t\t\t{1.2, 1.1},\n\t\t\t\t{1.2, 1.2},\n\t\t\t\t{1.1, 1.2},\n\t\t\t\t{1.1, 1.1},\n\t\t\t}},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"nested polygons\",\n\t\t\tExpected:     []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][]float64{{\n\t\t\t\t{1.0, 1.0},\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{2.0, 2.0},\n\t\t\t\t{1.0, 2.0},\n\t\t\t\t{1.0, 1.0},\n\t\t\t}},\n\t\t\tDocShapeVertices: [][][]float64{{\n\t\t\t\t{0.0, 1.0},\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{2.0, 2.0},\n\t\t\t\t{0.0, 2.0},\n\t\t\t\t{0.0, 1.0},\n\t\t\t}},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"intersecting polygons\",\n\t\t\tExpected:     []string{\"polygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][]float64{{{0, 0}, {5, 0}, {5, 5}, {0, 5}, {0, 0}}, {\n\t\t\t\t{1, 4},\n\t\t\t\t{4, 4},\n\t\t\t\t{4, 1},\n\t\t\t\t{1, 1},\n\t\t\t\t{1, 4},\n\t\t\t}},\n\t\t\tDocShapeVertices: [][][]float64{{{2, 2}, {3, 2}, {3, 3}, {2, 3}, {2, 2}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon inside hole of a larger polygon\",\n\t\t\tExpected:         nil,\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][]float64{{\n\t\t\t\t{1.0, 1.0},\n\t\t\t\t{2.0, 1.0},\n\t\t\t\t{2.0, 2.0},\n\t\t\t\t{1.0, 2.0},\n\t\t\t\t{1.0, 1.0},\n\t\t\t}},\n\t\t\tDocShapeVertices: [][][]float64{{\n\t\t\t\t{3.0, 3.0},\n\t\t\t\t{4.0, 3.0},\n\t\t\t\t{4.0, 4.0},\n\t\t\t\t{3.0, 4.0},\n\t\t\t\t{3.0, 3.0},\n\t\t\t}},\n\t\t\tDocShapeName: \"polygon1\",\n\t\t\tDesc:         \"disjoint polygons\",\n\t\t\tExpected:     nil,\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonLinestringIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{150, 85}, {160, 85}, {-20, 85}, {-30, 85}, {150, 85}}},\n\t\t\tDocShapeVertices: [][]float64{{150, 85}, {160, 85}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"polygon intersects line along edge\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][]float64{{150, 85}, {160, 85}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"polygon not intersecting line\",\n\t\t\tExpected:         nil,\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][]float64{{0.2, 0.2}, {0.4, 0.4}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"polygon completely encloses line\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][]float64{{-0.5, 0.5}, {1.5, 0.5}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"line cuts through entire polygon\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][]float64{{-0.439, -0.318}, {0.4339, 0.335}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"line partially cuts through polygon\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonMultiLinestringIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{150, 85}, {160, 85}, {-20, 85}, {-30, 85}, {150, 85}}},\n\t\t\tDocShapeVertices: [][][]float64{{{150, 85}, {160, 85}}, {{0, 1}, {5, 10}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"polygon intersects one line along edge\",\n\t\t\tExpected:         []string{\"multilinestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{150, 85}, {160, 85}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"polygon not intersecting any line\",\n\t\t\tExpected:         nil,\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.2, 0.2}, {0.4, 0.4}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"polygon completely encloses line\",\n\t\t\tExpected:         []string{\"multilinestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{-0.5, 0.5}, {1.5, 0.5}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"line cuts through entire polygon\",\n\t\t\tExpected:         []string{\"multilinestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{-0.439, -0.318}, {0.4339, 0.335}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"line partially cuts through polygon\",\n\t\t\tExpected:         []string{\"multilinestring1\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multilinestring\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonPointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{150, 85}, {160, 85}, {-20, 85}, {-30, 85}, {150, 85}}},\n\t\t\tDocShapeVertices: []float64{150, 88},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"polygon intersects point in latitudinal boundary\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{150, 85}, {160, 85}, {-20, 85}, {-30, 85}, {150, 85}}},\n\t\t\tDocShapeVertices: []float64{170, 88},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"polygon does not intersects point outside latitudinal boundary\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPolygonIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][]float64\n\t\tDocShapeVertices [][][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape: [][][][]float64{{{\n\t\t\t\t{15, 5},\n\t\t\t\t{40, 10},\n\t\t\t\t{10, 20},\n\t\t\t\t{5, 10},\n\t\t\t\t{15, 5},\n\t\t\t}}, {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}},\n\t\t\tDocShapeVertices: [][][][]float64{{{\n\t\t\t\t{0.0, 0.0},\n\t\t\t\t{1.0, 0.0},\n\t\t\t\t{1.0, 1.0},\n\t\t\t\t{0.0, 1.0},\n\t\t\t\t{0.0, 0.0},\n\t\t\t}, {{30, 20}, {45, 40}, {10, 40}, {30, 20}}}},\n\t\t\tDocShapeName: \"multipolygon1\",\n\t\t\tDesc:         \"intersecting multi polygons\",\n\t\t\tExpected:     []string{\"multipolygon1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][][]float64{{{\n\t\t\t\t{15, 5},\n\t\t\t\t{40, 10},\n\t\t\t\t{10, 20},\n\t\t\t\t{5, 10},\n\t\t\t\t{15, 5},\n\t\t\t}}, {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}},\n\t\t\tDocShapeVertices: [][][][]float64{{{\n\t\t\t\t{0.0, 0.0},\n\t\t\t\t{1.0, 0.0},\n\t\t\t\t{1.0, 1.0},\n\t\t\t\t{0.0, 1.0},\n\t\t\t\t{0.0, 0.0},\n\t\t\t}}},\n\t\t\tDocShapeName: \"multipolygon1\",\n\t\t\tDesc:         \"non intersecting multi polygons\",\n\t\t\tExpected:     nil,\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipolygon\",\n\t\t\ttest.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiPolygonQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipolygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPolygonMultiPointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{30, 20}, {45, 40}, {10, 40}, {30, 20}}},\n\t\t\t\t{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][]float64{{30, 20}, {30, 30}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tDesc:             \"multipolygon intersects multipoint at the vertex\",\n\t\t\tExpected:         []string{\"multipoint1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}},\n\t\t\t\t{{{30, 20}, {45, 50}, {10, 50}, {30, 20}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][]float64{{30, -20}, {-30, 30}, {45, 66}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tDesc:             \"multipolygon does not intersect multipoint\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipoint\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiPolygonQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipolygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPolygonMultiLinestringIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][]float64{{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}},\n\t\t\tDocShapeVertices: [][][]float64{{{65, 40}, {60, 40}}, {{45, 40}, {10, 40}, {30, 20}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"multipolygon intersects multilinestring\",\n\t\t\tExpected:         []string{\"multilinestring1\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][][]float64{{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}},\n\t\t\tDocShapeVertices: [][][]float64{{{45, 41}, {60, 80}}, {{-45, -40}, {-10, -40}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tDesc:             \"multipolygon does not intersect multilinestring\",\n\t\t\tExpected:         nil,\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multilinestring\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiPolygonQueryWithRelation(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipolygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestGeometryCollectionIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][][]float64\n\t\tDocShapeVertices [][][][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tTypes            []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{}}}},\n\t\t\tDocShapeVertices: [][][][][]float64{{{{}}}},\n\t\t\tDocShapeName:     \"geometrycollection1\",\n\t\t\tDesc:             \"empty geometry collections\",\n\t\t\tExpected:         nil,\n\t\t\tTypes:            []string{\"\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.Types,\n\t\t\ttest.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, test.Types, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for geometry collection: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestGeometryCollectionPointIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tTypes            []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{4, 5}}}}},\n\t\t\tDocShapeVertices: []float64{4, 5},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point coincident with point in geometry collection\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tTypes:            []string{\"point\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{4, 5}, {6, 7}}}}},\n\t\t\tDocShapeVertices: []float64{4, 5},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point on vertex of linestring in geometry collection\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tTypes:            []string{\"linestring\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{1, 1}, {2, 2}, {0, 2}, {1, 1}}}, {{{5, 6}}}}},\n\t\t\tDocShapeVertices: []float64{1.5, 1.9},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point inside polygon in geometry collection\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tTypes:            []string{\"polygon\", \"point\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Fatal(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, test.Types, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for geometry collection: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestGeometryCollectionLinestringIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tTypes            []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{4, 5}, {6, 7}, {7, 8}}}}},\n\t\t\tDocShapeVertices: [][]float64{{6, 7}, {7, 8}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring intersecting with linestring in geometry collection\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t\tTypes:            []string{\"linestring\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{1.5, 1.9}}}}},\n\t\t\tDocShapeVertices: [][]float64{{1.5, 1.9}, {2.5, 2.8}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring intersects point in geometry collection at vertex\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t\tTypes:            []string{\"point\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, test.Types, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for geometry collection: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestGeometryCollectionPolygonIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tTypes            []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{4, 5}, {6, 7}, {7, 8}, {4, 5}}}, {{{1, 2}, {2, 3}, {3, 4}, {1, 2}}}}},\n\t\t\tDocShapeVertices: [][][]float64{{{4, 5}, {6, 7}, {7, 8}, {4, 5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon coincides with one of the polygons in multipolygon in geometry collection\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tTypes:            []string{\"multipolygon\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{14, 15}}}}},\n\t\t\tDocShapeVertices: [][][]float64{{{4, 5}, {6, 7}, {7, 8}, {4, 5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon does not intersect point in geometry collection\",\n\t\t\tExpected:         nil,\n\t\t\tTypes:            []string{\"point\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(\"intersects\",\n\t\t\t\tindexReader, test.QueryShape, test.Types, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for geometry collection: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPointGeometryCollectionIntersects(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][][][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tTypes            []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{1.0, 2.0},\n\t\t\tDocShapeVertices: [][][][][]float64{{{{}}}},\n\t\t\tDocShapeName:     \"geometrycollection1\",\n\t\t\tDesc:             \"geometry collection does not intersect with a point\",\n\t\t\tExpected:         nil,\n\t\t\tTypes:            []string{\"\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.0, 2.0},\n\t\t\tDocShapeVertices: [][][][][]float64{{{{{1.0, 2.0}}}}},\n\t\t\tDocShapeName:     \"geometrycollection1\",\n\t\t\tDesc:             \"geometry collection intersects with a point\",\n\t\t\tExpected:         []string{\"geometrycollection1\"},\n\t\t\tTypes:            []string{\"point\"},\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.Types,\n\t\t\ttest.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/geoshape_within_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar (\n\tleftRect       [][][]float64 = [][][]float64{{{-1, 0}, {0, 0}, {0, 1}, {-1, 1}, {-1, 0}}}\n\tleftRectPoint  []float64     = []float64{-0.5, 0.5}\n\trightRect      [][][]float64 = [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}\n\trightRectPoint []float64     = []float64{0.5, 0.5}\n)\n\nfunc testCaseSetupGeometryCollection(t *testing.T, docShapeName string, types []string, docShapeVertices [][][][][]float64,\n\ti index.Index,\n) (index.IndexReader, func() error, error) {\n\tdoc := document.NewDocument(docShapeName)\n\tgcField := document.NewGeometryCollectionFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, docShapeVertices, types, document.DefaultGeoShapeIndexingOptions)\n\tif gcField == nil {\n\t\treturn nil, nil, fmt.Errorf(\"the GC field is nil\")\n\t}\n\tdoc.AddField(gcField)\n\tif doc == nil {\n\t\treturn nil, nil, fmt.Errorf(\"the doc is nil\")\n\t}\n\terr := i.Update(doc)\n\tif err != nil {\n\t\tt.Error(err.Error())\n\t}\n\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tcloseFn := func() error {\n\t\terr = i.Delete(doc.ID())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\n\treturn indexReader, closeFn, nil\n}\n\nfunc TestPointWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{1.0, 1.0},\n\t\t\tDocShapeVertices: []float64{1.0, 1.0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point contains itself\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{1.0, 1.0},\n\t\t\tDocShapeVertices: []float64{1.0, 1.1},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point does not contain a different point\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType,\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPointWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {2.0, 2.0}},\n\t\t\tDocShapeVertices: [][]float64{{1.0, 1.0}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         []string{\"multipoint1\"},\n\t\t\tDesc:             \"single multipoint common\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}},\n\t\t\tDocShapeVertices: [][]float64{{1.0, 1.0}, {2.0, 2.0}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multipoint not covered by multiple multipoints\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipoint\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType,\n\t\t\t\ttrue, indexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipoint: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopePointWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: []float64{0.5, 0.5},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point completely within bounded rectangle\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: []float64{0, 1},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point on vertex of bounded rectangle\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: []float64{10, 11},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tDesc:             \"point outside bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopeLinestringWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][]float64{{0.5, 0.5}, {0.75, 0.75}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring completely within bounded rectangle\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][]float64{{0.5, 0.5}, {1.75, 1.75}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring partially within bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][]float64{{1.5, 2.5}, {2.75, 2.75}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tDesc:             \"linestring completely outside bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestEnvelopePolygonWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.5, 0.5}, {1, 0.5}, {1, 1}, {0.5, 1}, {0.5, 0.5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon completely within bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.5, 0.5}, {1.5, 0.5}, {1.5, 1.5}, {0.5, 1.5}, {0.5, 0.5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon partially within bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{0, 1}, {1, 0}},\n\t\t\tDocShapeVertices: [][][]float64{{{10.5, 10.5}, {11.5, 10.5}, {11.5, 11.5}, {10.5, 11.5}, {10.5, 10.5}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tDesc:             \"polygon completely outside bounded rectangle\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeEnvelopeRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for Envelope: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPointLinestringWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{1.0, 1.0},\n\t\t\tDocShapeVertices: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point does not cover different linestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       []float64{179.1, 0.0},\n\t\t\tDocShapeVertices: [][]float64{{-179.0, 0.0}, {179.0, 0.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point across latitudinal boundary of linestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType,\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPointPolygonWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{1.0, 1.0},\n\t\t\tDocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point not within polygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{ // from binary predicates file\n\t\t\tQueryShape:       rightRectPoint,\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         nil, // will return nil since a point only returns non-nil for a coincident point\n\t\t\t// even if the point is on the polygon\n\t\t\tDesc:      \"point on rectangle vertex\",\n\t\t\tQueryType: \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType,\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestLinestringPointWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}},\n\t\t\tDocShapeVertices: []float64{1.0, 1.0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point at start of linestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}},\n\t\t\tDocShapeVertices: []float64{2.0, 2.0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point in the middle of linestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}},\n\t\t\tDocShapeVertices: []float64{3.0, 3.0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point at end of linestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}},\n\t\t\tDocShapeVertices: []float64{1.5, 1.50017},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point in between linestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}},\n\t\t\tDocShapeVertices: []float64{4, 5},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point not contained by linestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeLinestringQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for linestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPointMultiLinestringWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{2, 2}, {2.1, 2.1}},\n\t\t\tDocShapeVertices: [][][]float64{{{1, 1}, {1.1, 1.1}}, {{2, 2}, {2.1, 2.1}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tExpected:         nil, // nil since multipoint within multiline is always nil\n\t\t\tDesc:             \"multilinestring covering multipoint\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{2, 2}, {1, 1}, {3, 3}},\n\t\t\tDocShapeVertices: [][][]float64{{{1, 1}, {1.1, 1.1}}, {{2, 2}, {2.1, 2.1}}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multilinestring not covering multipoint\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multilinestring\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(test.QueryType,\n\t\t\t\ttrue, indexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multilinestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestLinestringWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1, 1}, {2, 2}, {3, 3}},\n\t\t\tDocShapeVertices: [][]float64{{1, 1}, {2, 2}, {3, 3}, {4, 4}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"longer linestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1, 1}, {2, 2}, {3, 3}},\n\t\t\tDocShapeVertices: [][]float64{{1, 1}, {2, 2}, {3, 3}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"coincident linestrings\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeLinestringQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for linestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestLinestringGeometryCollectionWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][]float64\n\t\tDocShapeVertices [][][][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t\tTypes            []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][]float64{{1, 1}, {2, 2}},\n\t\t\tDocShapeVertices: [][][][][]float64{{{{{1, 1}}}}},\n\t\t\tDocShapeName:     \"geometrycollection1\",\n\t\t\tExpected:         nil, // LS is not a closed shape\n\t\t\tDesc:             \"geometry collection with a point on vertex of linestring\",\n\t\t\tTypes:            []string{\"point\"},\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.Types,\n\t\t\ttest.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeLinestringQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for linestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonPointWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices []float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: []float64{0.5, 0.5},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point within polygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: []float64{5.5, 5.5},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point not within polygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][]float64{{\n\t\t\t\t{0, 0},\n\t\t\t\t{1, 0},\n\t\t\t\t{1, 1},\n\t\t\t\t{0, 1},\n\t\t\t\t{0, 0},\n\t\t\t\t{0.2, 0.2},\n\t\t\t\t{0.2, 0.4},\n\t\t\t\t{0.4, 0.4},\n\t\t\t\t{0.4, 0.2},\n\t\t\t\t{0.2, 0.2},\n\t\t\t}},\n\t\t\tDocShapeVertices: []float64{0.3, 0.3},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point within polygon hole\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}},\n\t\t\tDocShapeVertices: []float64{1.0, 0.0},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point on polygon vertex\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{1, 1}, {2, 2}, {0, 2}, {1, 1}}},\n\t\t\tDocShapeVertices: []float64{1.5, 1.5001714},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point inside polygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{150, 85}, {-20, -85}, {-30, 85}, {160, -85}, {150, 85}}},\n\t\t\tDocShapeVertices: []float64{170, 85},\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"point outside the polygon's latitudinal boundary\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\t// from binary predicates tests\n\t\t\tQueryShape:       leftRect,\n\t\t\tDocShapeVertices: leftRectPoint,\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point in left rectangle\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\t// from binary predicates tests\n\t\t\tQueryShape:       rightRect,\n\t\t\tDocShapeVertices: rightRectPoint,\n\t\t\tDocShapeName:     \"point1\",\n\t\t\tExpected:         []string{\"point1\"},\n\t\t\tDesc:             \"point in right rectangle\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"point\",\n\t\t\t[][][][]float64{{{test.DocShapeVertices}}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonLinestringWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][]float64{{0.1, 0.1}, {0.4, 0.4}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         []string{\"linestring1\"},\n\t\t\tDesc:             \"linestring within polygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][]float64{\n\t\t\t\t{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}},\n\t\t\t\t{{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][]float64{{0.3, 0.3}, {0.55, 0.55}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"linestring intersecting with polygon hole\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][]float64{\n\t\t\t\t{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}},\n\t\t\t\t{{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][]float64{{0.3, 0.3}, {4.0, 4.0}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"linestring intersecting with polygon hole and outside\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][]float64{{-1, -1}, {-2, -2}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"linestring outside polygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][]float64{{-0.5, -0.5}, {0.5, 0.5}},\n\t\t\tDocShapeName:     \"linestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"linestring intersecting polygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"linestring\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestPolygonWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tDesc:             \"coincident polygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.2, 0.2}, {1, 0}, {1, 1}, {0, 1}, {0.2, 0.2}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tDesc:             \"polygon covers an intersecting window of itself\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{0.1, 0.1}, {0.2, 0.1}, {0.2, 0.2}, {0.1, 0.2}, {0.1, 0.1}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         []string{\"polygon1\"},\n\t\t\tDesc:             \"polygon covers a nested version of itself\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{-1, 0}, {1, 0}, {1, 1}, {-1, 1}, {-1, 0}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"intersecting polygons\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\tDocShapeVertices: [][][]float64{{{3, 3}, {4, 3}, {4, 4}, {3, 4}, {3, 3}}},\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"polygon totally out of range\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       leftRect,\n\t\t\tDocShapeVertices: rightRect,\n\t\t\tDocShapeName:     \"polygon1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"left and right polygons,sharing an edge\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"polygon\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPolygonMultiPointWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][]float64\n\t\tDocShapeVertices [][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{30, 25}, {45, 40}, {10, 40}, {30, 20}, {30, 25}}},\n\t\t\t\t{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][]float64{{30, 20}, {15, 5}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         []string{\"multipoint1\"},\n\t\t\tDesc:             \"multipolygon covers multipoint\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}},\n\t\t\t\t{{{30, 20}, {45, 40}, {10, 40}, {30, 20}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][]float64{{30, 20}, {30, 30}, {45, 66}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multipolygon does not cover multipoint\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}},\n\t\t\t\t{{{1, 0}, {2, 0}, {2, 1}, {1, 1}, {1, 0}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][]float64{{0.5, 0.5}, {1.5, 0.5}},\n\t\t\tDocShapeName:     \"multipoint1\",\n\t\t\tExpected:         []string{\"multipoint1\"},\n\t\t\tDesc:             \"multiple multipolygons required to cover multipoint\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipoint\",\n\t\t\t[][][][]float64{{test.DocShapeVertices}}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiPolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiLinestringWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][]float64{{{1, 2}, {2, 3}, {3, 4}}, {{5, 6}, {6.5, 7.8}}},\n\t\t\tDocShapeVertices: [][][]float64{{{1, 2}, {2, 3}, {3, 4}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multilinestrings with common linestrings\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multilinestring\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiLinestringQueryWithRelation(\"within\",\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multilinestring: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPolygonMultiLinestringWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][]float64\n\t\tDocShapeVertices [][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}},\n\t\t\t\t{{{30, 20}, {45, 40}, {10, 40}, {30, 20}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][][]float64{{{45, 40}, {10, 40}}, {{45, 40}, {10, 40}, {30, 20}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tExpected:         []string{\"multilinestring1\"},\n\t\t\tDesc:             \"multilinestring intersecting at the edge of multipolygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}},\n\t\t\t\t{{{30, 20}, {45, 40}, {10, 40}, {30, 20}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][][]float64{{{48, 40}, {8, 40}}, {{48, 40}, {8, 40}, {30, 12}}},\n\t\t\tDocShapeName:     \"multilinestring1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multipolygon does not cover multilinestring\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multilinestring\",\n\t\t\t[][][][]float64{test.DocShapeVertices}, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiPolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipolygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestMultiPolygonWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][]float64\n\t\tDocShapeVertices [][][][]float64\n\t\tDocShapeName     string\n\t\tExpected         []string\n\t\tDesc             string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{16, 6}, {41, 11}, {11, 21}, {6, 11}, {16, 6}}},\n\t\t\t\t{{{31, 21}, {46, 41}, {11, 41}, {31, 21}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][][][]float64{{{{31, 21}, {46, 41}, {11, 41}, {31, 21}}}},\n\t\t\tDocShapeName:     \"multipolygon1\",\n\t\t\tExpected:         []string{\"multipolygon1\"},\n\t\t\tDesc:             \"multipolygon covers another multipolygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape: [][][][]float64{\n\t\t\t\t{{{16, 6}, {41, 11}, {11, 21}, {6, 11}, {16, 6}}},\n\t\t\t\t{{{31, 21}, {46, 41}, {11, 41}, {31, 21}}},\n\t\t\t},\n\t\t\tDocShapeVertices: [][][][]float64{{{{31, 21}, {46, 41}, {16, 46}, {31, 21}}}},\n\t\t\tDocShapeName:     \"multipolygon1\",\n\t\t\tExpected:         nil,\n\t\t\tDesc:             \"multipolygon does not cover multipolygon\",\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, \"multipolygon\",\n\t\t\ttest.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeMultiPolygonQueryWithRelation(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err.Error())\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for multipolygon: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestGeometryCollectionWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       [][][][][]float64\n\t\tDocShapeVertices [][][][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tQueryType        string\n\t\tQueryShapeTypes  []string\n\t\tDocShapeTypes    []string\n\t}{\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{}}}},\n\t\t\tDocShapeVertices: [][][][][]float64{{{{}}}},\n\t\t\tDocShapeName:     \"geometrycollection1\",\n\t\t\tDesc:             \"empty geometry collections\",\n\t\t\tExpected:         nil,\n\t\t\tQueryType:        \"within\",\n\t\t\tQueryShapeTypes:  []string{\"\"},\n\t\t\tDocShapeTypes:    []string{\"\"},\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{1, 2}, {2, 3}}}}},\n\t\t\tDocShapeVertices: [][][][][]float64{{{{{1, 2}}}}},\n\t\t\tDocShapeName:     \"geometrycollection1\",\n\t\t\tDesc:             \"geometry collection with a linestring\",\n\t\t\tExpected:         []string{\"geometrycollection1\"},\n\t\t\tQueryShapeTypes:  []string{\"linestring\"},\n\t\t\tDocShapeTypes:    []string{\"point\"},\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t\t{\n\t\t\tQueryShape:       [][][][][]float64{{{{{1, 2}, {2, 3}, {5, 6}}}}},\n\t\t\tDocShapeVertices: [][][][][]float64{{{{{1, 2}}}}},\n\t\t\tDocShapeName:     \"geometrycollection1\",\n\t\t\tDesc:             \"geometry collections with common points and multipoints\",\n\t\t\tExpected:         []string{\"geometrycollection1\"},\n\t\t\tQueryShapeTypes:  []string{\"multipoint\"},\n\t\t\tDocShapeTypes:    []string{\"point\"},\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.DocShapeTypes,\n\t\t\ttest.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(test.QueryType,\n\t\t\t\tindexReader, test.QueryShape, test.QueryShapeTypes, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for geometry collection: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestGeometryCollectionPointWithin(t *testing.T) {\n\ttests := []struct {\n\t\tQueryShape       []float64\n\t\tDocShapeVertices [][][][][]float64\n\t\tDocShapeName     string\n\t\tDesc             string\n\t\tExpected         []string\n\t\tTypes            []string\n\t\tQueryType        string\n\t}{\n\t\t{\n\t\t\tQueryShape:       []float64{1.0, 2.0},\n\t\t\tDocShapeVertices: [][][][][]float64{{{{}}}},\n\t\t\tDocShapeName:     \"geometrycollection1\",\n\t\t\tDesc:             \"empty geometry collection not within a point\",\n\t\t\tExpected:         nil,\n\t\t\tTypes:            []string{\"\"},\n\t\t\tQueryType:        \"within\",\n\t\t},\n\t}\n\n\ti := setupIndex(t)\n\n\tfor _, test := range tests {\n\t\tindexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.Types,\n\t\t\ttest.DocShapeVertices, i)\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\n\t\tt.Run(test.Desc, func(t *testing.T) {\n\t\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\t\tfalse, indexReader, [][]float64{test.QueryShape}, \"geometry\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, test.Expected) {\n\t\t\t\tt.Errorf(\"expected %v, got %v for point: %+v\",\n\t\t\t\t\ttest.Expected, got, test.QueryShape)\n\t\t\t}\n\t\t})\n\t\terr = closeFn()\n\t\tif err != nil {\n\t\t\tt.Error(err.Error())\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/optimize_knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage searcher\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc optimizeKNN(ctx context.Context, indexReader index.IndexReader,\n\tqsearchers []search.Searcher) error {\n\tvar octx index.VectorOptimizableContext\n\tvar err error\n\n\tfor _, searcher := range qsearchers {\n\t\t// Only applicable to KNN Searchers.\n\t\to, ok := searcher.(index.VectorOptimizable)\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\n\t\toctx, err = o.VectorOptimize(ctx, octx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// No KNN searchers.\n\tif octx == nil {\n\t\treturn nil\n\t}\n\n\t// Postings lists and iterators replaced in the pointer to the\n\t// vector reader\n\treturn octx.Finish()\n}\n"
  },
  {
    "path": "search/searcher/optimize_no_knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !vectors\n// +build !vectors\n\npackage searcher\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc optimizeKNN(ctx context.Context, indexReader index.IndexReader,\n\tqsearchers []search.Searcher) error {\n\t// No-op\n\treturn nil\n}\n"
  },
  {
    "path": "search/searcher/ordered_searchers_list.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\ntype OrderedSearcherList []search.Searcher\n\n// sort.Interface\n\nfunc (otrl OrderedSearcherList) Len() int {\n\treturn len(otrl)\n}\n\nfunc (otrl OrderedSearcherList) Less(i, j int) bool {\n\treturn otrl[i].Count() < otrl[j].Count()\n}\n\nfunc (otrl OrderedSearcherList) Swap(i, j int) {\n\totrl[i], otrl[j] = otrl[j], otrl[i]\n}\n\ntype OrderedPositionalSearcherList struct {\n\tsearchers []search.Searcher\n\tindex     []int\n}\n\n// sort.Interface\n\nfunc (otrl OrderedPositionalSearcherList) Len() int {\n\treturn len(otrl.searchers)\n}\n\nfunc (otrl OrderedPositionalSearcherList) Less(i, j int) bool {\n\treturn otrl.searchers[i].Count() < otrl.searchers[j].Count()\n}\n\nfunc (otrl OrderedPositionalSearcherList) Swap(i, j int) {\n\totrl.searchers[i], otrl.searchers[j] = otrl.searchers[j], otrl.searchers[i]\n\totrl.index[i], otrl.index[j] = otrl.index[j], otrl.index[i]\n}\n"
  },
  {
    "path": "search/searcher/search_boolean.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/scorer\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeBooleanSearcher int\n\nfunc init() {\n\tvar bs BooleanSearcher\n\treflectStaticSizeBooleanSearcher = int(reflect.TypeOf(bs).Size())\n}\n\ntype BooleanSearcher struct {\n\tindexReader     index.IndexReader\n\tmustSearcher    search.Searcher\n\tshouldSearcher  search.Searcher\n\tmustNotSearcher search.Searcher\n\tqueryNorm       float64\n\tcurrMust        *search.DocumentMatch\n\tcurrShould      *search.DocumentMatch\n\tcurrMustNot     *search.DocumentMatch\n\tcurrentID       index.IndexInternalID\n\tmin             uint64\n\tscorer          *scorer.ConjunctionQueryScorer\n\tmatches         []*search.DocumentMatch\n\tinitialized     bool\n\tdone            bool\n}\n\nfunc NewBooleanSearcher(ctx context.Context, indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, options search.SearcherOptions) (*BooleanSearcher, error) {\n\t// build our searcher\n\trv := BooleanSearcher{\n\t\tindexReader:     indexReader,\n\t\tmustSearcher:    mustSearcher,\n\t\tshouldSearcher:  shouldSearcher,\n\t\tmustNotSearcher: mustNotSearcher,\n\t\tscorer:          scorer.NewConjunctionQueryScorer(options),\n\t\tmatches:         make([]*search.DocumentMatch, 2),\n\t}\n\trv.computeQueryNorm()\n\treturn &rv, nil\n}\n\nfunc (s *BooleanSearcher) Size() int {\n\tsizeInBytes := reflectStaticSizeBooleanSearcher + size.SizeOfPtr\n\n\tif s.mustSearcher != nil {\n\t\tsizeInBytes += s.mustSearcher.Size()\n\t}\n\n\tif s.shouldSearcher != nil {\n\t\tsizeInBytes += s.shouldSearcher.Size()\n\t}\n\n\tif s.mustNotSearcher != nil {\n\t\tsizeInBytes += s.mustNotSearcher.Size()\n\t}\n\n\tsizeInBytes += s.scorer.Size()\n\n\tfor _, entry := range s.matches {\n\t\tif entry != nil {\n\t\t\tsizeInBytes += entry.Size()\n\t\t}\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (s *BooleanSearcher) computeQueryNorm() {\n\t// first calculate sum of squared weights\n\tsumOfSquaredWeights := 0.0\n\tif s.mustSearcher != nil {\n\t\tsumOfSquaredWeights += s.mustSearcher.Weight()\n\t}\n\tif s.shouldSearcher != nil {\n\t\tsumOfSquaredWeights += s.shouldSearcher.Weight()\n\t}\n\n\t// now compute query norm from this\n\ts.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)\n\t// finally tell all the downstream searchers the norm\n\tif s.mustSearcher != nil {\n\t\ts.mustSearcher.SetQueryNorm(s.queryNorm)\n\t}\n\tif s.shouldSearcher != nil {\n\t\ts.shouldSearcher.SetQueryNorm(s.queryNorm)\n\t}\n}\n\nfunc (s *BooleanSearcher) initSearchers(ctx *search.SearchContext) error {\n\tvar err error\n\t// get all searchers pointing at their first match\n\tif s.mustSearcher != nil {\n\t\tif s.currMust != nil {\n\t\t\tctx.DocumentMatchPool.Put(s.currMust)\n\t\t}\n\t\ts.currMust, err = s.mustSearcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif s.shouldSearcher != nil {\n\t\tif s.currShould != nil {\n\t\t\tctx.DocumentMatchPool.Put(s.currShould)\n\t\t}\n\t\ts.currShould, err = s.shouldSearcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif s.mustNotSearcher != nil {\n\t\tif s.currMustNot != nil {\n\t\t\tctx.DocumentMatchPool.Put(s.currMustNot)\n\t\t}\n\t\ts.currMustNot, err = s.mustNotSearcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif s.mustSearcher != nil && s.currMust != nil {\n\t\ts.currentID = s.currMust.IndexInternalID\n\t} else if s.mustSearcher == nil && s.currShould != nil {\n\t\ts.currentID = s.currShould.IndexInternalID\n\t} else {\n\t\ts.currentID = nil\n\t}\n\n\ts.initialized = true\n\treturn nil\n}\n\nfunc (s *BooleanSearcher) advanceNextMust(ctx *search.SearchContext, skipReturn *search.DocumentMatch) error {\n\tvar err error\n\n\tif s.mustSearcher != nil {\n\t\tif s.currMust != skipReturn {\n\t\t\tctx.DocumentMatchPool.Put(s.currMust)\n\t\t}\n\t\ts.currMust, err = s.mustSearcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tif s.currShould != skipReturn {\n\t\t\tctx.DocumentMatchPool.Put(s.currShould)\n\t\t}\n\t\ts.currShould, err = s.shouldSearcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif s.mustSearcher != nil && s.currMust != nil {\n\t\ts.currentID = s.currMust.IndexInternalID\n\t} else if s.mustSearcher == nil && s.currShould != nil {\n\t\ts.currentID = s.currShould.IndexInternalID\n\t} else {\n\t\ts.currentID = nil\n\t}\n\treturn nil\n}\n\nfunc (s *BooleanSearcher) Weight() float64 {\n\tvar rv float64\n\tif s.mustSearcher != nil {\n\t\trv += s.mustSearcher.Weight()\n\t}\n\tif s.shouldSearcher != nil {\n\t\trv += s.shouldSearcher.Weight()\n\t}\n\n\treturn rv\n}\n\nfunc (s *BooleanSearcher) SetQueryNorm(qnorm float64) {\n\tif s.mustSearcher != nil {\n\t\ts.mustSearcher.SetQueryNorm(qnorm)\n\t}\n\tif s.shouldSearcher != nil {\n\t\ts.shouldSearcher.SetQueryNorm(qnorm)\n\t}\n}\n\nfunc (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\n\tif s.done {\n\t\treturn nil, nil\n\t}\n\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tvar err error\n\tvar rv *search.DocumentMatch\n\n\tfor s.currentID != nil {\n\t\tif s.currMustNot != nil {\n\t\t\tcmp := s.currMustNot.IndexInternalID.Compare(s.currentID)\n\t\t\tif cmp < 0 {\n\t\t\t\tctx.DocumentMatchPool.Put(s.currMustNot)\n\t\t\t\t// advance must not searcher to our candidate entry\n\t\t\t\ts.currMustNot, err = s.mustNotSearcher.Advance(ctx, s.currentID)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tif s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) {\n\t\t\t\t\t// the candidate is excluded\n\t\t\t\t\terr = s.advanceNextMust(ctx, nil)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t} else if cmp == 0 {\n\t\t\t\t// the candidate is excluded\n\t\t\t\terr = s.advanceNextMust(ctx, nil)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tshouldCmpOrNil := 1 // NOTE: shouldCmp will also be 1 when currShould == nil.\n\t\tif s.currShould != nil {\n\t\t\tshouldCmpOrNil = s.currShould.IndexInternalID.Compare(s.currentID)\n\t\t}\n\n\t\tif shouldCmpOrNil < 0 {\n\t\t\tctx.DocumentMatchPool.Put(s.currShould)\n\t\t\t// advance should searcher to our candidate entry\n\t\t\ts.currShould, err = s.shouldSearcher.Advance(ctx, s.currentID)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) {\n\t\t\t\t// score bonus matches should\n\t\t\t\tvar cons []*search.DocumentMatch\n\t\t\t\tif s.currMust != nil {\n\t\t\t\t\tcons = s.matches\n\t\t\t\t\tcons[0] = s.currMust\n\t\t\t\t\tcons[1] = s.currShould\n\t\t\t\t} else {\n\t\t\t\t\tcons = s.matches[0:1]\n\t\t\t\t\tcons[0] = s.currShould\n\t\t\t\t}\n\t\t\t\trv = s.scorer.Score(ctx, cons)\n\t\t\t\terr = s.advanceNextMust(ctx, rv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t} else if s.shouldSearcher.Min() == 0 {\n\t\t\t\t// match is OK anyway\n\t\t\t\tcons := s.matches[0:1]\n\t\t\t\tcons[0] = s.currMust\n\t\t\t\trv = s.scorer.Score(ctx, cons)\n\t\t\t\terr = s.advanceNextMust(ctx, rv)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t} else if shouldCmpOrNil == 0 {\n\t\t\t// score bonus matches should\n\t\t\tvar cons []*search.DocumentMatch\n\t\t\tif s.currMust != nil {\n\t\t\t\tcons = s.matches\n\t\t\t\tcons[0] = s.currMust\n\t\t\t\tcons[1] = s.currShould\n\t\t\t} else {\n\t\t\t\tcons = s.matches[0:1]\n\t\t\t\tcons[0] = s.currShould\n\t\t\t}\n\t\t\trv = s.scorer.Score(ctx, cons)\n\t\t\terr = s.advanceNextMust(ctx, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tbreak\n\t\t} else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 {\n\t\t\t// match is OK anyway\n\t\t\tcons := s.matches[0:1]\n\t\t\tcons[0] = s.currMust\n\t\t\trv = s.scorer.Score(ctx, cons)\n\t\t\terr = s.advanceNextMust(ctx, rv)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\terr = s.advanceNextMust(ctx, nil)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif rv == nil {\n\t\ts.done = true\n\t}\n\n\treturn rv, nil\n}\n\nfunc (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\n\tif s.done {\n\t\treturn nil, nil\n\t}\n\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Advance the searcher only if the cursor is trailing the lookup ID\n\tif s.currentID == nil || s.currentID.Compare(ID) < 0 {\n\t\tvar err error\n\t\tif s.mustSearcher != nil {\n\t\t\tif s.currMust != nil {\n\t\t\t\tctx.DocumentMatchPool.Put(s.currMust)\n\t\t\t}\n\t\t\ts.currMust, err = s.mustSearcher.Advance(ctx, ID)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\n\t\tif s.shouldSearcher != nil {\n\t\t\tif s.currShould != nil {\n\t\t\t\tctx.DocumentMatchPool.Put(s.currShould)\n\t\t\t}\n\t\t\ts.currShould, err = s.shouldSearcher.Advance(ctx, ID)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\n\t\tif s.mustNotSearcher != nil {\n\t\t\t// Additional check for mustNotSearcher, whose cursor isn't tracked by\n\t\t\t// currentID to prevent it from moving when the searcher's tracked\n\t\t\t// position is already ahead of or at the requested ID.\n\t\t\tif s.currMustNot == nil || s.currMustNot.IndexInternalID.Compare(ID) < 0 {\n\t\t\t\tif s.currMustNot != nil {\n\t\t\t\t\tctx.DocumentMatchPool.Put(s.currMustNot)\n\t\t\t\t}\n\t\t\t\ts.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif s.mustSearcher != nil && s.currMust != nil {\n\t\t\ts.currentID = s.currMust.IndexInternalID\n\t\t} else if s.mustSearcher == nil && s.currShould != nil {\n\t\t\ts.currentID = s.currShould.IndexInternalID\n\t\t} else {\n\t\t\ts.currentID = nil\n\t\t}\n\t}\n\n\treturn s.Next(ctx)\n}\n\nfunc (s *BooleanSearcher) Count() uint64 {\n\n\t// for now return a worst case\n\tvar sum uint64\n\tif s.mustSearcher != nil {\n\t\tsum += s.mustSearcher.Count()\n\t}\n\tif s.shouldSearcher != nil {\n\t\tsum += s.shouldSearcher.Count()\n\t}\n\treturn sum\n}\n\nfunc (s *BooleanSearcher) Close() error {\n\tvar err0, err1, err2 error\n\tif s.mustSearcher != nil {\n\t\terr0 = s.mustSearcher.Close()\n\t}\n\tif s.shouldSearcher != nil {\n\t\terr1 = s.shouldSearcher.Close()\n\t}\n\tif s.mustNotSearcher != nil {\n\t\terr2 = s.mustNotSearcher.Close()\n\t}\n\tif err0 != nil {\n\t\treturn err0\n\t}\n\tif err1 != nil {\n\t\treturn err1\n\t}\n\tif err2 != nil {\n\t\treturn err2\n\t}\n\treturn nil\n}\n\nfunc (s *BooleanSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *BooleanSearcher) DocumentMatchPoolSize() int {\n\trv := 3\n\tif s.mustSearcher != nil {\n\t\trv += s.mustSearcher.DocumentMatchPoolSize()\n\t}\n\tif s.shouldSearcher != nil {\n\t\trv += s.shouldSearcher.DocumentMatchPoolSize()\n\t}\n\tif s.mustNotSearcher != nil {\n\t\trv += s.mustNotSearcher.DocumentMatchPoolSize()\n\t}\n\treturn rv\n}\n"
  },
  {
    "path": "search/searcher/search_boolean_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestBooleanSearch(t *testing.T) {\n\tif twoDocIndex == nil {\n\t\tt.Fatal(\"its null\")\n\t}\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\texplainTrue := search.SearcherOptions{Explain: true}\n\n\t// test 0\n\tbeerTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tshouldSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsteveTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"steve\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustNotSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbooleanSearcher, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher, shouldSearcher, mustNotSearcher, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 1\n\tmartyTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tshouldSearcher2, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher2, dustinTermSearcher2}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsteveTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"steve\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustNotSearcher2, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher2}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbooleanSearcher2, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, nil, shouldSearcher2, mustNotSearcher2, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 2\n\tsteveTermSearcher3, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"steve\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustNotSearcher3, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher3}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbooleanSearcher3, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, nil, nil, mustNotSearcher3, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 3\n\tbeerTermSearcher4, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustSearcher4, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher4}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsteveTermSearcher4, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"steve\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustNotSearcher4, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher4}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbooleanSearcher4, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher4, nil, mustNotSearcher4, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 4\n\tbeerTermSearcher5, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustSearcher5, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher5}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsteveTermSearcher5, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"steve\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyTermSearcher5, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustNotSearcher5, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher5, martyTermSearcher5}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbooleanSearcher5, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher5, nil, mustNotSearcher5, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 5\n\tbeerTermSearcher6, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustSearcher6, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher6}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyTermSearcher6, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher6, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tshouldSearcher6, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher6, dustinTermSearcher6}, 2, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbooleanSearcher6, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher6, shouldSearcher6, nil, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 6\n\tbeerTermSearcher7, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustSearcher7, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher7}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbooleanSearcher7, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher7, nil, nil, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyTermSearcher7, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 5.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tconjunctionSearcher7, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher7, booleanSearcher7}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 7\n\tbeerTermSearcher8, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustSearcher8, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher8}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyTermSearcher8, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher8, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tshouldSearcher8, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher8, dustinTermSearcher8}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsteveTermSearcher8, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"steve\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmustNotSearcher8, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher8}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbooleanSearcher8, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher8, shouldSearcher8, mustNotSearcher8, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher8a, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 5.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tconjunctionSearcher8, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{booleanSearcher8, dustinTermSearcher8a}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttests := []struct {\n\t\tsearcher search.Searcher\n\t\tresults  []*search.DocumentMatch\n\t}{\n\t\t{\n\t\t\tsearcher: booleanSearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           0.9818005051949021,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           0.808709699395535,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"4\"),\n\t\t\t\t\tScore:           0.34618161159873423,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: booleanSearcher2,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           0.6775110856165737,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           0.6775110856165737,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// no MUST or SHOULD clauses yields no results\n\t\t{\n\t\t\tsearcher: booleanSearcher3,\n\t\t\tresults:  []*search.DocumentMatch{},\n\t\t},\n\t\t{\n\t\t\tsearcher: booleanSearcher4,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           0.5,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"4\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: booleanSearcher5,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           0.5,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"4\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: booleanSearcher6,\n\t\t\tresults:  []*search.DocumentMatch{},\n\t\t},\n\t\t// test a conjunction query with a nested boolean\n\t\t{\n\t\t\tsearcher: conjunctionSearcher7,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           2.0097428702814377,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: conjunctionSearcher8,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           2.0681575785068107,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor testIndex, test := range tests {\n\t\tdefer func() {\n\t\t\terr := test.searcher.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}()\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := test.searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tif i < len(test.results) {\n\t\t\t\tif !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have id %s got %s for test %d\", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)\n\t\t\t\t}\n\t\t\t\tif !scoresCloseEnough(next.Score, test.results[i].Score) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have score %v got  %v for test %d\", i, test.results[i].Score, next.Score, testIndex)\n\t\t\t\t\tt.Logf(\"scoring explanation: %s\", next.Expl)\n\t\t\t\t}\n\t\t\t}\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = test.searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, testIndex)\n\t\t}\n\t\tif len(test.results) != i {\n\t\t\tt.Errorf(\"expected %d results got %d for test %d\", len(test.results), i, testIndex)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_conjunction.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"math\"\n\t\"reflect\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/scorer\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeConjunctionSearcher int\n\nfunc init() {\n\tvar cs ConjunctionSearcher\n\treflectStaticSizeConjunctionSearcher = int(reflect.TypeOf(cs).Size())\n}\n\ntype ConjunctionSearcher struct {\n\tindexReader index.IndexReader\n\tsearchers   []search.Searcher\n\tqueryNorm   float64\n\tcurrs       []*search.DocumentMatch\n\tmaxIDIdx    int\n\tscorer      *scorer.ConjunctionQueryScorer\n\tinitialized bool\n\toptions     search.SearcherOptions\n\tbytesRead   uint64\n}\n\nfunc NewConjunctionSearcher(ctx context.Context, indexReader index.IndexReader,\n\tqsearchers []search.Searcher, options search.SearcherOptions) (\n\tsearch.Searcher, error,\n) {\n\t// build the sorted downstream searchers\n\tsearchers := make(OrderedSearcherList, len(qsearchers))\n\tcopy(searchers, qsearchers)\n\tsort.Sort(searchers)\n\n\t// attempt the \"unadorned\" conjunction optimization only when we\n\t// do not need extra information like freq-norm's or term vectors\n\tif len(searchers) > 1 &&\n\t\toptions.Score == \"none\" && !options.IncludeTermVectors {\n\t\trv, err := optimizeCompositeSearcher(ctx, \"conjunction:unadorned\",\n\t\t\tindexReader, searchers, options)\n\t\tif err != nil || rv != nil {\n\t\t\treturn rv, err\n\t\t}\n\t}\n\n\t// build our searcher\n\trv := ConjunctionSearcher{\n\t\tindexReader: indexReader,\n\t\toptions:     options,\n\t\tsearchers:   searchers,\n\t\tcurrs:       make([]*search.DocumentMatch, len(searchers)),\n\t\tscorer:      scorer.NewConjunctionQueryScorer(options),\n\t}\n\trv.computeQueryNorm()\n\n\t// attempt push-down conjunction optimization when there's >1 searchers\n\tif len(searchers) > 1 {\n\t\trv, err := optimizeCompositeSearcher(ctx, \"conjunction\",\n\t\t\tindexReader, searchers, options)\n\t\tif err != nil || rv != nil {\n\t\t\treturn rv, err\n\t\t}\n\t}\n\n\treturn &rv, nil\n}\n\nfunc (s *ConjunctionSearcher) computeQueryNorm() {\n\t// first calculate sum of squared weights\n\tsumOfSquaredWeights := 0.0\n\tfor _, searcher := range s.searchers {\n\t\tsumOfSquaredWeights += searcher.Weight()\n\t}\n\t// now compute query norm from this\n\ts.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)\n\t// finally tell all the downstream searchers the norm\n\tfor _, searcher := range s.searchers {\n\t\tsearcher.SetQueryNorm(s.queryNorm)\n\t}\n}\n\nfunc (s *ConjunctionSearcher) Size() int {\n\tsizeInBytes := reflectStaticSizeConjunctionSearcher + size.SizeOfPtr +\n\t\ts.scorer.Size()\n\n\tfor _, entry := range s.searchers {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tfor _, entry := range s.currs {\n\t\tif entry != nil {\n\t\t\tsizeInBytes += entry.Size()\n\t\t}\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (s *ConjunctionSearcher) initSearchers(ctx *search.SearchContext) error {\n\tvar err error\n\t// get all searchers pointing at their first match\n\tfor i, searcher := range s.searchers {\n\t\tif s.currs[i] != nil {\n\t\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t\t}\n\t\ts.currs[i], err = searcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\ts.initialized = true\n\treturn nil\n}\n\nfunc (s *ConjunctionSearcher) Weight() float64 {\n\tvar rv float64\n\tfor _, searcher := range s.searchers {\n\t\trv += searcher.Weight()\n\t}\n\treturn rv\n}\n\nfunc (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) {\n\tfor _, searcher := range s.searchers {\n\t\tsearcher.SetQueryNorm(qnorm)\n\t}\n}\n\nfunc (s *ConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tvar rv *search.DocumentMatch\n\tvar err error\nOUTER:\n\tfor s.maxIDIdx < len(s.currs) && s.currs[s.maxIDIdx] != nil {\n\t\tmaxID := s.currs[s.maxIDIdx].IndexInternalID\n\n\t\ti := 0\n\t\tfor i < len(s.currs) {\n\t\t\tif s.currs[i] == nil {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\n\t\t\tif i == s.maxIDIdx {\n\t\t\t\ti++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tcmp := maxID.Compare(s.currs[i].IndexInternalID)\n\t\t\tif cmp == 0 {\n\t\t\t\ti++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif cmp < 0 {\n\t\t\t\t// maxID < currs[i], so we found a new maxIDIdx\n\t\t\t\ts.maxIDIdx = i\n\n\t\t\t\t// advance the positions where [0 <= x < i], since we\n\t\t\t\t// know they were equal to the former max entry\n\t\t\t\tmaxID = s.currs[s.maxIDIdx].IndexInternalID\n\t\t\t\tfor x := 0; x < i; x++ {\n\t\t\t\t\terr = s.advanceChild(ctx, x, maxID)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tcontinue OUTER\n\t\t\t}\n\n\t\t\t// maxID > currs[i], so need to advance searchers[i]\n\t\t\terr = s.advanceChild(ctx, i, maxID)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\t// don't bump i, so that we'll examine the just-advanced\n\t\t\t// currs[i] again\n\t\t}\n\n\t\t// if we get here, a doc matched all readers, so score and add it\n\t\trv = s.scorer.Score(ctx, s.currs)\n\n\t\t// we know all the searchers are pointing at the same thing\n\t\t// so they all need to be bumped\n\t\tfor i, searcher := range s.searchers {\n\t\t\tif s.currs[i] != rv {\n\t\t\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t\t\t}\n\t\t\ts.currs[i], err = searcher.Next(ctx)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\n\t\t// don't continue now, wait for the next call to Next()\n\t\tbreak\n\t}\n\treturn rv, nil\n}\n\nfunc (s *ConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tfor i := range s.searchers {\n\t\tif s.currs[i] != nil && s.currs[i].IndexInternalID.Compare(ID) >= 0 {\n\t\t\tcontinue\n\t\t}\n\t\terr := s.advanceChild(ctx, i, ID)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn s.Next(ctx)\n}\n\nfunc (s *ConjunctionSearcher) advanceChild(ctx *search.SearchContext, i int, ID index.IndexInternalID) (err error) {\n\tif s.currs[i] != nil {\n\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t}\n\ts.currs[i], err = s.searchers[i].Advance(ctx, ID)\n\treturn err\n}\n\nfunc (s *ConjunctionSearcher) Count() uint64 {\n\t// for now return a worst case\n\tvar sum uint64\n\tfor _, searcher := range s.searchers {\n\t\tsum += searcher.Count()\n\t}\n\treturn sum\n}\n\nfunc (s *ConjunctionSearcher) Close() (rv error) {\n\tfor _, searcher := range s.searchers {\n\t\terr := searcher.Close()\n\t\tif err != nil && rv == nil {\n\t\t\trv = err\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (s *ConjunctionSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *ConjunctionSearcher) DocumentMatchPoolSize() int {\n\trv := len(s.currs)\n\tfor _, s := range s.searchers {\n\t\trv += s.DocumentMatchPoolSize()\n\t}\n\treturn rv\n}\n"
  },
  {
    "path": "search/searcher/search_conjunction_nested.go",
    "content": "//  Copyright (c) 2026 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"slices\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeNestedConjunctionSearcher int\n\nfunc init() {\n\tvar ncs NestedConjunctionSearcher\n\treflectStaticSizeNestedConjunctionSearcher = int(reflect.TypeOf(ncs).Size())\n}\n\ntype NestedConjunctionSearcher struct {\n\tnestedReader  index.NestedReader\n\tsearchers     []search.Searcher\n\tqueryNorm     float64\n\tcurrs         []*search.DocumentMatch\n\tcurrAncestors [][]index.AncestorID\n\tcurrKeys      []index.AncestorID\n\tinitialized   bool\n\tjoinIdx       int\n\toptions       search.SearcherOptions\n\tdocQueue      *CoalesceQueue\n\t// reusable ID buffer for Advance() calls\n\tadvanceID index.IndexInternalID\n\t// reusable buffer for Advance() calls\n\tancestors []index.AncestorID\n}\n\nfunc NewNestedConjunctionSearcher(ctx context.Context, indexReader index.IndexReader,\n\tsearchers []search.Searcher, joinIdx int, options search.SearcherOptions) (search.Searcher, error) {\n\n\tvar nr index.NestedReader\n\tvar ok bool\n\tif nr, ok = indexReader.(index.NestedReader); !ok {\n\t\treturn nil, fmt.Errorf(\"indexReader does not support nested documents\")\n\t}\n\n\t// build our searcher\n\trv := NestedConjunctionSearcher{\n\t\tnestedReader:  nr,\n\t\toptions:       options,\n\t\tsearchers:     searchers,\n\t\tcurrs:         make([]*search.DocumentMatch, len(searchers)),\n\t\tcurrAncestors: make([][]index.AncestorID, len(searchers)),\n\t\tcurrKeys:      make([]index.AncestorID, len(searchers)),\n\t\tjoinIdx:       joinIdx,\n\t\tdocQueue:      NewCoalesceQueue(),\n\t}\n\trv.computeQueryNorm()\n\n\treturn &rv, nil\n}\n\nfunc (s *NestedConjunctionSearcher) computeQueryNorm() {\n\t// first calculate sum of squared weights\n\tsumOfSquaredWeights := 0.0\n\tfor _, searcher := range s.searchers {\n\t\tsumOfSquaredWeights += searcher.Weight()\n\t}\n\t// now compute query norm from this\n\ts.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)\n\t// finally tell all the downstream searchers the norm\n\tfor _, searcher := range s.searchers {\n\t\tsearcher.SetQueryNorm(s.queryNorm)\n\t}\n}\n\nfunc (s *NestedConjunctionSearcher) Size() int {\n\tsizeInBytes := reflectStaticSizeNestedConjunctionSearcher + size.SizeOfPtr\n\n\tfor _, entry := range s.searchers {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tfor _, entry := range s.currs {\n\t\tif entry != nil {\n\t\t\tsizeInBytes += entry.Size()\n\t\t}\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (s *NestedConjunctionSearcher) Weight() float64 {\n\tvar rv float64\n\tfor _, searcher := range s.searchers {\n\t\trv += searcher.Weight()\n\t}\n\treturn rv\n}\n\nfunc (s *NestedConjunctionSearcher) SetQueryNorm(qnorm float64) {\n\tfor _, searcher := range s.searchers {\n\t\tsearcher.SetQueryNorm(qnorm)\n\t}\n}\n\nfunc (s *NestedConjunctionSearcher) Count() uint64 {\n\t// for now return a worst case\n\tvar sum uint64\n\tfor _, searcher := range s.searchers {\n\t\tsum += searcher.Count()\n\t}\n\treturn sum\n}\n\nfunc (s *NestedConjunctionSearcher) Close() (rv error) {\n\tfor _, searcher := range s.searchers {\n\t\terr := searcher.Close()\n\t\tif err != nil && rv == nil {\n\t\t\trv = err\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (s *NestedConjunctionSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *NestedConjunctionSearcher) DocumentMatchPoolSize() int {\n\trv := len(s.currs)\n\tfor _, s := range s.searchers {\n\t\trv += s.DocumentMatchPoolSize()\n\t}\n\treturn rv\n}\n\nfunc (s *NestedConjunctionSearcher) initialize(ctx *search.SearchContext) (bool, error) {\n\tvar err error\n\tfor i, searcher := range s.searchers {\n\t\tif s.currs[i] != nil {\n\t\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t\t}\n\t\ts.currs[i], err = searcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tif s.currs[i] == nil {\n\t\t\t// one of the searchers is exhausted, so we are done\n\t\t\treturn true, nil\n\t\t}\n\t\t// get the ancestry chain for this match\n\t\ts.currAncestors[i], err = s.nestedReader.Ancestors(s.currs[i].IndexInternalID, s.currAncestors[i][:0])\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\t// check if the ancestry chain is > joinIdx, if not we reset the joinIdx\n\t\t// to the minimum possible value across all searchers, ideally this will be\n\t\t// done in query construction time itself, by using the covering depth across\n\t\t// all sub-queries, but we do this here as a fallback\n\t\tif s.joinIdx >= len(s.currAncestors[i]) {\n\t\t\ts.joinIdx = len(s.currAncestors[i]) - 1\n\t\t}\n\t}\n\t// build currKeys for each searcher, do it here as we may have adjusted joinIdx\n\tfor i := range s.searchers {\n\t\ts.currKeys[i] = ancestorFromRoot(s.currAncestors[i], s.joinIdx)\n\t}\n\ts.initialized = true\n\treturn false, nil\n}\n\nfunc (s *NestedConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\t// initialize on first call to Next, by getting first match\n\t// from each searcher and their ancestry chains\n\tif !s.initialized {\n\t\tdone, err := s.initialize(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif done {\n\t\t\treturn nil, nil\n\t\t}\n\t}\n\t// check if the docQueue has any buffered matches\n\tif s.docQueue.Len() > 0 {\n\t\treturn s.docQueue.Dequeue(ctx), nil\n\t}\n\t// now enter the main alignment loop\n\tn := len(s.searchers)\nOUTER:\n\tfor {\n\t\t// pick the pivot searcher with the highest key (ancestor at joinIdx level)\n\t\tif s.currs[0] == nil {\n\t\t\treturn nil, nil\n\t\t}\n\t\tmaxKey := s.currKeys[0]\n\t\tfor i := 1; i < n; i++ {\n\t\t\t// currs[i] is nil means one of the searchers is exhausted\n\t\t\tif s.currs[i] == nil {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\t\t\tcurrKey := s.currKeys[i]\n\t\t\tif maxKey.Compare(currKey) < 0 {\n\t\t\t\tmaxKey = currKey\n\t\t\t}\n\t\t}\n\t\t// store maxkey as advanceID only once only if needed\n\t\tvar advanceID index.IndexInternalID\n\t\t// flag to track if all searchers are aligned\n\t\tvar aligned bool = true\n\t\t// now try to align all other searchers to the\n\t\t// we check if the a searchers key matches maxKey\n\t\t// if not, we advance the pivot searcher to maxKey\n\t\t// else do nothing and move to the next searcher\n\t\tfor i := 0; i < n; i++ {\n\t\t\tcmp := s.currKeys[i].Compare(maxKey)\n\t\t\tif cmp < 0 {\n\t\t\t\t// not aligned, so advance this searcher to maxKey\n\t\t\t\t// convert maxKey to advanceID only once\n\t\t\t\tif advanceID == nil {\n\t\t\t\t\tadvanceID = s.toAdvanceID(maxKey)\n\t\t\t\t}\n\t\t\t\tvar err error\n\t\t\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t\t\t\ts.currs[i], err = s.searchers[i].Advance(ctx, advanceID)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tif s.currs[i] == nil {\n\t\t\t\t\t// one of the searchers is exhausted, so we are done\n\t\t\t\t\treturn nil, nil\n\t\t\t\t}\n\t\t\t\t// recalc ancestors\n\t\t\t\ts.currAncestors[i], err = s.nestedReader.Ancestors(s.currs[i].IndexInternalID, s.currAncestors[i][:0])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\t// recalc key\n\t\t\t\ts.currKeys[i] = ancestorFromRoot(s.currAncestors[i], s.joinIdx)\n\t\t\t\t// recalc cmp\n\t\t\t\tcmp = s.currKeys[i].Compare(maxKey)\n\t\t\t}\n\t\t\tif cmp != 0 {\n\t\t\t\t// not aligned\n\t\t\t\taligned = false\n\t\t\t}\n\t\t}\n\t\t// now check if all the searchers are aligned at the same maxKey\n\t\t// if they are not aligned, we need to restart the loop of picking\n\t\t// the pivot searcher with the highest key\n\t\tif !aligned {\n\t\t\tcontinue OUTER\n\t\t}\n\t\t// if we are here, all the searchers are aligned at maxKey\n\t\t// now we need to buffer all the intermediate matches for every\n\t\t// searcher at this key, until either the searcher's key changes\n\t\t// or the searcher is exhausted\n\t\tvar err error\n\t\tfor i := 0; i < n; i++ {\n\t\t\tfor {\n\t\t\t\t// buffer the current match\n\t\t\t\ts.docQueue.Enqueue(s.currs[i])\n\t\t\t\t// advance to next match\n\t\t\t\ts.currs[i], err = s.searchers[i].Next(ctx)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tif s.currs[i] == nil {\n\t\t\t\t\t// searcher exhausted, break out\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\t// recalc ancestors\n\t\t\t\ts.currAncestors[i], err = s.nestedReader.Ancestors(s.currs[i].IndexInternalID, s.currAncestors[i][:0])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\t// recalc key\n\t\t\t\ts.currKeys[i] = ancestorFromRoot(s.currAncestors[i], s.joinIdx)\n\t\t\t\t// check if key has changed\n\t\t\t\tif !s.currKeys[i].Equals(maxKey) {\n\t\t\t\t\t// key changed, break out\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// finalize the docQueue for dequeueing\n\t\ts.docQueue.Finalize()\n\t\t// finally return the first buffered match\n\t\treturn s.docQueue.Dequeue(ctx), nil\n\t}\n}\n\n// ancestorFromRoot gets the AncestorID at the given position from the root\n// if pos is 0, it returns the root AncestorID, and so on\nfunc ancestorFromRoot(ancestors []index.AncestorID, pos int) index.AncestorID {\n\treturn ancestors[len(ancestors)-pos-1]\n}\n\n// toAdvanceID converts an AncestorID to IndexInternalID, reusing the advanceID buffer.\n// The returned ID is safe to pass to Advance() since Advance() never retains references.\nfunc (s *NestedConjunctionSearcher) toAdvanceID(key index.AncestorID) index.IndexInternalID {\n\t// Reset length to 0 while preserving capacity for buffer reuse\n\ts.advanceID = s.advanceID[:0]\n\t// Convert key to IndexInternalID, reusing the underlying buffer\n\ts.advanceID = key.ToIndexInternalID(s.advanceID)\n\treturn s.advanceID\n}\n\nfunc (s *NestedConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\tif !s.initialized {\n\t\tdone, err := s.initialize(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif done {\n\t\t\treturn nil, nil\n\t\t}\n\t}\n\t// first check if the docQueue has any buffered matches\n\t// if so we first check if any of them can satisfy the Advance(ID)\n\tfor s.docQueue.Len() > 0 {\n\t\tdm := s.docQueue.Dequeue(ctx)\n\t\tif dm.IndexInternalID.Compare(ID) >= 0 {\n\t\t\treturn dm, nil\n\t\t}\n\t\t// otherwise recycle this match\n\t\tctx.DocumentMatchPool.Put(dm)\n\t}\n\tvar err error\n\t// now we first get the ancestry chain for the given ID\n\ts.ancestors, err = s.nestedReader.Ancestors(ID, s.ancestors[:0])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// we now follow the the following logic for each searcher:\n\t// let S be the length of the ancestry chain for the searcher\n\t// let I be the length of the ancestry chain for the given ID\n\t// 1. if S > I:\n\t//    then we just Advance() the searcher to the given ID if required\n\t// 2. else if S <= I:\n\t//    then we get the AncestorID at position (S - 1) from the root of\n\t//    the given ID's ancestry chain, and Advance() the searcher to\n\t//    it if required\n\tfor i, searcher := range s.searchers {\n\t\tif s.currs[i] == nil {\n\t\t\treturn nil, nil // already exhausted, nothing to do\n\t\t}\n\t\tvar targetID index.IndexInternalID\n\t\tS := len(s.currAncestors[i])\n\t\tI := len(s.ancestors)\n\t\tif S > I {\n\t\t\t// case 1: S > I\n\t\t\ttargetID = ID\n\t\t} else {\n\t\t\t// case 2: S <= I\n\t\t\ttargetID = s.toAdvanceID(ancestorFromRoot(s.ancestors, S-1))\n\t\t}\n\t\tif s.currs[i].IndexInternalID.Compare(targetID) < 0 {\n\t\t\t// need to advance this searcher\n\t\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t\t\ts.currs[i], err = searcher.Advance(ctx, targetID)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif s.currs[i] == nil {\n\t\t\t\t// one of the searchers is exhausted, so we are done\n\t\t\t\treturn nil, nil\n\t\t\t}\n\t\t\t// recalc ancestors\n\t\t\ts.currAncestors[i], err = s.nestedReader.Ancestors(s.currs[i].IndexInternalID, s.currAncestors[i][:0])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\t// recalc key\n\t\t\ts.currKeys[i] = ancestorFromRoot(s.currAncestors[i], s.joinIdx)\n\t\t}\n\t}\n\t// we need to call Next() in a loop until we reach or exceed the given ID\n\t// the Next() call basically gives us a match that is aligned correctly, but\n\t// if joinIdx < I, we can have multiple matches for the same joinIdx ancestor\n\t// and they may be < ID, so we need to loop\n\tfor {\n\t\tnext, err := s.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif next == nil {\n\t\t\treturn nil, nil\n\t\t}\n\t\tif next.IndexInternalID.Compare(ID) >= 0 {\n\t\t\treturn next, nil\n\t\t}\n\t\tctx.DocumentMatchPool.Put(next)\n\t}\n}\n\n// ------------------------------------------------------------------------------------------\ntype CoalesceQueue struct {\n\torder []*search.DocumentMatch // queue of DocumentMatch\n}\n\nfunc NewCoalesceQueue() *CoalesceQueue {\n\tcq := &CoalesceQueue{\n\t\torder: make([]*search.DocumentMatch, 0),\n\t}\n\treturn cq\n}\n\n// Enqueue appends the given DocumentMatch to the queue. Coalescing of duplicates\n// is deferred until Dequeue, after Finalize has sorted items by IndexInternalID.\nfunc (cq *CoalesceQueue) Enqueue(it *search.DocumentMatch) {\n\t// append to order slice (this is a stack)\n\tcq.order = append(cq.order, it)\n}\n\n// Finalize prepares the queue for dequeue operations by sorting the items based on\n// their IndexInternalID values. This MUST be called before any Dequeue operations,\n// and after all Enqueue operations are complete. The sort is done in descending order\n// so that dequeueing will basically be popping from the end of the slice, allowing for\n// slice reuse.\nfunc (cq *CoalesceQueue) Finalize() {\n\tslices.SortFunc(cq.order, func(a, b *search.DocumentMatch) int {\n\t\treturn b.IndexInternalID.Compare(a.IndexInternalID)\n\t})\n}\n\n// Dequeue removes and returns the next DocumentMatch in sorted order, merging any\n// consecutive duplicates. Merged items are recycled via ctx.DocumentMatchPool.\n// Returns nil when the queue is empty.\nfunc (cq *CoalesceQueue) Dequeue(ctx *search.SearchContext) *search.DocumentMatch {\n\tif cq.Len() == 0 {\n\t\treturn nil\n\t}\n\n\t// pop from end of slice\n\trv := cq.order[len(cq.order)-1]\n\tcq.order = cq.order[:len(cq.order)-1]\n\n\t// merge duplicates\n\tfor cq.Len() > 0 {\n\t\t// peek at next item\n\t\tnext := cq.order[len(cq.order)-1]\n\t\tif !rv.IndexInternalID.Equals(next.IndexInternalID) {\n\t\t\t// different ID, stop merging\n\t\t\tbreak\n\t\t}\n\t\t// pop the next item\n\t\tcq.order = cq.order[:len(cq.order)-1]\n\t\t// same ID, merge\n\t\trv.Score += next.Score\n\t\trv.Expl = rv.Expl.MergeWith(next.Expl)\n\t\trv.FieldTermLocations = search.MergeFieldTermLocationsFromMatch(\n\t\t\trv.FieldTermLocations, next)\n\t\t// recycle the merged item\n\t\tctx.DocumentMatchPool.Put(next)\n\t}\n\n\treturn rv\n}\n\n// Len returns the number of DocumentMatch items currently in the queue.\nfunc (cq *CoalesceQueue) Len() int {\n\treturn len(cq.order)\n}\n"
  },
  {
    "path": "search/searcher/search_conjunction_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestConjunctionSearch(t *testing.T) {\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\texplainTrue := search.SearcherOptions{Explain: true}\n\n\t// test 0\n\tbeerTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 5.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbeerAndMartySearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher, martyTermSearcher}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 1\n\tangstTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"angst\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbeerTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tangstAndBeerSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{angstTermSearcher, beerTermSearcher2}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 2\n\tbeerTermSearcher3, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tjackTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"jack\", \"name\", 5.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbeerAndJackSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher3, jackTermSearcher}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 3\n\tbeerTermSearcher4, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmisterTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"mister\", \"title\", 5.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbeerAndMisterSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher4, misterTermSearcher}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 4\n\tcouchbaseTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"couchbase\", \"street\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmisterTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"mister\", \"title\", 5.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcouchbaseAndMisterSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{couchbaseTermSearcher, misterTermSearcher2}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// test 5\n\tbeerTermSearcher5, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"beer\", \"desc\", 5.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcouchbaseTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"couchbase\", \"street\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmisterTermSearcher3, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"mister\", \"title\", 5.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcouchbaseAndMisterSearcher2, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{couchbaseTermSearcher2, misterTermSearcher3}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tbeerAndCouchbaseAndMisterSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher5, couchbaseAndMisterSearcher2}, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttests := []struct {\n\t\tsearcher search.Searcher\n\t\tresults  []*search.DocumentMatch\n\t}{\n\t\t{\n\t\t\tsearcher: beerAndMartySearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           2.0097428702814377,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: angstAndBeerSearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"2\"),\n\t\t\t\t\tScore:           1.0807601687084403,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: beerAndJackSearcher,\n\t\t\tresults:  []*search.DocumentMatch{},\n\t\t},\n\t\t{\n\t\t\tsearcher: beerAndMisterSearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"2\"),\n\t\t\t\t\tScore:           1.2877980334016337,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           1.2877980334016337,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: couchbaseAndMisterSearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"2\"),\n\t\t\t\t\tScore:           1.4436599157093672,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: beerAndCouchbaseAndMisterSearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"2\"),\n\t\t\t\t\tScore:           1.441614953806971,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor testIndex, test := range tests {\n\t\tdefer func() {\n\t\t\terr := test.searcher.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}()\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(10, 0),\n\t\t}\n\t\tnext, err := test.searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tif i < len(test.results) {\n\t\t\t\tif !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have id %s got %s for test %d\", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)\n\t\t\t\t}\n\t\t\t\tif !scoresCloseEnough(next.Score, test.results[i].Score) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have score %v got  %v for test %d\", i, test.results[i].Score, next.Score, testIndex)\n\t\t\t\t\tt.Logf(\"scoring explanation: %s\", next.Expl)\n\t\t\t\t}\n\t\t\t}\n\t\t\tnext, err = test.searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, testIndex)\n\t\t}\n\t\tif len(test.results) != i {\n\t\t\tt.Errorf(\"expected %d results got %d for test %d\", len(test.results), i, testIndex)\n\t\t}\n\t}\n}\n\ntype compositeSearchOptimizationTest struct {\n\tfieldTerms  []string\n\texpectEmpty string\n}\n\nfunc TestScorchCompositeSearchOptimizations(t *testing.T) {\n\tdir, _ := os.MkdirTemp(\"\", \"scorchTwoDoc\")\n\tdefer func() {\n\t\t_ = os.RemoveAll(dir)\n\t}()\n\n\ttwoDocIndex := initTwoDocScorch(dir)\n\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttests := []compositeSearchOptimizationTest{\n\t\t{\n\t\t\tfieldTerms:  []string{},\n\t\t\texpectEmpty: \"conjunction,disjunction\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:marty\"},\n\t\t\texpectEmpty: \"\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:marty\", \"desc:beer\"},\n\t\t\texpectEmpty: \"\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:marty\", \"name:marty\"},\n\t\t\texpectEmpty: \"\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:marty\", \"desc:beer\", \"title:mister\", \"street:couchbase\"},\n\t\t\texpectEmpty: \"conjunction\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:steve\", \"desc:beer\", \"title:mister\", \"street:couchbase\"},\n\t\t\texpectEmpty: \"\",\n\t\t},\n\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:NotARealName\"},\n\t\t\texpectEmpty: \"conjunction,disjunction\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:NotARealName\", \"name:marty\"},\n\t\t\texpectEmpty: \"conjunction\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:NotARealName\", \"name:marty\", \"desc:beer\"},\n\t\t\texpectEmpty: \"conjunction\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:NotARealName\", \"name:marty\", \"name:marty\"},\n\t\t\texpectEmpty: \"conjunction\",\n\t\t},\n\t\t{\n\t\t\tfieldTerms:  []string{\"name:NotARealName\", \"name:marty\", \"desc:beer\", \"title:mister\", \"street:couchbase\"},\n\t\t\texpectEmpty: \"conjunction\",\n\t\t},\n\t}\n\n\t// The theme of this unit test is that given one of the above\n\t// search test cases -- no matter what searcher options we\n\t// provide, across either conjunctions or disjunctions, whether we\n\t// have optimizations that are enabled or disabled, the set of doc\n\t// ID's from the search results from any of those combinations\n\t// should be the same.\n\tsearcherOptionsToCompare := []search.SearcherOptions{\n\t\t{},\n\t\t{Explain: true},\n\t\t{IncludeTermVectors: true},\n\t\t{IncludeTermVectors: true, Explain: true},\n\t\t{Score: \"none\"},\n\t\t{Score: \"none\", IncludeTermVectors: true},\n\t\t{Score: \"none\", IncludeTermVectors: true, Explain: true},\n\t\t{Score: \"none\", Explain: true},\n\t}\n\n\ttestScorchCompositeSearchOptimizations(t, twoDocIndexReader, tests,\n\t\tsearcherOptionsToCompare, \"conjunction\")\n\n\ttestScorchCompositeSearchOptimizations(t, twoDocIndexReader, tests,\n\t\tsearcherOptionsToCompare, \"disjunction\")\n}\n\nfunc testScorchCompositeSearchOptimizations(t *testing.T, indexReader index.IndexReader,\n\ttests []compositeSearchOptimizationTest,\n\tsearcherOptionsToCompare []search.SearcherOptions,\n\tcompositeKind string,\n) {\n\tfor testi := range tests {\n\t\tresultsToCompare := map[string]bool{}\n\n\t\ttestScorchCompositeSearchOptimizationsHelper(t, indexReader, tests, testi,\n\t\t\tsearcherOptionsToCompare, compositeKind, false, resultsToCompare)\n\n\t\ttestScorchCompositeSearchOptimizationsHelper(t, indexReader, tests, testi,\n\t\t\tsearcherOptionsToCompare, compositeKind, true, resultsToCompare)\n\t}\n}\n\nfunc testScorchCompositeSearchOptimizationsHelper(\n\tt *testing.T, indexReader index.IndexReader,\n\ttests []compositeSearchOptimizationTest, testi int,\n\tsearcherOptionsToCompare []search.SearcherOptions,\n\tcompositeKind string, allowOptimizations bool, resultsToCompare map[string]bool,\n) {\n\t// Save the global allowed optimization settings to restore later.\n\toptimizeConjunction := scorch.OptimizeConjunction\n\toptimizeConjunctionUnadorned := scorch.OptimizeConjunctionUnadorned\n\toptimizeDisjunctionUnadorned := scorch.OptimizeDisjunctionUnadorned\n\toptimizeDisjunctionUnadornedMinChildCardinality := scorch.OptimizeDisjunctionUnadornedMinChildCardinality\n\n\tscorch.OptimizeConjunction = allowOptimizations\n\tscorch.OptimizeConjunctionUnadorned = allowOptimizations\n\tscorch.OptimizeDisjunctionUnadorned = allowOptimizations\n\n\tif allowOptimizations {\n\t\tscorch.OptimizeDisjunctionUnadornedMinChildCardinality = uint64(0)\n\t}\n\n\tdefer func() {\n\t\tscorch.OptimizeConjunction = optimizeConjunction\n\t\tscorch.OptimizeConjunctionUnadorned = optimizeConjunctionUnadorned\n\t\tscorch.OptimizeDisjunctionUnadorned = optimizeDisjunctionUnadorned\n\t\tscorch.OptimizeDisjunctionUnadornedMinChildCardinality = optimizeDisjunctionUnadornedMinChildCardinality\n\t}()\n\n\ttest := tests[testi]\n\n\tfor searcherOptionsI, searcherOptions := range searcherOptionsToCompare {\n\t\t// Construct the leaf term searchers.\n\t\tvar searchers []search.Searcher\n\n\t\tfor _, fieldTerm := range test.fieldTerms {\n\t\t\tft := strings.Split(fieldTerm, \":\")\n\t\t\tfield := ft[0]\n\t\t\tterm := ft[1]\n\n\t\t\tsearcher, err := NewTermSearcher(context.TODO(), indexReader, term, field, 1.0, searcherOptions)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tsearchers = append(searchers, searcher)\n\t\t}\n\n\t\t// Construct the composite searcher.\n\t\tvar cs search.Searcher\n\t\tvar err error\n\t\tif compositeKind == \"conjunction\" {\n\t\t\tcs, err = NewConjunctionSearcher(context.TODO(), indexReader, searchers, searcherOptions)\n\t\t} else {\n\t\t\tcs, err = NewDisjunctionSearcher(context.TODO(), indexReader, searchers, 0, searcherOptions)\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(10, 0),\n\t\t}\n\n\t\tnext, err := cs.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tdocID, err := indexReader.ExternalID(next.IndexInternalID)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif searcherOptionsI == 0 && allowOptimizations == false {\n\t\t\t\tresultsToCompare[string(docID)] = true\n\t\t\t} else {\n\t\t\t\tif !resultsToCompare[string(docID)] {\n\t\t\t\t\tt.Errorf(\"missing %s\", string(docID))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tnext, err = cs.Next(ctx)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"error iterating searcher: %v\", err)\n\t\t\t}\n\n\t\t\ti++\n\t\t}\n\n\t\tif i != len(resultsToCompare) {\n\t\t\tt.Errorf(\"mismatched count, %d vs %d\", i, len(resultsToCompare))\n\t\t}\n\n\t\tif i == 0 && !strings.Contains(test.expectEmpty, compositeKind) {\n\t\t\tt.Errorf(\"testi: %d, compositeKind: %s, allowOptimizations: %t,\"+\n\t\t\t\t\" searcherOptionsI: %d, searcherOptions: %#v,\"+\n\t\t\t\t\" expected some results but got no results on test: %#v\",\n\t\t\t\ttesti, compositeKind, allowOptimizations,\n\t\t\t\tsearcherOptionsI, searcherOptions, test)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_disjunction.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\n// DisjunctionMaxClauseCount is a compile time setting that applications can\n// adjust to non-zero value to cause the DisjunctionSearcher to return an\n// error instead of executing searches when the size exceeds this value.\nvar DisjunctionMaxClauseCount = 0\n\n// DisjunctionHeapTakeover is a compile time setting that applications can\n// adjust to control when the DisjunctionSearcher will switch from a simple\n// slice implementation to a heap implementation.\nvar DisjunctionHeapTakeover = 10\n\nfunc NewDisjunctionSearcher(ctx context.Context, indexReader index.IndexReader,\n\tqsearchers []search.Searcher, min float64, options search.SearcherOptions) (\n\tsearch.Searcher, error) {\n\treturn newDisjunctionSearcher(ctx, indexReader, qsearchers, min, options, true)\n}\n\nfunc optionsDisjunctionOptimizable(options search.SearcherOptions) bool {\n\trv := options.Score == \"none\" && !options.IncludeTermVectors\n\treturn rv\n}\n\nfunc newDisjunctionSearcher(ctx context.Context, indexReader index.IndexReader,\n\tqsearchers []search.Searcher, min float64, options search.SearcherOptions,\n\tlimit bool) (search.Searcher, error) {\n\n\tvar disjOverKNN bool\n\tif ctx != nil {\n\t\tdisjOverKNN, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool)\n\t}\n\tif disjOverKNN {\n\t\t// The KNN Searcher optimization is a necessary pre-req for the KNN Searchers,\n\t\t// not an optional optimization like for, say term searchers.\n\t\t// It's an optimization to repeat search an open vector index when applicable,\n\t\t// rather than individually opening and searching a vector index.\n\t\terr := optimizeKNN(ctx, indexReader, qsearchers)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\t// attempt the \"unadorned\" disjunction optimization only when we\n\t\t// do not need extra information like freq-norm's or term vectors\n\t\t// and the requested min is simple\n\t\tif len(qsearchers) > 1 && min <= 1 &&\n\t\t\toptionsDisjunctionOptimizable(options) {\n\t\t\trv, err := optimizeCompositeSearcher(ctx, \"disjunction:unadorned\",\n\t\t\t\tindexReader, qsearchers, options)\n\t\t\tif err != nil || rv != nil {\n\t\t\t\treturn rv, err\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(qsearchers) > DisjunctionHeapTakeover {\n\t\treturn newDisjunctionHeapSearcher(ctx, indexReader, qsearchers, min, options,\n\t\t\tlimit)\n\t}\n\treturn newDisjunctionSliceSearcher(ctx, indexReader, qsearchers, min, options,\n\t\tlimit)\n}\n\nfunc optimizeCompositeSearcher(ctx context.Context, optimizationKind string,\n\tindexReader index.IndexReader, qsearchers []search.Searcher,\n\toptions search.SearcherOptions) (search.Searcher, error) {\n\tvar octx index.OptimizableContext\n\n\tfor _, searcher := range qsearchers {\n\t\to, ok := searcher.(index.Optimizable)\n\t\tif !ok {\n\t\t\treturn nil, nil\n\t\t}\n\n\t\tvar err error\n\t\toctx, err = o.Optimize(optimizationKind, octx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif octx == nil {\n\t\t\treturn nil, nil\n\t\t}\n\t}\n\n\toptimized, err := octx.Finish()\n\tif err != nil || optimized == nil {\n\t\treturn nil, err\n\t}\n\n\ttfr, ok := optimized.(index.TermFieldReader)\n\tif !ok {\n\t\treturn nil, nil\n\t}\n\n\treturn newTermSearcherFromReader(ctx, indexReader, tfr,\n\t\t[]byte(optimizationKind), \"*\", 1.0, options)\n}\n\nfunc tooManyClauses(count int) bool {\n\tif DisjunctionMaxClauseCount != 0 && count > DisjunctionMaxClauseCount {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc tooManyClausesErr(field string, count int) error {\n\treturn fmt.Errorf(\"TooManyClauses over field: `%s` [%d > maxClauseCount,\"+\n\t\t\" which is set to %d]\", field, count, DisjunctionMaxClauseCount)\n}\n"
  },
  {
    "path": "search/searcher/search_disjunction_heap.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"container/heap\"\n\t\"context\"\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/scorer\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeDisjunctionHeapSearcher int\nvar reflectStaticSizeSearcherCurr int\n\nfunc init() {\n\tvar dhs DisjunctionHeapSearcher\n\treflectStaticSizeDisjunctionHeapSearcher = int(reflect.TypeOf(dhs).Size())\n\n\tvar sc SearcherCurr\n\treflectStaticSizeSearcherCurr = int(reflect.TypeOf(sc).Size())\n}\n\ntype SearcherCurr struct {\n\tsearcher    search.Searcher\n\tcurr        *search.DocumentMatch\n\tmatchingIdx int\n}\n\ntype DisjunctionHeapSearcher struct {\n\tindexReader index.IndexReader\n\n\tnumSearchers           int\n\tscorer                 *scorer.DisjunctionQueryScorer\n\tmin                    int\n\tqueryNorm              float64\n\tretrieveScoreBreakdown bool\n\tinitialized            bool\n\tsearchers              []search.Searcher\n\theap                   []*SearcherCurr\n\n\tmatching      []*search.DocumentMatch\n\tmatchingIdxs  []int\n\tmatchingCurrs []*SearcherCurr\n\n\tbytesRead uint64\n}\n\nfunc newDisjunctionHeapSearcher(ctx context.Context, indexReader index.IndexReader,\n\tsearchers []search.Searcher, min float64, options search.SearcherOptions,\n\tlimit bool) (\n\t*DisjunctionHeapSearcher, error) {\n\tif limit && tooManyClauses(len(searchers)) {\n\t\treturn nil, tooManyClausesErr(\"\", len(searchers))\n\t}\n\tvar retrieveScoreBreakdown bool\n\tif ctx != nil {\n\t\tretrieveScoreBreakdown, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool)\n\t}\n\n\t// build our searcher\n\trv := DisjunctionHeapSearcher{\n\t\tindexReader:            indexReader,\n\t\tsearchers:              searchers,\n\t\tnumSearchers:           len(searchers),\n\t\tscorer:                 scorer.NewDisjunctionQueryScorer(options),\n\t\tmin:                    int(min),\n\t\tmatching:               make([]*search.DocumentMatch, len(searchers)),\n\t\tmatchingCurrs:          make([]*SearcherCurr, len(searchers)),\n\t\tmatchingIdxs:           make([]int, len(searchers)),\n\t\tretrieveScoreBreakdown: retrieveScoreBreakdown,\n\t\theap:                   make([]*SearcherCurr, 0, len(searchers)),\n\t}\n\trv.computeQueryNorm()\n\treturn &rv, nil\n}\n\nfunc (s *DisjunctionHeapSearcher) computeQueryNorm() {\n\t// first calculate sum of squared weights\n\tsumOfSquaredWeights := 0.0\n\tfor _, searcher := range s.searchers {\n\t\tsumOfSquaredWeights += searcher.Weight()\n\t}\n\t// now compute query norm from this\n\ts.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)\n\t// finally tell all the downstream searchers the norm\n\tfor _, searcher := range s.searchers {\n\t\tsearcher.SetQueryNorm(s.queryNorm)\n\t}\n}\n\nfunc (s *DisjunctionHeapSearcher) Size() int {\n\tsizeInBytes := reflectStaticSizeDisjunctionHeapSearcher + size.SizeOfPtr +\n\t\ts.scorer.Size()\n\n\tfor _, entry := range s.searchers {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tfor _, entry := range s.matching {\n\t\tif entry != nil {\n\t\t\tsizeInBytes += entry.Size()\n\t\t}\n\t}\n\n\t// for matchingCurrs and heap, just use static size * len\n\t// since searchers and document matches already counted above\n\tsizeInBytes += len(s.matchingCurrs) * reflectStaticSizeSearcherCurr\n\tsizeInBytes += len(s.heap) * reflectStaticSizeSearcherCurr\n\tsizeInBytes += len(s.matchingIdxs) * size.SizeOfInt\n\n\treturn sizeInBytes\n}\n\nfunc (s *DisjunctionHeapSearcher) initSearchers(ctx *search.SearchContext) error {\n\t// alloc a single block of SearcherCurrs\n\tblock := make([]SearcherCurr, len(s.searchers))\n\n\t// get all searchers pointing at their first match\n\tfor i, searcher := range s.searchers {\n\t\tcurr, err := searcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif curr != nil {\n\t\t\tblock[i].searcher = searcher\n\t\t\tblock[i].curr = curr\n\t\t\tblock[i].matchingIdx = i\n\t\t\theap.Push(s, &block[i])\n\t\t}\n\t}\n\n\terr := s.updateMatches()\n\tif err != nil {\n\t\treturn err\n\t}\n\ts.initialized = true\n\treturn nil\n}\n\nfunc (s *DisjunctionHeapSearcher) updateMatches() error {\n\tmatching := s.matching[:0]\n\tmatchingCurrs := s.matchingCurrs[:0]\n\tmatchingIdxs := s.matchingIdxs[:0]\n\n\tif len(s.heap) > 0 {\n\n\t\t// top of the heap is our next hit\n\t\tnext := heap.Pop(s).(*SearcherCurr)\n\t\tmatching = append(matching, next.curr)\n\t\tmatchingCurrs = append(matchingCurrs, next)\n\t\tmatchingIdxs = append(matchingIdxs, next.matchingIdx)\n\n\t\t// now as long as top of heap matches, keep popping\n\t\tfor len(s.heap) > 0 && next.curr.IndexInternalID.Equals(s.heap[0].curr.IndexInternalID) {\n\t\t\tnext = heap.Pop(s).(*SearcherCurr)\n\t\t\tmatching = append(matching, next.curr)\n\t\t\tmatchingCurrs = append(matchingCurrs, next)\n\t\t\tmatchingIdxs = append(matchingIdxs, next.matchingIdx)\n\t\t}\n\t}\n\n\ts.matching = matching\n\ts.matchingCurrs = matchingCurrs\n\ts.matchingIdxs = matchingIdxs\n\n\treturn nil\n}\n\nfunc (s *DisjunctionHeapSearcher) Weight() float64 {\n\tvar rv float64\n\tfor _, searcher := range s.searchers {\n\t\trv += searcher.Weight()\n\t}\n\treturn rv\n}\n\nfunc (s *DisjunctionHeapSearcher) SetQueryNorm(qnorm float64) {\n\tfor _, searcher := range s.searchers {\n\t\tsearcher.SetQueryNorm(qnorm)\n\t}\n}\n\nfunc (s *DisjunctionHeapSearcher) Next(ctx *search.SearchContext) (\n\t*search.DocumentMatch, error) {\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tvar rv *search.DocumentMatch\n\tfound := false\n\tfor !found && len(s.matching) > 0 {\n\t\tif len(s.matching) >= s.min {\n\t\t\tfound = true\n\t\t\tif s.retrieveScoreBreakdown {\n\t\t\t\t// just return score and expl breakdown here, since it is a disjunction over knn searchers,\n\t\t\t\t// and the final score and expl is calculated in the knn collector\n\t\t\t\trv = s.scorer.ScoreAndExplBreakdown(ctx, s.matching, s.matchingIdxs, nil, s.numSearchers)\n\t\t\t} else {\n\t\t\t\t// score this match\n\t\t\t\trv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers)\n\t\t\t}\n\t\t}\n\n\t\t// invoke next on all the matching searchers\n\t\tfor _, matchingCurr := range s.matchingCurrs {\n\t\t\tif matchingCurr.curr != rv {\n\t\t\t\tctx.DocumentMatchPool.Put(matchingCurr.curr)\n\t\t\t}\n\t\t\tcurr, err := matchingCurr.searcher.Next(ctx)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif curr != nil {\n\t\t\t\tmatchingCurr.curr = curr\n\t\t\t\theap.Push(s, matchingCurr)\n\t\t\t}\n\t\t}\n\n\t\terr := s.updateMatches()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn rv, nil\n}\n\nfunc (s *DisjunctionHeapSearcher) Advance(ctx *search.SearchContext,\n\tID index.IndexInternalID) (*search.DocumentMatch, error) {\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// if there is anything in matching, toss it back onto the heap\n\tfor _, matchingCurr := range s.matchingCurrs {\n\t\theap.Push(s, matchingCurr)\n\t}\n\ts.matching = s.matching[:0]\n\ts.matchingCurrs = s.matchingCurrs[:0]\n\n\t// find all searchers that actually need to be advanced\n\t// advance them, using s.matchingCurrs as temp storage\n\tfor len(s.heap) > 0 && s.heap[0].curr.IndexInternalID.Compare(ID) < 0 {\n\t\tsearcherCurr := heap.Pop(s).(*SearcherCurr)\n\t\tctx.DocumentMatchPool.Put(searcherCurr.curr)\n\t\tcurr, err := searcherCurr.searcher.Advance(ctx, ID)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif curr != nil {\n\t\t\tsearcherCurr.curr = curr\n\t\t\ts.matchingCurrs = append(s.matchingCurrs, searcherCurr)\n\t\t}\n\t}\n\t// now all of the searchers that we advanced have to be pushed back\n\tfor _, matchingCurr := range s.matchingCurrs {\n\t\theap.Push(s, matchingCurr)\n\t}\n\t// reset our temp space\n\ts.matchingCurrs = s.matchingCurrs[:0]\n\n\terr := s.updateMatches()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn s.Next(ctx)\n}\n\nfunc (s *DisjunctionHeapSearcher) Count() uint64 {\n\t// for now return a worst case\n\tvar sum uint64\n\tfor _, searcher := range s.searchers {\n\t\tsum += searcher.Count()\n\t}\n\treturn sum\n}\n\nfunc (s *DisjunctionHeapSearcher) Close() (rv error) {\n\tfor _, searcher := range s.searchers {\n\t\terr := searcher.Close()\n\t\tif err != nil && rv == nil {\n\t\t\trv = err\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (s *DisjunctionHeapSearcher) Min() int {\n\treturn s.min\n}\n\nfunc (s *DisjunctionHeapSearcher) DocumentMatchPoolSize() int {\n\trv := len(s.searchers)\n\tfor _, s := range s.searchers {\n\t\trv += s.DocumentMatchPoolSize()\n\t}\n\treturn rv\n}\n\n// a disjunction searcher implements the index.Optimizable interface\n// but only activates on an edge case where the disjunction is a\n// wrapper around a single Optimizable child searcher\nfunc (s *DisjunctionHeapSearcher) Optimize(kind string, octx index.OptimizableContext) (\n\tindex.OptimizableContext, error) {\n\tif len(s.searchers) == 1 {\n\t\to, ok := s.searchers[0].(index.Optimizable)\n\t\tif ok {\n\t\t\treturn o.Optimize(kind, octx)\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\n// heap impl\n\nfunc (s *DisjunctionHeapSearcher) Len() int { return len(s.heap) }\n\nfunc (s *DisjunctionHeapSearcher) Less(i, j int) bool {\n\tif s.heap[i].curr == nil {\n\t\treturn true\n\t} else if s.heap[j].curr == nil {\n\t\treturn false\n\t}\n\treturn s.heap[i].curr.IndexInternalID.Compare(s.heap[j].curr.IndexInternalID) < 0\n}\n\nfunc (s *DisjunctionHeapSearcher) Swap(i, j int) {\n\ts.heap[i], s.heap[j] = s.heap[j], s.heap[i]\n}\n\nfunc (s *DisjunctionHeapSearcher) Push(x interface{}) {\n\ts.heap = append(s.heap, x.(*SearcherCurr))\n}\n\nfunc (s *DisjunctionHeapSearcher) Pop() interface{} {\n\told := s.heap\n\tn := len(old)\n\tx := old[n-1]\n\ts.heap = old[0 : n-1]\n\treturn x\n}\n"
  },
  {
    "path": "search/searcher/search_disjunction_slice.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"math\"\n\t\"reflect\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/scorer\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeDisjunctionSliceSearcher int\n\nfunc init() {\n\tvar ds DisjunctionSliceSearcher\n\treflectStaticSizeDisjunctionSliceSearcher = int(reflect.TypeOf(ds).Size())\n}\n\ntype DisjunctionSliceSearcher struct {\n\tindexReader            index.IndexReader\n\tsearchers              []search.Searcher\n\toriginalPos            []int\n\tnumSearchers           int\n\tqueryNorm              float64\n\tretrieveScoreBreakdown bool\n\tcurrs                  []*search.DocumentMatch\n\tscorer                 *scorer.DisjunctionQueryScorer\n\tmin                    int\n\tmatching               []*search.DocumentMatch\n\tmatchingIdxs           []int\n\tinitialized            bool\n\tbytesRead              uint64\n}\n\nfunc newDisjunctionSliceSearcher(ctx context.Context, indexReader index.IndexReader,\n\tqsearchers []search.Searcher, min float64, options search.SearcherOptions,\n\tlimit bool) (\n\t*DisjunctionSliceSearcher, error,\n) {\n\tif limit && tooManyClauses(len(qsearchers)) {\n\t\treturn nil, tooManyClausesErr(\"\", len(qsearchers))\n\t}\n\n\tvar searchers OrderedSearcherList\n\tvar originalPos []int\n\tvar retrieveScoreBreakdown bool\n\tif ctx != nil {\n\t\tretrieveScoreBreakdown, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool)\n\t}\n\n\tif retrieveScoreBreakdown {\n\t\t// needed only when kNN is in picture\n\t\tsortedSearchers := &OrderedPositionalSearcherList{\n\t\t\tsearchers: make([]search.Searcher, len(qsearchers)),\n\t\t\tindex:     make([]int, len(qsearchers)),\n\t\t}\n\t\tfor i, searcher := range qsearchers {\n\t\t\tsortedSearchers.searchers[i] = searcher\n\t\t\tsortedSearchers.index[i] = i\n\t\t}\n\t\tsort.Sort(sortedSearchers)\n\t\tsearchers = sortedSearchers.searchers\n\t\toriginalPos = sortedSearchers.index\n\t} else {\n\t\tsearchers = make(OrderedSearcherList, len(qsearchers))\n\t\tcopy(searchers, qsearchers)\n\t\tsort.Sort(searchers)\n\t}\n\n\trv := DisjunctionSliceSearcher{\n\t\tindexReader:            indexReader,\n\t\tsearchers:              searchers,\n\t\toriginalPos:            originalPos,\n\t\tnumSearchers:           len(searchers),\n\t\tcurrs:                  make([]*search.DocumentMatch, len(searchers)),\n\t\tscorer:                 scorer.NewDisjunctionQueryScorer(options),\n\t\tmin:                    int(min),\n\t\tretrieveScoreBreakdown: retrieveScoreBreakdown,\n\n\t\tmatching:     make([]*search.DocumentMatch, len(searchers)),\n\t\tmatchingIdxs: make([]int, len(searchers)),\n\t}\n\trv.computeQueryNorm()\n\treturn &rv, nil\n}\n\nfunc (s *DisjunctionSliceSearcher) computeQueryNorm() {\n\t// first calculate sum of squared weights\n\tsumOfSquaredWeights := 0.0\n\tfor _, searcher := range s.searchers {\n\t\tsumOfSquaredWeights += searcher.Weight()\n\t}\n\t// now compute query norm from this\n\ts.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)\n\t// finally tell all the downstream searchers the norm\n\tfor _, searcher := range s.searchers {\n\t\tsearcher.SetQueryNorm(s.queryNorm)\n\t}\n}\n\nfunc (s *DisjunctionSliceSearcher) Size() int {\n\tsizeInBytes := reflectStaticSizeDisjunctionSliceSearcher + size.SizeOfPtr +\n\t\ts.scorer.Size()\n\n\tfor _, entry := range s.searchers {\n\t\tsizeInBytes += entry.Size()\n\t}\n\n\tfor _, entry := range s.currs {\n\t\tif entry != nil {\n\t\t\tsizeInBytes += entry.Size()\n\t\t}\n\t}\n\n\tfor _, entry := range s.matching {\n\t\tif entry != nil {\n\t\t\tsizeInBytes += entry.Size()\n\t\t}\n\t}\n\n\tsizeInBytes += len(s.matchingIdxs) * size.SizeOfInt\n\tsizeInBytes += len(s.originalPos) * size.SizeOfInt\n\n\treturn sizeInBytes\n}\n\nfunc (s *DisjunctionSliceSearcher) initSearchers(ctx *search.SearchContext) error {\n\tvar err error\n\t// get all searchers pointing at their first match\n\tfor i, searcher := range s.searchers {\n\t\tif s.currs[i] != nil {\n\t\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t\t}\n\t\ts.currs[i], err = searcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\terr = s.updateMatches()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ts.initialized = true\n\treturn nil\n}\n\nfunc (s *DisjunctionSliceSearcher) updateMatches() error {\n\tmatching := s.matching[:0]\n\tmatchingIdxs := s.matchingIdxs[:0]\n\n\tfor i := 0; i < len(s.currs); i++ {\n\t\tcurr := s.currs[i]\n\t\tif curr == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(matching) > 0 {\n\t\t\tcmp := curr.IndexInternalID.Compare(matching[0].IndexInternalID)\n\t\t\tif cmp > 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif cmp < 0 {\n\t\t\t\tmatching = matching[:0]\n\t\t\t\tmatchingIdxs = matchingIdxs[:0]\n\t\t\t}\n\t\t}\n\t\tmatching = append(matching, curr)\n\t\tmatchingIdxs = append(matchingIdxs, i)\n\t}\n\n\ts.matching = matching\n\ts.matchingIdxs = matchingIdxs\n\n\treturn nil\n}\n\nfunc (s *DisjunctionSliceSearcher) Weight() float64 {\n\tvar rv float64\n\tfor _, searcher := range s.searchers {\n\t\trv += searcher.Weight()\n\t}\n\treturn rv\n}\n\nfunc (s *DisjunctionSliceSearcher) SetQueryNorm(qnorm float64) {\n\tfor _, searcher := range s.searchers {\n\t\tsearcher.SetQueryNorm(qnorm)\n\t}\n}\n\nfunc (s *DisjunctionSliceSearcher) Next(ctx *search.SearchContext) (\n\t*search.DocumentMatch, error,\n) {\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tvar err error\n\tvar rv *search.DocumentMatch\n\n\tfound := false\n\tfor !found && len(s.matching) > 0 {\n\t\tif len(s.matching) >= s.min {\n\t\t\tfound = true\n\t\t\tif s.retrieveScoreBreakdown {\n\t\t\t\t// just return score and expl breakdown here, since it is a disjunction over knn searchers,\n\t\t\t\t// and the final score and expl is calculated in the knn collector\n\t\t\t\trv = s.scorer.ScoreAndExplBreakdown(ctx, s.matching, s.matchingIdxs, s.originalPos, s.numSearchers)\n\t\t\t} else {\n\t\t\t\t// score this match\n\t\t\t\trv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers)\n\t\t\t}\n\t\t}\n\n\t\t// invoke next on all the matching searchers\n\t\tfor _, i := range s.matchingIdxs {\n\t\t\tsearcher := s.searchers[i]\n\t\t\tif s.currs[i] != rv {\n\t\t\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t\t\t}\n\t\t\ts.currs[i], err = searcher.Next(ctx)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\n\t\terr = s.updateMatches()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn rv, nil\n}\n\nfunc (s *DisjunctionSliceSearcher) Advance(ctx *search.SearchContext,\n\tID index.IndexInternalID,\n) (*search.DocumentMatch, error) {\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\t// get all searchers pointing at their first match\n\tvar err error\n\tfor i, searcher := range s.searchers {\n\t\tif s.currs[i] != nil {\n\t\t\tif s.currs[i].IndexInternalID.Compare(ID) >= 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tctx.DocumentMatchPool.Put(s.currs[i])\n\t\t}\n\t\ts.currs[i], err = searcher.Advance(ctx, ID)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\terr = s.updateMatches()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn s.Next(ctx)\n}\n\nfunc (s *DisjunctionSliceSearcher) Count() uint64 {\n\t// for now return a worst case\n\tvar sum uint64\n\tfor _, searcher := range s.searchers {\n\t\tsum += searcher.Count()\n\t}\n\treturn sum\n}\n\nfunc (s *DisjunctionSliceSearcher) Close() (rv error) {\n\tfor _, searcher := range s.searchers {\n\t\terr := searcher.Close()\n\t\tif err != nil && rv == nil {\n\t\t\trv = err\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc (s *DisjunctionSliceSearcher) Min() int {\n\treturn s.min\n}\n\nfunc (s *DisjunctionSliceSearcher) DocumentMatchPoolSize() int {\n\trv := len(s.currs)\n\tfor _, s := range s.searchers {\n\t\trv += s.DocumentMatchPoolSize()\n\t}\n\treturn rv\n}\n\n// a disjunction searcher implements the index.Optimizable interface\n// but only activates on an edge case where the disjunction is a\n// wrapper around a single Optimizable child searcher\nfunc (s *DisjunctionSliceSearcher) Optimize(kind string, octx index.OptimizableContext) (\n\tindex.OptimizableContext, error,\n) {\n\tif len(s.searchers) == 1 {\n\t\to, ok := s.searchers[0].(index.Optimizable)\n\t\tif ok {\n\t\t\treturn o.Optimize(kind, octx)\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n"
  },
  {
    "path": "search/searcher/search_disjunction_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestDisjunctionSearch(t *testing.T) {\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\texplainTrue := search.SearcherOptions{Explain: true}\n\n\tmartyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyOrDustinSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmartyTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyOrDustinSearcher2, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher2, dustinTermSearcher2}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\traviTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"ravi\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnestedRaviOrMartyOrDustinSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{raviTermSearcher, martyOrDustinSearcher2}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttests := []struct {\n\t\tsearcher search.Searcher\n\t\tresults  []*search.DocumentMatch\n\t}{\n\t\t{\n\t\t\tsearcher: martyOrDustinSearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           0.6775110856165737,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           0.6775110856165737,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test a nested disjunction\n\t\t{\n\t\t\tsearcher: nestedRaviOrMartyOrDustinSearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           0.2765927424732821,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           0.2765927424732821,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"4\"),\n\t\t\t\t\tScore:           0.5531854849465642,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor testIndex, test := range tests {\n\t\tdefer func() {\n\t\t\terr := test.searcher.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}()\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := test.searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tif i < len(test.results) {\n\t\t\t\tif !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have id %s got %s for test %d\", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)\n\t\t\t\t}\n\t\t\t\tif !scoresCloseEnough(next.Score, test.results[i].Score) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have score %v got  %v for test %d\", i, test.results[i].Score, next.Score, testIndex)\n\t\t\t\t\tt.Logf(\"scoring explanation: %s\", next.Expl)\n\t\t\t\t}\n\t\t\t}\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = test.searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, testIndex)\n\t\t}\n\t\tif len(test.results) != i {\n\t\t\tt.Errorf(\"expected %d results got %d for test %d\", len(test.results), i, testIndex)\n\t\t}\n\t}\n}\n\nfunc TestDisjunctionAdvance(t *testing.T) {\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\texplainTrue := search.SearcherOptions{Explain: true}\n\n\tmartyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmartyOrDustinSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(martyOrDustinSearcher.DocumentMatchPoolSize(), 0),\n\t}\n\tmatch, err := martyOrDustinSearcher.Advance(ctx, index.IndexInternalID(\"3\"))\n\tif err != nil {\n\t\tt.Errorf(\"unexpected error: %v\", err)\n\t}\n\tif match == nil {\n\t\tt.Errorf(\"expected 3, got nil\")\n\t}\n}\n\nfunc TestDisjunctionSearchTooMany(t *testing.T) {\n\t// set to max to a low non-zero value\n\tDisjunctionMaxClauseCount = 2\n\tdefer func() {\n\t\t// reset it after the test\n\t\tDisjunctionMaxClauseCount = 0\n\t}()\n\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\texplainTrue := search.SearcherOptions{Explain: true}\n\n\tmartyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"marty\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdustinTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"dustin\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tsteveTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, \"steve\", \"name\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t_, err = NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher, steveTermSearcher}, 0, explainTrue)\n\tif err == nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestUnadornedDisjunctionAdvance(t *testing.T) {\n\tdir, _ := os.MkdirTemp(\"\", \"scorchTwoDoc\")\n\tdefer func() {\n\t\t_ = os.RemoveAll(dir)\n\t}()\n\ttwoDocIndex := initTwoDocScorch(dir)\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\tgetNewOptimizedCompositeSearcher := func(t *testing.T) search.Searcher {\n\t\toptimizedCompositeSearcherOptions := search.SearcherOptions{Explain: false, IncludeTermVectors: false, Score: \"none\"}\n\t\tmartyTermSearcher, err := NewTermSearcher(context.Background(), twoDocIndexReader, \"marty\", \"name\", 1.0, optimizedCompositeSearcherOptions)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tdustinTermSearcher, err := NewTermSearcher(context.Background(), twoDocIndexReader, \"dustin\", \"name\", 1.0, optimizedCompositeSearcherOptions)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tsteveTermSearcher, err := NewTermSearcher(context.Background(), twoDocIndexReader, \"steve\", \"name\", 1.0, optimizedCompositeSearcherOptions)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tmartyOrDustinOrSteveSearcher, err := NewDisjunctionSearcher(context.Background(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher, steveTermSearcher}, 0, optimizedCompositeSearcherOptions)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\treturn martyOrDustinOrSteveSearcher\n\t}\n\tmartyOrDustinOrSteveSearcher := getNewOptimizedCompositeSearcher(t)\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(martyOrDustinOrSteveSearcher.DocumentMatchPoolSize(), 0),\n\t}\n\t// get the correct order using only next calls\n\tdm, err := martyOrDustinOrSteveSearcher.Next(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texpectedDocIDs := []index.IndexInternalID{}\n\tfor dm != nil && err == nil {\n\t\texpectedDocIDs = append(expectedDocIDs, dm.IndexInternalID)\n\t\tdm, err = martyOrDustinOrSteveSearcher.Next(ctx)\n\t}\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(expectedDocIDs) != 3 {\n\t\tt.Fatalf(\"expected 3 results, got %d\", len(expectedDocIDs))\n\t}\n\t// Test 1 - Advance in reverse direction after getting the correct order using only next calls\n\t// Next(->) - Next(->) - Next(->) - Advance(<-) - Advance(<-)\n\tfor i := len(expectedDocIDs) - 1; i >= 0; i-- {\n\t\txID := expectedDocIDs[i]\n\t\tdm, err = martyOrDustinOrSteveSearcher.Advance(ctx, xID)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif dm == nil {\n\t\t\tt.Fatalf(\"expected to find %v\", xID)\n\t\t}\n\t\tif !dm.IndexInternalID.Equals(xID) {\n\t\t\tt.Fatalf(\"expected %v, got %v\", xID, dm.IndexInternalID)\n\t\t}\n\t}\n\t// Test 2 - Advance in forward direction after getting the correct order using only next calls\n\t// Next(->) - Next(->) - Next(->) - Advance(ResetTo0) - Advance(->) - Advance(->)\n\tmartyOrDustinOrSteveSearcher = getNewOptimizedCompositeSearcher(t)\n\tfor i := 0; i < len(expectedDocIDs); i++ {\n\t\txID := expectedDocIDs[i]\n\t\tdm, err = martyOrDustinOrSteveSearcher.Advance(ctx, xID)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif dm == nil {\n\t\t\tt.Fatalf(\"expected to find %v\", xID)\n\t\t}\n\t\tif !dm.IndexInternalID.Equals(xID) {\n\t\t\tt.Fatalf(\"expected %v, got %v\", xID, dm.IndexInternalID)\n\t\t}\n\t}\n\t// Test 3 - Alternate Next and Advance calls\n\t// Next(->) -> Next(->) -> Advance(<-) -> Next(->) -> Next(->) -> Advance(<-) -> Advance(<-) -> Next(->)\n\tmartyOrDustinOrSteveSearcher = getNewOptimizedCompositeSearcher(t)\n\tgoNext := func(expectedDocID index.IndexInternalID) {\n\t\tdm, err = martyOrDustinOrSteveSearcher.Next(ctx)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif dm == nil {\n\t\t\tt.Fatal(\"expected a document, got nil\")\n\t\t}\n\t\tif !dm.IndexInternalID.Equals(expectedDocID) {\n\t\t\tt.Fatalf(\"expected %v, got %v\", expectedDocID, dm.IndexInternalID)\n\t\t}\n\t}\n\tgoBack := func(goTo index.IndexInternalID) {\n\t\tdm, err = martyOrDustinOrSteveSearcher.Advance(ctx, goTo)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif dm == nil {\n\t\t\tt.Fatalf(\"expected to find %v\", goTo)\n\t\t}\n\t\tif !dm.IndexInternalID.Equals(goTo) {\n\t\t\tt.Fatalf(\"expected %v, got %v\", goTo, dm.IndexInternalID)\n\t\t}\n\t}\n\t// Next\t\t(->)\n\tgoNext(expectedDocIDs[0])\n\t// Next\t\t(->)\n\tgoNext(expectedDocIDs[1])\n\t// Advance\t(<-)\n\tgoBack(expectedDocIDs[0])\n\t// Next\t\t(->)\n\tgoNext(expectedDocIDs[1])\n\t// Next\t\t(->)\n\tgoNext(expectedDocIDs[2])\n\t// Advance\t(<-)\n\tgoBack(expectedDocIDs[1])\n\t// Advance\t(<-)\n\tgoBack(expectedDocIDs[0])\n\t// Next\t\t(->)\n\tgoNext(expectedDocIDs[1])\n}\n"
  },
  {
    "path": "search/searcher/search_docid.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/scorer\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeDocIDSearcher int\n\nfunc init() {\n\tvar ds DocIDSearcher\n\treflectStaticSizeDocIDSearcher = int(reflect.TypeOf(ds).Size())\n}\n\n// DocIDSearcher returns documents matching a predefined set of identifiers.\ntype DocIDSearcher struct {\n\treader index.DocIDReader\n\tscorer *scorer.ConstantScorer\n\tcount  int\n}\n\nfunc NewDocIDSearcher(ctx context.Context, indexReader index.IndexReader, ids []string, boost float64,\n\toptions search.SearcherOptions) (searcher *DocIDSearcher, err error) {\n\n\treader, err := indexReader.DocIDReaderOnly(ids)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tscorer := scorer.NewConstantScorer(1.0, boost, options)\n\treturn &DocIDSearcher{\n\t\tscorer: scorer,\n\t\treader: reader,\n\t\tcount:  len(ids),\n\t}, nil\n}\n\nfunc (s *DocIDSearcher) Size() int {\n\treturn reflectStaticSizeDocIDSearcher + size.SizeOfPtr +\n\t\ts.reader.Size() +\n\t\ts.scorer.Size()\n}\n\nfunc (s *DocIDSearcher) Count() uint64 {\n\treturn uint64(s.count)\n}\n\nfunc (s *DocIDSearcher) Weight() float64 {\n\treturn s.scorer.Weight()\n}\n\nfunc (s *DocIDSearcher) SetQueryNorm(qnorm float64) {\n\ts.scorer.SetQueryNorm(qnorm)\n}\n\nfunc (s *DocIDSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\tdocidMatch, err := s.reader.Next()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif docidMatch == nil {\n\t\treturn nil, nil\n\t}\n\n\tdocMatch := s.scorer.Score(ctx, docidMatch)\n\treturn docMatch, nil\n}\n\nfunc (s *DocIDSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\tdocidMatch, err := s.reader.Advance(ID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif docidMatch == nil {\n\t\treturn nil, nil\n\t}\n\n\tdocMatch := s.scorer.Score(ctx, docidMatch)\n\treturn docMatch, nil\n}\n\nfunc (s *DocIDSearcher) Close() error {\n\treturn s.reader.Close()\n}\n\nfunc (s *DocIDSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *DocIDSearcher) DocumentMatchPoolSize() int {\n\treturn 1\n}\n"
  },
  {
    "path": "search/searcher/search_docid_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := upsidedown.NewUpsideDownCouch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": \"\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, id := range indexed {\n\t\tdoc := document.NewDocument(id)\n\t\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\t\terr = i.Update(doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\texplainOff := search.SearcherOptions{Explain: false}\n\n\tsearcher, err := NewDocIDSearcher(context.TODO(), indexReader, searched, 1.0, explainOff)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := searcher.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0),\n\t}\n\n\t// Check the sequence\n\tfor i, id := range wanted {\n\t\tm, err := searcher.Next(ctx)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !index.IndexInternalID(id).Equals(m.IndexInternalID) {\n\t\t\tt.Fatalf(\"expected %v at position %v, got %v\", id, i, m.IndexInternalID)\n\t\t}\n\t\tctx.DocumentMatchPool.Put(m)\n\t}\n\tm, err := searcher.Next(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif m != nil {\n\t\tt.Fatalf(\"expected nil past the end of the sequence, got %v\", m.IndexInternalID)\n\t}\n\tctx.DocumentMatchPool.Put(m)\n\n\t// Check seeking\n\tfor _, id := range wanted {\n\t\tif len(id) != 2 {\n\t\t\tt.Fatalf(\"expected identifier must be 2 characters long, got %v\", id)\n\t\t}\n\t\tbefore := id[:1]\n\t\tfor _, target := range []string{before, id} {\n\t\t\tm, err := searcher.Advance(ctx, index.IndexInternalID(target))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) {\n\t\t\t\tt.Fatalf(\"advancing to %v returned %v instead of %v\", before, m, id)\n\t\t\t}\n\t\t\tctx.DocumentMatchPool.Put(m)\n\t\t}\n\t}\n\t// Seek after the end of the sequence\n\tafter := \"zzz\"\n\tm, err = searcher.Advance(ctx, index.IndexInternalID(after))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif m != nil {\n\t\tt.Fatalf(\"advancing past the end of the sequence should return nil, got %v\", m)\n\t}\n\tctx.DocumentMatchPool.Put(m)\n}\n\nfunc TestDocIDSearcherEmptySearchEmptyIndex(t *testing.T) {\n\ttestDocIDSearcher(t, nil, nil, nil)\n}\n\nfunc TestDocIDSearcherEmptyIndex(t *testing.T) {\n\ttestDocIDSearcher(t, nil, []string{\"aa\", \"bb\"}, nil)\n}\n\nfunc TestDocIDSearcherEmptySearch(t *testing.T) {\n\ttestDocIDSearcher(t, []string{\"aa\", \"bb\"}, nil, nil)\n}\n\nfunc TestDocIDSearcherValid(t *testing.T) {\n\t// Test missing, out of order and duplicate inputs\n\ttestDocIDSearcher(t, []string{\"aa\", \"bb\", \"cc\"},\n\t\t[]string{\"ee\", \"bb\", \"aa\", \"bb\"},\n\t\t[]string{\"aa\", \"bb\"})\n}\n"
  },
  {
    "path": "search/searcher/search_filter.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeFilteringSearcher int\n\nfunc init() {\n\tvar fs FilteringSearcher\n\treflectStaticSizeFilteringSearcher = int(reflect.TypeOf(fs).Size())\n}\n\n// FilterFunc defines a function which can filter documents\n// returning true means keep the document\n// returning false means do not keep the document\ntype FilterFunc func(sctx *search.SearchContext, d *search.DocumentMatch) bool\n\n// FilteringSearcher wraps any other searcher, but checks any Next/Advance\n// call against the supplied FilterFunc\ntype FilteringSearcher struct {\n\tchild  search.Searcher\n\taccept FilterFunc\n}\n\nfunc NewFilteringSearcher(ctx context.Context, s search.Searcher, filter FilterFunc) *FilteringSearcher {\n\treturn &FilteringSearcher{\n\t\tchild:  s,\n\t\taccept: filter,\n\t}\n}\n\nfunc (f *FilteringSearcher) Size() int {\n\treturn reflectStaticSizeFilteringSearcher + size.SizeOfPtr +\n\t\tf.child.Size()\n}\n\nfunc (f *FilteringSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\tnext, err := f.child.Next(ctx)\n\tfor next != nil && err == nil {\n\t\tif f.accept(ctx, next) {\n\t\t\treturn next, nil\n\t\t}\n\t\t// recycle this document match now, since\n\t\t// we do not need it anymore\n\t\tctx.DocumentMatchPool.Put(next)\n\t\tnext, err = f.child.Next(ctx)\n\t}\n\treturn nil, err\n}\n\nfunc (f *FilteringSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\tadv, err := f.child.Advance(ctx, ID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif adv == nil {\n\t\treturn nil, nil\n\t}\n\tif f.accept(ctx, adv) {\n\t\treturn adv, nil\n\t}\n\t// recycle this document match now, since\n\t// we do not need it anymore\n\tctx.DocumentMatchPool.Put(adv)\n\treturn f.Next(ctx)\n}\n\nfunc (f *FilteringSearcher) Close() error {\n\treturn f.child.Close()\n}\n\nfunc (f *FilteringSearcher) Weight() float64 {\n\treturn f.child.Weight()\n}\n\nfunc (f *FilteringSearcher) SetQueryNorm(n float64) {\n\tf.child.SetQueryNorm(n)\n}\n\nfunc (f *FilteringSearcher) Count() uint64 {\n\treturn f.child.Count()\n}\n\nfunc (f *FilteringSearcher) Min() int {\n\treturn f.child.Min()\n}\n\nfunc (f *FilteringSearcher) DocumentMatchPoolSize() int {\n\treturn f.child.DocumentMatchPoolSize()\n}\n"
  },
  {
    "path": "search/searcher/search_fuzzy.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar MaxFuzziness = 2\n\n// AutoFuzzinessHighThreshold is the threshold for the term length\n// above which the fuzziness is set to MaxFuzziness when the fuzziness\n// mode is set to AutoFuzziness.\nvar AutoFuzzinessHighThreshold = 5\n\n// AutoFuzzinessLowThreshold is the threshold for the term length\n// below which the fuzziness is set to zero when the fuzziness mode\n// is set to AutoFuzziness.\n// For terms with length between AutoFuzzinessLowThreshold and\n// AutoFuzzinessHighThreshold, the fuzziness is set to\n// MaxFuzziness - 1.\nvar AutoFuzzinessLowThreshold = 2\n\nfunc NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term string,\n\tprefix, fuzziness int, field string, boost float64,\n\toptions search.SearcherOptions) (search.Searcher, error) {\n\n\tif fuzziness > MaxFuzziness {\n\t\treturn nil, fmt.Errorf(\"fuzziness exceeds max (%d)\", MaxFuzziness)\n\t}\n\n\tif fuzziness < 0 {\n\t\treturn nil, fmt.Errorf(\"invalid fuzziness, negative\")\n\t}\n\tif fuzziness == 0 {\n\t\t// no fuzziness, just do a term search\n\t\t// check if the call is made from a phrase searcher\n\t\t// and if so, add the term to the fuzzy term matches\n\t\t// since the fuzzy candidate terms are not collected\n\t\t// for a term search, and the only candidate term is\n\t\t// the term itself\n\t\tif ctx != nil {\n\t\t\tfuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey)\n\t\t\tif fuzzyTermMatches != nil {\n\t\t\t\tfuzzyTermMatches.(map[string][]string)[term] = []string{term}\n\t\t\t}\n\t\t}\n\t\treturn NewTermSearcher(ctx, indexReader, term, field, boost, options)\n\t}\n\n\t// Note: we don't byte slice the term for a prefix because of runes.\n\tprefixTerm := \"\"\n\tfor i, r := range term {\n\t\tif i < prefix {\n\t\t\tprefixTerm += string(r)\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\tfuzzyCandidates, err := findFuzzyCandidateTerms(ctx, indexReader, term, fuzziness,\n\t\tfield, prefixTerm)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar candidates []string\n\tvar editDistances []uint8\n\tvar dictBytesRead uint64\n\tif fuzzyCandidates != nil {\n\t\tcandidates = fuzzyCandidates.candidates\n\t\teditDistances = fuzzyCandidates.editDistances\n\t\tdictBytesRead = fuzzyCandidates.bytesRead\n\t}\n\n\tif ctx != nil {\n\t\treportIOStats(ctx, dictBytesRead)\n\t\tsearch.RecordSearchCost(ctx, search.AddM, dictBytesRead)\n\t\tfuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey)\n\t\tif fuzzyTermMatches != nil {\n\t\t\tfuzzyTermMatches.(map[string][]string)[term] = candidates\n\t\t}\n\t}\n\t// check if the candidates are empty or have one term which is the term itself\n\tif len(candidates) == 0 || (len(candidates) == 1 && candidates[0] == term) {\n\t\tif ctx != nil {\n\t\t\tfuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey)\n\t\t\tif fuzzyTermMatches != nil {\n\t\t\t\tfuzzyTermMatches.(map[string][]string)[term] = []string{term}\n\t\t\t}\n\t\t}\n\t\treturn NewTermSearcher(ctx, indexReader, term, field, boost, options)\n\t}\n\n\treturn NewMultiTermSearcherBoosted(ctx, indexReader, candidates, field,\n\t\tboost, editDistances, options, true)\n}\n\nfunc GetAutoFuzziness(term string) int {\n\ttermLength := len(term)\n\tif termLength > AutoFuzzinessHighThreshold {\n\t\treturn MaxFuzziness\n\t} else if termLength > AutoFuzzinessLowThreshold {\n\t\treturn MaxFuzziness - 1\n\t}\n\treturn 0\n}\n\nfunc NewAutoFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term string,\n\tprefix int, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {\n\treturn NewFuzzySearcher(ctx, indexReader, term, prefix, GetAutoFuzziness(term), field, boost, options)\n}\n\ntype fuzzyCandidates struct {\n\tcandidates    []string\n\teditDistances []uint8\n\tbytesRead     uint64\n}\n\nfunc reportIOStats(ctx context.Context, bytesRead uint64) {\n\t// The fuzzy, regexp like queries essentially load a dictionary,\n\t// which potentially incurs a cost that must be accounted by\n\t// using the callback to report the value.\n\tif ctx != nil {\n\t\tstatsCallbackFn := ctx.Value(search.SearchIOStatsCallbackKey)\n\t\tif statsCallbackFn != nil {\n\t\t\tstatsCallbackFn.(search.SearchIOStatsCallbackFunc)(bytesRead)\n\t\t}\n\t}\n}\n\nfunc findFuzzyCandidateTerms(ctx context.Context, indexReader index.IndexReader, term string,\n\tfuzziness int, field, prefixTerm string) (rv *fuzzyCandidates, err error) {\n\trv = &fuzzyCandidates{\n\t\tcandidates:    make([]string, 0),\n\t\teditDistances: make([]uint8, 0),\n\t}\n\n\t// in case of advanced reader implementations directly call\n\t// the levenshtein automaton based iterator to collect the\n\t// candidate terms\n\tif ir, ok := indexReader.(index.IndexReaderFuzzy); ok {\n\t\ttermSet := make(map[string]struct{})\n\t\taddCandidateTerm := func(term string, editDistance uint8) error {\n\t\t\tif _, exists := termSet[term]; !exists {\n\t\t\t\ttermSet[term] = struct{}{}\n\t\t\t\trv.candidates = append(rv.candidates, term)\n\t\t\t\trv.editDistances = append(rv.editDistances, editDistance)\n\t\t\t\tif tooManyClauses(len(rv.candidates)) {\n\t\t\t\t\treturn tooManyClausesErr(field, len(rv.candidates))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\tfieldDict, a, err := ir.FieldDictFuzzyAutomaton(field, term, fuzziness, prefixTerm)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer func() {\n\t\t\tif cerr := fieldDict.Close(); cerr != nil && err == nil {\n\t\t\t\terr = cerr\n\t\t\t}\n\t\t}()\n\t\ttfd, err := fieldDict.Next()\n\t\tfor err == nil && tfd != nil {\n\t\t\terr = addCandidateTerm(tfd.Term, tfd.EditDistance)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\ttfd, err = fieldDict.Next()\n\t\t}\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif ctx != nil {\n\t\t\tif fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {\n\t\t\t\tif ts, exists := fts[field]; exists {\n\t\t\t\t\tfor term := range ts {\n\t\t\t\t\t\tif _, exists := termSet[term]; exists {\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif !strings.HasPrefix(term, prefixTerm) {\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t\tmatch, editDistance := a.MatchAndDistance(term)\n\t\t\t\t\t\tif match {\n\t\t\t\t\t\t\terr = addCandidateTerm(term, editDistance)\n\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\trv.bytesRead = fieldDict.BytesRead()\n\t\treturn rv, nil\n\t}\n\n\tvar fieldDict index.FieldDict\n\tif len(prefixTerm) > 0 {\n\t\tfieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm))\n\t} else {\n\t\tfieldDict, err = indexReader.FieldDict(field)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := fieldDict.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\t// enumerate terms and check levenshtein distance\n\tvar reuse []int\n\ttfd, err := fieldDict.Next()\n\tfor err == nil && tfd != nil {\n\t\tvar ld int\n\t\tvar exceeded bool\n\t\tld, exceeded, reuse = search.LevenshteinDistanceMaxReuseSlice(term, tfd.Term, fuzziness, reuse)\n\t\tif !exceeded && ld <= fuzziness {\n\t\t\trv.candidates = append(rv.candidates, tfd.Term)\n\t\t\trv.editDistances = append(rv.editDistances, uint8(ld))\n\t\t\tif tooManyClauses(len(rv.candidates)) {\n\t\t\t\treturn nil, tooManyClausesErr(field, len(rv.candidates))\n\t\t\t}\n\t\t}\n\t\ttfd, err = fieldDict.Next()\n\t}\n\n\trv.bytesRead = fieldDict.BytesRead()\n\treturn rv, err\n}\n"
  },
  {
    "path": "search/searcher/search_fuzzy_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestFuzzySearch(t *testing.T) {\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\texplainTrue := search.SearcherOptions{Explain: true}\n\n\tfuzzySearcherbeet, err := NewFuzzySearcher(context.TODO(), twoDocIndexReader, \"beet\", 0, 1, \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfuzzySearcherdouches, err := NewFuzzySearcher(context.TODO(), twoDocIndexReader, \"douches\", 0, 2, \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfuzzySearcheraplee, err := NewFuzzySearcher(context.TODO(), twoDocIndexReader, \"aplee\", 0, 2, \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfuzzySearcherprefix, err := NewFuzzySearcher(context.TODO(), twoDocIndexReader, \"water\", 3, 2, \"desc\", 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttests := []struct {\n\t\tsearcher search.Searcher\n\t\tresults  []*search.DocumentMatch\n\t}{\n\t\t{\n\t\t\tsearcher: fuzzySearcherbeet,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"2\"),\n\t\t\t\t\tScore:           0.5,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           0.5,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"4\"),\n\t\t\t\t\tScore:           0.9999999838027345,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: fuzzySearcherdouches,\n\t\t\tresults:  []*search.DocumentMatch{},\n\t\t},\n\t\t{\n\t\t\tsearcher: fuzzySearcheraplee,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           0.9581453659370776,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: fuzzySearcherprefix,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"5\"),\n\t\t\t\t\tScore:           1.916290731874155,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor testIndex, test := range tests {\n\t\tdefer func() {\n\t\t\terr := test.searcher.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}()\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := test.searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tif i < len(test.results) {\n\t\t\t\tif !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have id %s got %s for test %d\", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)\n\t\t\t\t}\n\t\t\t\tif next.Score != test.results[i].Score {\n\t\t\t\t\tt.Errorf(\"expected result %d to have score %v got %v for test %d\", i, test.results[i].Score, next.Score, testIndex)\n\t\t\t\t\tt.Logf(\"scoring explanation: %s\", next.Expl)\n\t\t\t\t}\n\t\t\t}\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = test.searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, testIndex)\n\t\t}\n\t\tif len(test.results) != i {\n\t\t\tt.Errorf(\"expected %d results got %d for test %d\", len(test.results), i, testIndex)\n\t\t}\n\t}\n}\n\nfunc TestFuzzySearchLimitErrors(t *testing.T) {\n\texplainTrue := search.SearcherOptions{Explain: true}\n\t_, err := NewFuzzySearcher(context.TODO(), nil, \"water\", 3, 3, \"desc\", 1.0, explainTrue)\n\tif err == nil {\n\t\tt.Fatal(\"`fuzziness exceeds max (2)` error expected\")\n\t}\n\n\t_, err = NewFuzzySearcher(context.TODO(), nil, \"water\", 3, -1, \"desc\", 1.0, explainTrue)\n\tif err == nil {\n\t\tt.Fatal(\"`invalid fuzziness, negative` error expected\")\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_geoboundingbox.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\ntype filterFunc func(key []byte) bool\n\nvar (\n\tGeoBitsShift1       = geo.GeoBits << 1\n\tGeoBitsShift1Minus1 = GeoBitsShift1 - 1\n)\n\nfunc NewGeoBoundingBoxSearcher(ctx context.Context, indexReader index.IndexReader, minLon, minLat,\n\tmaxLon, maxLat float64, field string, boost float64,\n\toptions search.SearcherOptions, checkBoundaries bool) (\n\tsearch.Searcher, error,\n) {\n\tif tp, ok := indexReader.(index.SpatialIndexPlugin); ok {\n\t\tsp, err := tp.GetSpatialAnalyzerPlugin(\"s2\")\n\t\tif err == nil {\n\t\t\tterms := sp.GetQueryTokens(geo.NewBoundedRectangle(minLat,\n\t\t\t\tminLon, maxLat, maxLon))\n\t\t\tboxSearcher, err := NewMultiTermSearcher(ctx, indexReader,\n\t\t\t\tterms, field, boost, options, false)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tdvReader, err := indexReader.DocValueReader([]string{field})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\treturn NewFilteringSearcher(ctx, boxSearcher, buildRectFilter(ctx, dvReader,\n\t\t\t\tminLon, minLat, maxLon, maxLat)), nil\n\t\t}\n\t}\n\n\t// indexes without the spatial plugin override would continue here.\n\n\t// track list of opened searchers, for cleanup on early exit\n\tvar openedSearchers []search.Searcher\n\tcleanupOpenedSearchers := func() {\n\t\tfor _, s := range openedSearchers {\n\t\t\t_ = s.Close()\n\t\t}\n\t}\n\n\t// do math to produce list of terms needed for this search\n\tonBoundaryTerms, notOnBoundaryTerms, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1,\n\t\tminLon, minLat, maxLon, maxLat, checkBoundaries, indexReader, field)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar onBoundarySearcher search.Searcher\n\tdvReader, err := indexReader.DocValueReader([]string{field})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(onBoundaryTerms) > 0 {\n\t\trawOnBoundarySearcher, err := NewMultiTermSearcherBytes(ctx, indexReader,\n\t\t\tonBoundaryTerms, field, boost, options, false)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// add filter to check points near the boundary\n\t\tonBoundarySearcher = NewFilteringSearcher(ctx, rawOnBoundarySearcher,\n\t\t\tbuildRectFilter(ctx, dvReader, minLon, minLat, maxLon, maxLat))\n\t\topenedSearchers = append(openedSearchers, onBoundarySearcher)\n\t}\n\n\tvar notOnBoundarySearcher search.Searcher\n\tif len(notOnBoundaryTerms) > 0 {\n\t\tvar err error\n\t\tnotOnBoundarySearcher, err = NewMultiTermSearcherBytes(ctx, indexReader,\n\t\t\tnotOnBoundaryTerms, field, boost, options, false)\n\t\tif err != nil {\n\t\t\tcleanupOpenedSearchers()\n\t\t\treturn nil, err\n\t\t}\n\t\topenedSearchers = append(openedSearchers, notOnBoundarySearcher)\n\t}\n\n\tif onBoundarySearcher != nil && notOnBoundarySearcher != nil {\n\t\trv, err := NewDisjunctionSearcher(ctx, indexReader,\n\t\t\t[]search.Searcher{\n\t\t\t\tonBoundarySearcher,\n\t\t\t\tnotOnBoundarySearcher,\n\t\t\t},\n\t\t\t0, options)\n\t\tif err != nil {\n\t\t\tcleanupOpenedSearchers()\n\t\t\treturn nil, err\n\t\t}\n\t\treturn rv, nil\n\t} else if onBoundarySearcher != nil {\n\t\treturn onBoundarySearcher, nil\n\t} else if notOnBoundarySearcher != nil {\n\t\treturn notOnBoundarySearcher, nil\n\t}\n\n\treturn NewMatchNoneSearcher(indexReader)\n}\n\nvar (\n\tgeoMaxShift    = document.GeoPrecisionStep * 4\n\tgeoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2\n)\n\ntype closeFunc func() error\n\nfunc ComputeGeoRange(ctx context.Context, term uint64, shift uint,\n\tsminLon, sminLat, smaxLon, smaxLat float64, checkBoundaries bool,\n\tindexReader index.IndexReader, field string) (\n\tonBoundary [][]byte, notOnBoundary [][]byte, err error,\n) {\n\tisIndexed, closeF, err := buildIsIndexedFunc(ctx, indexReader, field)\n\tif closeF != nil {\n\t\tdefer func() {\n\t\t\tcerr := closeF()\n\t\t\tif cerr != nil {\n\t\t\t\terr = cerr\n\t\t\t}\n\t\t}()\n\t}\n\n\tgrc := &geoRangeCompute{\n\t\tpreallocBytesLen: 32,\n\t\tpreallocBytes:    make([]byte, 32),\n\t\tsminLon:          sminLon,\n\t\tsminLat:          sminLat,\n\t\tsmaxLon:          smaxLon,\n\t\tsmaxLat:          smaxLat,\n\t\tcheckBoundaries:  checkBoundaries,\n\t\tisIndexed:        isIndexed,\n\t}\n\n\tgrc.computeGeoRange(term, shift)\n\n\treturn grc.onBoundary, grc.notOnBoundary, nil\n}\n\nfunc buildIsIndexedFunc(ctx context.Context, indexReader index.IndexReader, field string) (isIndexed filterFunc, closeF closeFunc, err error) {\n\tif irr, ok := indexReader.(index.IndexReaderContains); ok {\n\t\tfieldDict, err := irr.FieldDictContains(field)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\tisIndexed = func(term []byte) bool {\n\t\t\tfound, err := fieldDict.Contains(term)\n\t\t\treturn err == nil && found\n\t\t}\n\n\t\tcloseF = func() error {\n\t\t\tif fd, ok := fieldDict.(index.FieldDict); ok {\n\t\t\t\terr := fd.Close()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t} else if indexReader != nil {\n\t\tisIndexed = func(term []byte) bool {\n\t\t\treader, err := indexReader.TermFieldReader(ctx, term, field, false, false, false)\n\t\t\tif err != nil || reader == nil {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif reader.Count() == 0 {\n\t\t\t\t_ = reader.Close()\n\t\t\t\treturn false\n\t\t\t}\n\t\t\t_ = reader.Close()\n\t\t\treturn true\n\t\t}\n\t} else {\n\t\tisIndexed = func([]byte) bool {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn isIndexed, closeF, err\n}\n\nfunc buildRectFilter(ctx context.Context, dvReader index.DocValueReader,\n\tminLon, minLat, maxLon, maxLat float64,\n) FilterFunc {\n\t// reuse the following for each document match that is checked using the filter\n\tvar lons, lats []float64\n\tvar found bool\n\tdvVisitor := func(_ string, term []byte) {\n\t\tif found {\n\t\t\t// avoid redundant work if already found\n\t\t\treturn\n\t\t}\n\t\t// only consider the values which are shifted 0\n\t\tprefixCoded := numeric.PrefixCoded(term)\n\t\tshift, err := prefixCoded.Shift()\n\t\tif err == nil && shift == 0 {\n\t\t\tvar i64 int64\n\t\t\ti64, err = prefixCoded.Int64()\n\t\t\tif err == nil {\n\t\t\t\tlons = append(lons, geo.MortonUnhashLon(uint64(i64)))\n\t\t\t\tlats = append(lats, geo.MortonUnhashLat(uint64(i64)))\n\t\t\t\tfound = true\n\t\t\t}\n\t\t}\n\t}\n\treturn func(sctx *search.SearchContext, d *search.DocumentMatch) bool {\n\t\t// check geo matches against all numeric type terms indexed\n\t\tlons, lats = lons[:0], lats[:0]\n\t\tfound = false\n\t\tif err := dvReader.VisitDocValues(d.IndexInternalID, dvVisitor); err == nil && found {\n\t\t\tbytes := dvReader.BytesRead()\n\t\t\tif bytes > 0 {\n\t\t\t\treportIOStats(ctx, bytes)\n\t\t\t\tsearch.RecordSearchCost(ctx, search.AddM, bytes)\n\t\t\t}\n\t\t\tfor i := range lons {\n\t\t\t\tif geo.BoundingBoxContains(lons[i], lats[i],\n\t\t\t\t\tminLon, minLat, maxLon, maxLat) {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n}\n\ntype geoRangeCompute struct {\n\tpreallocBytesLen                   int\n\tpreallocBytes                      []byte\n\tsminLon, sminLat, smaxLon, smaxLat float64\n\tcheckBoundaries                    bool\n\tonBoundary, notOnBoundary          [][]byte\n\tisIndexed                          func(term []byte) bool\n}\n\nfunc (grc *geoRangeCompute) makePrefixCoded(in int64, shift uint) (rv numeric.PrefixCoded) {\n\tif len(grc.preallocBytes) <= 0 {\n\t\tgrc.preallocBytesLen = grc.preallocBytesLen * 2\n\t\tgrc.preallocBytes = make([]byte, grc.preallocBytesLen)\n\t}\n\n\trv, grc.preallocBytes, _ = numeric.NewPrefixCodedInt64Prealloc(in, shift, grc.preallocBytes)\n\n\treturn rv\n}\n\nfunc (grc *geoRangeCompute) computeGeoRange(term uint64, shift uint) {\n\tsplit := term | uint64(0x1)<<shift\n\tvar upperMax uint64\n\tif shift < 63 {\n\t\tupperMax = term | ((uint64(1) << (shift + 1)) - 1)\n\t} else {\n\t\tupperMax = 0xffffffffffffffff\n\t}\n\tlowerMax := split - 1\n\tgrc.relateAndRecurse(term, lowerMax, shift)\n\tgrc.relateAndRecurse(split, upperMax, shift)\n}\n\nfunc (grc *geoRangeCompute) relateAndRecurse(start, end uint64, res uint) {\n\tminLon := geo.MortonUnhashLon(start)\n\tminLat := geo.MortonUnhashLat(start)\n\tmaxLon := geo.MortonUnhashLon(end)\n\tmaxLat := geo.MortonUnhashLat(end)\n\n\tlevel := (GeoBitsShift1 - res) >> 1\n\n\twithin := res%document.GeoPrecisionStep == 0 &&\n\t\tgeo.RectWithin(minLon, minLat, maxLon, maxLat,\n\t\t\tgrc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat)\n\tif within || (level == geoDetailLevel &&\n\t\tgeo.RectIntersects(minLon, minLat, maxLon, maxLat,\n\t\t\tgrc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat)) {\n\t\tcodedTerm := grc.makePrefixCoded(int64(start), res)\n\t\tif grc.isIndexed(codedTerm) {\n\t\t\tif !within && grc.checkBoundaries {\n\t\t\t\tgrc.onBoundary = append(grc.onBoundary, codedTerm)\n\t\t\t} else {\n\t\t\t\tgrc.notOnBoundary = append(grc.notOnBoundary, codedTerm)\n\t\t\t}\n\t\t}\n\t} else if level < geoDetailLevel &&\n\t\tgeo.RectIntersects(minLon, minLat, maxLon, maxLat,\n\t\t\tgrc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) {\n\t\tgrc.computeGeoRange(start, res-1)\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_geoboundingbox_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestGeoBoundingBox(t *testing.T) {\n\ttests := []struct {\n\t\tminLon float64\n\t\tminLat float64\n\t\tmaxLon float64\n\t\tmaxLat float64\n\t\tfield  string\n\t\twant   []string\n\t}{\n\t\t{10.001, 10.001, 20.002, 20.002, \"loc\", nil},\n\t\t{0.001, 0.001, 0.002, 0.002, \"loc\", []string{\"a\"}},\n\t\t{0.001, 0.001, 1.002, 1.002, \"loc\", []string{\"a\", \"b\"}},\n\t\t{0.001, 0.001, 9.002, 9.002, \"loc\", []string{\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\"}},\n\t\t// same upper-left, bottom-right point\n\t\t{25, 25, 25, 25, \"loc\", nil},\n\t\t// box that would return points, but points reversed\n\t\t{0.002, 0.002, 0.001, 0.001, \"loc\", nil},\n\t}\n\n\ti := setupGeo(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor _, test := range tests {\n\t\tgot, err := testGeoBoundingBoxSearch(indexReader, test.minLon, test.minLat, test.maxLon, test.maxLat, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"expected %v, got %v for %f %f %f %f %s\", test.want, got, test.minLon, test.minLat, test.maxLon, test.maxLat, test.field)\n\t\t}\n\n\t}\n}\n\nfunc testGeoBoundingBoxSearch(i index.IndexReader, minLon, minLat, maxLon, maxLat float64, field string) ([]string, error) {\n\tvar rv []string\n\tgbs, err := NewGeoBoundingBoxSearcher(context.TODO(), i, minLon, minLat, maxLon, maxLat, field, 1.0, search.SearcherOptions{}, true)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\trv = append(rv, string(docMatch.IndexInternalID))\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc setupGeo(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := upsidedown.NewUpsideDownCouch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": \"\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc := document.NewDocument(\"a\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 0.0015, 0.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"b\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 1.0015, 1.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"c\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 2.0015, 2.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"d\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 3.0015, 3.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"e\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 4.0015, 4.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"f\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 5.0015, 5.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"g\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 6.0015, 6.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"h\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 7.0015, 7.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"i\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 8.0015, 8.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"j\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 9.0015, 9.0015))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n\nfunc TestComputeGeoRange(t *testing.T) {\n\ttests := []struct {\n\t\tdegs        float64\n\t\tonBoundary  int\n\t\toffBoundary int\n\t\terr         string\n\t}{\n\t\t{0.01, 4, 0, \"\"},\n\t\t{0.1, 56, 144, \"\"},\n\t\t{100.0, 32768, 258560, \"\"},\n\t}\n\n\tfor testi, test := range tests {\n\t\tonBoundaryRes, offBoundaryRes, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1,\n\t\t\t-1.0*test.degs, -1.0*test.degs, test.degs, test.degs, true, nil, \"\")\n\t\tif (err != nil) != (test.err != \"\") {\n\t\t\tt.Errorf(\"test: %+v, err: %v\", test, err)\n\t\t}\n\t\tif len(onBoundaryRes) != test.onBoundary {\n\t\t\tt.Errorf(\"test: %+v, onBoundaryRes: %v\", test, len(onBoundaryRes))\n\t\t}\n\t\tif len(offBoundaryRes) != test.offBoundary {\n\t\t\tt.Errorf(\"test: %+v, offBoundaryRes: %v\", test, len(offBoundaryRes))\n\t\t}\n\n\t\tonBROrig, offBROrig := origComputeGeoRange(0, GeoBitsShift1Minus1,\n\t\t\t-1.0*test.degs, -1.0*test.degs, test.degs, test.degs, true)\n\t\tif !reflect.DeepEqual(onBoundaryRes, onBROrig) {\n\t\t\tt.Errorf(\"testi: %d, test: %+v, onBoundaryRes != onBROrig,\\n onBoundaryRes:%v,\\n onBROrig: %v\",\n\t\t\t\ttesti, test, onBoundaryRes, onBROrig)\n\t\t}\n\t\tif !reflect.DeepEqual(offBoundaryRes, offBROrig) {\n\t\t\tt.Errorf(\"testi: %d, test: %+v, offBoundaryRes, offBROrig,\\n offBoundaryRes: %v,\\n offBROrig: %v\",\n\t\t\t\ttesti, test, offBoundaryRes, offBROrig)\n\t\t}\n\t}\n}\n\n// --------------------------------------------------------------------\n\nfunc BenchmarkComputeGeoRangePt01(b *testing.B) {\n\tonBoundary := 4\n\toffBoundary := 0\n\tbenchmarkComputeGeoRange(b, -0.01, -0.01, 0.01, 0.01, onBoundary, offBoundary)\n}\n\nfunc BenchmarkComputeGeoRangePt1(b *testing.B) {\n\tonBoundary := 56\n\toffBoundary := 144\n\tbenchmarkComputeGeoRange(b, -0.1, -0.1, 0.1, 0.1, onBoundary, offBoundary)\n}\n\nfunc BenchmarkComputeGeoRange10(b *testing.B) {\n\tonBoundary := 5464\n\toffBoundary := 53704\n\tbenchmarkComputeGeoRange(b, -10.0, -10.0, 10.0, 10.0, onBoundary, offBoundary)\n}\n\nfunc BenchmarkComputeGeoRange100(b *testing.B) {\n\tonBoundary := 32768\n\toffBoundary := 258560\n\tbenchmarkComputeGeoRange(b, -100.0, -100.0, 100.0, 100.0, onBoundary, offBoundary)\n}\n\n// --------------------------------------------------------------------\n\nfunc benchmarkComputeGeoRange(b *testing.B,\n\tminLon, minLat, maxLon, maxLat float64, onBoundary, offBoundary int,\n) {\n\tcheckBoundaries := true\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tonBoundaryRes, offBoundaryRes, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1, minLon, minLat, maxLon, maxLat, checkBoundaries, nil, \"\")\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"expected no err\")\n\t\t}\n\t\tif len(onBoundaryRes) != onBoundary || len(offBoundaryRes) != offBoundary {\n\t\t\tb.Fatalf(\"boundaries not matching\")\n\t\t}\n\t}\n}\n\n// --------------------------------------------------------------------\n\n// original, non-optimized implementation of ComputeGeoRange\nfunc origComputeGeoRange(term uint64, shift uint,\n\tsminLon, sminLat, smaxLon, smaxLat float64,\n\tcheckBoundaries bool) (\n\tonBoundary [][]byte, notOnBoundary [][]byte,\n) {\n\tsplit := term | uint64(0x1)<<shift\n\tvar upperMax uint64\n\tif shift < 63 {\n\t\tupperMax = term | ((uint64(1) << (shift + 1)) - 1)\n\t} else {\n\t\tupperMax = 0xffffffffffffffff\n\t}\n\tlowerMax := split - 1\n\tonBoundary, notOnBoundary = origRelateAndRecurse(term, lowerMax, shift,\n\t\tsminLon, sminLat, smaxLon, smaxLat, checkBoundaries)\n\tplusOnBoundary, plusNotOnBoundary := origRelateAndRecurse(split, upperMax, shift,\n\t\tsminLon, sminLat, smaxLon, smaxLat, checkBoundaries)\n\tonBoundary = append(onBoundary, plusOnBoundary...)\n\tnotOnBoundary = append(notOnBoundary, plusNotOnBoundary...)\n\treturn\n}\n\n// original, non-optimized implementation of relateAndRecurse\nfunc origRelateAndRecurse(start, end uint64, res uint,\n\tsminLon, sminLat, smaxLon, smaxLat float64,\n\tcheckBoundaries bool) (\n\tonBoundary [][]byte, notOnBoundary [][]byte,\n) {\n\tminLon := geo.MortonUnhashLon(start)\n\tminLat := geo.MortonUnhashLat(start)\n\tmaxLon := geo.MortonUnhashLon(end)\n\tmaxLat := geo.MortonUnhashLat(end)\n\n\tlevel := ((geo.GeoBits << 1) - res) >> 1\n\n\twithin := res%document.GeoPrecisionStep == 0 &&\n\t\tgeo.RectWithin(minLon, minLat, maxLon, maxLat,\n\t\t\tsminLon, sminLat, smaxLon, smaxLat)\n\tif within || (level == geoDetailLevel &&\n\t\tgeo.RectIntersects(minLon, minLat, maxLon, maxLat,\n\t\t\tsminLon, sminLat, smaxLon, smaxLat)) {\n\t\tif !within && checkBoundaries {\n\t\t\treturn [][]byte{\n\t\t\t\tnumeric.MustNewPrefixCodedInt64(int64(start), res),\n\t\t\t}, nil\n\t\t}\n\t\treturn nil,\n\t\t\t[][]byte{\n\t\t\t\tnumeric.MustNewPrefixCodedInt64(int64(start), res),\n\t\t\t}\n\t} else if level < geoDetailLevel &&\n\t\tgeo.RectIntersects(minLon, minLat, maxLon, maxLat,\n\t\t\tsminLon, sminLat, smaxLon, smaxLat) {\n\t\treturn origComputeGeoRange(start, res-1, sminLon, sminLat, smaxLon, smaxLat,\n\t\t\tcheckBoundaries)\n\t}\n\treturn nil, nil\n}\n"
  },
  {
    "path": "search/searcher/search_geopointdistance.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc NewGeoPointDistanceSearcher(ctx context.Context, indexReader index.IndexReader, centerLon,\n\tcenterLat, dist float64, field string, boost float64,\n\toptions search.SearcherOptions) (search.Searcher, error) {\n\tvar rectSearcher search.Searcher\n\tif tp, ok := indexReader.(index.SpatialIndexPlugin); ok {\n\t\tsp, err := tp.GetSpatialAnalyzerPlugin(\"s2\")\n\t\tif err == nil {\n\t\t\tterms := sp.GetQueryTokens(geo.NewPointDistance(centerLat,\n\t\t\t\tcenterLon, dist))\n\t\t\trectSearcher, err = NewMultiTermSearcher(ctx, indexReader, terms,\n\t\t\t\tfield, boost, options, false)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\t// indexes without the spatial plugin override would get\n\t// initialized here.\n\tif rectSearcher == nil {\n\t\t// compute bounding box containing the circle\n\t\ttopLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err :=\n\t\t\tgeo.RectFromPointDistance(centerLon, centerLat, dist)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// build a searcher for the box\n\t\trectSearcher, err = boxSearcher(ctx, indexReader,\n\t\t\ttopLeftLon, topLeftLat, bottomRightLon, bottomRightLat,\n\t\t\tfield, boost, options, false)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tdvReader, err := indexReader.DocValueReader([]string{field})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// wrap it in a filtering searcher which checks the actual distance\n\treturn NewFilteringSearcher(ctx, rectSearcher,\n\t\tbuildDistFilter(ctx, dvReader, centerLon, centerLat, dist)), nil\n}\n\n// boxSearcher builds a searcher for the described bounding box\n// if the desired box crosses the dateline, it is automatically split into\n// two boxes joined through a disjunction searcher\nfunc boxSearcher(ctx context.Context, indexReader index.IndexReader,\n\ttopLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64,\n\tfield string, boost float64, options search.SearcherOptions, checkBoundaries bool) (\n\tsearch.Searcher, error) {\n\tif bottomRightLon < topLeftLon {\n\t\t// cross date line, rewrite as two parts\n\n\t\tleftSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader,\n\t\t\t-180, bottomRightLat, bottomRightLon, topLeftLat,\n\t\t\tfield, boost, options, checkBoundaries)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trightSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader,\n\t\t\ttopLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options,\n\t\t\tcheckBoundaries)\n\t\tif err != nil {\n\t\t\t_ = leftSearcher.Close()\n\t\t\treturn nil, err\n\t\t}\n\n\t\tboxSearcher, err := NewDisjunctionSearcher(ctx, indexReader,\n\t\t\t[]search.Searcher{leftSearcher, rightSearcher}, 0, options)\n\t\tif err != nil {\n\t\t\t_ = leftSearcher.Close()\n\t\t\t_ = rightSearcher.Close()\n\t\t\treturn nil, err\n\t\t}\n\t\treturn boxSearcher, nil\n\t}\n\n\t// build geoboundingbox searcher for that bounding box\n\tboxSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader,\n\t\ttopLeftLon, bottomRightLat, bottomRightLon, topLeftLat, field, boost,\n\t\toptions, checkBoundaries)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn boxSearcher, nil\n}\n\nfunc buildDistFilter(ctx context.Context, dvReader index.DocValueReader,\n\tcenterLon, centerLat, maxDist float64) FilterFunc {\n\t// reuse the following for each document match that is checked using the filter\n\tvar lons, lats []float64\n\tvar found bool\n\tdvVisitor := func(_ string, term []byte) {\n\t\tif found {\n\t\t\t// avoid redundant work if already found\n\t\t\treturn\n\t\t}\n\t\t// only consider the values which are shifted 0\n\t\tprefixCoded := numeric.PrefixCoded(term)\n\t\tshift, err := prefixCoded.Shift()\n\t\tif err == nil && shift == 0 {\n\t\t\ti64, err := prefixCoded.Int64()\n\t\t\tif err == nil {\n\t\t\t\tlons = append(lons, geo.MortonUnhashLon(uint64(i64)))\n\t\t\t\tlats = append(lats, geo.MortonUnhashLat(uint64(i64)))\n\t\t\t\tfound = true\n\t\t\t}\n\t\t}\n\t}\n\treturn func(sctx *search.SearchContext, d *search.DocumentMatch) bool {\n\t\t// check geo matches against all numeric type terms indexed\n\t\tlons, lats = lons[:0], lats[:0]\n\t\tfound = false\n\t\tif err := dvReader.VisitDocValues(d.IndexInternalID, dvVisitor); err == nil && found {\n\t\t\tbytes := dvReader.BytesRead()\n\t\t\tif bytes > 0 {\n\t\t\t\treportIOStats(ctx, bytes)\n\t\t\t\tsearch.RecordSearchCost(ctx, search.AddM, bytes)\n\t\t\t}\n\t\t\tfor i := range lons {\n\t\t\t\tdist := geo.Haversin(lons[i], lats[i], centerLon, centerLat)\n\t\t\t\tif dist <= maxDist/1000 {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_geopointdistance_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestGeoPointDistanceSearcher(t *testing.T) {\n\ttests := []struct {\n\t\tcenterLon float64\n\t\tcenterLat float64\n\t\tdist      float64\n\t\tfield     string\n\t\twant      []string\n\t}{\n\t\t// approx 110567m per degree at equator\n\t\t{0.0, 0.0, 0, \"loc\", nil},\n\t\t{0.0, 0.0, 110567, \"loc\", []string{\"a\"}},\n\t\t{0.0, 0.0, 2 * 110567, \"loc\", []string{\"a\", \"b\"}},\n\t\t// stretching our approximation here\n\t\t{0.0, 0.0, 15 * 110567, \"loc\", []string{\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\"}},\n\t}\n\n\ti := setupGeo(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor _, test := range tests {\n\t\tgot, err := testGeoPointDistanceSearch(indexReader, test.centerLon, test.centerLat, test.dist, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"expected %v, got %v for %f %f %f %s\", test.want, got, test.centerLon, test.centerLat, test.dist, test.field)\n\t\t}\n\n\t}\n}\n\nfunc testGeoPointDistanceSearch(i index.IndexReader, centerLon, centerLat, dist float64, field string) ([]string, error) {\n\tvar rv []string\n\tgds, err := NewGeoPointDistanceSearcher(context.TODO(), i, centerLon, centerLat, dist, field, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gds.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gds.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\trv = append(rv, string(docMatch.IndexInternalID))\n\t\tdocMatch, err = gds.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc TestGeoPointDistanceCompare(t *testing.T) {\n\ttests := []struct {\n\t\tdocLat, docLon       float64\n\t\tcenterLat, centerLon float64\n\t\tdistance             string\n\t}{\n\t\t// Data points originally from MB-33454.\n\t\t{\n\t\t\tdocLat:    33.718,\n\t\t\tdocLon:    -116.8293,\n\t\t\tcenterLat: 39.59000587,\n\t\t\tcenterLon: -119.22998428,\n\t\t\tdistance:  \"10000mi\",\n\t\t},\n\t\t{\n\t\t\tdocLat:    41.1305,\n\t\t\tdocLon:    -121.6587,\n\t\t\tcenterLat: 61.28,\n\t\t\tcenterLon: -149.34,\n\t\t\tdistance:  \"10000mi\",\n\t\t},\n\t}\n\n\tfor testi, test := range tests {\n\t\t// compares the results from ComputeGeoRange with original, non-optimized version\n\t\tcompare := func(desc string,\n\t\t\tminLon, minLat, maxLon, maxLat float64, checkBoundaries bool,\n\t\t) {\n\t\t\t// do math to produce list of terms needed for this search\n\t\t\tonBoundaryRes, offBoundaryRes, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1,\n\t\t\t\tminLon, minLat, maxLon, maxLat, checkBoundaries, nil, \"\")\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tonBROrig, offBROrig := origComputeGeoRange(0, GeoBitsShift1Minus1,\n\t\t\t\tminLon, minLat, maxLon, maxLat, checkBoundaries)\n\t\t\tif !reflect.DeepEqual(onBoundaryRes, onBROrig) {\n\t\t\t\tt.Fatalf(\"testi: %d, test: %+v, desc: %s, onBoundaryRes != onBROrig,\\n onBoundaryRes:%v,\\n onBROrig: %v\",\n\t\t\t\t\ttesti, test, desc, onBoundaryRes, onBROrig)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(offBoundaryRes, offBROrig) {\n\t\t\t\tt.Fatalf(\"testi: %d, test: %+v, desc: %s, offBoundaryRes, offBROrig,\\n offBoundaryRes: %v,\\n offBROrig: %v\",\n\t\t\t\t\ttesti, test, desc, offBoundaryRes, offBROrig)\n\t\t\t}\n\t\t}\n\n\t\t// follow the general approach of the GeoPointDistanceSearcher...\n\t\tdist, err := geo.ParseDistance(test.distance)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\ttopLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := geo.RectFromPointDistance(test.centerLon, test.centerLat, dist)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif bottomRightLon < topLeftLon {\n\t\t\t// crosses date line, rewrite as two parts\n\t\t\tcompare(\"-180/f\", -180, bottomRightLat, bottomRightLon, topLeftLat, false)\n\t\t\tcompare(\"-180/t\", -180, bottomRightLat, bottomRightLon, topLeftLat, true)\n\n\t\t\tcompare(\"180/f\", topLeftLon, bottomRightLat, 180, topLeftLat, false)\n\t\t\tcompare(\"180/t\", topLeftLon, bottomRightLat, 180, topLeftLat, true)\n\t\t} else {\n\t\t\tcompare(\"reg/f\", topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, false)\n\t\t\tcompare(\"reg/t\", topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, true)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_geopolygon.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc NewGeoBoundedPolygonSearcher(ctx context.Context, indexReader index.IndexReader,\n\tcoordinates []geo.Point, field string, boost float64,\n\toptions search.SearcherOptions) (search.Searcher, error) {\n\tif len(coordinates) < 3 {\n\t\treturn nil, fmt.Errorf(\"Too few points specified for the polygon boundary\")\n\t}\n\n\tvar rectSearcher search.Searcher\n\tif sr, ok := indexReader.(index.SpatialIndexPlugin); ok {\n\t\ttp, err := sr.GetSpatialAnalyzerPlugin(\"s2\")\n\t\tif err == nil {\n\t\t\tterms := tp.GetQueryTokens(geo.NewBoundedPolygon(coordinates))\n\t\t\trectSearcher, err = NewMultiTermSearcher(ctx, indexReader, terms,\n\t\t\t\tfield, boost, options, false)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\t// indexes without the spatial plugin override would get\n\t// initialized here.\n\tif rectSearcher == nil {\n\t\t// compute the bounding box enclosing the polygon\n\t\ttopLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err :=\n\t\t\tgeo.BoundingRectangleForPolygon(coordinates)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// build a searcher for the bounding box on the polygon\n\t\trectSearcher, err = boxSearcher(ctx, indexReader,\n\t\t\ttopLeftLon, topLeftLat, bottomRightLon, bottomRightLat,\n\t\t\tfield, boost, options, true)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tdvReader, err := indexReader.DocValueReader([]string{field})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// wrap it in a filtering searcher that checks for the polygon inclusivity\n\treturn NewFilteringSearcher(ctx, rectSearcher,\n\t\tbuildPolygonFilter(ctx, dvReader, field, coordinates)), nil\n}\n\nconst float64EqualityThreshold = 1e-6\n\nfunc almostEqual(a, b float64) bool {\n\treturn math.Abs(a-b) <= float64EqualityThreshold\n}\n\n// buildPolygonFilter returns true if the point lies inside the\n// polygon. It is based on the ray-casting technique as referred\n// here: https://wrf.ecse.rpi.edu/nikola/pubdetails/pnpoly.html\nfunc buildPolygonFilter(ctx context.Context, dvReader index.DocValueReader, field string,\n\tcoordinates []geo.Point) FilterFunc {\n\t// reuse the following for each document match that is checked using the filter\n\tvar lons, lats []float64\n\tvar found bool\n\tdvVisitor := func(_ string, term []byte) {\n\t\tif found {\n\t\t\t// avoid redundant work if already found\n\t\t\treturn\n\t\t}\n\t\t// only consider the values which are shifted 0\n\t\tprefixCoded := numeric.PrefixCoded(term)\n\t\tshift, err := prefixCoded.Shift()\n\t\tif err == nil && shift == 0 {\n\t\t\ti64, err := prefixCoded.Int64()\n\t\t\tif err == nil {\n\t\t\t\tlons = append(lons, geo.MortonUnhashLon(uint64(i64)))\n\t\t\t\tlats = append(lats, geo.MortonUnhashLat(uint64(i64)))\n\t\t\t\tfound = true\n\t\t\t}\n\t\t}\n\t}\n\trayIntersectsSegment := func(point, a, b geo.Point) bool {\n\t\treturn (a.Lat > point.Lat) != (b.Lat > point.Lat) &&\n\t\t\tpoint.Lon < (b.Lon-a.Lon)*(point.Lat-a.Lat)/(b.Lat-a.Lat)+a.Lon\n\t}\n\treturn func(sctx *search.SearchContext, d *search.DocumentMatch) bool {\n\t\t// check geo matches against all numeric type terms indexed\n\t\tlons, lats = lons[:0], lats[:0]\n\t\tfound = false\n\t\t// Note: this approach works for points which are strictly inside\n\t\t// the polygon. ie it might fail for certain points on the polygon boundaries.\n\t\tif err := dvReader.VisitDocValues(d.IndexInternalID, dvVisitor); err == nil && found {\n\t\t\tbytes := dvReader.BytesRead()\n\t\t\tif bytes > 0 {\n\t\t\t\treportIOStats(ctx, bytes)\n\t\t\t\tsearch.RecordSearchCost(ctx, search.AddM, bytes)\n\t\t\t}\n\t\t\tnVertices := len(coordinates)\n\t\t\tif len(coordinates) < 3 {\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tfor i := range lons {\n\t\t\t\tpt := geo.Point{Lon: lons[i], Lat: lats[i]}\n\t\t\t\tinside := rayIntersectsSegment(pt, coordinates[len(coordinates)-1], coordinates[0])\n\t\t\t\t// check for a direct vertex match\n\t\t\t\tif almostEqual(coordinates[0].Lat, lats[i]) &&\n\t\t\t\t\talmostEqual(coordinates[0].Lon, lons[i]) {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\n\t\t\t\tfor j := 1; j < nVertices; j++ {\n\t\t\t\t\tif almostEqual(coordinates[j].Lat, lats[i]) &&\n\t\t\t\t\t\talmostEqual(coordinates[j].Lon, lons[i]) {\n\t\t\t\t\t\treturn true\n\t\t\t\t\t}\n\t\t\t\t\tif rayIntersectsSegment(pt, coordinates[j-1], coordinates[j]) {\n\t\t\t\t\t\tinside = !inside\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif inside {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_geopolygon_test.go",
    "content": "//  Copyright (c) 2019 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestSimpleGeoPolygons(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon []geo.Point\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t// test points inside a triangle & on vertices\n\t\t// r, s - inside and t,u - on vertices.\n\t\t{[]geo.Point{{Lon: 1.0, Lat: 1.0}, {Lon: 2.0, Lat: 1.9}, {Lon: 2.0, Lat: 1.0}}, \"loc\", []string{\"r\", \"s\", \"t\", \"u\"}},\n\t\t// non overlapping polygon for the indexed documents\n\t\t{[]geo.Point{{Lon: 3.0, Lat: 1.0}, {Lon: 4.0, Lat: 2.5}, {Lon: 3.0, Lat: 2}}, \"loc\", nil},\n\t}\n\ti := setupGeoPolygonPoints(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor _, test := range tests {\n\t\tgot, err := testGeoPolygonSearch(indexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\", test.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc TestRealGeoPolygons(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon []geo.Point\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t{[]geo.Point{\n\t\t\t{Lon: -80.881, Lat: 35.282},\n\t\t\t{Lon: -80.858, Lat: 35.281},\n\t\t\t{Lon: -80.864, Lat: 35.270},\n\t\t}, \"loc\", []string{\"k\", \"l\"}},\n\t\t{[]geo.Point{\n\t\t\t{Lon: -82.467, Lat: 36.356},\n\t\t\t{Lon: -78.127, Lat: 36.321},\n\t\t\t{Lon: -80.555, Lat: 32.932},\n\t\t\t{Lon: -84.807, Lat: 33.111},\n\t\t}, \"loc\", []string{\"k\", \"l\", \"m\"}},\n\t\t// same polygon vertices\n\t\t{[]geo.Point{{Lon: -82.467, Lat: 36.356}, {Lon: -82.467, Lat: 36.356}, {Lon: -82.467, Lat: 36.356}, {Lon: -82.467, Lat: 36.356}}, \"loc\", nil},\n\t\t// non-overlaping polygon\n\t\t{[]geo.Point{{Lon: -89.113, Lat: 36.400}, {Lon: -93.947, Lat: 36.471}, {Lon: -93.947, Lat: 34.031}}, \"loc\", nil},\n\t\t// concave polygon with a document `n` residing inside the hands, but outside the polygon\n\t\t{[]geo.Point{{Lon: -71.65, Lat: 42.446}, {Lon: -71.649, Lat: 42.428}, {Lon: -71.640, Lat: 42.445}, {Lon: -71.649, Lat: 42.435}}, \"loc\", nil},\n\t\t// V like concave polygon with a document 'p' residing inside the bottom corner\n\t\t{[]geo.Point{{Lon: -80.304, Lat: 40.740}, {Lon: -80.038, Lat: 40.239}, {Lon: -79.562, Lat: 40.786}, {Lon: -80.018, Lat: 40.328}}, \"loc\", []string{\"p\"}},\n\t\t{[]geo.Point{\n\t\t\t{Lon: -111.918, Lat: 33.515},\n\t\t\t{Lon: -111.938, Lat: 33.494},\n\t\t\t{Lon: -111.944, Lat: 33.481},\n\t\t\t{Lon: -111.886, Lat: 33.517},\n\t\t\t{Lon: -111.919, Lat: 33.468},\n\t\t\t{Lon: -111.929, Lat: 33.508},\n\t\t}, \"loc\", []string{\"q\"}},\n\t\t// real points near cb bangalore\n\t\t{[]geo.Point{\n\t\t\t{Lat: 12.974872, Lon: 77.607749},\n\t\t\t{Lat: 12.971725, Lon: 77.610110},\n\t\t\t{Lat: 12.972530, Lon: 77.606912},\n\t\t\t{Lat: 12.975112, Lon: 77.603780},\n\t\t}, \"loc\", []string{\"amoeba\", \"communiti\"}},\n\t}\n\n\ti := setupGeoPolygonPoints(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor _, test := range tests {\n\t\tgot, err := testGeoPolygonSearch(indexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\", test.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc TestGeoRectanglePolygon(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon []geo.Point\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t{\n\t\t\t[]geo.Point{{Lon: 0, Lat: 0}, {Lon: 0, Lat: 50}, {Lon: 50, Lat: 50}, {Lon: 50, Lat: 0}, {Lon: 0, Lat: 0}},\n\t\t\t\"loc\",\n\t\t\t[]string{\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\"},\n\t\t},\n\t}\n\n\ti := setupGeo(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor _, test := range tests {\n\t\tgot, err := testGeoPolygonSearch(indexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\", test.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc testGeoPolygonSearch(i index.IndexReader, polygon []geo.Point, field string) ([]string, error) {\n\tvar rv []string\n\tgbs, err := NewGeoBoundedPolygonSearcher(context.TODO(), i, polygon, field, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\trv = append(rv, string(docMatch.IndexInternalID))\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc setupGeoPolygonPoints(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := upsidedown.NewUpsideDownCouch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": \"\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc := document.NewDocument(\"k\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, -80.86469327, 35.2782))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"l\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, -80.8713, 35.28138))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"m\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, -84.25, 33.153))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"n\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, -89.992, 35.063))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"o\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, -71.648, 42.437))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"p\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, -80.016, 40.314))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"q\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, -111.919, 33.494))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"r\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 1.5, 1.1))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"s\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 2, 1.5))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"t\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 2.0, 1.9))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"u\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 2.0, 1.0))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"amoeba\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 77.60490, 12.97467))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"communiti\")\n\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, 77.608237, 12.97237))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n\ntype geoPoint struct {\n\ttitle string\n\tlon   float64\n\tlat   float64\n}\n\n// Test points inside a complex self intersecting polygon\nfunc TestComplexGeoPolygons(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon []geo.Point\n\t\tpoints  []geoPoint\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t/*\n\t\t\t /\\      /\\\n\t\t\t/__\\____/__\\\n\t\t\t    \\  /\n\t\t\t     \\/\n\t\t*/\n\t\t// a, b, c - inside and d - on vertices.\n\t\t{\n\t\t\t[]geo.Point{\n\t\t\t\t{Lon: 6.0, Lat: 2.0},\n\t\t\t\t{Lon: 3.0, Lat: 4.0},\n\t\t\t\t{Lon: 9.0, Lat: 6.0},\n\t\t\t\t{Lon: 3.0, Lat: 8.0},\n\t\t\t\t{Lon: 6.0, Lat: 10.0},\n\t\t\t\t{Lon: 6.0, Lat: 2.0},\n\t\t\t},\n\t\t\t[]geoPoint{\n\t\t\t\t{title: \"a\", lon: 3, lat: 4},\n\t\t\t\t{title: \"b\", lon: 7, lat: 6},\n\t\t\t\t{title: \"c\", lon: 4, lat: 8.1},\n\t\t\t\t{title: \"d\", lon: 6, lat: 10.0},\n\t\t\t\t{title: \"e\", lon: 5, lat: 6},\n\t\t\t\t{title: \"f\", lon: 7, lat: 5},\n\t\t\t},\n\t\t\t\"loc\",\n\t\t\t[]string{\"a\", \"b\", \"c\", \"d\"},\n\t\t},\n\t\t/*\n\t\t\t____\n\t\t\t\\  /\n\t\t\t \\/\n\t\t\t /\\\n\t\t\t/__\\\n\t\t*/\n\t\t{\n\t\t\t[]geo.Point{\n\t\t\t\t{Lon: 7.0, Lat: 2.0},\n\t\t\t\t{Lon: 1.0, Lat: 8.0},\n\t\t\t\t{Lon: 1.0, Lat: 2.0},\n\t\t\t\t{Lon: 7.0, Lat: 8.0},\n\t\t\t\t{Lon: 7.0, Lat: 2.0},\n\t\t\t},\n\t\t\t[]geoPoint{\n\t\t\t\t{title: \"a\", lon: 6, lat: 5},\n\t\t\t\t{title: \"b\", lon: 5, lat: 5},\n\t\t\t\t{title: \"c\", lon: 3, lat: 5.0},\n\t\t\t\t{title: \"d\", lon: 2, lat: 4.0},\n\t\t\t\t{title: \"e\", lon: 5, lat: 3},\n\t\t\t\t{title: \"f\", lon: 4, lat: 4},\n\t\t\t},\n\t\t\t\"loc\",\n\t\t\t[]string{\"a\", \"b\", \"c\", \"d\"},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ti := setupComplexGeoPolygonPoints(t, test.points)\n\t\tindexReader, err := i.Reader()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tgot, err := testGeoPolygonSearch(indexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"expected %v, got %v for polygon: %+v\", test.want, got, test.polygon)\n\t\t}\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc setupComplexGeoPolygonPoints(t *testing.T, points []geoPoint) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := upsidedown.NewUpsideDownCouch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": \"\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, point := range points {\n\t\tdoc := document.NewDocument(point.title)\n\t\tdoc.AddField(document.NewGeoPointField(\"loc\", []uint64{}, point.lon, point.lat))\n\t\terr = i.Update(doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\treturn i\n}\n"
  },
  {
    "path": "search/searcher/search_geoshape.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n\t\"github.com/blevesearch/geo/geojson\"\n\t\"github.com/blevesearch/geo/s2\"\n)\n\nfunc NewGeoShapeSearcher(ctx context.Context, indexReader index.IndexReader, shape index.GeoJSON,\n\trelation string, field string, boost float64,\n\toptions search.SearcherOptions,\n) (search.Searcher, error) {\n\tvar err error\n\tvar spatialPlugin index.SpatialAnalyzerPlugin\n\n\t// check for the spatial plugin from the index.\n\tif sr, ok := indexReader.(index.SpatialIndexPlugin); ok {\n\t\tspatialPlugin, _ = sr.GetSpatialAnalyzerPlugin(\"s2\")\n\t}\n\n\tif spatialPlugin == nil {\n\t\t// fallback to the default spatial plugin(s2).\n\t\tspatialPlugin = geo.GetSpatialAnalyzerPlugin(\"s2\")\n\t}\n\n\t// obtain the query tokens.\n\tterms := spatialPlugin.GetQueryTokens(shape)\n\tmSearcher, err := NewMultiTermSearcher(ctx, indexReader, terms,\n\t\tfield, boost, options, false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdvReader, err := indexReader.DocValueReader([]string{field})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn NewFilteringSearcher(ctx, mSearcher, buildRelationFilterOnShapes(ctx, dvReader, field, relation, shape)), nil\n}\n\nfunc buildRelationFilterOnShapes(ctx context.Context, dvReader index.DocValueReader, field string,\n\trelation string, shape index.GeoJSON,\n) FilterFunc {\n\t// this is for accumulating the shape's actual complete value\n\t// spread across multiple docvalue visitor callbacks.\n\tvar dvShapeValue []byte\n\tvar startReading, finishReading, found bool\n\tvar reader *bytes.Reader\n\n\tvar bufPool *s2.GeoBufferPool\n\tif bufPoolCallback, ok := ctx.Value(search.GeoBufferPoolCallbackKey).(search.GeoBufferPoolCallbackFunc); ok {\n\t\tbufPool = bufPoolCallback()\n\t}\n\n\tdvVisitor := func(_ string, term []byte) {\n\t\tif found {\n\t\t\t// avoid redundant work if already found\n\t\t\treturn\n\t\t}\n\t\ttl := len(term)\n\t\t// only consider the values which are GlueBytes prefixed or\n\t\t// if it had already started reading the shape bytes from previous callbacks.\n\t\tif startReading || tl > geo.GlueBytesOffset {\n\n\t\t\tif !startReading && bytes.Equal(geo.GlueBytes, term[:geo.GlueBytesOffset]) {\n\t\t\t\tstartReading = true\n\n\t\t\t\tif bytes.Equal(geo.GlueBytes, term[tl-geo.GlueBytesOffset:]) {\n\t\t\t\t\tterm = term[:tl-geo.GlueBytesOffset]\n\t\t\t\t\tfinishReading = true\n\t\t\t\t}\n\n\t\t\t\tdvShapeValue = append(dvShapeValue, term[geo.GlueBytesOffset:]...)\n\n\t\t\t} else if startReading && !finishReading {\n\t\t\t\tif tl > geo.GlueBytesOffset &&\n\t\t\t\t\tbytes.Equal(geo.GlueBytes, term[tl-geo.GlueBytesOffset:]) {\n\t\t\t\t\tterm = term[:tl-geo.GlueBytesOffset]\n\t\t\t\t\tfinishReading = true\n\t\t\t\t}\n\n\t\t\t\tdvShapeValue = append(dvShapeValue, index.DocValueTermSeparator)\n\t\t\t\tdvShapeValue = append(dvShapeValue, term...)\n\t\t\t}\n\n\t\t\t// apply the filter once the entire docvalue is finished reading.\n\t\t\tif finishReading {\n\t\t\t\tv, err := geojson.FilterGeoShapesOnRelation(shape, dvShapeValue, relation, &reader, bufPool)\n\t\t\t\tif err == nil && v {\n\t\t\t\t\tfound = true\n\t\t\t\t}\n\n\t\t\t\tdvShapeValue = dvShapeValue[:0]\n\t\t\t\tstartReading = false\n\t\t\t\tfinishReading = false\n\t\t\t}\n\t\t}\n\t}\n\n\treturn func(sctx *search.SearchContext, d *search.DocumentMatch) bool {\n\t\t// reset state variables for each document\n\t\tfound = false\n\t\tstartReading = false\n\t\tfinishReading = false\n\t\tdvShapeValue = dvShapeValue[:0]\n\t\tif err := dvReader.VisitDocValues(d.IndexInternalID, dvVisitor); err == nil && found {\n\t\t\tbytes := dvReader.BytesRead()\n\t\t\tif bytes > 0 {\n\t\t\t\treportIOStats(ctx, bytes)\n\t\t\t\tsearch.RecordSearchCost(ctx, search.AddM, bytes)\n\t\t\t}\n\t\t\treturn found\n\t\t}\n\n\t\treturn false\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_geoshape_circle_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestGeoJsonCircleIntersectsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tcentrePoint    []float64\n\t\tradiusInMeters string\n\t\tfield          string\n\t\twant           []string\n\t}{\n\t\t// test intersecting query circle for polygon1.\n\t\t{\n\t\t\t[]float64{77.68115043640137, 12.94663769274367},\n\t\t\t\"200m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test intersecting query circle for polygon1, circle1 and linestring1.\n\t\t{\n\t\t\t[]float64{77.68115043640137, 12.94663769274367},\n\t\t\t\"750m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"circle1\", \"linestring1\"},\n\t\t},\n\n\t\t// test intersecting query circle for linestring2.\n\t\t{\n\t\t\t[]float64{77.69591331481932, 12.92756503709986},\n\t\t\t\"250m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring2\"},\n\t\t},\n\n\t\t// test intersecting query circle for circle1.\n\t\t{[]float64{77.6767, 12.9422}, \"250m\", \"geometry\", []string{\"circle1\"}},\n\n\t\t// test intersecting query circle for point1, envelope1 and linestring3.\n\t\t{\n\t\t\t[]float64{81.243896484375, 26.22444694563432},\n\t\t\t\"90000m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"point1\", \"envelope1\", \"linestring3\"},\n\t\t},\n\n\t\t// test intersecting query circle for envelope.\n\t\t{\n\t\t\t[]float64{79.98458862304688, 25.339061458818374},\n\t\t\t\"1250m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\"},\n\t\t},\n\n\t\t// test intersecting query circle for multipoint.\n\t\t{\n\t\t\t[]float64{81.87346458435059, 25.41505910223247},\n\t\t\t\"200m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipoint1\"},\n\t\t},\n\n\t\t// test intersecting query circle for multilinestring.\n\t\t{\n\t\t\t[]float64{81.8669843673706, 25.512661276952272},\n\t\t\t\"90m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring1\"},\n\t\t},\n\t}\n\n\ti := setupGeoJsonShapesIndexForCircleQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeCircleRelationQuery(\"intersects\",\n\t\t\tindexReader, test.centrePoint, test.radiusInMeters, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.centrePoint)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonCircleWithInQuery(t *testing.T) {\n\ttests := []struct {\n\t\tcentrePoint    []float64\n\t\tradiusInMeters string\n\t\tfield          string\n\t\twant           []string\n\t}{\n\t\t// test query circle containing polygon2 and multilinestring2.\n\t\t{\n\t\t\t[]float64{81.85981750488281, 25.546778150624146},\n\t\t\t\"3700m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon2\", \"multilinestring2\"},\n\t\t},\n\n\t\t// test query circle containing multilinestring2.\n\t\t{\n\t\t\t[]float64{81.85981750488281, 25.546778150624146},\n\t\t\t\"3250m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring2\"},\n\t\t},\n\n\t\t// test query circle containing multipoint1.\n\t\t{\n\t\t\t[]float64{81.88599586486816, 25.425756968727935},\n\t\t\t\"1650m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipoint1\"},\n\t\t},\n\n\t\t// test query circle containing circle2.\n\t\t{\n\t\t\t[]float64{82.09362030029297, 25.546313513788725},\n\t\t\t\"1280m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope2\", \"circle2\"},\n\t\t},\n\n\t\t// test query circle containing envelope2 and circle2.\n\t\t{\n\t\t\t[]float64{82.10289001464844, 25.544919592476727},\n\t\t\t\"700m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope2\", \"circle2\"},\n\t\t},\n\n\t\t// test query circle containing point1 and linestring3.\n\t\t{\n\t\t\t[]float64{81.27685546875, 26.1899475672235},\n\t\t\t\"5600m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"point1\", \"linestring3\"},\n\t\t},\n\t}\n\n\ti := setupGeoJsonShapesIndexForCircleQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeCircleRelationQuery(\"within\", indexReader, test.centrePoint, test.radiusInMeters, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\", n, test.want, got, test.centrePoint)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonCircleContainsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tcentrePoint    []float64\n\t\tradiusInMeters string\n\t\tfield          string\n\t\twant           []string\n\t}{\n\t\t// test query circle within polygon3.\n\t\t{\n\t\t\t[]float64{8.549551963806152, 47.3759038562437},\n\t\t\t\"180m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon3\"},\n\t\t},\n\n\t\t// test query circle containing envelope3.\n\t\t{\n\t\t\t[]float64{8.551011085510254, 47.380117626829275},\n\t\t\t\"75m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope3\"},\n\t\t},\n\n\t\t// test query circle exceeding envelope3 with a few meters.\n\t\t{\n\t\t\t[]float64{8.551011085510254, 47.380117626829275},\n\t\t\t\"78m\",\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test query circle containing circle3.\n\t\t{\n\t\t\t[]float64{8.535819053649902, 47.38297989270074},\n\t\t\t\"185m\",\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle3\"},\n\t\t},\n\t}\n\n\ti := setupGeoJsonShapesIndexForCircleQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeCircleRelationQuery(\"contains\",\n\t\t\tindexReader, test.centrePoint, test.radiusInMeters, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.centrePoint)\n\t\t}\n\t}\n}\n\nfunc runGeoShapeCircleRelationQuery(relation string, i index.IndexReader,\n\tpoints []float64, radius string, field string,\n) ([]string, error) {\n\tvar rv []string\n\ts := geo.NewGeoCircle(points, radius)\n\n\tgbs, err := NewGeoShapeSearcher(context.TODO(), i, s, relation, field, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\tdocID, _ := i.ExternalID(docMatch.IndexInternalID)\n\t\trv = append(rv, docID)\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc setupGeoJsonShapesIndexForCircleQuery(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon1 := [][][][]float64{{{\n\t\t{77.67248153686523, 12.957679089615821},\n\t\t{77.67956256866455, 12.948101542434257},\n\t\t{77.68908977508545, 12.948896200093982},\n\t\t{77.68934726715086, 12.955211547173878},\n\t\t{77.68016338348389, 12.954291440344619},\n\t\t{77.67248153686523, 12.957679089615821},\n\t}}}\n\tdoc := document.NewDocument(\"polygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tpolygon2 := [][][][]float64{{{\n\t\t{81.84951782226561, 25.522692102524033},\n\t\t{81.8557834625244, 25.521762640415535},\n\t\t{81.86264991760254, 25.521762640415535},\n\t\t{81.86676979064941, 25.521607729364224},\n\t\t{81.89560890197754, 25.542673796271302},\n\t\t{81.88977241516113, 25.543293330460937},\n\t\t{81.84951782226561, 25.522692102524033},\n\t}}}\n\tdoc = document.NewDocument(\"polygon2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon2, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon3 := [][][][]float64{{{\n\t\t{8.548071384429932, 47.379216780040124},\n\t\t{8.547642230987549, 47.3771680227784},\n\t\t{8.545818328857422, 47.37677569847655},\n\t\t{8.546290397644043, 47.37417465983494},\n\t\t{8.551719188690186, 47.37417465983494},\n\t\t{8.553242683410645, 47.37679022905829},\n\t\t{8.548071384429932, 47.379216780040124},\n\t}}}\n\tdoc = document.NewDocument(\"polygon3\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon3, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpoint1 := [][][][]float64{{{{81.2439, 26.2244}}}}\n\tdoc = document.NewDocument(\"point1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpoint1, \"point\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tenvelope1 := [][][][]float64{{{\n\t\t{79.9969482421875, 23.895882703682627},\n\t\t{80.7220458984375, 25.750424835909385},\n\t}}}\n\tdoc = document.NewDocument(\"envelope1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tenvelope1, \"envelope\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tenvelope2 := [][][][]float64{{{\n\t\t{82.10409164428711, 25.54360309635522},\n\t\t{82.10537910461424, 25.544609829984058},\n\t}}}\n\tdoc = document.NewDocument(\"envelope2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tenvelope2, \"envelope\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tenvelope3 := [][][][]float64{{{\n\t\t{8.545668125152588, 47.37942019840244},\n\t\t{8.552148342132568, 47.383778974713124},\n\t}}}\n\tdoc = document.NewDocument(\"envelope3\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tenvelope3, \"envelope\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle1\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\t[]float64{77.67252445220947, 12.936348678099293}, \"900m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle2\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\t[]float64{82.10289001464844, 25.544919592476727}, \"100m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle3\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\t[]float64{\n\t\t\t8.53363037109375,\n\t\t\t47.38191927423153,\n\t\t}, \"400m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring := [][][][]float64{{{\n\t\t{77.68715858459473, 12.944755587650944},\n\t\t{77.69213676452637, 12.945090185150542},\n\t}}}\n\tdoc = document.NewDocument(\"linestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring1 := [][][][]float64{{{\n\t\t{77.68913269042969, 12.929614580987227},\n\t\t{77.70252227783203, 12.929698235482276},\n\t}}}\n\tdoc = document.NewDocument(\"linestring2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring1, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring2 := [][][][]float64{{{\n\t\t{81.26792907714844, 26.170845301716813},\n\t\t{81.30157470703125, 26.18440207077121},\n\t}}}\n\tdoc = document.NewDocument(\"linestring3\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring2, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring := [][][][]float64{{{\n\t\t{81.86170578002928, 25.430407918899984},\n\t\t{81.86273574829102, 25.421958559611397},\n\t}, {\n\t\t{81.88230514526367, 25.437616536907512},\n\t\t{81.90084457397461, 25.431415601111418},\n\t}, {\n\t\t{81.86805725097656, 25.514868905100244},\n\t\t{81.86702728271484, 25.502474677473746},\n\t}}}\n\tdoc = document.NewDocument(\"multilinestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultilinestring, \"multilinestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring1 := [][][][]float64{{\n\t\t{\n\t\t\t{81.84642791748047, 25.561335859046192},\n\t\t\t{81.84230804443358, 25.550495180470026},\n\t\t},\n\t\t{{81.87423706054688, 25.55142441992021}, {81.88453674316406, 25.555141305670045}},\n\t\t{{81.8642807006836, 25.572175556682115}, {81.87458038330078, 25.567839795359724}},\n\t}}\n\tdoc = document.NewDocument(\"multilinestring2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultilinestring1, \"multilinestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipoint1 := [][][][]float64{{{\n\t\t{81.87337875366211, 25.432268248708212},\n\t\t{81.87355041503906, 25.416299483230368},\n\t\t{81.90118789672852, 25.426067037656946},\n\t}}}\n\tdoc = document.NewDocument(\"multipoint1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultipoint1, \"multipoint\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygonWithHole1 := [][][][]float64{{\n\t\t{\n\t\t\t{77.59991168975829, 12.972232910164502},\n\t\t\t{77.6039457321167, 12.97582941279006},\n\t\t\t{77.60424613952637, 12.98168407323241},\n\t\t\t{77.59974002838135, 12.985489528568463},\n\t\t\t{77.59321689605713, 12.979300406693417},\n\t\t\t{77.59991168975829, 12.972232910164502},\n\t\t},\n\t\t{\n\t\t\t{77.59682178497314, 12.975787593290978},\n\t\t\t{77.60295867919922, 12.975787593290978},\n\t\t\t{77.60295867919922, 12.98143316204164},\n\t\t\t{77.59682178497314, 12.98143316204164},\n\t\t\t{77.59682178497314, 12.975787593290978},\n\t\t},\n\t}}\n\n\tdoc = document.NewDocument(\"polygonWithHole1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygonWithHole1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n"
  },
  {
    "path": "search/searcher/search_geoshape_envelope_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestGeoJsonEnvelopeWithInQuery(t *testing.T) {\n\ttests := []struct {\n\t\tpoints [][]float64\n\t\tfield  string\n\t\twant   []string\n\t}{\n\t\t// test within query envelope for point1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{76.256103515625, 16.76772739719064},\n\t\t\t\t{76.35772705078125, 16.872890378907783},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"point1\"},\n\t\t},\n\n\t\t// test within query envelope for multipoint1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.046142578125, 17.156537255486093},\n\t\t\t\t{81.331787109375, 17.96305758238804},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipoint1\"},\n\t\t},\n\n\t\t// test within query envelope for partial points in a multipoint1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.05987548828125, 17.16178591271515},\n\t\t\t\t{81.36199951171875, 17.861132899477624},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test within query envelope for polygon2 and point1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{76.00341796875, 16.573022719182777},\n\t\t\t\t{76.717529296875, 17.006888277600524},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon2\", \"point1\"},\n\t\t},\n\n\t\t// test within query envelope for linestring1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{76.84112548828125, 16.86500518090961},\n\t\t\t\t{77.62115478515625, 17.531439701706244},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring1\"},\n\t\t},\n\n\t\t// test within query envelope for multilinestring1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.683349609375, 17.104042525557904},\n\t\t\t\t{81.99234008789062, 17.66495983051931},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring1\"},\n\t\t},\n\n\t\t// test within query envelope that is intersecting multilinestring1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.65725708007812, 17.2601707001208},\n\t\t\t\t{81.95114135742186, 17.66495983051931},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test within query envelope for envelope1 and circle1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{74.75372314453125, 17.36636733709516},\n\t\t\t\t{75.509033203125, 18.038809662036805},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\", \"circle1\"},\n\t\t},\n\n\t\t// test within query envelope for envelope1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{74.783935546875, 17.38209494787749},\n\t\t\t\t{75.96221923828125, 17.727758609852284},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\"},\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndexForEnvelopeQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeEnvelopeRelationQuery(\"within\",\n\t\t\tindexReader, test.points, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.points)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonEnvelopeIntersectsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tpoints [][]float64\n\t\tfield  string\n\t\twant   []string\n\t}{\n\t\t// test intersecting query envelope for partial points in a multipoint1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.00769042968749, 17.80622614478282},\n\t\t\t\t{81.199951171875, 17.983957957423037},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipoint1\"},\n\t\t},\n\n\t\t// test intersecting query envelope that is intersecting multilinestring1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.65725708007812, 17.2601707001208},\n\t\t\t\t{81.95114135742186, 17.66495983051931},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring1\"},\n\t\t},\n\n\t\t// test intersecting query envelope for linestring2.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.9854736328125, 18.27369419984127},\n\t\t\t\t{82.14752197265625, 18.633232565431218},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring2\"},\n\t\t},\n\n\t\t// test intersecting query envelope for circle2.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{82.6336669921875, 17.82714499951342},\n\t\t\t\t{82.66387939453125, 17.861132899477624},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle2\"},\n\t\t},\n\n\t\t// test intersecting query envelope for polygon3.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{82.92343139648438, 17.739530934289657},\n\t\t\t\t{82.98797607421874, 17.79184300887134},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon3\"},\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndexForEnvelopeQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeEnvelopeRelationQuery(\"intersects\", indexReader, test.points, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\", n, test.want, got, test.points)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonEnvelopeContainsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tpoints [][]float64\n\t\tfield  string\n\t\twant   []string\n\t}{\n\t\t// test envelope contained within polygon1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{8.548285961151123, 47.376092756617446},\n\t\t\t\t{8.551225662231445, 47.37764752629426},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test envelope partially contained within polygon1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{8.549273014068604, 47.376194471922986},\n\t\t\t\t{8.551654815673828, 47.37827232736301},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test envelope partially contained within polygon1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{8.549273014068604, 47.376194471922986},\n\t\t\t\t{8.551654815673828, 47.37827232736301},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test envelope fully contained within circle3.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{8.532772064208984, 47.380379160110856},\n\t\t\t\t{8.534531593322752, 47.38299442157271},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle3\"},\n\t\t},\n\n\t\t// test envelope partially contained within circle3.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{8.532836437225342, 47.38010309716447},\n\t\t\t\t{8.538415431976318, 47.383081594720466},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndexForEnvelopeQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeEnvelopeRelationQuery(\"contains\",\n\t\t\tindexReader, test.points, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.points)\n\t\t}\n\t}\n}\n\nfunc runGeoShapeEnvelopeRelationQuery(relation string, i index.IndexReader,\n\tpoints [][]float64, field string,\n) ([]string, error) {\n\tvar rv []string\n\ts := geo.NewGeoEnvelope(points)\n\n\tgbs, err := NewGeoShapeSearcher(context.TODO(), i, s, relation, field, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\tdocID, _ := i.ExternalID(docMatch.IndexInternalID)\n\t\trv = append(rv, docID)\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc setupGeoJsonShapesIndexForEnvelopeQuery(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon1 := [][][][]float64{{{\n\t\t{8.548071384429932, 47.379216780040124},\n\t\t{8.547642230987549, 47.3771680227784},\n\t\t{8.545818328857422, 47.37677569847655},\n\t\t{8.546290397644043, 47.37417465983494},\n\t\t{8.551719188690186, 47.37417465983494},\n\t\t{8.553242683410645, 47.37679022905829},\n\t\t{8.548071384429932, 47.379216780040124},\n\t}}}\n\tdoc := document.NewDocument(\"polygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon2 := [][][][]float64{{{\n\t\t{76.70379638671874, 16.828203242420393},\n\t\t{76.36322021484375, 16.58881695544584},\n\t\t{76.70928955078125, 16.720385051694},\n\t\t{76.70379638671874, 16.828203242420393},\n\t}}}\n\tdoc = document.NewDocument(\"polygon2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon2, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon3 := [][][][]float64{{{\n\t\t{82.9522705078125, 17.749994573141873},\n\t\t{82.94952392578125, 17.692436998627272},\n\t\t{82.87673950195312, 17.64009591883757},\n\t\t{82.76412963867188, 17.58643052828743},\n\t\t{82.8094482421875, 17.522272941245202},\n\t\t{82.99621582031249, 17.64009591883757},\n\t\t{82.9522705078125, 17.749994573141873},\n\t}}}\n\tdoc = document.NewDocument(\"polygon3\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon3, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tenvelope1 := [][][][]float64{{{\n\t\t{74.89654541015625, 17.403062993328923},\n\t\t{74.92401123046875, 17.66495983051931},\n\t}}}\n\tdoc = document.NewDocument(\"envelope1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tenvelope1, \"envelope\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle1\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\t[]float64{75.0531005859375, 17.675427818339383}, \"12900m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle2\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\t[]float64{82.69683837890625, 17.902955242676995}, \"6000m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle3\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\t[]float64{\n\t\t\t8.53363037109375,\n\t\t\t47.38191927423153,\n\t\t}, \"400m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpoint1 := [][][][]float64{{{{76.29730224609375, 16.796653031618053}}}}\n\tdoc = document.NewDocument(\"point1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpoint1, \"point\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring1 := [][][][]float64{{{\n\t\t{76.85211181640624, 17.51048642597462},\n\t\t{77.24212646484374, 16.93070509876554},\n\t}}}\n\tdoc = document.NewDocument(\"linestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring1, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring2 := [][][][]float64{{{\n\t\t{81.89208984375, 18.555136195095105},\n\t\t{82.21343994140625, 18.059701055000478},\n\t}}}\n\tdoc = document.NewDocument(\"linestring2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring2, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipoint1 := [][][][]float64{{{\n\t\t{81.24938964843749, 17.602139123350838},\n\t\t{81.30432128906249, 17.56548361143177},\n\t\t{81.29058837890625, 17.180155043474496},\n\t\t{81.09283447265625, 17.87681743233167},\n\t}}}\n\tdoc = document.NewDocument(\"multipoint1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultipoint1, \"multipoint\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring := [][][][]float64{{\n\t\t{\n\t\t\t{81.69708251953125, 17.641404631355755},\n\t\t\t{81.90994262695312, 17.642713334367667},\n\t\t},\n\t\t{{81.6998291015625, 17.620464090732245}, {81.69708251953125, 17.468572623463153}},\n\t\t{{81.70120239257811, 17.458092664041494}, {81.81243896484375, 17.311310073048123}},\n\t\t{{81.815185546875, 17.3034434020238}, {81.81243896484375, 17.109292665395643}},\n\t}}\n\tdoc = document.NewDocument(\"multilinestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultilinestring, \"multilinestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring1 := [][][][]float64{{\n\t\t{\n\t\t\t{77.6015853881836, 12.990089451715061},\n\t\t\t{77.60476112365723, 12.987747683302153},\n\t\t},\n\t\t{{77.59875297546387, 12.988751301039581}, {77.59446144104004, 12.98197680263484}},\n\t\t{{77.60188579559325, 12.982604078764705}, {77.60557651519775, 12.987329508048184}},\n\t}}\n\tdoc = document.NewDocument(\"multilinestring2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultilinestring1, \"multilinestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n"
  },
  {
    "path": "search/searcher/search_geoshape_geometrycollection_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestGeoJSONIntersectsQueryAgainstGeometryCollection(t *testing.T) {\n\ttests := []struct {\n\t\tpoints [][][][][]float64\n\t\ttypes  []string\n\t\tfield  string\n\t\twant   []string\n\t}{\n\t\t// test intersects geometrycollection query for gc_polygon1_linestring1.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{{\n\t\t\t\t\t{-120.80017089843749, 36.54053616262899},\n\t\t\t\t\t{-120.67932128906249, 36.33725319397006},\n\t\t\t\t\t{-120.30578613281251, 36.90597988519294},\n\t\t\t\t\t{-120.80017089843749, 36.54053616262899},\n\t\t\t\t}}},\n\t\t\t\t{{{{-118.24584960937499, 35.32184842037683}, {-117.8668212890625, 35.06597313798418}}}},\n\t\t\t},\n\t\t\t[]string{\"polygon\", \"linestring\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"gc_polygon1_linestring1\"},\n\t\t},\n\n\t\t// test intersects geometrycollection query for gc_polygon1_linestring1.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{\n\t\t\t\t\t{{-118.3172607421875, 35.250105158539355}, {-117.50976562499999, 35.37561413174875}},\n\t\t\t\t\t{{-118.69628906249999, 34.6241677899049}, {-118.3172607421875, 35.03899204678081}},\n\t\t\t\t\t{{-117.94921874999999, 35.146862906756304}, {-117.674560546875, 34.41144164327245}},\n\t\t\t\t}},\n\t\t\t\t{{{\n\t\t\t\t\t{-117.04284667968749, 35.263561862152095},\n\t\t\t\t\t{-116.8505859375, 35.263561862152095},\n\t\t\t\t\t{-116.8505859375, 35.33529320309328},\n\t\t\t\t\t{-117.04284667968749, 35.33529320309328},\n\t\t\t\t\t{-117.04284667968749, 35.263561862152095},\n\t\t\t\t}}},\n\t\t\t},\n\t\t\t[]string{\"multilinestring\", \"polygon\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"gc_polygon1_linestring1\"},\n\t\t},\n\n\t\t// test intersects geometrycollection query for gc_multipolygon1_multilinestring1.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{\n\t\t\t\t\t{{-115.8563232421875, 38.53957267203905}, {-115.58166503906251, 38.54816542304656}},\n\t\t\t\t\t{{-115.8343505859375, 38.45789034424927}, {-115.81237792968749, 38.19502155795575}},\n\t\t\t\t}},\n\t\t\t\t{{{{-116.64905548095702, 37.94920616351679}}}},\n\t\t\t},\n\t\t\t[]string{\"multilinestring\", \"point\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"gc_multipolygon1_multilinestring1\"},\n\t\t},\n\n\t\t// test intersects geometrycollection query for gc_polygon1_linestring1 and gc_multipolygon1_multilinestring1.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{{{-116.64905548095702, 37.94920616351679}, {-118.29528808593751, 34.52466147177172}}}},\n\t\t\t\t{{\n\t\t\t\t\t{{-115.8563232421875, 38.53957267203905}, {-115.58166503906251, 38.54816542304656}},\n\t\t\t\t\t{{-115.8343505859375, 38.45789034424927}, {-115.81237792968749, 38.19502155795575}},\n\t\t\t\t}},\n\t\t\t},\n\t\t\t[]string{\"multipoint\", \"multilinestring\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\n\t\t\t\t\"gc_polygon1_linestring1\",\n\t\t\t\t\"gc_multipolygon1_multilinestring1\",\n\t\t\t},\n\t\t},\n\n\t\t// test intersects geometrycollection query for gc_polygon1_linestring1 and gc_multipolygon1_multilinestring1.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{{\n\t\t\t\t\t{-117.46582031249999, 36.146746777814364},\n\t\t\t\t\t{-116.70227050781249, 36.146746777814364},\n\t\t\t\t\t{-116.70227050781249, 36.69485094156225},\n\t\t\t\t\t{-117.46582031249999, 36.69485094156225},\n\t\t\t\t\t{-117.46582031249999, 36.146746777814364},\n\t\t\t\t}}, {{\n\t\t\t\t\t{-115.5267333984375, 38.06106741381201},\n\t\t\t\t\t{-115.4937744140625, 37.18220222107978},\n\t\t\t\t\t{-114.93896484374999, 37.304644804751106},\n\t\t\t\t\t{-115.5267333984375, 38.06106741381201},\n\t\t\t\t}}},\n\t\t\t\t{{\n\t\t\t\t\t{{-115.8563232421875, 38.53957267203905}, {-115.58166503906251, 38.54816542304656}},\n\t\t\t\t\t{{-115.8343505859375, 38.45789034424927}, {-115.81237792968749, 38.19502155795575}},\n\t\t\t\t}},\n\t\t\t},\n\t\t\t[]string{\"multipolygon\", \"multilinestring\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"gc_point1_multipoint1\"},\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndexForGeometryCollectionQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(\"intersects\",\n\t\t\tindexReader, test.points, test.types, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.points)\n\t\t}\n\t}\n}\n\nfunc TestGeoJSONWithInQueryAgainstGeometryCollection(t *testing.T) {\n\ttests := []struct {\n\t\tpoints [][][][][]float64\n\t\ttypes  []string\n\t\tfield  string\n\t\twant   []string\n\t}{\n\t\t// test within geometrycollection query for gc_multipoint2_multipolygon2_multiline2.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{{{-122.40434646606444, 37.73400071182758}, {-122.39730834960938, 37.73691949864062}}}},\n\t\t\t\t{{{\n\t\t\t\t\t{-122.42511749267578, 37.760808496517235},\n\t\t\t\t\t{-122.42314338684082, 37.74248523826606},\n\t\t\t\t\t{-122.40082740783691, 37.756669194195815},\n\t\t\t\t\t{-122.42511749267578, 37.760808496517235},\n\t\t\t\t}}},\n\t\t\t\t{\n\t\t\t\t\t{{\n\t\t\t\t\t\t{-122.46339797973633, 37.76637243960179},\n\t\t\t\t\t\t{-122.46176719665527, 37.7502901437285},\n\t\t\t\t\t\t{-122.43644714355469, 37.75911208915015},\n\t\t\t\t\t\t{-122.46339797973633, 37.76637243960179},\n\t\t\t\t\t}},\n\t\t\t\t\t{{\n\t\t\t\t\t\t{-122.43653297424315, 37.714720253587004},\n\t\t\t\t\t\t{-122.40563392639159, 37.714720253587004},\n\t\t\t\t\t\t{-122.40563392639159, 37.72904529863455},\n\t\t\t\t\t\t{-122.43653297424315, 37.72904529863455},\n\t\t\t\t\t\t{-122.43653297424315, 37.714720253587004},\n\t\t\t\t\t}},\n\t\t\t\t},\n\t\t\t},\n\t\t\t[]string{\"linestring\", \"polygon\", \"multipolygon\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"gc_multipoint2_multipolygon2_multiline2\"},\n\t\t},\n\n\t\t// test within geometrycollection query.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{{{-122.40434646606444, 37.73400071182758}, {-122.39730834960938, 37.73691949864062}}}},\n\t\t\t\t{\n\t\t\t\t\t{{\n\t\t\t\t\t\t{-122.46339797973633, 37.76637243960179},\n\t\t\t\t\t\t{-122.46176719665527, 37.7502901437285},\n\t\t\t\t\t\t{-122.43644714355469, 37.75911208915015},\n\t\t\t\t\t\t{-122.46339797973633, 37.76637243960179},\n\t\t\t\t\t}},\n\t\t\t\t\t{{\n\t\t\t\t\t\t{-122.43653297424315, 37.714720253587004},\n\t\t\t\t\t\t{-122.40563392639159, 37.714720253587004},\n\t\t\t\t\t\t{-122.40563392639159, 37.72904529863455},\n\t\t\t\t\t\t{-122.43653297424315, 37.72904529863455},\n\t\t\t\t\t\t{-122.43653297424315, 37.714720253587004},\n\t\t\t\t\t}},\n\t\t\t\t},\n\t\t\t},\n\t\t\t[]string{\"linestring\", \"multipolygon\"},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test within geometrycollection for gc_multipoint2_multipolygon2_multiline2.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{{\n\t\t\t\t\t{-122.4491500854492, 37.78170504295941},\n\t\t\t\t\t{-122.4862289428711, 37.747371884118664},\n\t\t\t\t\t{-122.43078231811525, 37.6949593672454},\n\t\t\t\t\t{-122.3799705505371, 37.72945260537779},\n\t\t\t\t\t{-122.3928451538086, 37.78007695280165},\n\t\t\t\t\t{-122.4491500854492, 37.78170504295941},\n\t\t\t\t}}},\n\t\t\t},\n\t\t\t[]string{\"polygon\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"gc_multipoint2_multipolygon2_multiline2\"},\n\t\t},\n\n\t\t// test within geometrycollection for gc_multipolygon3\n\t\t// gc_multipolygon3's multipolygons within the geometrycollection is covered by the\n\t\t// query's geometric collection of a polygon and a multipolygon.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t{{{\n\t\t\t\t\t{86.6162109375, 57.26716357153586},\n\t\t\t\t\t{85.1220703125, 8119},\n\t\t\t\t\t{84.462890625, 56.27996083172844},\n\t\t\t\t\t{86.98974609375, 55.70235509327093},\n\t\t\t\t\t{87.802734375, 56.77680831656842},\n\t\t\t\t\t{86.6162109375, 57.26716357153586},\n\t\t\t\t}}},\n\t\t\t\t{\n\t\t\t\t\t{{\n\t\t\t\t\t\t{75.1025390625, 54.3549556895541},\n\t\t\t\t\t\t{73.1689453125, 54.29088164657006},\n\t\t\t\t\t\t{72.7294921875, 53.08082737207479},\n\t\t\t\t\t\t{74.091796875, 51.998410382390325},\n\t\t\t\t\t\t{76.79443359375, 53.396432127095984},\n\t\t\t\t\t\t{75.1025390625, 54.3549556895541},\n\t\t\t\t\t}},\n\t\t\t\t\t{{\n\t\t\t\t\t\t{80.1123046875, 55.57834467218206},\n\t\t\t\t\t\t{78.9697265625, 55.65279803318956},\n\t\t\t\t\t\t{78.5302734375, 54.635697306063854},\n\t\t\t\t\t\t{79.87060546875, 54.18815548107151},\n\t\t\t\t\t\t{80.96923828125, 54.80068486732233},\n\t\t\t\t\t\t{80.1123046875, 55.57834467218206},\n\t\t\t\t\t}},\n\t\t\t\t},\n\t\t\t},\n\t\t\t[]string{\"polygon\", \"multipolygon\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"gc_multipolygon3\"},\n\t\t},\n\t}\n\n\ti := setupGeoJsonShapesIndexForGeometryCollectionQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(\"within\", indexReader, test.points, test.types, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\", n, test.want, got, test.points)\n\t\t}\n\t}\n}\n\nfunc TestGeoJSONContainsQueryAgainstGeometryCollection(t *testing.T) {\n\ttests := []struct {\n\t\tpoints [][][][][]float64\n\t\ttypes  []string\n\t\tfield  string\n\t\twant   []string\n\t}{\n\t\t// test contains for a geometrycollection that comprises of a linestring,\n\t\t// polygon, multipolygon, point and multipoint for polygon2.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t// linestring\n\t\t\t\t{{{{7.457013130187988, 46.966401589723894}, {7.482891082763671, 46.94554547022893}}}},\n\t\t\t\t// polygon\n\t\t\t\t{{{\n\t\t\t\t\t{7.466454505920409, 46.965054389418476},\n\t\t\t\t\t{7.46143341064453, 46.9641171865865},\n\t\t\t\t\t{7.466325759887694, 46.96101258493027},\n\t\t\t\t\t{7.466454505920409, 46.965054389418476},\n\t\t\t\t}}},\n\t\t\t\t// multipolygon\n\t\t\t\t{\n\t\t\t\t\t{{\n\t\t\t\t\t\t{7.4811744689941415, 46.957966385567474},\n\t\t\t\t\t\t{7.478899955749511, 46.95492001277476},\n\t\t\t\t\t\t{7.484478950500488, 46.95509576976545},\n\t\t\t\t\t\t{7.4811744689941415, 46.957966385567474},\n\t\t\t\t\t}},\n\t\t\t\t\t{{\n\t\t\t\t\t\t{7.466540336608888, 46.94753769790697},\n\t\t\t\t\t\t{7.464609146118165, 46.946219320241674},\n\t\t\t\t\t\t{7.468342781066894, 46.94592634301753},\n\t\t\t\t\t\t{7.466540336608888, 46.94753769790697},\n\t\t\t\t\t}},\n\t\t\t\t\t{{\n\t\t\t\t\t\t{7.504348754882812, 47.00425575323296},\n\t\t\t\t\t\t{7.501087188720703, 47.001680295206874},\n\t\t\t\t\t\t{7.507266998291015, 47.00191443288521},\n\t\t\t\t\t\t{7.504348754882812, 47.00425575323296},\n\t\t\t\t\t}},\n\t\t\t\t},\n\t\t\t\t// point\n\t\t\t\t{{{{7.449932098388673, 46.95817142366062}}}},\n\t\t\t\t// multipoint\n\t\t\t\t{{{{7.479157447814942, 46.96370715518446}, {7.4532365798950195, 46.96657730900153}}}},\n\t\t\t},\n\t\t\t[]string{\"linestring\", \"polygon\", \"multipolygon\", \"point\", \"multipoint\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipolygon4\"},\n\t\t},\n\n\t\t// test contains for a geometrycollection query with one point inside the multipoint lying outside\n\t\t// polygon2.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t// linestring\n\t\t\t\t{{{{7.457013130187988, 46.966401589723894}, {7.482891082763671, 46.94554547022893}}}},\n\t\t\t\t// polygon\n\t\t\t\t{{{\n\t\t\t\t\t{7.466454505920409, 46.965054389418476},\n\t\t\t\t\t{7.46143341064453, 46.9641171865865},\n\t\t\t\t\t{7.466325759887694, 46.96101258493027},\n\t\t\t\t\t{7.466454505920409, 46.965054389418476},\n\t\t\t\t}}},\n\t\t\t\t// multipolygon\n\t\t\t\t{\n\t\t\t\t\t{{\n\t\t\t\t\t\t{7.4811744689941415, 46.957966385567474},\n\t\t\t\t\t\t{7.478899955749511, 46.95492001277476},\n\t\t\t\t\t\t{7.484478950500488, 46.95509576976545},\n\t\t\t\t\t\t{7.4811744689941415, 46.957966385567474},\n\t\t\t\t\t}},\n\t\t\t\t\t{{\n\t\t\t\t\t\t{7.466540336608888, 46.94753769790697},\n\t\t\t\t\t\t{7.464609146118165, 46.946219320241674},\n\t\t\t\t\t\t{7.468342781066894, 46.94592634301753},\n\t\t\t\t\t\t{7.466540336608888, 46.94753769790697},\n\t\t\t\t\t}},\n\t\t\t\t},\n\t\t\t\t// point\n\t\t\t\t{{{{7.449932098388673, 46.95817142366062}}}},\n\t\t\t\t// multipoint\n\t\t\t\t{{{{7.479157447814942, 46.96370715518446}, {7.475638389587402, 46.965200825877794}}}},\n\t\t\t},\n\t\t\t[]string{\"linestring\", \"polygon\", \"multipolygon\", \"point\", \"multipoint\"},\n\t\t\t\"geometry\",\n\t\t\tnil,\n\t\t},\n\n\t\t// test contains for a geometrycollection query with one point inside the multipoint lying outside\n\t\t// polygon2.\n\t\t{\n\t\t\t[][][][][]float64{\n\t\t\t\t// linestring\n\t\t\t\t{{{{7.457013130187988, 46.966401589723894}, {7.482891082763671, 46.94554547022893}}}},\n\t\t\t\t// polygon\n\t\t\t\t{{{\n\t\t\t\t\t{7.466454505920409, 46.965054389418476},\n\t\t\t\t\t{7.46143341064453, 46.9641171865865},\n\t\t\t\t\t{7.466325759887694, 46.96101258493027},\n\t\t\t\t\t{7.466454505920409, 46.965054389418476},\n\t\t\t\t}}},\n\t\t\t\t// multipolygon\n\t\t\t\t{\n\t\t\t\t\t{{\n\t\t\t\t\t\t{7.4811744689941415, 46.957966385567474},\n\t\t\t\t\t\t{7.478899955749511, 46.95492001277476},\n\t\t\t\t\t\t{7.484478950500488, 46.95509576976545},\n\t\t\t\t\t\t{7.4811744689941415, 46.957966385567474},\n\t\t\t\t\t}},\n\t\t\t\t\t{{\n\t\t\t\t\t\t{7.466540336608888, 46.94753769790697},\n\t\t\t\t\t\t{7.464609146118165, 46.946219320241674},\n\t\t\t\t\t\t{7.468342781066894, 46.94592634301753},\n\t\t\t\t\t\t{7.466540336608888, 46.94753769790697},\n\t\t\t\t\t}},\n\t\t\t\t},\n\t\t\t\t// point\n\t\t\t\t{{{{7.449932098388673, 46.95817142366062}}}},\n\t\t\t\t// multipoint\n\t\t\t\t{{{{7.479157447814942, 46.96370715518446}, {7.4532365798950195, 46.96657730900153}}}},\n\t\t\t},\n\t\t\t[]string{\"linestring\", \"polygon\", \"multipolygon\", \"point\", \"multipoint\"},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon2\", \"multipolygon4\"},\n\t\t},\n\t}\n\n\ti := setupGeoJsonShapesIndexForGeometryCollectionQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeGeometryCollectionRelationQuery(\"contains\",\n\t\t\tindexReader, test.points, test.types, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.points)\n\t\t}\n\t}\n}\n\nfunc runGeoShapeGeometryCollectionRelationQuery(relation string, i index.IndexReader,\n\tpoints [][][][][]float64, types []string, field string,\n) ([]string, error) {\n\tvar rv []string\n\ts, _, err := geo.NewGeometryCollection(points, types)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tgbs, err := NewGeoShapeSearcher(context.TODO(), i, s, relation, field, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\tdocID, _ := i.ExternalID(docMatch.IndexInternalID)\n\t\trv = append(rv, docID)\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc setupGeoJsonShapesIndexForGeometryCollectionQuery(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// document gc_polygon1_linestring1\n\tpolygon1 := [][][][]float64{{{\n\t\t{-118.15246582031249, 34.876918445772084},\n\t\t{-118.46557617187499, 34.773203753940734},\n\t\t{-118.3172607421875, 34.50655662164561},\n\t\t{-117.91625976562499, 34.4793919710481},\n\t\t{-117.76245117187499, 34.76417891445512},\n\t\t{-118.15246582031249, 34.876918445772084},\n\t}}}\n\n\tlinestring1 := [][][][]float64{{{\n\t\t{-120.78918457031251, 36.87522650673951},\n\t\t{-118.9215087890625, 34.95349314197422},\n\t}}}\n\n\tcoordinates := [][][][][]float64{polygon1, linestring1}\n\ttypes := []string{\"polygon\", \"linestring\"}\n\n\tdoc := document.NewDocument(\"gc_polygon1_linestring1\")\n\tdoc.AddField(document.NewGeometryCollectionFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// document gc_multipolygon1_multilinestring1\n\tmultipolygon1 := [][][][]float64{\n\t\t{{\n\t\t\t{-117.24609374999999, 37.67512527892127},\n\t\t\t{-117.61962890624999, 37.26530995561875},\n\t\t\t{-116.597900390625, 37.56199695314352},\n\t\t\t{-117.24609374999999, 37.67512527892127},\n\t\t}},\n\t\t{{\n\t\t\t{-117.60864257812501, 38.71123253895224},\n\t\t\t{-117.41638183593749, 38.36750215395045},\n\t\t\t{-117.66357421875, 37.93986540897977},\n\t\t\t{-116.6473388671875, 37.94852933714952},\n\t\t\t{-117.1307373046875, 38.363195134453846},\n\t\t\t{-116.75170898437501, 38.7283759182398},\n\t\t\t{-117.60864257812501, 38.71123253895224},\n\t\t}},\n\t}\n\tmultilinestring1 := [][][][]float64{{\n\t\t{{-118.9215087890625, 38.74123075381228}, {-118.78967285156249, 38.43207668538207}},\n\t\t{{-118.57543945312501, 38.8225909761771}, {-118.45458984375, 38.522384090200845}},\n\t\t{{-118.94897460937499, 38.788345355085625}, {-118.61938476562499, 38.86965182408357}},\n\t}}\n\n\tcoordinates = [][][][][]float64{multipolygon1, multilinestring1}\n\ttypes = []string{\"multipolygon\", \"multilinestring\"}\n\tdoc = document.NewDocument(\"gc_multipolygon1_multilinestring1\")\n\tdoc.AddField(document.NewGeometryCollectionFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// document gc_point1_multipoint1\n\tpoint1 := [][][][]float64{{{{-115.10925292968749, 36.20882309283712}}}}\n\tmultipoint1 := [][][][]float64{{{\n\t\t{-117.13623046874999, 36.474306755095235},\n\t\t{-118.57543945312501, 36.518465989675875},\n\t\t{-118.58642578124999, 36.90597988519294},\n\t\t{-119.5477294921875, 37.85316995894978},\n\t}}}\n\n\tcoordinates = [][][][][]float64{point1, multipoint1}\n\ttypes = []string{\"point\", \"multipoint\"}\n\n\tdoc = document.NewDocument(\"gc_point1_multipoint1\")\n\tdoc.AddField(document.NewGeometryCollectionFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// document gc_multipoint2_multipolygon2_multiline2\n\tmultipoint2 := [][][][]float64{{{\n\t\t{-122.4052906036377, 37.75626203719391},\n\t\t{-122.42091178894044, 37.74757548736071},\n\t}}}\n\tmultipolygon2 := [][][][]float64{\n\t\t{{\n\t\t\t{-122.46168136596681, 37.765151122096945},\n\t\t\t{-122.46168136596681, 37.754972691904946},\n\t\t\t{-122.45103836059569, 37.754972691904946},\n\t\t\t{-122.451810836792, 37.7624370109886},\n\t\t\t{-122.46168136596681, 37.765151122096945},\n\t\t}},\n\t\t{{\n\t\t\t{-122.41902351379395, 37.726194088705576},\n\t\t\t{-122.43533134460448, 37.71668926284967},\n\t\t\t{-122.40777969360353, 37.71634978222733},\n\t\t\t{-122.41902351379395, 37.726194088705576},\n\t\t}},\n\t}\n\tmultilinestring2 := [][][][]float64{{\n\t\t{{-122.41284370422362, 37.73155698786267}, {-122.40700721740721, 37.73338978839743}},\n\t\t{{-122.40434646606444, 37.73400071182758}, {-122.39730834960938, 37.73691949864062}},\n\t}}\n\n\tcoordinates = [][][][][]float64{multipoint2, multipolygon2, multilinestring2}\n\ttypes = []string{\"multipoint\", \"multipolygon\", \"multiline\"}\n\n\tdoc = document.NewDocument(\"gc_multipoint2_multipolygon2_multiline2\")\n\tdoc.AddField(document.NewGeometryCollectionFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// document gc_multipolygon3\n\tmultipolygon3 := [][][][]float64{\n\t\t{{\n\t\t\t{85.60546875, 57.20771009775018},\n\t\t\t{86.396484375, 55.99838095535963},\n\t\t\t{87.03369140625, 56.71656572651468},\n\t\t\t{85.60546875, 57.20771009775018},\n\t\t}},\n\t\t{{\n\t\t\t{79.56298828125, 55.3915921070334},\n\t\t\t{79.60693359375, 54.43171285946844},\n\t\t\t{80.39794921875, 54.85131525968606},\n\t\t\t{79.56298828125, 55.3915921070334},\n\t\t}},\n\t\t{{\n\t\t\t{74.35546875, 54.13669645687002},\n\t\t\t{74.1796875, 52.802761415419674},\n\t\t\t{75.87158203125, 53.44880683542759},\n\t\t\t{74.35546875, 54.13669645687002},\n\t\t}},\n\t}\n\n\tcoordinates = [][][][][]float64{multipolygon3}\n\ttypes = []string{\"multipolygon\"}\n\n\tdoc = document.NewDocument(\"gc_multipolygon3\")\n\tdoc.AddField(document.NewGeometryCollectionFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon2 := [][][][]float64{{{\n\t\t{7.452635765075683, 46.96692874582506},\n\t\t{7.449803352355956, 46.95817142366062},\n\t\t{7.4573564529418945, 46.95149263607834},\n\t\t{7.462162971496582, 46.945955640812095},\n\t\t{7.483148574829102, 46.945311085627445},\n\t\t{7.487225532531738, 46.957029058564686},\n\t\t{7.4793291091918945, 46.96388288331302},\n\t\t{7.464480400085448, 46.96903731827891},\n\t\t{7.452635765075683, 46.96692874582506},\n\t}}}\n\n\tdoc = document.NewDocument(\"polygon2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon2, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipolygon4 := [][][][]float64{\n\t\t{{\n\t\t\t{7.452635765075683, 46.96692874582506},\n\t\t\t{7.449803352355956, 46.95817142366062},\n\t\t\t{7.4573564529418945, 46.95149263607834},\n\t\t\t{7.462162971496582, 46.945955640812095},\n\t\t\t{7.483148574829102, 46.945311085627445},\n\t\t\t{7.487225532531738, 46.957029058564686},\n\t\t\t{7.4793291091918945, 46.96388288331302},\n\t\t\t{7.464480400085448, 46.96903731827891},\n\t\t\t{7.452635765075683, 46.96692874582506},\n\t\t}},\n\t\t{{\n\t\t\t{7.4478721618652335, 47.00015837528636},\n\t\t\t{7.5110435485839835, 47.00015837528636},\n\t\t\t{7.5110435485839835, 47.00683108710118},\n\t\t\t{7.4478721618652335, 47.00683108710118},\n\t\t\t{7.4478721618652335, 47.00015837528636},\n\t\t}},\n\t}\n\n\tdoc = document.NewDocument(\"multipolygon4\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultipolygon4, \"multipolygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n"
  },
  {
    "path": "search/searcher/search_geoshape_linestring_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestGeoJsonLinestringIntersectsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tline  [][]float64\n\t\tfield string\n\t\twant  []string\n\t}{\n\t\t// test intersecting linestring query for polygon1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{74.85860824584961, 22.407219759334023},\n\t\t\t\t{74.8663330078125, 22.382936446589863},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test intersecting linestring query for polygon1 and polygon2.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{74.82461929321289, 22.393729553598526},\n\t\t\t\t{74.93671417236328, 22.356743809494784},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"polygon2\"},\n\t\t},\n\n\t\t// test intersecting linestring query for envelope1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{74.83938217163086, 22.325782524687973},\n\t\t\t\t{74.8692512512207, 22.311172762889516},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\"},\n\t\t},\n\n\t\t// test intersecting linestring query for circle.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{74.94546890258789, 22.310815439776572},\n\t\t\t\t{74.93276596069336, 22.303708490145645},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle1\"},\n\t\t},\n\n\t\t// test intersecting linestring query for linestring1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{74.938645362854, 22.321614134448936},\n\t\t\t\t{74.94070529937744, 22.320224643365446},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring1\"},\n\t\t},\n\n\t\t// test intersecting linestring query for multilinestring1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{74.9241828918457, 22.307996525380194},\n\t\t\t\t{74.94100570678711, 22.293781977618558},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring1\"},\n\t\t},\n\n\t\t// test intersecting linestring query for multipolygon1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{36.22072219848633, 50.007132228568786},\n\t\t\t\t{36.22218132019043, 49.99791917183082},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipolygon1\"},\n\t\t},\n\n\t\t// test intersecting linestring query for envelope2, circle2,\n\t\t// multipolygon1 and gc_polygonInGc_multipolygonInGc.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{36.19840621948242, 50.03834418692451},\n\t\t\t\t{36.25720024108887, 50.02136210283289},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope2\", \"circle2\", \"multipolygon1\", \"gc_polygonInGc_multipolygonInGc\"},\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndexForLinestringQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeLinestringQueryWithRelation(\"intersects\",\n\t\t\tindexReader, test.line, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.line)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonLinestringContainsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tline  [][]float64\n\t\tfield string\n\t\twant  []string\n\t}{\n\t\t// test a linestring query for multipolygon1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{36.21668815612793, 50.040494087443996},\n\t\t\t\t{36.226301193237305, 50.03861982057644},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipolygon1\"},\n\t\t},\n\n\t\t// test a linestring query with endspoints on two\n\t\t// different polygons in a multipolygon.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{36.19746208190918, 50.038564693972646},\n\t\t\t\t{36.21565818786621, 50.03718650830641},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test a linestring query for envelope2.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{36.25290870666503, 50.03018471417061},\n\t\t\t\t{36.23110771179199, 50.01854955486945},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope2\"},\n\t\t},\n\n\t\t// test a linestring query for circle2.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{36.220550537109375, 50.02930252595981},\n\t\t\t\t{36.224327087402344, 50.02847545979485},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle2\"},\n\t\t},\n\n\t\t// test a linestring query for polygonWithHole2.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{36.27367973327637, 49.89883638369706},\n\t\t\t\t{36.27445220947265, 49.89596137883285},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygonWithHole2\"},\n\t\t},\n\n\t\t// test a linestring query within the hole of polygonWithHole2.\n\t\t{[][]float64{\n\t\t\t{36.261234283447266, 49.89540847364305},\n\t\t\t{36.26243591308594, 49.89087441212101},\n\t\t}, \"geometry\", nil},\n\t}\n\ti := setupGeoJsonShapesIndexForLinestringQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeLinestringQueryWithRelation(\"contains\",\n\t\t\tindexReader, test.line, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.line)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonMultiLinestringContainsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tline  [][][]float64\n\t\tfield string\n\t\twant  []string\n\t}{\n\t\t// test a multilinestring query for multipolygon1.\n\t\t{\n\t\t\t[][][]float64{\n\t\t\t\t{\n\t\t\t\t\t{36.21668815612793, 50.040494087443996},\n\t\t\t\t\t{36.226301193237305, 50.03861982057644},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t{36.226816177368164, 49.999463999158},\n\t\t\t\t\t{36.234025955200195, 50.00271900853649},\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipolygon1\"},\n\t\t},\n\n\t\t// test a multilinestring query that is covered by the geometryCollection.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{36.28664016723633, 49.96574238290487},\n\t\t\t\t{36.30251884460449, 49.96369956194569},\n\t\t\t}, {\n\t\t\t\t{36.19179725646973, 50.03983258984584},\n\t\t\t\t{36.19420051574707, 50.03801342445342},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"gc_polygonInGc_multipolygonInGc\"},\n\t\t},\n\n\t\t// test a multilinestring query for envelope2.\n\t\t{\n\t\t\t[][][]float64{\n\t\t\t\t{\n\t\t\t\t\t{36.23213768005371, 50.02913711386621},\n\t\t\t\t\t{36.25187873840332, 50.02902683882067},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t{36.231794357299805, 50.018935600613254},\n\t\t\t\t\t{36.2314510345459, 50.025883893582055},\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope2\"},\n\t\t},\n\n\t\t// test a multilinestring query with one linestring outside of envelope2.\n\t\t{\n\t\t\t[][][]float64{\n\t\t\t\t{\n\t\t\t\t\t{36.23213768005371, 50.02913711386621},\n\t\t\t\t\t{36.25187873840332, 50.02902683882067},\n\t\t\t\t},\n\t\t\t\t{{36.231794357299805, 50.018935600613254}, {36.2314510345459, 50.025883893582055}},\n\t\t\t\t{{36.25659942626953, 50.024284772330844}, {36.24406814575195, 50.01518531066489}},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test a multilinestring query with one linestring\n\t\t// inside the whole of a polygonWithHole2.\n\t\t{\n\t\t\t[][][]float64{\n\t\t\t\t{\n\t\t\t\t\t{36.27367973327637, 49.89883638369706},\n\t\t\t\t\t{36.27445220947265, 49.89596137883285},\n\t\t\t\t},\n\t\t\t\t{{36.261234283447266, 49.89540847364305}, {36.26243591308594, 49.89087441212101}},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test a multilinestring query for polygonWithHole2.\n\t\t{\n\t\t\t[][][]float64{\n\t\t\t\t{\n\t\t\t\t\t{36.27367973327637, 49.89883638369706},\n\t\t\t\t\t{36.27445220947265, 49.89596137883285},\n\t\t\t\t},\n\t\t\t\t{{36.279258728027344, 49.894302644257856}, {36.28166198730469, 49.887335336408235}},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygonWithHole2\"},\n\t\t},\n\n\t\t// test a multilinestring query for polygonWithHole2 with last line cross the hole.\n\t\t{\n\t\t\t[][][]float64{\n\t\t\t\t{\n\t\t\t\t\t{36.27367973327637, 49.89883638369706},\n\t\t\t\t\t{36.27445220947265, 49.89596137883285},\n\t\t\t\t},\n\t\t\t\t{{36.279258728027344, 49.894302644257856}, {36.28166198730469, 49.887335336408235}},\n\t\t\t\t{{36.254024505615234, 49.89839408640621}, {36.27016067504883, 49.90038439228633}},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndexForLinestringQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeMultiLinestringQueryWithRelation(\"contains\",\n\t\t\tindexReader, test.line, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.line)\n\t\t}\n\t}\n}\n\nfunc runGeoShapeMultiLinestringQueryWithRelation(relation string, i index.IndexReader,\n\tpoints [][][]float64, field string,\n) ([]string, error) {\n\ts := geo.NewGeoJsonMultilinestring(points)\n\treturn executeSearch(relation, i, s, field)\n}\n\nfunc runGeoShapeLinestringQueryWithRelation(relation string, i index.IndexReader,\n\tpoints [][]float64, field string,\n) ([]string, error) {\n\ts := geo.NewGeoJsonLinestring(points)\n\treturn executeSearch(relation, i, s, field)\n}\n\nfunc executeSearch(relation string, i index.IndexReader,\n\ts index.GeoJSON, field string,\n) ([]string, error) {\n\tvar rv []string\n\tgbs, err := NewGeoShapeSearcher(context.TODO(), i, s, relation, field, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\tdocID, _ := i.ExternalID(docMatch.IndexInternalID)\n\t\trv = append(rv, docID)\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc setupGeoJsonShapesIndexForLinestringQuery(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon1 := [][][][]float64{{{\n\t\t{74.84642028808594, 22.402776071459712},\n\t\t{74.83234405517578, 22.39039647758608},\n\t\t{74.86719131469727, 22.38801566009795},\n\t\t{74.85139846801758, 22.39103135536648},\n\t\t{74.86461639404297, 22.394840561182853},\n\t\t{74.8495101928711, 22.397697397065034},\n\t\t{74.86186981201172, 22.401982540816856},\n\t\t{74.84642028808594, 22.402776071459712},\n\t}}}\n\tdoc := document.NewDocument(\"polygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, polygon1, \"polygon\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon2 := [][][][]float64{{{\n\t\t{74.93431091308592, 22.376428433285266},\n\t\t{74.92898941040039, 22.39103135536648},\n\t\t{74.9241828918457, 22.37722210974017},\n\t\t{74.90821838378906, 22.37388863821397},\n\t\t{74.92504119873047, 22.369920115637292},\n\t\t{74.92864608764648, 22.355632497760894},\n\t\t{74.93207931518555, 22.370396344320053},\n\t\t{74.94855880737305, 22.3743648533201},\n\t\t{74.93431091308592, 22.376428433285266},\n\t}}}\n\tdoc = document.NewDocument(\"polygon2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, polygon2, \"polygon\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tenvelope1 := [][][][]float64{{{\n\t\t{74.86736297607422, 22.307361269208684},\n\t\t{74.87028121948242, 22.345471522338478},\n\t}}}\n\tdoc = document.NewDocument(\"envelope1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, envelope1, \"envelope\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tenvelope2 := [][][][]float64{{{\n\t\t{36.23007774353027, 50.01810835593541},\n\t\t{36.25333786010742, 50.03068093791795},\n\t}}}\n\tdoc = document.NewDocument(\"envelope2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, envelope2, \"envelope\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle1\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, []float64{74.93671417236328, 22.308314152382284}, \"300m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle2\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, []float64{36.22243881225586, 50.02941280037234}, \"600m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring := [][][][]float64{{{\n\t\t{74.92697238922119, 22.320343743143248},\n\t\t{74.94036197662354, 22.32054224254707},\n\t}}}\n\tdoc = document.NewDocument(\"linestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, linestring, \"linestring\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring1 := [][][][]float64{{{\n\t\t{77.60188579559325, 12.982604078764705},\n\t\t{77.60557651519775, 12.987329508048184},\n\t}}}\n\tdoc = document.NewDocument(\"linestring2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, linestring1, \"linestring\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring := [][][][]float64{{\n\t\t{\n\t\t\t{74.92203712463379, 22.3113315728684},\n\t\t\t{74.92323875427246, 22.307798008137024},\n\t\t},\n\t\t{{74.92405414581299, 22.307559787072712}, {74.92735862731934, 22.310021385140573}},\n\t\t{{74.9223804473877, 22.311688894660474}, {74.92534160614014, 22.30930673210729}},\n\t}}\n\tdoc = document.NewDocument(\"multilinestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, multilinestring, \"multilinestring\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring1 := [][][][]float64{{\n\t\t{\n\t\t\t{77.6015853881836, 12.990089451715061},\n\t\t\t{77.60476112365723, 12.987747683302153},\n\t\t},\n\t\t{{77.59875297546387, 12.988751301039581}, {77.59446144104004, 12.98197680263484}},\n\t\t{{77.60188579559325, 12.982604078764705}, {77.60557651519775, 12.987329508048184}},\n\t}}\n\tdoc = document.NewDocument(\"multilinestring2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, multilinestring1, \"multilinestring\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipoint1 := [][][][]float64{{{\n\t\t{77.56618022918701, 12.958180959662695},\n\t\t{77.56407737731932, 12.951614746607163},\n\t\t{77.56922721862793, 12.956173473406446},\n\t}}}\n\tdoc = document.NewDocument(\"multipoint1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, multipoint1, \"multipoint\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygonWithHole1 := [][][][]float64{{\n\t\t{\n\t\t\t{77.59991168975829, 12.972232910164502},\n\t\t\t{77.6039457321167, 12.97582941279006},\n\t\t\t{77.60424613952637, 12.98168407323241},\n\t\t\t{77.59974002838135, 12.985489528568463},\n\t\t\t{77.59321689605713, 12.979300406693417},\n\t\t\t{77.59991168975829, 12.972232910164502},\n\t\t},\n\t\t{\n\t\t\t{77.59682178497314, 12.975787593290978},\n\t\t\t{77.60295867919922, 12.975787593290978},\n\t\t\t{77.60295867919922, 12.98143316204164},\n\t\t\t{77.59682178497314, 12.98143316204164},\n\t\t\t{77.59682178497314, 12.975787593290978},\n\t\t},\n\t}}\n\n\tdoc = document.NewDocument(\"polygonWithHole1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, polygonWithHole1, \"polygon\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygonWithHole2 := [][][][]float64{{\n\t\t{\n\t\t\t{36.261234283447266, 49.90712870720605},\n\t\t\t{36.2479305267334, 49.89480027061714},\n\t\t\t{36.254539489746094, 49.883408870659736},\n\t\t\t{36.280717849731445, 49.883408870659736},\n\t\t\t{36.28741264343262, 49.890432041848264},\n\t\t\t{36.27788543701172, 49.90276159448742},\n\t\t\t{36.261234283447266, 49.90712870720605},\n\t\t},\n\n\t\t{\n\t\t\t{36.264581680297844, 49.905249238801304},\n\t\t\t{36.25368118286133, 49.89673543545543},\n\t\t\t{36.253509521484375, 49.88578690918283},\n\t\t\t{36.270332336425774, 49.886174020645804},\n\t\t\t{36.27127647399902, 49.89579550794111},\n\t\t\t{36.264581680297844, 49.905249238801304},\n\t\t},\n\t}}\n\n\tdoc = document.NewDocument(\"polygonWithHole2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, polygonWithHole2, \"polygon\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipolygon1 := [][][][]float64{{{\n\t\t{36.1875057220459, 50.04363607656457},\n\t\t{36.192398071289055, 50.034871067327856},\n\t\t{36.20218276977539, 50.03955696315653},\n\t\t{36.1875057220459, 50.04363607656457},\n\t}}, // polygon1\n\t\t{{\n\t\t\t{36.2123966217041, 50.03795829715335},\n\t\t\t{36.218318939208984, 50.0333273779768},\n\t\t\t{36.226558685302734, 50.03867494711694},\n\t\t\t{36.217031478881836, 50.04286437899031},\n\t\t\t{36.2123966217041, 50.03795829715335},\n\t\t}}, // polygon2\n\t\t{{\n\t\t\t{36.221065521240234, 50.00365685169585},\n\t\t\t{36.226301193237305, 49.998029518286025},\n\t\t\t{36.23342514038086, 49.9995743420677},\n\t\t\t{36.23531341552734, 50.002994846659156},\n\t\t\t{36.231021881103516, 50.00630478067617},\n\t\t\t{36.22810363769531, 50.00663576154257},\n\t\t\t{36.226043701171875, 50.004815338573046},\n\t\t\t{36.221065521240234, 50.00365685169585},\n\t\t}}}\n\tdoc = document.NewDocument(\"multipolygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, multipolygon1, \"multipolygon\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygonInGc := [][][][]float64{{{\n\t\t{36.1875057220459, 50.04363607656457},\n\t\t{36.192398071289055, 50.034871067327856},\n\t\t{36.20218276977539, 50.03955696315653},\n\t\t{36.1875057220459, 50.04363607656457},\n\t}}}\n\tmultipolygonInGc := [][][][]float64{{{\n\t\t{36.29015922546387, 49.980150089789376},\n\t\t{36.28337860107422, 49.961656654293485},\n\t\t{36.307411193847656, 49.96033147865059},\n\t\t{36.29015922546387, 49.980150089789376},\n\t}}, // polygon1\n\t\t{{\n\t\t\t{36.16106986999512, 50.00387751801547},\n\t\t\t{36.161842346191406, 49.9908012905034},\n\t\t\t{36.17900848388672, 49.99841572888488},\n\t\t\t{36.16106986999512, 50.00387751801547},\n\t\t}}}\n\tcoordinates := [][][][][]float64{polygonInGc, multipolygonInGc}\n\ttypes := []string{\"polygon\", \"multipolygon\"}\n\tdoc = document.NewDocument(\"gc_polygonInGc_multipolygonInGc\")\n\tdoc.AddField(document.NewGeometryCollectionFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, coordinates, types,\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n"
  },
  {
    "path": "search/searcher/search_geoshape_points_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestGeoJsonPointContainsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tpoint []float64\n\t\tfield string\n\t\twant  []string\n\t}{\n\t\t// test points inside the polygon1.\n\t\t{\n\t\t\t[]float64{77.58334636688232, 12.948268838994263},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test points inside the circle1.\n\t\t{\n\t\t\t[]float64{77.58553504943848, 12.954040501528555},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle1\"},\n\t\t},\n\n\t\t// test points inside the polygon1 and the circle.\n\t\t{\n\t\t\t[]float64{77.59293794631958, 12.948896200093982},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"circle1\"},\n\t\t},\n\n\t\t// test points outside the polygon1 and the circle1.\n\t\t{\n\t\t\t[]float64{77.5614595413208, 12.953287683563568},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test point within the envelope1.\n\t\t{\n\t\t\t[]float64{81.28166198730469, 26.34203746601541},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\"},\n\t\t},\n\n\t\t// test point on the linestring vertex.\n\t\t{\n\t\t\t[]float64{77.57776737213135, 12.952074805390097},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring1\"},\n\t\t},\n\n\t\t// test point on the multilinestring vertex.\n\t\t{\n\t\t\t[]float64{77.5779390335083, 12.945006535817749},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring1\"},\n\t\t},\n\n\t\t// test point on the multipoint vertex.\n\t\t{\n\t\t\t[]float64{77.56407737731932, 12.951614746607163},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipoint1\"},\n\t\t},\n\n\t\t// test point within the polygonWithHole1.\n\t\t{\n\t\t\t[]float64{77.60334491729736, 12.979844051951334},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygonWithHole1\"},\n\t\t},\n\n\t\t// test point within the hole of the polygonWithHole1.\n\t\t{\n\t\t\t[]float64{77.60244369506836, 12.976247607394027},\n\t\t\t\"geometry\", nil,\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndex(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapePointRelationQuery(\"contains\",\n\t\t\tfalse, indexReader, [][]float64{test.point}, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.point)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonMultiPointWithInQuery(t *testing.T) {\n\ttests := []struct {\n\t\tmultipoint [][]float64\n\t\tfield      string\n\t\twant       []string\n\t}{\n\t\t// test multipoint inside the polygon1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.58334636688232, 12.948268838994263},\n\t\t\t\t{77.58467674255371, 12.944295515355652},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test multipoint inside the circle1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.58553504943848, 12.954040501528555},\n\t\t\t\t{77.58643627166747, 12.956089827794571},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle1\"},\n\t\t},\n\n\t\t// test multipoint inside the envelope1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.28166198730469, 26.34203746601541},\n\t\t\t\t{80.94314575195312, 26.346960121309415},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\"},\n\t\t},\n\n\t\t// test multipoint inside the polygon1 and the circle.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.59293794631958, 12.948896200093982},\n\t\t\t\t{77.58532047271729, 12.953789562459688},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"circle1\"},\n\t\t},\n\n\t\t// test multipoint (only 1 point outside) outside.\n\t\t{[][]float64{\n\t\t\t{77.58334636688232, 12.948268838994263},\n\t\t\t{77.58643627166747, 12.956089827794571},\n\t\t\t{77.5615, 12.9533},\n\t\t}, \"geometry\", nil},\n\n\t\t// test multipoint on the linestring vertex.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.5841188430786, 12.957093573282744},\n\t\t\t\t{77.57776737213135, 12.952074805390097},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring1\"},\n\t\t},\n\n\t\t// test multipoint outside the linestring vertex.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.5841188430786, 12.957093573282744},\n\t\t\t\t{77.57776737213135, 12.952074805390097},\n\t\t\t\t{77.58334636688232, 12.948268838994263},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test multipoint on the multilinestring vertex.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.5779390335083, 12.94471376293191},\n\t\t\t\t{77.57218837738037, 12.948268838994263},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring1\"},\n\t\t},\n\n\t\t// test multipoint outside the multilinestring vertex.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.5779390335083, 12.94471376293191},\n\t\t\t\t{77.57218837738037, 12.948268838994263},\n\t\t\t\t{77.58532047271729, 12.953789562459688},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test multipoint with one inside the hole within the polygonWithHole1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.60334491729736, 12.979844051951334},\n\t\t\t\t{77.60244369506836, 12.976247607394027},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test multipoint with all inside the hole within the polygonWithHole1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.59656429290771, 12.981767710239714},\n\t\t\t\t{77.59888172149658, 12.979969508380469},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test multipoint with all inside the polygonWithHole1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.60334491729736, 12.979844051951334},\n\t\t\t\t{77.59656429290771, 12.981767710239714},\n\t\t\t\t{77.59802341461182, 12.9751602999608},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygonWithHole1\"},\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndex(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapePointRelationQuery(\"contains\",\n\t\t\ttrue, indexReader, test.multipoint, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.multipoint)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonMultiPointIntersectsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tmultipoint [][]float64\n\t\tfield      string\n\t\twant       []string\n\t}{\n\t\t// test multipoint inside the polygon1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.58334636688232, 12.948268838994263},\n\t\t\t\t{77.58467674255371, 12.944295515355652},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test multipoint inside the circle1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.58553504943848, 12.954040501528555},\n\t\t\t\t{77.58643627166747, 12.956089827794571},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle1\"},\n\t\t},\n\n\t\t// test multipoint inside the envelope1. (1 point outside)\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{81.28166198730469, 26.34203746601541},\n\t\t\t\t{80.94314575195312, 26.346960121309415},\n\t\t\t\t{81.12716674804688, 26.353728430338332},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\"},\n\t\t},\n\n\t\t// test multipoint inside the polygon1 and the circle.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.59293794631958, 12.948896200093982},\n\t\t\t\t{77.58532047271729, 12.953789562459688},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"circle1\"},\n\t\t},\n\n\t\t// test multipoint (only 1 point outside) intersects.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.58334636688232, 12.948268838994263},\n\t\t\t\t{77.58643627166747, 12.956089827794571},\n\t\t\t\t{77.5615, 12.9533},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"circle1\"},\n\t\t},\n\n\t\t// test multipoint on the linestring vertex.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.5841188430786, 12.957093573282744},\n\t\t\t\t{77.57776737213135, 12.952074805390097},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring1\"},\n\t\t},\n\n\t\t// test multipoint outside the linestring vertex.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.5841188430786, 12.957093573282744},\n\t\t\t\t{77.57776737213135, 12.952074805390097},\n\t\t\t\t{77.58334636688232, 12.948268838994263},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"linestring1\"},\n\t\t},\n\n\t\t// test multipoint on the multilinestring vertex.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.5779390335083, 12.94471376293191},\n\t\t\t\t{77.57218837738037, 12.948268838994263},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring1\"},\n\t\t},\n\n\t\t// test multipoint outside the multilinestring vertex.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.5779390335083, 12.94471376293191},\n\t\t\t\t{77.57218837738037, 12.948268838994263},\n\t\t\t\t{77.58532047271729, 12.953789562459688},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"circle1\", \"multilinestring1\"},\n\t\t},\n\n\t\t// test multipoint with one inside the hole within the polygonWithHole1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.60334491729736, 12.979844051951334},\n\t\t\t\t{77.60244369506836, 12.976247607394027},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygonWithHole1\"},\n\t\t},\n\n\t\t// test multipoint with all inside the hole within the polygonWithHole1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.60244369506836, 12.976247607394027},\n\t\t\t\t{77.59888172149658, 12.979969508380469},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test multipoint with all inside the polygonWithHole1.\n\t\t{\n\t\t\t[][]float64{\n\t\t\t\t{77.60334491729736, 12.979844051951334},\n\t\t\t\t{77.59656429290771, 12.981767710239714},\n\t\t\t\t{77.59802341461182, 12.9751602999608},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygonWithHole1\"},\n\t\t},\n\t}\n\ti := setupGeoJsonShapesIndex(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapePointRelationQuery(\"intersects\",\n\t\t\ttrue, indexReader, test.multipoint, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.multipoint)\n\t\t}\n\t}\n}\n\nfunc runGeoShapePointRelationQuery(relation string, multi bool,\n\ti index.IndexReader, points [][]float64, field string,\n) ([]string, error) {\n\tvar rv []string\n\tvar s index.GeoJSON\n\tif multi {\n\t\ts = geo.NewGeoJsonMultiPoint(points)\n\t} else {\n\t\ts = geo.NewGeoJsonPoint(points[0])\n\t}\n\n\tgbs, err := NewGeoShapeSearcher(context.TODO(), i, s, relation, field, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\tdocID, _ := i.ExternalID(docMatch.IndexInternalID)\n\t\trv = append(rv, docID)\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\ntype Fatalfable interface {\n\tFatalf(format string, args ...interface{})\n}\n\nfunc setupGeoJsonShapesIndex(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon1 := [][][][]float64{{{\n\t\t{77.5853419303894, 12.953977766785052},\n\t\t{77.58405447006226, 12.95393594361393},\n\t\t{77.5819730758667, 12.9495026476557},\n\t\t{77.58068561553955, 12.94883346405509},\n\t\t{77.58019208908081, 12.948331575175299},\n\t\t{77.57991313934326, 12.943814529775414},\n\t\t{77.58497714996338, 12.94394000436408},\n\t\t{77.58517026901245, 12.9446301134728},\n\t\t{77.58572816848755, 12.945508431393435},\n\t\t{77.58785247802734, 12.946365833997325},\n\t\t{77.58967638015747, 12.946428570657417},\n\t\t{77.59070634841918, 12.947474179333993},\n\t\t{77.59317398071289, 12.948875288082773},\n\t\t{77.59167194366454, 12.949962710338657},\n\t\t{77.59077072143555, 12.950276388953625},\n\t\t{77.59098529815674, 12.951196510612728},\n\t\t{77.58729457855225, 12.952472128200755},\n\t\t{77.5853419303894, 12.953977766785052},\n\t}}}\n\tdoc := document.NewDocument(\"polygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// not working envelope\n\tenvelope1 := [][][][]float64{{{\n\t\t{80.93696594238281, 26.33957605983274},\n\t\t{81.28440856933594, 26.351267272877074},\n\t}}}\n\tdoc = document.NewDocument(\"envelope1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tenvelope1, \"envelope\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle1\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\t[]float64{77.59137153625487, 12.952660333521468}, \"900m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring := [][][][]float64{{{\n\t\t{77.5841188430786, 12.957093573282744},\n\t\t{77.57776737213135, 12.952074805390097},\n\t}}}\n\tdoc = document.NewDocument(\"linestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring := [][][][]float64{{{\n\t\t{77.57227420806883, 12.948687079902895},\n\t\t{77.57600784301758, 12.954165970968194},\n\t\t{77.5779390335083, 12.94471376293191},\n\t\t{77.57218837738037, 12.948268838994263},\n\t\t{77.57781028747559, 12.951740217268595},\n\t\t{77.5779390335083, 12.945006535817749},\n\t}}}\n\tdoc = document.NewDocument(\"multilinestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultilinestring, \"multilinestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipoint1 := [][][][]float64{{{\n\t\t{77.56618022918701, 12.958180959662695},\n\t\t{77.56407737731932, 12.951614746607163},\n\t\t{77.56922721862793, 12.956173473406446},\n\t}}}\n\tdoc = document.NewDocument(\"multipoint1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultipoint1, \"multipoint\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygonWithHole1 := [][][][]float64{{\n\t\t{\n\t\t\t{77.59991168975829, 12.972232910164502},\n\t\t\t{77.6039457321167, 12.97582941279006},\n\t\t\t{77.60424613952637, 12.98168407323241},\n\t\t\t{77.59974002838135, 12.985489528568463},\n\t\t\t{77.59321689605713, 12.979300406693417},\n\t\t\t{77.59991168975829, 12.972232910164502},\n\t\t},\n\t\t{\n\t\t\t{77.59682178497314, 12.975787593290978},\n\t\t\t{77.60295867919922, 12.975787593290978},\n\t\t\t{77.60295867919922, 12.98143316204164},\n\t\t\t{77.59682178497314, 12.98143316204164},\n\t\t\t{77.59682178497314, 12.975787593290978},\n\t\t},\n\t}}\n\n\tdoc = document.NewDocument(\"polygonWithHole1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygonWithHole1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n"
  },
  {
    "path": "search/searcher/search_geoshape_polygon_test.go",
    "content": "//  Copyright (c) 2022 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestGeoJsonPolygonIntersectsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon [][][]float64\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t// test intersecting query polygon for polygon1.\n\t\t{[][][]float64{{\n\t\t\t{77.57926940917969, 12.945257483731918},\n\t\t\t{77.57875442504883, 12.942036966318216},\n\t\t\t{77.58278846740721, 12.9424970427816},\n\t\t\t{77.57926940917969, 12.945257483731918},\n\t\t}}, \"geometry\", []string{\"polygon1\"}},\n\n\t\t// test intersecting query polygon for polygon1, polygon2, circle1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.59562015533446, 12.94099133483504},\n\t\t\t\t{77.59665012359619, 12.949356263896634},\n\t\t\t\t{77.59313106536865, 12.951321981484776},\n\t\t\t\t{77.59085655212402, 12.948477959536318},\n\t\t\t\t{77.59562015533446, 12.94099133483504},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\", \"polygon2\", \"circle1\"},\n\t\t},\n\n\t\t// test intersecting query polygon for polygon1, polygon2 and polygon3.\n\t\t{[][][]float64{{\n\t\t\t{77.5929594039917, 12.939151012774925},\n\t\t\t{77.58321762084961, 12.94546660680072},\n\t\t\t{77.59737968444824, 12.931998723107322},\n\t\t\t{77.60111331939697, 12.955169724209911},\n\t\t\t{77.59592056274414, 12.936265025833965},\n\t\t\t{77.5929594039917, 12.939151012774925},\n\t\t}}, \"geometry\", []string{\"polygon1\", \"polygon2\", \"polygon3\"}},\n\n\t\t// test intersecting query polygon for polygon2 and the circle1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.59012699127197, 12.959853852513307},\n\t\t\t\t{77.59836673736572, 12.959853852513307},\n\t\t\t\t{77.59836673736572, 12.965541604118611},\n\t\t\t\t{77.59012699127197, 12.965541604118611},\n\t\t\t\t{77.59012699127197, 12.959853852513307},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon2\", \"circle1\"},\n\t\t},\n\n\t\t// test intersecting query polygon for linestring2 and multilinestring2.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.59669303894043, 12.989504011681609},\n\t\t\t\t{77.60699272155762, 12.983231353311314},\n\t\t\t\t{77.60115623474121, 12.993183897537897},\n\t\t\t\t{77.59669303894043, 12.989504011681609},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring2\", \"multilinestring2\"},\n\t\t},\n\n\t\t// test intersecting query polygon for multilinestring2.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.60124206542969, 12.987162237749484},\n\t\t\t\t{77.60330200195312, 12.992849364713313},\n\t\t\t\t{77.59514808654785, 12.989671280403403},\n\t\t\t\t{77.60124206542969, 12.987162237749484},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multilinestring2\"},\n\t\t},\n\n\t\t// test intersecting query polygon for multipoint1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.56648063659668, 12.956382587313202},\n\t\t\t\t{77.56819725036621, 12.949523559614263},\n\t\t\t\t{77.5718879699707, 12.958222782120954},\n\t\t\t\t{77.56648063659668, 12.956382587313202},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipoint1\"},\n\t\t},\n\n\t\t// test intersecting query polygon for envelope1.\n\t\t{[][][]float64{{\n\t\t\t{36.19986534118652, 50.00034673534484},\n\t\t\t{36.19351387023926, 50.00464984215712},\n\t\t\t{36.178321838378906, 49.991573824716205},\n\t\t\t{36.19986534118652, 50.00034673534484},\n\t\t}}, \"geometry\", []string{\"envelope1\"}},\n\n\t\t// test intersecting query polygon for envelope1.\n\t\t{[][][]float64{{\n\t\t\t{36.170082092285156, 49.99229116680205},\n\t\t\t{36.14982604980469, 49.99002874388075},\n\t\t\t{36.227073669433594, 49.98754547425633},\n\t\t\t{36.170082092285156, 49.99229116680205},\n\t\t}}, \"geometry\", []string{\"envelope1\"}},\n\t}\n\ti := setupGeoJsonShapesIndexForPolygonQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"intersects\",\n\t\t\tindexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\", n,\n\t\t\t\ttest.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonPolygonContainsQuery(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon [][][]float64\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t// test containment query polygon for polygon1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.5843334197998, 12.952702156906767},\n\t\t\t\t{77.58510589599608, 12.952702156906767},\n\t\t\t\t{77.58510589599608, 12.953622269606669},\n\t\t\t\t{77.5843334197998, 12.953622269606669},\n\t\t\t\t{77.5843334197998, 12.952702156906767},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test containment query polygon for circle1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.59025573730469, 12.953810474058429},\n\t\t\t\t{77.59145736694336, 12.953810474058429},\n\t\t\t\t{77.59145736694336, 12.954918786278716},\n\t\t\t\t{77.59025573730469, 12.954918786278716},\n\t\t\t\t{77.59025573730469, 12.953810474058429},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"circle1\"},\n\t\t},\n\n\t\t// test containment query polygon for polygon2, polygon3.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.60235786437988, 12.956884459972992},\n\t\t\t\t{77.60124206542969, 12.956800814599926},\n\t\t\t\t{77.6008129119873, 12.955713422193524},\n\t\t\t\t{77.60244369506836, 12.955211547173878},\n\t\t\t\t{77.60313034057617, 12.955880713641998},\n\t\t\t\t{77.60235786437988, 12.956884459972992},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon2\", \"polygon3\"},\n\t\t},\n\n\t\t// test containment query polygon which resides within a hole in polygonWithHole1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.60012626647949, 12.97963495776207},\n\t\t\t\t{77.5978946685791, 12.978213112610835},\n\t\t\t\t{77.60089874267577, 12.977962197916442},\n\t\t\t\t{77.60012626647949, 12.97963495776207},\n\t\t\t}},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test containment query polygon which resides within polygonWithHole1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.59978294372559, 12.984067716910454},\n\t\t\t\t{77.59780883789062, 12.982227713276774},\n\t\t\t\t{77.60089874267577, 12.982227713276774},\n\t\t\t\t{77.59978294372559, 12.984067716910454},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygonWithHole1\"},\n\t\t},\n\n\t\t// test with query polygon for polygon4 with a single vertex lying outside.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{-121.48138761520384, 38.50964107572585},\n\t\t\t\t{-121.48226737976073, 38.509238097766875},\n\t\t\t\t{-121.48115158081055, 38.50781086602439},\n\t\t\t\t{-121.48014307022095, 38.50806273250507},\n\t\t\t\t{-121.48138761520384, 38.50964107572585},\n\t\t\t}},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test with query polygon for polygon4.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{-121.48381233215332, 38.507974579337045},\n\t\t\t\t{-121.48361384868622, 38.507869634948676},\n\t\t\t\t{-121.48361921310425, 38.50765135013098},\n\t\t\t\t{-121.48343682289122, 38.50797038156446},\n\t\t\t\t{-121.48381233215332, 38.507974579337045},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon4\"},\n\t\t},\n\n\t\t// test with query polygon for multipolygon1.\n\t\t{[][][]float64{{\n\t\t\t{-121.47578716278075, 38.51617236229197},\n\t\t\t{-121.47578716278075, 38.51566868518406},\n\t\t\t{-121.47546529769896, 38.516105205547866},\n\t\t\t{-121.47578716278075, 38.51617236229197},\n\t\t}}, \"geometry\", []string{\"multipolygon1\"}},\n\n\t\t// test with query polygon for envelope1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{36.197547912597656, 49.99642946989866},\n\t\t\t\t{36.18939399719238, 49.988649165474},\n\t\t\t\t{36.20201110839844, 49.98853879749191},\n\t\t\t\t{36.1970329284668, 49.980150089789376},\n\t\t\t\t{\n\t\t\t\t\t36.205787658691406,\n\t\t\t\t\t49.9885939815146,\n\t\t\t\t},\n\t\t\t\t{36.197547912597656, 49.99642946989866},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\"},\n\t\t},\n\n\t\t// test with query polygon for no hits. (envelope1 has one vertex outside the polygon)\n\t\t{[][][]float64{{\n\t\t\t{36.19832038879394, 49.99626394461266},\n\t\t\t{36.19016647338867, 49.98439981533724},\n\t\t\t{36.20698928833008, 49.98158510403259},\n\t\t\t{36.19832038879394, 49.99626394461266},\n\t\t}}, \"geometry\", nil},\n\t}\n\ti := setupGeoJsonShapesIndexForPolygonQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"contains\",\n\t\t\tindexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonPolygonWithInQuery(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon [][][]float64\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t// test with query polygon for polygon1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.58407592773438, 12.956382587313202},\n\t\t\t\t{77.57746696472168, 12.943249893344905},\n\t\t\t\t{77.5920581817627, 12.944086391304364},\n\t\t\t\t{77.59454727172852, 12.95353862313803},\n\t\t\t\t{77.58407592773438, 12.956382587313202},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test with query polygon for circle1 and polygon3.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.59248733520508, 12.967841760870071},\n\t\t\t\t{77.58261680603027, 12.968594534825176},\n\t\t\t\t{77.57789611816406, 12.957302686416881},\n\t\t\t\t{77.58896827697754, 12.945341132980488},\n\t\t\t\t{77.60450363159178, 12.947599652080394},\n\t\t\t\t{77.60673522949219, 12.96483064227584},\n\t\t\t\t{77.59248733520508, 12.967841760870071},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon3\", \"circle1\"},\n\t\t},\n\n\t\t// test with query polygon for linestring2, multilinestring2.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{77.59909629821777, 12.998118204343788},\n\t\t\t\t{77.58931159973145, 12.978882217224443},\n\t\t\t\t{77.61128425598145, 12.983565899088745},\n\t\t\t\t{77.59909629821777, 12.998118204343788},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"linestring2\", \"multilinestring2\"},\n\t\t},\n\n\t\t// test with query polygon for multipoint1.\n\t\t{[][][]float64{{\n\t\t\t{77.55703926086426, 12.964245142762644},\n\t\t\t{77.5631332397461, 12.944253690559432},\n\t\t\t{77.57429122924805, 12.957720912158363},\n\t\t\t{77.55703926086426, 12.964245142762644},\n\t\t}}, \"geometry\", []string{\"multipoint1\"}},\n\n\t\t// test with query polygon with no results.\n\t\t// (polygon4 has one vertex lying outside the query polygon).\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{-121.48812532424927, 38.51058134885975},\n\t\t\t\t{-121.48258924484252, 38.500153704565065},\n\t\t\t\t{-121.47492885589598, 38.50799556819636},\n\t\t\t\t{-121.48630142211913, 38.51147123890908},\n\t\t\t\t{-121.48812532424927, 38.51058134885975},\n\t\t\t}},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test with query polygon for polygon4.\n\t\t{[][][]float64{{\n\t\t\t{-121.48366212844849, 38.510161585585045},\n\t\t\t{-121.48533582687377, 38.50841534409804},\n\t\t\t{-121.48376941680908, 38.507777283760426},\n\t\t\t{-121.48370504379272, 38.50250467407243},\n\t\t\t{-121.48010015487672, 38.50253825879518},\n\t\t\t{-121.48018598556519, 38.504502937819765},\n\t\t\t{-121.47756814956665, 38.50755899866278},\n\t\t\t{-121.48113012313843, 38.50866720846446},\n\t\t\t{-121.48115158081055, 38.51017837616302},\n\t\t\t{-121.48366212844849, 38.510161585585045},\n\t\t}}, \"geometry\", []string{\"polygon4\"}},\n\n\t\t// test with query polygon for envelope1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{36.20587348937988, 50.00470500769241},\n\t\t\t\t{36.17969512939453, 49.993946530777606},\n\t\t\t\t{36.19368553161621, 49.971870325635074},\n\t\t\t\t{36.21119499206543, 49.983075265826656},\n\t\t\t\t{36.20587348937988, 50.00470500769241},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"envelope1\"},\n\t\t},\n\n\t\t// test with query polygon for linestring2 which lies outside except the endpoints.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{8.515305519104004, 47.392597129887},\n\t\t\t\t{8.514232635498047, 47.38896544894171},\n\t\t\t\t{8.507537841796875, 47.38815191810328},\n\t\t\t\t{8.514318466186523, 47.38725120859953},\n\t\t\t\t{8.516035079956053, 47.383357642070706},\n\t\t\t\t{8.516979217529295, 47.38733837470806},\n\t\t\t\t{8.522472381591797, 47.38794853343167},\n\t\t\t\t{8.516507148742676, 47.388994503382285},\n\t\t\t\t{8.515305519104004, 47.392597129887},\n\t\t\t}},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test with query polygon for all the shapes.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{-135.0, -38.0},\n\t\t\t\t{149.0, -38.0},\n\t\t\t\t{149.0, 77.0},\n\t\t\t\t{-135.0, 77.0},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\n\t\t\t\t\"polygon1\", \"polygon2\", \"polygon3\", \"envelope1\", \"circle1\", \"linestring1\",\n\t\t\t\t\"linestring2\", \"linestring3\", \"multilinestring1\", \"multilinestring2\", \"multipoint1\",\n\t\t\t\t\"polygonWithHole1\", \"polygon4\", \"multipolygon1\",\n\t\t\t},\n\t\t},\n\t}\n\n\ti := setupGeoJsonShapesIndexForPolygonQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"within\",\n\t\t\tindexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc runGeoShapePolygonQueryWithRelation(relation string, i index.IndexReader,\n\tpoints [][][]float64, field string,\n) ([]string, error) {\n\tvar rv []string\n\ts := geo.NewGeoJsonPolygon(points)\n\n\tgbs, err := NewGeoShapeSearcher(context.TODO(), i, s, relation, field, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\tdocID, _ := i.ExternalID(docMatch.IndexInternalID)\n\t\trv = append(rv, docID)\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc setupGeoJsonShapesIndexForPolygonQuery(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon1 := [][][][]float64{{{\n\t\t{77.5853419303894, 12.953977766785052},\n\t\t{77.58405447006226, 12.95393594361393},\n\t\t{77.5819730758667, 12.9495026476557},\n\t\t{77.58068561553955, 12.94883346405509},\n\t\t{77.58019208908081, 12.948331575175299},\n\t\t{77.57991313934326, 12.943814529775414},\n\t\t{77.58497714996338, 12.94394000436408},\n\t\t{77.58517026901245, 12.9446301134728},\n\t\t{77.58572816848755, 12.945508431393435},\n\t\t{77.58785247802734, 12.946365833997325},\n\t\t{77.58967638015747, 12.946428570657417},\n\t\t{77.59070634841918, 12.947474179333993},\n\t\t{77.59317398071289, 12.948875288082773},\n\t\t{77.59167194366454, 12.949962710338657},\n\t\t{77.59077072143555, 12.950276388953625},\n\t\t{77.59098529815674, 12.951196510612728},\n\t\t{77.58729457855225, 12.952472128200755},\n\t\t{77.5853419303894, 12.953977766785052},\n\t}}}\n\tdoc := document.NewDocument(\"polygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon2 := [][][][]float64{{{\n\t\t{77.59527683258057, 12.951112863329588},\n\t\t{77.59420394897461, 12.947976069940545},\n\t\t{77.59579181671143, 12.946010325958518},\n\t\t{77.60347366333008, 12.950401860289055},\n\t\t{77.60673522949219, 12.95600618215462},\n\t\t{77.60107040405273, 12.96345053407734},\n\t\t{77.5984525680542, 12.961861309096507},\n\t\t{77.59527683258057, 12.951112863329588},\n\t}}}\n\tdoc = document.NewDocument(\"polygon2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon2, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon3 := [][][][]float64{{{\n\t\t{77.59974002838135, 12.953789562459688},\n\t\t{77.60347366333008, 12.953789562459688},\n\t\t{77.60347366333008, 12.957720912158363},\n\t\t{77.59974002838135, 12.957720912158363},\n\t\t{77.59974002838135, 12.953789562459688},\n\t}}}\n\tdoc = document.NewDocument(\"polygon3\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon3, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t/*polygon4 := [][][][]float64{{{{8.515305519104004, 47.392597129887},\n\t\t{8.514232635498047, 47.38896544894171}, {8.507537841796875, 47.38815191810328},\n\t\t{8.514318466186523, 47.38725120859953}, {8.516035079956053, 47.383357642070706},\n\t\t{8.516979217529295, 47.38733837470806}, {8.522472381591797, 47.38794853343167},\n\t\t{8.516507148742676, 47.388994503382285}, {8.515305519104004, 47.392597129887}}}}\n\tdoc = document.NewDocument(\"polygon4\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon4, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}*/\n\n\t// not working envelope\n\tenvelope1 := [][][][]float64{{{\n\t\t{36.18896484375, 49.9799293145682},\n\t\t{36.20613098144531, 49.99714673955337},\n\t}}}\n\tdoc = document.NewDocument(\"envelope1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tenvelope1, \"envelope\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdoc = document.NewDocument(\"circle1\")\n\tdoc.AddField(document.NewGeoCircleFieldWithIndexingOptions(\"geometry\",\n\t\t[]uint64{}, []float64{77.59253025054932, 12.955587953533424}, \"900m\",\n\t\tdocument.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring := [][][][]float64{{{\n\t\t{77.5841188430786, 12.957093573282744},\n\t\t{77.57776737213135, 12.952074805390097},\n\t}}}\n\tdoc = document.NewDocument(\"linestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring1 := [][][][]float64{{{\n\t\t{77.60188579559325, 12.982604078764705},\n\t\t{77.60557651519775, 12.987329508048184},\n\t}}}\n\tdoc = document.NewDocument(\"linestring2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring1, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tlinestring3 := [][][][]float64{{{\n\t\t{8.51539134979248, 47.390592472948434},\n\t\t{8.520884513854979, 47.388006643417924},\n\t}}}\n\tdoc = document.NewDocument(\"linestring3\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tlinestring3, \"linestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring := [][][][]float64{{\n\t\t{\n\t\t\t{77.57227420806883, 12.948687079902895},\n\t\t\t{77.57600784301758, 12.954165970968194},\n\t\t},\n\t\t{{77.5779390335083, 12.94471376293191}, {77.57218837738037, 12.948268838994263}},\n\t\t{{77.57781028747559, 12.951740217268595}, {77.5779390335083, 12.945006535817749}},\n\t}}\n\tdoc = document.NewDocument(\"multilinestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultilinestring, \"multilinestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring1 := [][][][]float64{{\n\t\t{\n\t\t\t{77.6015853881836, 12.990089451715061},\n\t\t\t{77.60476112365723, 12.987747683302153},\n\t\t},\n\t\t{{77.59875297546387, 12.988751301039581}, {77.59446144104004, 12.98197680263484}},\n\t\t{{77.60188579559325, 12.982604078764705}, {77.60557651519775, 12.987329508048184}},\n\t}}\n\tdoc = document.NewDocument(\"multilinestring2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultilinestring1, \"multilinestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipoint1 := [][][][]float64{{{\n\t\t{77.56618022918701, 12.958180959662695},\n\t\t{77.56407737731932, 12.951614746607163},\n\t\t{77.56922721862793, 12.956173473406446},\n\t}}}\n\tdoc = document.NewDocument(\"multipoint1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultipoint1, \"multipoint\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygonWithHole1 := [][][][]float64{{\n\t\t{\n\t\t\t{77.59991168975829, 12.972232910164502},\n\t\t\t{77.6039457321167, 12.97582941279006},\n\t\t\t{77.60424613952637, 12.98168407323241},\n\t\t\t{77.59974002838135, 12.985489528568463},\n\t\t\t{77.59321689605713, 12.979300406693417},\n\t\t\t{77.59991168975829, 12.972232910164502},\n\t\t},\n\t\t{\n\t\t\t{77.59682178497314, 12.975787593290978},\n\t\t\t{77.60295867919922, 12.975787593290978},\n\t\t\t{77.60295867919922, 12.98143316204164},\n\t\t\t{77.59682178497314, 12.98143316204164},\n\t\t\t{77.59682178497314, 12.975787593290978},\n\t\t},\n\t}}\n\n\tdoc = document.NewDocument(\"polygonWithHole1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygonWithHole1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon4 := [][][][]float64{{{\n\t\t{-121.48125886917113, 38.51009442323401},\n\t\t{-121.48361921310425, 38.51012800441735},\n\t\t{-121.48497104644774, 38.50858325377352},\n\t\t{-121.48366212844849, 38.507861239391026},\n\t\t{-121.48353338241577, 38.50277335141579},\n\t\t{-121.4803147315979, 38.50267259752949},\n\t\t{-121.48033618927, 38.5046204810195},\n\t\t{-121.47771835327147, 38.50754220747402},\n\t\t{-121.48123741149902, 38.508616835661655},\n\t\t{-121.48125886917113, 38.51009442323401},\n\t}}}\n\n\tdoc = document.NewDocument(\"polygon4\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon4, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipolygon1 := [][][][]float64{\n\t\t{{\n\t\t\t{-121.49104356765746, 38.52149433504263},\n\t\t\t{-121.47857666015625, 38.51592052417851},\n\t\t\t{-121.47688150405884, 38.515970891871696},\n\t\t\t{-121.4770746231079, 38.51714612804143},\n\t\t\t{-121.49033546447754, 38.52221621271097},\n\t\t\t{-121.49104356765746, 38.52149433504263},\n\t\t}},\n\t\t{{\n\t\t\t{-121.47647380828859, 38.51714612804143},\n\t\t\t{-121.47658109664916, 38.51477884701455},\n\t\t\t{-121.4741563796997, 38.5159876810949},\n\t\t\t{-121.47647380828859, 38.51714612804143},\n\t\t}},\n\t}\n\n\tdoc = document.NewDocument(\"multipolygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultipolygon1, \"multipolygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n\nfunc TestGeoJsonMultiPolygonWithInQuery(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon [][][][]float64\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t// test within multipolygon query for multipolygon1.\n\t\t// (where each query polygon contains each of the indexed polygons)\n\t\t{\n\t\t\t[][][][]float64{\n\t\t\t\t{\n\t\t\t\t\t{\n\t\t\t\t\t\t{-121.49458408355713, 38.53270780324851},\n\t\t\t\t\t\t{-121.48823261260985, 38.52533866992879},\n\t\t\t\t\t\t{-121.48048639297485, 38.53253994984147},\n\t\t\t\t\t\t{-121.49458408355713, 38.53270780324851},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{{\n\t\t\t\t\t{-121.48700952529907, 38.53306029412857},\n\t\t\t\t\t{-121.48160219192505, 38.53306029412857},\n\t\t\t\t\t{-121.48160219192505, 38.53829709805414},\n\t\t\t\t\t{-121.48700952529907, 38.53829709805414},\n\t\t\t\t\t{-121.48700952529907, 38.53306029412857},\n\t\t\t\t}},\n\t\t\t\t{{\n\t\t\t\t\t{-121.47344827651976, 38.54475865436684},\n\t\t\t\t\t{-121.46396398544312, 38.54475865436684},\n\t\t\t\t\t{-121.46396398544312, 38.55366961462033},\n\t\t\t\t\t{-121.47344827651976, 38.55366961462033},\n\t\t\t\t\t{-121.47344827651976, 38.54475865436684},\n\t\t\t\t}},\n\t\t\t},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"multipolygon1\"},\n\t\t},\n\n\t\t// test within multipolygon query. (only partial containment of the three\n\t\t// indexed polygons by the two query polygons)\n\t\t{\n\t\t\t[][][][]float64{\n\t\t\t\t{\n\t\t\t\t\t{\n\t\t\t\t\t\t{-121.49458408355713, 38.53270780324851},\n\t\t\t\t\t\t{-121.48823261260985, 38.52533866992879},\n\t\t\t\t\t\t{-121.48048639297485, 38.53253994984147},\n\t\t\t\t\t\t{-121.49458408355713, 38.53270780324851},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{{\n\t\t\t\t\t{-121.48700952529907, 38.53306029412857},\n\t\t\t\t\t{-121.48160219192505, 38.53306029412857},\n\t\t\t\t\t{-121.48160219192505, 38.53829709805414},\n\t\t\t\t\t{-121.48700952529907, 38.53829709805414},\n\t\t\t\t\t{-121.48700952529907, 38.53306029412857},\n\t\t\t\t}},\n\t\t\t\t{{\n\t\t\t\t\t{-121.4734697341919, 38.544825784372485},\n\t\t\t\t\t{-121.4644145965576, 38.544825784372485},\n\t\t\t\t\t{-121.4644145965576, 38.5537199558913},\n\t\t\t\t\t{-121.4734697341919, 38.5537199558913},\n\t\t\t\t\t{-121.4734697341919, 38.544825784372485},\n\t\t\t\t}},\n\t\t\t},\n\t\t\t\"geometry\", nil,\n\t\t},\n\n\t\t// test within multipolygon query for multilinestring1.\n\t\t{[][][][]float64{\n\t\t\t{{\n\t\t\t\t{-121.49876832962036, 38.551739839324334},\n\t\t\t\t{-121.49814605712889, 38.54553064564853},\n\t\t\t\t{-121.49158000946044, 38.54908841140355},\n\t\t\t\t{-121.49876832962036, 38.551739839324334},\n\t\t\t}},\n\t\t\t{\n\t\t\t\t{\n\t\t\t\t\t{-121.49258852005006, 38.54294612052762},\n\t\t\t\t\t{-121.49117231369017, 38.54294612052762},\n\t\t\t\t\t{-121.49117231369017, 38.54526212788182},\n\t\t\t\t\t{-121.49258852005006, 38.54526212788182},\n\t\t\t\t\t{-121.49258852005006, 38.54294612052762},\n\t\t\t\t},\n\t\t\t},\n\t\t}, \"geometry\", []string{\"multilinestring1\"}},\n\n\t\t// test within multipolygon query for multipoint1.\n\t\t{[][][][]float64{\n\t\t\t{{\n\t\t\t\t{-121.50286674499512, 38.564810956372185},\n\t\t\t\t{-121.49694442749023, 38.56226068115802},\n\t\t\t\t{-121.48406982421875, 38.5675624676039},\n\t\t\t\t{-121.4875030517578, 38.57514535565976},\n\t\t\t\t{-121.50286674499512, 38.564810956372185},\n\t\t\t}},\n\t\t\t{{\n\t\t\t\t{-121.48685932159422, 38.565163289911425},\n\t\t\t\t{-121.48623704910278, 38.56283114531348},\n\t\t\t\t{-121.48357629776001, 38.565129734410704},\n\t\t\t\t{-121.48685932159422, 38.565163289911425},\n\t\t\t}},\n\t\t\t{\n\t\t\t\t{\n\t\t\t\t\t{-121.49430513381958, 38.56195866888961},\n\t\t\t\t\t{-121.4899492263794, 38.5584518779682},\n\t\t\t\t\t{-121.48842573165892, 38.56194189039304},\n\t\t\t\t\t{-121.49430513381958, 38.56195866888961},\n\t\t\t\t},\n\t\t\t},\n\t\t}, \"geometry\", []string{\"multipoint1\"}},\n\t}\n\ti := setupGeoJsonShapesIndexForMultiPolygonQuery(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapeMultiPolygonQueryWithRelation(\"within\",\n\t\t\tindexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc runGeoShapeMultiPolygonQueryWithRelation(relation string,\n\ti index.IndexReader,\n\tpoints [][][][]float64, field string,\n) ([]string, error) {\n\tvar rv []string\n\ts := geo.NewGeoJsonMultiPolygon(points)\n\n\tgbs, err := NewGeoShapeSearcher(context.TODO(), i, s, relation,\n\t\tfield, 1.0, search.SearcherOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0),\n\t}\n\tdocMatch, err := gbs.Next(ctx)\n\tfor docMatch != nil && err == nil {\n\t\tdocID, _ := i.ExternalID(docMatch.IndexInternalID)\n\t\trv = append(rv, docID)\n\t\tdocMatch, err = gbs.Next(ctx)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc setupGeoJsonShapesIndexForMultiPolygonQuery(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipolygon1 := [][][][]float64{{{\n\t\t{-121.49140834808348, 38.5320028163074},\n\t\t{-121.49112939834593, 38.52916601331889},\n\t\t{-121.48889780044556, 38.52913244101627},\n\t\t{-121.4887261390686, 38.527655244193205},\n\t\t{-121.48559331893921, 38.52794061412457},\n\t\t{-121.48638725280762, 38.53213710006686},\n\t\t{-121.49140834808348, 38.5320028163074},\n\t}}, // polygon1\n\t\t{{\n\t\t\t{-121.48677349090575, 38.533194575914315},\n\t\t\t{-121.48179531097412, 38.533194575914315},\n\t\t\t{-121.48179531097412, 38.53814604174215},\n\t\t\t{-121.48677349090575, 38.53814604174215},\n\t\t\t{-121.48677349090575, 38.533194575914315},\n\t\t}}, // polygon2\n\t\t{{\n\t\t\t{-121.47334098815918, 38.553485029658475},\n\t\t\t{-121.47329807281494, 38.54485934935182},\n\t\t\t{-121.46415710449219, 38.54526212788182},\n\t\t\t{-121.47334098815918, 38.553485029658475},\n\t\t}}}\n\tdoc := document.NewDocument(\"multipolygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultipolygon1, \"multipolygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultilinestring1 := [][][][]float64{{\n\t\t{\n\t\t\t{-121.4983820915222, 38.55081688500274},\n\t\t\t{-121.49649381637572, 38.550447699956685},\n\t\t},\n\t\t{{-121.49655818939209, 38.548635309508775}, {-121.49370431900023, 38.54811507788636}},\n\t\t{{-121.49134397506714, 38.54490969679143}, {-121.4919662475586, 38.54304681805045}},\n\t}}\n\tdoc = document.NewDocument(\"multilinestring1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultilinestring1, \"multilinestring\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmultipoint1 := [][][][]float64{{{\n\t\t{-121.48960590362547, 38.56066671319285},\n\t\t{-121.4933180809021, 38.56157276247755},\n\t\t{-121.4973521232605, 38.56318348855919},\n\t\t{-121.48582935333252, 38.56736114108619},\n\t\t{-121.50104284286498, 38.56449217691959},\n\t\t{-121.4881682395935, 38.57158887950165},\n\t}}}\n\tdoc = document.NewDocument(\"multipoint1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tmultipoint1, \"multipoint\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n\nfunc setupGeoJsonPolygonS2LoopPortingIssue(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon1 := [][][][]float64{{{\n\t\t{-135.0, -38.0},\n\t\t{149.0, -38.0},\n\t\t{149.0, 77.0},\n\t\t{-135.0, 77.0},\n\t}}}\n\tdoc := document.NewDocument(\"polygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn i\n}\n\nfunc TestGeoJsonPolygonContainsQueryS2LoopPortingIssue(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon [][][]float64\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t// test containment query polygon for polygon1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{13.007812500000002, 37.99616267972809},\n\t\t\t\t{13.559375000000002, 37.99616267972809},\n\t\t\t\t{13.559375000000002, 38.472819658516866},\n\t\t\t\t{13.007812500000002, 38.472819658516866},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\n\t\t// test containment query polygon for polygon1.\n\t\t{\n\t\t\t[][][]float64{{\n\t\t\t\t{13.007812500000002, 37.99616267972809},\n\t\t\t\t{13.359375000000002, 37.99616267972809},\n\t\t\t\t{13.359375000000002, 38.272819658516866},\n\t\t\t\t{13.007812500000002, 38.272819658516866},\n\t\t\t}},\n\t\t\t\"geometry\",\n\t\t\t[]string{\"polygon1\"},\n\t\t},\n\t}\n\ti := setupGeoJsonPolygonS2LoopPortingIssue(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"contains\",\n\t\t\tindexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\",\n\t\t\t\tn, test.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc TestGeoJsonPolygonIntersectsQuery1(t *testing.T) {\n\ttests := []struct {\n\t\tpolygon [][][]float64\n\t\tfield   string\n\t\twant    []string\n\t}{\n\t\t// test non-intersecting query polygon.\n\t\t{[][][]float64{{\n\t\t\t{\n\t\t\t\t97.745361328125,\n\t\t\t\t68.21644657802169,\n\t\t\t},\n\t\t\t{\n\t\t\t\t97.701416015625,\n\t\t\t\t67.97051353559428,\n\t\t\t},\n\t\t\t{\n\t\t\t\t97.80029296875,\n\t\t\t\t67.97875365614591,\n\t\t\t},\n\t\t\t{\n\t\t\t\t97.745361328125,\n\t\t\t\t68.21644657802169,\n\t\t\t},\n\t\t}}, \"geometry\", nil},\n\n\t\t// test intersecting query polygon.\n\t\t{[][][]float64{{\n\t\t\t{\n\t\t\t\t77.59214401245117,\n\t\t\t\t12.966043458314124,\n\t\t\t},\n\t\t\t{\n\t\t\t\t77.58853912353516,\n\t\t\t\t12.95232574618635,\n\t\t\t},\n\t\t\t{\n\t\t\t\t77.60943889617919,\n\t\t\t\t12.956466232826733,\n\t\t\t},\n\t\t\t{\n\t\t\t\t77.59214401245117,\n\t\t\t\t12.966043458314124,\n\t\t\t},\n\t\t}}, \"geometry\", nil},\n\n\t\t// test intersecting query polygon for polygon1.\n\t\t{[][][]float64{{\n\t\t\t{97.0806884765625, 61.61423180712503},\n\t\t\t{96.7510986328125, 61.54625879879804},\n\t\t\t{97.305908203125, 61.367777577924},\n\t\t\t{97.0806884765625, 61.61423180712503},\n\t\t}}, \"geometry\", []string{\"polygon1\"}},\n\t}\n\ti := setupGeoJsonShapesIndexForPolygonQuery1(t)\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr = indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor n, test := range tests {\n\t\tgot, err := runGeoShapePolygonQueryWithRelation(\"intersects\",\n\t\t\tindexReader, test.polygon, test.field)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"test %d, expected %v, got %v for polygon: %+v\", n,\n\t\t\t\ttest.want, got, test.polygon)\n\t\t}\n\t}\n}\n\nfunc setupGeoJsonShapesIndexForPolygonQuery1(t *testing.T) index.Index {\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := scorch.NewScorch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\":          \"\",\n\t\t\t\"spatialPlugin\": \"s2\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon1 := [][][][]float64{{{\n\t\t{96.69202458735312, 61.59480859768306},\n\t\t{96.79202458735311, 61.39480859768306},\n\t\t{96.79202458735311, 61.59480859768306},\n\t\t{96.69202458735312, 61.59480859768306},\n\t}}}\n\tdoc := document.NewDocument(\"polygon1\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon1, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpolygon2 := [][][][]float64{{{\n\t\t{91.35604953911839, 65.11164029408492},\n\t\t{91.45604953911838, 64.91164029408492},\n\t\t{91.45604953911838, 65.11164029408492},\n\t\t{91.35604953911839, 65.11164029408492},\n\t}}}\n\tdoc = document.NewDocument(\"polygon2\")\n\tdoc.AddField(document.NewGeoShapeFieldWithIndexingOptions(\"geometry\", []uint64{},\n\t\tpolygon2, \"polygon\", document.DefaultGeoShapeIndexingOptions))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn i\n}\n"
  },
  {
    "path": "search/searcher/search_ip_range.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\n// netLimits returns the lo and hi bounds inside the network.\nfunc netLimits(n *net.IPNet) (lo net.IP, hi net.IP) {\n\tones, bits := n.Mask.Size()\n\tnetNum := n.IP\n\tif bits == net.IPv4len*8 {\n\t\tnetNum = netNum.To16()\n\t\tones += 8 * (net.IPv6len - net.IPv4len)\n\t}\n\tmask := net.CIDRMask(ones, 8*net.IPv6len)\n\tlo = make(net.IP, net.IPv6len)\n\thi = make(net.IP, net.IPv6len)\n\tfor i := 0; i < net.IPv6len; i++ {\n\t\tlo[i] = netNum[i] & mask[i]\n\t\thi[i] = lo[i] | ^mask[i]\n\t}\n\treturn lo, hi\n}\n\nfunc NewIPRangeSearcher(ctx context.Context, indexReader index.IndexReader, ipNet *net.IPNet,\n\tfield string, boost float64, options search.SearcherOptions) (\n\tsearch.Searcher, error) {\n\n\tlo, hi := netLimits(ipNet)\n\tfieldDict, err := indexReader.FieldDictRange(field, lo, hi)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer fieldDict.Close()\n\n\tvar terms []string\n\ttfd, err := fieldDict.Next()\n\tfor err == nil && tfd != nil {\n\t\tterms = append(terms, tfd.Term)\n\t\tif tooManyClauses(len(terms)) {\n\t\t\treturn nil, tooManyClausesErr(field, len(terms))\n\t\t}\n\t\ttfd, err = fieldDict.Next()\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true)\n}\n"
  },
  {
    "path": "search/searcher/search_ip_range_test.go",
    "content": "//  Copyright (c) 2021 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"net\"\n\t\"testing\"\n)\n\nfunc Test_netLimits(t *testing.T) {\n\ttests := []struct {\n\t\targ string\n\t\tlo  string\n\t\thi  string\n\t}{\n\t\t{\"128.0.0.0/1\", \"128.0.0.0\", \"255.255.255.255\"},\n\t\t{\"128.0.0.0/7\", \"128.0.0.0\", \"129.255.255.255\"},\n\t\t{\"1.1.1.1/8\", \"1.0.0.0\", \"1.255.255.255\"},\n\t\t{\"1.2.3.0/24\", \"1.2.3.0\", \"1.2.3.255\"},\n\t\t{\"1.2.2.0/23\", \"1.2.2.0\", \"1.2.3.255\"},\n\t\t{\"1.2.3.128/25\", \"1.2.3.128\", \"1.2.3.255\"},\n\t\t{\"1.2.3.0/25\", \"1.2.3.0\", \"1.2.3.127\"},\n\t\t{\"1.2.3.4/31\", \"1.2.3.4\", \"1.2.3.5\"},\n\t\t{\"1.2.3.4/32\", \"1.2.3.4\", \"1.2.3.4\"},\n\t\t{\"2a00:23c8:7283:ff00:1fa8:0:0:0/80\", \"2a00:23c8:7283:ff00:1fa8::\", \"2a00:23c8:7283:ff00:1fa8:ffff:ffff:ffff\"},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.arg, func(t *testing.T) {\n\t\t\t_, net, err := net.ParseCIDR(tt.arg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tlo, hi := netLimits(net)\n\t\t\tif lo.String() != tt.lo || hi.String() != tt.hi {\n\t\t\t\tt.Errorf(\"netLimits(%q) = %s %s, want %s %s\", tt.arg, lo, hi, tt.lo, tt.hi)\n\t\t\t}\n\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/scorer\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeKNNSearcher int\n\nfunc init() {\n\tvar ks KNNSearcher\n\treflectStaticSizeKNNSearcher = int(reflect.TypeOf(ks).Size())\n}\n\ntype KNNSearcher struct {\n\tfield        string\n\tvector       []float32\n\tk            int64\n\tindexReader  index.IndexReader\n\tvectorReader index.VectorReader\n\tscorer       *scorer.KNNQueryScorer\n\tcount        uint64\n\tvd           index.VectorDoc\n}\n\nfunc NewKNNSearcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping,\n\toptions search.SearcherOptions, field string, vector []float32, k int64,\n\tboost float64, similarityMetric string, searchParams json.RawMessage,\n\teligibleSelector index.EligibleDocumentSelector) (\n\tsearch.Searcher, error) {\n\n\tif vr, ok := i.(index.VectorIndexReader); ok {\n\t\tvectorReader, err := vr.VectorReader(ctx, vector, field, k, searchParams, eligibleSelector)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tknnScorer := scorer.NewKNNQueryScorer(vector, field, boost,\n\t\t\toptions, similarityMetric)\n\t\treturn &KNNSearcher{\n\t\t\tindexReader:  i,\n\t\t\tvectorReader: vectorReader,\n\t\t\tfield:        field,\n\t\t\tvector:       vector,\n\t\t\tk:            k,\n\t\t\tscorer:       knnScorer,\n\t\t}, nil\n\t}\n\treturn nil, nil\n}\n\nfunc (s *KNNSearcher) VectorOptimize(ctx context.Context, octx index.VectorOptimizableContext) (\n\tindex.VectorOptimizableContext, error) {\n\to, ok := s.vectorReader.(index.VectorOptimizable)\n\tif ok {\n\t\treturn o.VectorOptimize(ctx, octx)\n\t}\n\n\treturn nil, nil\n}\n\nfunc (s *KNNSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (\n\t*search.DocumentMatch, error) {\n\tknnMatch, err := s.vectorReader.Advance(ID, s.vd.Reset())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif knnMatch == nil {\n\t\treturn nil, nil\n\t}\n\n\tdocMatch := s.scorer.Score(ctx, knnMatch)\n\n\treturn docMatch, nil\n}\n\nfunc (s *KNNSearcher) Close() error {\n\treturn s.vectorReader.Close()\n}\n\nfunc (s *KNNSearcher) Count() uint64 {\n\treturn s.vectorReader.Count()\n}\n\nfunc (s *KNNSearcher) DocumentMatchPoolSize() int {\n\treturn 1\n}\n\nfunc (s *KNNSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *KNNSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\tknnMatch, err := s.vectorReader.Next(s.vd.Reset())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif knnMatch == nil {\n\t\treturn nil, nil\n\t}\n\n\tdocMatch := s.scorer.Score(ctx, knnMatch)\n\n\treturn docMatch, nil\n}\n\nfunc (s *KNNSearcher) SetQueryNorm(qnorm float64) {\n\ts.scorer.SetQueryNorm(qnorm)\n}\n\nfunc (s *KNNSearcher) Size() int {\n\treturn reflectStaticSizeKNNSearcher + size.SizeOfPtr +\n\t\ts.vectorReader.Size() +\n\t\ts.vd.Size() +\n\t\ts.scorer.Size()\n}\n\nfunc (s *KNNSearcher) Weight() float64 {\n\treturn s.scorer.Weight()\n}\n"
  },
  {
    "path": "search/searcher/search_match_all.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/scorer\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeMatchAllSearcher int\n\nfunc init() {\n\tvar mas MatchAllSearcher\n\treflectStaticSizeMatchAllSearcher = int(reflect.TypeOf(mas).Size())\n}\n\ntype MatchAllSearcher struct {\n\tindexReader index.IndexReader\n\treader      index.DocIDReader\n\tscorer      *scorer.ConstantScorer\n\tcount       uint64\n}\n\nfunc NewMatchAllSearcher(ctx context.Context, indexReader index.IndexReader, boost float64, options search.SearcherOptions) (*MatchAllSearcher, error) {\n\treader, err := indexReader.DocIDReaderAll()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcount, err := indexReader.DocCount()\n\tif err != nil {\n\t\t_ = reader.Close()\n\t\treturn nil, err\n\t}\n\tscorer := scorer.NewConstantScorer(1.0, boost, options)\n\n\treturn &MatchAllSearcher{\n\t\tindexReader: indexReader,\n\t\treader:      reader,\n\t\tscorer:      scorer,\n\t\tcount:       count,\n\t}, nil\n}\n\nfunc (s *MatchAllSearcher) Size() int {\n\treturn reflectStaticSizeMatchAllSearcher + size.SizeOfPtr +\n\t\ts.reader.Size() +\n\t\ts.scorer.Size()\n}\n\nfunc (s *MatchAllSearcher) Count() uint64 {\n\treturn s.count\n}\n\nfunc (s *MatchAllSearcher) Weight() float64 {\n\treturn s.scorer.Weight()\n}\n\nfunc (s *MatchAllSearcher) SetQueryNorm(qnorm float64) {\n\ts.scorer.SetQueryNorm(qnorm)\n}\n\nfunc (s *MatchAllSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\tid, err := s.reader.Next()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif id == nil {\n\t\treturn nil, nil\n\t}\n\n\t// score match\n\tdocMatch := s.scorer.Score(ctx, id)\n\t// return doc match\n\treturn docMatch, nil\n\n}\n\nfunc (s *MatchAllSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\tid, err := s.reader.Advance(ID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif id == nil {\n\t\treturn nil, nil\n\t}\n\n\t// score match\n\tdocMatch := s.scorer.Score(ctx, id)\n\n\t// return doc match\n\treturn docMatch, nil\n}\n\nfunc (s *MatchAllSearcher) Close() error {\n\treturn s.reader.Close()\n}\n\nfunc (s *MatchAllSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *MatchAllSearcher) DocumentMatchPoolSize() int {\n\treturn 1\n}\n"
  },
  {
    "path": "search/searcher/search_match_all_test.go",
    "content": "//  Copyright (c) 2013 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestMatchAllSearch(t *testing.T) {\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\texplainTrue := search.SearcherOptions{Explain: true}\n\n\tallSearcher, err := NewMatchAllSearcher(context.TODO(), twoDocIndexReader, 1.0, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tallSearcher2, err := NewMatchAllSearcher(context.TODO(), twoDocIndexReader, 1.2, explainTrue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttests := []struct {\n\t\tsearcher  search.Searcher\n\t\tqueryNorm float64\n\t\tresults   []*search.DocumentMatch\n\t}{\n\t\t{\n\t\t\tsearcher:  allSearcher,\n\t\t\tqueryNorm: 1.0,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"2\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"4\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"5\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher:  allSearcher2,\n\t\t\tqueryNorm: 0.8333333,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"1\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"2\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"3\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"4\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"5\"),\n\t\t\t\t\tScore:           1.0,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor testIndex, test := range tests {\n\n\t\tif test.queryNorm != 1.0 {\n\t\t\ttest.searcher.SetQueryNorm(test.queryNorm)\n\t\t}\n\t\tdefer func() {\n\t\t\terr := test.searcher.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}()\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := test.searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tif i < len(test.results) {\n\t\t\t\tif !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have id %s got %s for test %d\", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)\n\t\t\t\t}\n\t\t\t\tif !scoresCloseEnough(next.Score, test.results[i].Score) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have score %v got  %v for test %d\", i, test.results[i].Score, next.Score, testIndex)\n\t\t\t\t\tt.Logf(\"scoring explanation: %s\", next.Expl)\n\t\t\t\t}\n\t\t\t}\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = test.searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, testIndex)\n\t\t}\n\t\tif len(test.results) != i {\n\t\t\tt.Errorf(\"expected %d results got %d for test %d\", len(test.results), i, testIndex)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_match_none.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeMatchNoneSearcher int\n\nfunc init() {\n\tvar mns MatchNoneSearcher\n\treflectStaticSizeMatchNoneSearcher = int(reflect.TypeOf(mns).Size())\n}\n\ntype MatchNoneSearcher struct {\n\tindexReader index.IndexReader\n}\n\nfunc NewMatchNoneSearcher(indexReader index.IndexReader) (*MatchNoneSearcher, error) {\n\treturn &MatchNoneSearcher{\n\t\tindexReader: indexReader,\n\t}, nil\n}\n\nfunc (s *MatchNoneSearcher) Size() int {\n\treturn reflectStaticSizeMatchNoneSearcher + size.SizeOfPtr\n}\n\nfunc (s *MatchNoneSearcher) Count() uint64 {\n\treturn uint64(0)\n}\n\nfunc (s *MatchNoneSearcher) Weight() float64 {\n\treturn 0.0\n}\n\nfunc (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) {\n\n}\n\nfunc (s *MatchNoneSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\treturn nil, nil\n}\n\nfunc (s *MatchNoneSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\treturn nil, nil\n}\n\nfunc (s *MatchNoneSearcher) Close() error {\n\treturn nil\n}\n\nfunc (s *MatchNoneSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *MatchNoneSearcher) DocumentMatchPoolSize() int {\n\treturn 0\n}\n"
  },
  {
    "path": "search/searcher/search_match_none_test.go",
    "content": "//  Copyright (c) 2013 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\nfunc TestMatchNoneSearch(t *testing.T) {\n\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tnoneSearcher, err := NewMatchNoneSearcher(twoDocIndexReader)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttests := []struct {\n\t\tsearcher search.Searcher\n\t\tresults  []*search.DocumentMatch\n\t}{\n\t\t{\n\t\t\tsearcher: noneSearcher,\n\t\t\tresults:  []*search.DocumentMatch{},\n\t\t},\n\t}\n\n\tfor testIndex, test := range tests {\n\t\tdefer func() {\n\t\t\terr := test.searcher.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}()\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := test.searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tif i < len(test.results) {\n\t\t\t\tif !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have id %s got %s for test %d\", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)\n\t\t\t\t}\n\t\t\t\tif !scoresCloseEnough(next.Score, test.results[i].Score) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have score %v got  %v for test %d\", i, test.results[i].Score, next.Score, testIndex)\n\t\t\t\t\tt.Logf(\"scoring explanation: %s\", next.Expl)\n\t\t\t\t}\n\t\t\t}\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = test.searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, testIndex)\n\t\t}\n\t\tif len(test.results) != i {\n\t\t\tt.Errorf(\"expected %d results got %d for test %d\", len(test.results), i, testIndex)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_multi_term.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc NewMultiTermSearcher(ctx context.Context, indexReader index.IndexReader, terms []string,\n\tfield string, boost float64, options search.SearcherOptions, limit bool) (\n\tsearch.Searcher, error) {\n\n\tif tooManyClauses(len(terms)) {\n\t\tif optionsDisjunctionOptimizable(options) {\n\t\t\treturn optimizeMultiTermSearcher(ctx, indexReader, terms, field, boost, options)\n\t\t}\n\t\tif limit {\n\t\t\treturn nil, tooManyClausesErr(field, len(terms))\n\t\t}\n\t}\n\n\tqsearchers, err := makeBatchSearchers(ctx, indexReader, terms, field, boost, options)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// build disjunction searcher of these ranges\n\treturn newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost,\n\t\toptions, limit)\n}\n\n// Works similarly to the multi term searcher but additionally boosts individual terms based on\n// their edit distance from the query terms\nfunc NewMultiTermSearcherBoosted(ctx context.Context, indexReader index.IndexReader, terms []string,\n\tfield string, boost float64, editDistances []uint8, options search.SearcherOptions, limit bool) (\n\tsearch.Searcher, error) {\n\n\tif tooManyClauses(len(terms)) {\n\t\tif optionsDisjunctionOptimizable(options) {\n\t\t\treturn optimizeMultiTermSearcher(ctx, indexReader, terms, field, boost, options)\n\t\t}\n\t\tif limit {\n\t\t\treturn nil, tooManyClausesErr(field, len(terms))\n\t\t}\n\t}\n\n\tqsearchers, err := makeBatchSearchersBoosted(ctx, indexReader, terms, field, boost, editDistances, options)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// build disjunction searcher of these ranges\n\treturn newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost,\n\t\toptions, limit)\n}\n\nfunc NewMultiTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte,\n\tfield string, boost float64, options search.SearcherOptions, limit bool) (\n\tsearch.Searcher, error) {\n\n\tif tooManyClauses(len(terms)) {\n\t\tif optionsDisjunctionOptimizable(options) {\n\t\t\treturn optimizeMultiTermSearcherBytes(ctx, indexReader, terms, field, boost, options)\n\t\t}\n\n\t\tif limit {\n\t\t\treturn nil, tooManyClausesErr(field, len(terms))\n\t\t}\n\t}\n\n\tqsearchers, err := makeBatchSearchersBytes(ctx, indexReader, terms, field, boost, options)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// build disjunction searcher of these ranges\n\treturn newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost,\n\t\toptions, limit)\n}\n\nfunc newMultiTermSearcherInternal(ctx context.Context, indexReader index.IndexReader,\n\tsearchers []search.Searcher, field string, boost float64,\n\toptions search.SearcherOptions, limit bool) (\n\tsearch.Searcher, error) {\n\n\t// build disjunction searcher of these ranges\n\tsearcher, err := newDisjunctionSearcher(ctx, indexReader, searchers, 0, options,\n\t\tlimit)\n\tif err != nil {\n\t\tfor _, s := range searchers {\n\t\t\t_ = s.Close()\n\t\t}\n\t\treturn nil, err\n\t}\n\n\treturn searcher, nil\n}\n\nfunc optimizeMultiTermSearcher(ctx context.Context, indexReader index.IndexReader, terms []string,\n\tfield string, boost float64, options search.SearcherOptions) (\n\tsearch.Searcher, error) {\n\tvar finalSearcher search.Searcher\n\tfor len(terms) > 0 {\n\t\tvar batchTerms []string\n\t\tif len(terms) > DisjunctionMaxClauseCount {\n\t\t\tbatchTerms = terms[:DisjunctionMaxClauseCount]\n\t\t\tterms = terms[DisjunctionMaxClauseCount:]\n\t\t} else {\n\t\t\tbatchTerms = terms\n\t\t\tterms = nil\n\t\t}\n\t\tbatch, err := makeBatchSearchers(ctx, indexReader, batchTerms, field, boost, options)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif finalSearcher != nil {\n\t\t\tbatch = append(batch, finalSearcher)\n\t\t}\n\t\tcleanup := func() {\n\t\t\tfor _, searcher := range batch {\n\t\t\t\tif searcher != nil {\n\t\t\t\t\t_ = searcher.Close()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfinalSearcher, err = optimizeCompositeSearcher(ctx, \"disjunction:unadorned\",\n\t\t\tindexReader, batch, options)\n\t\t// all searchers in batch should be closed, regardless of error or optimization failure\n\t\t// either we're returning, or continuing and only finalSearcher is needed for next loop\n\t\tcleanup()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif finalSearcher == nil {\n\t\t\treturn nil, fmt.Errorf(\"unable to optimize\")\n\t\t}\n\t}\n\treturn finalSearcher, nil\n}\n\nfunc makeBatchSearchers(ctx context.Context, indexReader index.IndexReader, terms []string, field string,\n\tboost float64, options search.SearcherOptions) ([]search.Searcher, error) {\n\n\tqsearchers := make([]search.Searcher, len(terms))\n\tqsearchersClose := func() {\n\t\tfor _, searcher := range qsearchers {\n\t\t\tif searcher != nil {\n\t\t\t\t_ = searcher.Close()\n\t\t\t}\n\t\t}\n\t}\n\tfor i, term := range terms {\n\t\tvar err error\n\t\tqsearchers[i], err = NewTermSearcher(ctx, indexReader, term, field, boost, options)\n\t\tif err != nil {\n\t\t\tqsearchersClose()\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn qsearchers, nil\n}\n\nfunc makeBatchSearchersBoosted(ctx context.Context, indexReader index.IndexReader, terms []string, field string,\n\tboost float64, editDistances []uint8, options search.SearcherOptions) ([]search.Searcher, error) {\n\n\tqsearchers := make([]search.Searcher, len(terms))\n\tqsearchersClose := func() {\n\t\tfor _, searcher := range qsearchers {\n\t\t\tif searcher != nil {\n\t\t\t\t_ = searcher.Close()\n\t\t\t}\n\t\t}\n\t}\n\tfor i, term := range terms {\n\t\tvar err error\n\t\tvar editMultiplier float64\n\t\tif editDistances != nil {\n\t\t\teditMultiplier = 1 / float64(editDistances[i]+1)\n\t\t}\n\t\tqsearchers[i], err = NewTermSearcher(ctx, indexReader, term, field, boost*editMultiplier, options)\n\t\tif err != nil {\n\t\t\tqsearchersClose()\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn qsearchers, nil\n}\n\nfunc optimizeMultiTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte,\n\tfield string, boost float64, options search.SearcherOptions) (\n\tsearch.Searcher, error) {\n\n\tvar finalSearcher search.Searcher\n\tfor len(terms) > 0 {\n\t\tvar batchTerms [][]byte\n\t\tif len(terms) > DisjunctionMaxClauseCount {\n\t\t\tbatchTerms = terms[:DisjunctionMaxClauseCount]\n\t\t\tterms = terms[DisjunctionMaxClauseCount:]\n\t\t} else {\n\t\t\tbatchTerms = terms\n\t\t\tterms = nil\n\t\t}\n\t\tbatch, err := makeBatchSearchersBytes(ctx, indexReader, batchTerms, field, boost, options)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif finalSearcher != nil {\n\t\t\tbatch = append(batch, finalSearcher)\n\t\t}\n\t\tcleanup := func() {\n\t\t\tfor _, searcher := range batch {\n\t\t\t\tif searcher != nil {\n\t\t\t\t\t_ = searcher.Close()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfinalSearcher, err = optimizeCompositeSearcher(ctx, \"disjunction:unadorned\",\n\t\t\tindexReader, batch, options)\n\t\t// all searchers in batch should be closed, regardless of error or optimization failure\n\t\t// either we're returning, or continuing and only finalSearcher is needed for next loop\n\t\tcleanup()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif finalSearcher == nil {\n\t\t\treturn nil, fmt.Errorf(\"unable to optimize\")\n\t\t}\n\t}\n\treturn finalSearcher, nil\n}\n\nfunc makeBatchSearchersBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte, field string,\n\tboost float64, options search.SearcherOptions) ([]search.Searcher, error) {\n\n\tqsearchers := make([]search.Searcher, len(terms))\n\tqsearchersClose := func() {\n\t\tfor _, searcher := range qsearchers {\n\t\t\tif searcher != nil {\n\t\t\t\t_ = searcher.Close()\n\t\t\t}\n\t\t}\n\t}\n\tfor i, term := range terms {\n\t\tvar err error\n\t\tqsearchers[i], err = NewTermSearcherBytes(ctx, indexReader, term, field, boost, options)\n\t\tif err != nil {\n\t\t\tqsearchersClose()\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn qsearchers, nil\n}\n"
  },
  {
    "path": "search/searcher/search_numeric_range.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"math\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc NewNumericRangeSearcher(ctx context.Context, indexReader index.IndexReader,\n\tmin *float64, max *float64, inclusiveMin, inclusiveMax *bool, field string,\n\tboost float64, options search.SearcherOptions) (search.Searcher, error) {\n\t// account for unbounded edges\n\tif min == nil {\n\t\tnegInf := math.Inf(-1)\n\t\tmin = &negInf\n\t}\n\tif max == nil {\n\t\tInf := math.Inf(1)\n\t\tmax = &Inf\n\t}\n\tif inclusiveMin == nil {\n\t\tdefaultInclusiveMin := true\n\t\tinclusiveMin = &defaultInclusiveMin\n\t}\n\tif inclusiveMax == nil {\n\t\tdefaultInclusiveMax := false\n\t\tinclusiveMax = &defaultInclusiveMax\n\t}\n\t// find all the ranges\n\tminInt64 := numeric.Float64ToInt64(*min)\n\tif !*inclusiveMin && minInt64 != math.MaxInt64 {\n\t\tminInt64++\n\t}\n\tmaxInt64 := numeric.Float64ToInt64(*max)\n\tif !*inclusiveMax && maxInt64 != math.MinInt64 {\n\t\tmaxInt64--\n\t}\n\n\tvar fieldDict index.FieldDictContains\n\tvar dictBytesRead uint64\n\tvar isIndexed filterFunc\n\tvar err error\n\tif irr, ok := indexReader.(index.IndexReaderContains); ok {\n\t\tfieldDict, err = irr.FieldDictContains(field)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tisIndexed = func(term []byte) bool {\n\t\t\tfound, err := fieldDict.Contains(term)\n\t\t\treturn err == nil && found\n\t\t}\n\n\t\tdictBytesRead = fieldDict.BytesRead()\n\t}\n\n\t// FIXME hard-coded precision, should match field declaration\n\ttermRanges := splitInt64Range(minInt64, maxInt64, 4)\n\tterms := termRanges.Enumerate(isIndexed)\n\tif fieldDict != nil {\n\t\tif fd, ok := fieldDict.(index.FieldDict); ok {\n\t\t\tif err = fd.Close(); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(terms) < 1 {\n\t\t// reporting back the IO stats with respect to the dictionary\n\t\t// loaded, using the context\n\t\tif ctx != nil {\n\t\t\treportIOStats(ctx, dictBytesRead)\n\t\t\tsearch.RecordSearchCost(ctx, search.AddM, dictBytesRead)\n\t\t}\n\n\t\t// cannot return MatchNoneSearcher because of interaction with\n\t\t// commit f391b991c20f02681bacd197afc6d8aed444e132\n\t\treturn NewMultiTermSearcherBytes(ctx, indexReader, terms, field,\n\t\t\tboost, options, true)\n\t}\n\n\t// for upside_down\n\tif isIndexed == nil {\n\t\tterms, err = filterCandidateTerms(indexReader, terms, field)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif tooManyClauses(len(terms)) {\n\t\treturn nil, tooManyClausesErr(field, len(terms))\n\t}\n\n\tif ctx != nil {\n\t\treportIOStats(ctx, dictBytesRead)\n\t\tsearch.RecordSearchCost(ctx, search.AddM, dictBytesRead)\n\t}\n\n\treturn NewMultiTermSearcherBytes(ctx, indexReader, terms, field,\n\t\tboost, options, true)\n}\n\nfunc filterCandidateTerms(indexReader index.IndexReader,\n\tterms [][]byte, field string) (rv [][]byte, err error) {\n\n\tfieldDict, err := indexReader.FieldDictRange(field, terms[0], terms[len(terms)-1])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// enumerate the terms and check against list of terms\n\ttfd, err := fieldDict.Next()\n\tfor err == nil && tfd != nil {\n\t\ttermBytes := []byte(tfd.Term)\n\t\ti := sort.Search(len(terms), func(i int) bool { return bytes.Compare(terms[i], termBytes) >= 0 })\n\t\tif i < len(terms) && bytes.Equal(terms[i], termBytes) {\n\t\t\trv = append(rv, terms[i])\n\t\t}\n\t\tterms = terms[i:]\n\t\ttfd, err = fieldDict.Next()\n\t}\n\n\tif cerr := fieldDict.Close(); cerr != nil && err == nil {\n\t\terr = cerr\n\t}\n\n\treturn rv, err\n}\n\ntype termRange struct {\n\tstartTerm []byte\n\tendTerm   []byte\n}\n\nfunc (t *termRange) Enumerate(filter filterFunc) [][]byte {\n\tvar rv [][]byte\n\tnext := t.startTerm\n\tfor bytes.Compare(next, t.endTerm) <= 0 {\n\t\tif filter != nil {\n\t\t\tif filter(next) {\n\t\t\t\trv = append(rv, next)\n\t\t\t}\n\t\t} else {\n\t\t\trv = append(rv, next)\n\t\t}\n\t\tnext = incrementBytes(next)\n\t}\n\treturn rv\n}\n\nfunc incrementBytes(in []byte) []byte {\n\trv := make([]byte, len(in))\n\tcopy(rv, in)\n\tfor i := len(rv) - 1; i >= 0; i-- {\n\t\trv[i] = rv[i] + 1\n\t\tif rv[i] != 0 {\n\t\t\t// didn't overflow, so stop\n\t\t\tbreak\n\t\t}\n\t}\n\treturn rv\n}\n\ntype termRanges []*termRange\n\nfunc (tr termRanges) Enumerate(filter filterFunc) [][]byte {\n\tvar rv [][]byte\n\tfor _, tri := range tr {\n\t\ttrie := tri.Enumerate(filter)\n\t\trv = append(rv, trie...)\n\t}\n\treturn rv\n}\n\nfunc splitInt64Range(minBound, maxBound int64, precisionStep uint) termRanges {\n\trv := make(termRanges, 0)\n\tif minBound > maxBound {\n\t\treturn rv\n\t}\n\n\tfor shift := uint(0); ; shift += precisionStep {\n\n\t\tdiff := int64(1) << (shift + precisionStep)\n\t\tmask := ((int64(1) << precisionStep) - int64(1)) << shift\n\t\thasLower := (minBound & mask) != int64(0)\n\t\thasUpper := (maxBound & mask) != mask\n\n\t\tvar nextMinBound int64\n\t\tif hasLower {\n\t\t\tnextMinBound = (minBound + diff) &^ mask\n\t\t} else {\n\t\t\tnextMinBound = minBound &^ mask\n\t\t}\n\t\tvar nextMaxBound int64\n\t\tif hasUpper {\n\t\t\tnextMaxBound = (maxBound - diff) &^ mask\n\t\t} else {\n\t\t\tnextMaxBound = maxBound &^ mask\n\t\t}\n\n\t\tlowerWrapped := nextMinBound < minBound\n\t\tupperWrapped := nextMaxBound > maxBound\n\n\t\tif shift+precisionStep >= 64 || nextMinBound > nextMaxBound ||\n\t\t\tlowerWrapped || upperWrapped {\n\t\t\t// We are in the lowest precision or the next precision is not available.\n\t\t\trv = append(rv, newRange(minBound, maxBound, shift))\n\t\t\t// exit the split recursion loop\n\t\t\tbreak\n\t\t}\n\n\t\tif hasLower {\n\t\t\trv = append(rv, newRange(minBound, minBound|mask, shift))\n\t\t}\n\t\tif hasUpper {\n\t\t\trv = append(rv, newRange(maxBound&^mask, maxBound, shift))\n\t\t}\n\n\t\t// recurse to next precision\n\t\tminBound = nextMinBound\n\t\tmaxBound = nextMaxBound\n\t}\n\n\treturn rv\n}\n\nfunc newRange(minBound, maxBound int64, shift uint) *termRange {\n\tmaxBound |= (int64(1) << shift) - int64(1)\n\tminBytes := numeric.MustNewPrefixCodedInt64(minBound, shift)\n\tmaxBytes := numeric.MustNewPrefixCodedInt64(maxBound, shift)\n\treturn newRangeBytes(minBytes, maxBytes)\n}\n\nfunc newRangeBytes(minBytes, maxBytes []byte) *termRange {\n\treturn &termRange{\n\t\tstartTerm: minBytes,\n\t\tendTerm:   maxBytes,\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_numeric_range_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n)\n\nfunc TestSplitRange(t *testing.T) {\n\tmin := numeric.Float64ToInt64(1.0)\n\tmax := numeric.Float64ToInt64(5.0)\n\tranges := splitInt64Range(min, max, 4)\n\tenumerated := ranges.Enumerate(nil)\n\tif len(enumerated) != 135 {\n\t\tt.Errorf(\"expected 135 terms, got %d\", len(enumerated))\n\t}\n\n}\n\nfunc TestIncrementBytes(t *testing.T) {\n\ttests := []struct {\n\t\tin  []byte\n\t\tout []byte\n\t}{\n\t\t{\n\t\t\tin:  []byte{0},\n\t\t\tout: []byte{1},\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 0},\n\t\t\tout: []byte{0, 1},\n\t\t},\n\t\t{\n\t\t\tin:  []byte{0, 255},\n\t\t\tout: []byte{1, 0},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tactual := incrementBytes(test.in)\n\t\tif !reflect.DeepEqual(actual, test.out) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.out, actual)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_phrase.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizePhraseSearcher int\n\nfunc init() {\n\tvar ps PhraseSearcher\n\treflectStaticSizePhraseSearcher = int(reflect.TypeOf(ps).Size())\n}\n\ntype PhraseSearcher struct {\n\tmustSearcher search.Searcher\n\tqueryNorm    float64\n\tcurrMust     *search.DocumentMatch\n\tterms        [][]string\n\tpath         phrasePath\n\tpaths        []phrasePath\n\tlocations    []search.Location\n\tinitialized  bool\n\t// map a term to a list of fuzzy terms that match it\n\tfuzzyTermMatches map[string][]string\n}\n\nfunc (s *PhraseSearcher) Size() int {\n\tsizeInBytes := reflectStaticSizePhraseSearcher + size.SizeOfPtr\n\n\tif s.mustSearcher != nil {\n\t\tsizeInBytes += s.mustSearcher.Size()\n\t}\n\n\tif s.currMust != nil {\n\t\tsizeInBytes += s.currMust.Size()\n\t}\n\n\tfor _, entry := range s.terms {\n\t\tsizeInBytes += size.SizeOfSlice\n\t\tfor _, entry1 := range entry {\n\t\t\tsizeInBytes += size.SizeOfString + len(entry1)\n\t\t}\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc NewPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms []string,\n\tfuzziness int, autoFuzzy bool, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) {\n\n\t// turn flat terms []string into [][]string\n\tmterms := make([][]string, len(terms))\n\tfor i, term := range terms {\n\t\tmterms[i] = []string{term}\n\t}\n\treturn NewMultiPhraseSearcher(ctx, indexReader, mterms, fuzziness, autoFuzzy, field, boost, options)\n}\n\nfunc NewMultiPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms [][]string,\n\tfuzziness int, autoFuzzy bool, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) {\n\n\toptions.IncludeTermVectors = true\n\tvar termPositionSearchers []search.Searcher\n\tvar err error\n\tvar ts search.Searcher\n\t// The following logic checks if fuzziness is enabled.\n\t// Fuzziness is considered enabled if either:\n\t//    a. `fuzziness` is greater than 0, or\n\t//    b. `autoFuzzy` is set to true.\n\t// if both conditions are true, `autoFuzzy` takes precedence.\n\t// If enabled, a map will be created to store the matches for fuzzy terms.\n\tfuzzinessEnabled := autoFuzzy || fuzziness > 0\n\tvar fuzzyTermMatches map[string][]string\n\tif fuzzinessEnabled {\n\t\tfuzzyTermMatches = make(map[string][]string)\n\t\tctx = context.WithValue(ctx, search.FuzzyMatchPhraseKey, fuzzyTermMatches)\n\t}\n\t// in case of fuzzy multi-phrase, phrase and match-phrase queries we hardcode the\n\t// prefix length to 0, as setting a per word matching prefix length would not\n\t// make sense from a user perspective.\n\tfor _, termPos := range terms {\n\t\tif len(termPos) == 1 && termPos[0] != \"\" {\n\t\t\t// single term\n\t\t\tif fuzzinessEnabled {\n\t\t\t\t// fuzzy\n\t\t\t\tif autoFuzzy {\n\t\t\t\t\t// auto fuzzy\n\t\t\t\t\tts, err = NewAutoFuzzySearcher(ctx, indexReader, termPos[0], 0, field, boost, options)\n\t\t\t\t} else {\n\t\t\t\t\t// non-auto fuzzy\n\t\t\t\t\tts, err = NewFuzzySearcher(ctx, indexReader, termPos[0], 0, fuzziness, field, boost, options)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// non-fuzzy\n\t\t\t\tts, err = NewTermSearcher(ctx, indexReader, termPos[0], field, boost, options)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\t// close any searchers already opened\n\t\t\t\tfor _, ts := range termPositionSearchers {\n\t\t\t\t\t_ = ts.Close()\n\t\t\t\t}\n\t\t\t\treturn nil, fmt.Errorf(\"phrase searcher error building term searcher: %v\", err)\n\t\t\t}\n\t\t\ttermPositionSearchers = append(termPositionSearchers, ts)\n\t\t} else if len(termPos) > 1 {\n\t\t\t// multiple terms\n\t\t\tvar termSearchers []search.Searcher\n\t\t\tfor _, term := range termPos {\n\t\t\t\tif term == \"\" {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif fuzzinessEnabled {\n\t\t\t\t\t// fuzzy\n\t\t\t\t\tif autoFuzzy {\n\t\t\t\t\t\t// auto fuzzy\n\t\t\t\t\t\tts, err = NewAutoFuzzySearcher(ctx, indexReader, term, 0, field, boost, options)\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// non-auto fuzzy\n\t\t\t\t\t\tts, err = NewFuzzySearcher(ctx, indexReader, term, 0, fuzziness, field, boost, options)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// non-fuzzy\n\t\t\t\t\tts, err = NewTermSearcher(ctx, indexReader, term, field, boost, options)\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\t// close any searchers already opened\n\t\t\t\t\tfor _, ts := range termPositionSearchers {\n\t\t\t\t\t\t_ = ts.Close()\n\t\t\t\t\t}\n\t\t\t\t\treturn nil, fmt.Errorf(\"phrase searcher error building term searcher: %v\", err)\n\t\t\t\t}\n\t\t\t\ttermSearchers = append(termSearchers, ts)\n\t\t\t}\n\t\t\tdisjunction, err := NewDisjunctionSearcher(ctx, indexReader, termSearchers, 1, options)\n\t\t\tif err != nil {\n\t\t\t\t// close any searchers already opened\n\t\t\t\tfor _, ts := range termPositionSearchers {\n\t\t\t\t\t_ = ts.Close()\n\t\t\t\t}\n\t\t\t\treturn nil, fmt.Errorf(\"phrase searcher error building term position disjunction searcher: %v\", err)\n\t\t\t}\n\t\t\ttermPositionSearchers = append(termPositionSearchers, disjunction)\n\t\t}\n\t}\n\n\tif ctx != nil {\n\t\tif fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {\n\t\t\tif ts, exists := fts[field]; exists {\n\t\t\t\tif fuzzinessEnabled {\n\t\t\t\t\tfor term, fuzzyTerms := range fuzzyTermMatches {\n\t\t\t\t\t\tfuzzySynonymTerms := make([]string, 0, len(fuzzyTerms))\n\t\t\t\t\t\tif s, found := ts[term]; found {\n\t\t\t\t\t\t\tfuzzySynonymTerms = append(fuzzySynonymTerms, s...)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tfor _, fuzzyTerm := range fuzzyTerms {\n\t\t\t\t\t\t\tif fuzzyTerm == term {\n\t\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif s, found := ts[fuzzyTerm]; found {\n\t\t\t\t\t\t\t\tfuzzySynonymTerms = append(fuzzySynonymTerms, s...)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif len(fuzzySynonymTerms) > 0 {\n\t\t\t\t\t\t\tfuzzyTermMatches[term] = append(fuzzyTermMatches[term], fuzzySynonymTerms...)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tfor _, termPos := range terms {\n\t\t\t\t\t\tfor _, term := range termPos {\n\t\t\t\t\t\t\tif s, found := ts[term]; found {\n\t\t\t\t\t\t\t\tif fuzzyTermMatches == nil {\n\t\t\t\t\t\t\t\t\tfuzzyTermMatches = make(map[string][]string)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tfuzzyTermMatches[term] = s\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tmustSearcher, err := NewConjunctionSearcher(ctx, indexReader, termPositionSearchers, options)\n\tif err != nil {\n\t\t// close any searchers already opened\n\t\tfor _, ts := range termPositionSearchers {\n\t\t\t_ = ts.Close()\n\t\t}\n\t\treturn nil, fmt.Errorf(\"phrase searcher error building conjunction searcher: %v\", err)\n\t}\n\n\t// build our searcher\n\trv := PhraseSearcher{\n\t\tmustSearcher:     mustSearcher,\n\t\tterms:            terms,\n\t\tfuzzyTermMatches: fuzzyTermMatches,\n\t}\n\trv.computeQueryNorm()\n\treturn &rv, nil\n}\n\nfunc (s *PhraseSearcher) computeQueryNorm() {\n\t// first calculate sum of squared weights\n\tsumOfSquaredWeights := 0.0\n\tif s.mustSearcher != nil {\n\t\tsumOfSquaredWeights += s.mustSearcher.Weight()\n\t}\n\n\t// now compute query norm from this\n\ts.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)\n\t// finally tell all the downstream searchers the norm\n\tif s.mustSearcher != nil {\n\t\ts.mustSearcher.SetQueryNorm(s.queryNorm)\n\t}\n}\n\nfunc (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error {\n\terr := s.advanceNextMust(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ts.initialized = true\n\treturn nil\n}\n\nfunc (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error {\n\tvar err error\n\n\tif s.mustSearcher != nil {\n\t\tif s.currMust != nil {\n\t\t\tctx.DocumentMatchPool.Put(s.currMust)\n\t\t}\n\t\ts.currMust, err = s.mustSearcher.Next(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (s *PhraseSearcher) Weight() float64 {\n\treturn s.mustSearcher.Weight()\n}\n\nfunc (s *PhraseSearcher) SetQueryNorm(qnorm float64) {\n\ts.mustSearcher.SetQueryNorm(qnorm)\n}\n\nfunc (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tfor s.currMust != nil {\n\t\t// check this match against phrase constraints\n\t\trv := s.checkCurrMustMatch(ctx)\n\n\t\t// prepare for next iteration (either loop or subsequent call to Next())\n\t\terr := s.advanceNextMust(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// if match satisfied phrase constraints return it as a hit\n\t\tif rv != nil {\n\t\t\treturn rv, nil\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\n// checkCurrMustMatch is solely concerned with determining if the DocumentMatch\n// pointed to by s.currMust (which satisfies the pre-condition searcher)\n// also satisfies the phrase constraints.  if so, it returns a DocumentMatch\n// for this document, otherwise nil\nfunc (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch {\n\ts.locations = s.currMust.Complete(s.locations)\n\n\tlocations := s.currMust.Locations\n\ts.currMust.Locations = nil\n\n\tftls := s.currMust.FieldTermLocations\n\n\t// typically we would expect there to only actually be results in\n\t// one field, but we allow for this to not be the case\n\t// but, we note that phrase constraints can only be satisfied within\n\t// a single field, so we can check them each independently\n\tfor field, tlm := range locations {\n\t\tftls = s.checkCurrMustMatchField(ctx, field, tlm, ftls)\n\t}\n\n\tif len(ftls) > 0 {\n\t\t// return match\n\t\trv := s.currMust\n\t\ts.currMust = nil\n\t\trv.FieldTermLocations = ftls\n\t\treturn rv\n\t}\n\n\treturn nil\n}\n\n// checkCurrMustMatchField is solely concerned with determining if one\n// particular field within the currMust DocumentMatch Locations\n// satisfies the phrase constraints (possibly more than once).  if so,\n// the matching field term locations are appended to the provided\n// slice\nfunc (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext,\n\tfield string, tlm search.TermLocationMap,\n\tftls []search.FieldTermLocation) []search.FieldTermLocation {\n\tif s.path == nil {\n\t\ts.path = make(phrasePath, 0, len(s.terms))\n\t}\n\tvar tlmPtr *search.TermLocationMap = &tlm\n\tif s.fuzzyTermMatches != nil {\n\t\t// if fuzzy search, we need to expand the tlm to include all the fuzzy matches\n\t\t// Example - term is \"foo\" and fuzzy matches are \"foo\", \"fool\", \"food\"\n\t\t// the non expanded tlm will be:\n\t\t//\tfoo  -> Locations[foo]\n\t\t//\tfool -> Locations[fool]\n\t\t//\tfood -> Locations[food]\n\t\t// the expanded tlm will be:\n\t\t//   foo -> [Locations[foo], Locations[fool], Locations[food]]\n\t\texpandedTlm := make(search.TermLocationMap)\n\t\ts.expandFuzzyMatches(tlm, expandedTlm)\n\t\ttlmPtr = &expandedTlm\n\t}\n\ts.paths = findPhrasePaths(0, nil, s.terms, *tlmPtr, s.path[:0], 0, s.paths[:0])\n\tfor _, p := range s.paths {\n\t\tfor _, pp := range p {\n\t\t\tftls = append(ftls, search.FieldTermLocation{\n\t\t\t\tField: field,\n\t\t\t\tTerm:  pp.term,\n\t\t\t\tLocation: search.Location{\n\t\t\t\t\tPos:            pp.loc.Pos,\n\t\t\t\t\tStart:          pp.loc.Start,\n\t\t\t\t\tEnd:            pp.loc.End,\n\t\t\t\t\tArrayPositions: pp.loc.ArrayPositions,\n\t\t\t\t},\n\t\t\t})\n\t\t}\n\t}\n\treturn ftls\n}\n\nfunc (s *PhraseSearcher) expandFuzzyMatches(tlm search.TermLocationMap, expandedTlm search.TermLocationMap) {\n\tfor term, fuzzyMatches := range s.fuzzyTermMatches {\n\t\tlocations := tlm[term]\n\t\tfor _, fuzzyMatch := range fuzzyMatches {\n\t\t\tif fuzzyMatch == term {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tlocations = append(locations, tlm[fuzzyMatch]...)\n\t\t}\n\t\texpandedTlm[term] = locations\n\t}\n}\n\ntype phrasePart struct {\n\tterm string\n\tloc  *search.Location\n}\n\nfunc (p *phrasePart) String() string {\n\treturn fmt.Sprintf(\"[%s %v]\", p.term, p.loc)\n}\n\ntype phrasePath []phrasePart\n\nfunc (p phrasePath) MergeInto(in search.TermLocationMap) {\n\tfor _, pp := range p {\n\t\tin[pp.term] = append(in[pp.term], pp.loc)\n\t}\n}\n\nfunc (p phrasePath) String() string {\n\trv := \"[\"\n\tfor i, pp := range p {\n\t\tif i > 0 {\n\t\t\trv += \", \"\n\t\t}\n\t\trv += pp.String()\n\t}\n\trv += \"]\"\n\treturn rv\n}\n\n// findPhrasePaths is a function to identify phrase matches from a set\n// of known term locations.  it recursive so care must be taken with\n// arguments and return values.\n//\n// prevPos - the previous location, 0 on first invocation\n//\n// ap - array positions of the first candidate phrase part to\n// which further recursive phrase parts must match,\n// nil on initial invocation or when there are no array positions\n//\n// phraseTerms - slice containing the phrase terms,\n// may contain empty string as placeholder (don't care)\n//\n// tlm - the Term Location Map containing all relevant term locations\n//\n// p - the current path being explored (appended to in recursive calls)\n// this is the primary state being built during the traversal\n//\n// remainingSlop - amount of sloppiness that's allowed, which is the\n// sum of the editDistances from each matching phrase part, where 0 means no\n// sloppiness allowed (all editDistances must be 0), decremented during recursion\n//\n// rv - the final result being appended to by all the recursive calls\n//\n// returns slice of paths, or nil if invocation did not find any successful paths\nfunc findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string,\n\ttlm search.TermLocationMap, p phrasePath, remainingSlop int, rv []phrasePath) []phrasePath {\n\t// no more terms\n\tif len(phraseTerms) < 1 {\n\t\t// snapshot or copy the recursively built phrasePath p and\n\t\t// append it to the rv, also optimizing by checking if next\n\t\t// phrasePath item in the rv (which we're about to overwrite)\n\t\t// is available for reuse\n\t\tvar pcopy phrasePath\n\t\tif len(rv) < cap(rv) {\n\t\t\tpcopy = rv[:len(rv)+1][len(rv)][:0]\n\t\t}\n\t\treturn append(rv, append(pcopy, p...))\n\t}\n\n\tcar := phraseTerms[0]\n\tcdr := phraseTerms[1:]\n\n\t// empty term is treated as match (continue)\n\tif len(car) == 0 || (len(car) == 1 && car[0] == \"\") {\n\t\tnextPos := prevPos + 1\n\t\tif prevPos == 0 {\n\t\t\t// if prevPos was 0, don't set it to 1 (as that's not a real abs pos)\n\t\t\tnextPos = 0 // don't advance nextPos if prevPos was 0\n\t\t}\n\t\treturn findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop, rv)\n\t}\n\n\t// locations for this term\n\tfor _, carTerm := range car {\n\t\tlocations := tlm[carTerm]\n\tLOCATIONS_LOOP:\n\t\tfor _, loc := range locations {\n\t\t\tif prevPos != 0 && !loc.ArrayPositions.Equals(ap) {\n\t\t\t\t// if the array positions are wrong, can't match, try next location\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// compute distance from previous phrase term\n\t\t\tdist := 0\n\t\t\tif prevPos != 0 {\n\t\t\t\tdist = editDistance(prevPos+1, loc.Pos)\n\t\t\t}\n\n\t\t\t// if enough slop remaining, continue recursively\n\t\t\tif prevPos == 0 || (remainingSlop-dist) >= 0 {\n\t\t\t\t// skip if we've already used this term+loc already\n\t\t\t\tfor _, ppart := range p {\n\t\t\t\t\tif ppart.term == carTerm && ppart.loc == loc {\n\t\t\t\t\t\tcontinue LOCATIONS_LOOP\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// this location works, add it to the path (but not for empty term)\n\t\t\t\tpx := append(p, phrasePart{term: carTerm, loc: loc})\n\t\t\t\trv = findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist, rv)\n\t\t\t}\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc editDistance(p1, p2 uint64) int {\n\tdist := int(p1 - p2)\n\tif dist < 0 {\n\t\treturn -dist\n\t}\n\treturn dist\n}\n\nfunc (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\tif !s.initialized {\n\t\terr := s.initSearchers(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tif s.currMust != nil {\n\t\tif s.currMust.IndexInternalID.Compare(ID) >= 0 {\n\t\t\treturn s.Next(ctx)\n\t\t}\n\t\tctx.DocumentMatchPool.Put(s.currMust)\n\t}\n\tif s.currMust == nil {\n\t\treturn nil, nil\n\t}\n\tvar err error\n\ts.currMust, err = s.mustSearcher.Advance(ctx, ID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn s.Next(ctx)\n}\n\nfunc (s *PhraseSearcher) Count() uint64 {\n\t// for now return a worst case\n\treturn s.mustSearcher.Count()\n}\n\nfunc (s *PhraseSearcher) Close() error {\n\tif s.mustSearcher != nil {\n\t\terr := s.mustSearcher.Close()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (s *PhraseSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *PhraseSearcher) DocumentMatchPoolSize() int {\n\treturn s.mustSearcher.DocumentMatchPoolSize() + 1\n}\n"
  },
  {
    "path": "search/searcher/search_phrase_test.go",
    "content": "//  Copyright (c) 2013 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestPhraseSearch(t *testing.T) {\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tsoptions := search.SearcherOptions{Explain: true, IncludeTermVectors: true}\n\tphraseSearcher, err := NewPhraseSearcher(context.TODO(), twoDocIndexReader, []string{\"angst\", \"beer\"}, 0, false, \"desc\", 1.0, soptions)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttests := []struct {\n\t\tsearcher   search.Searcher\n\t\tresults    []*search.DocumentMatch\n\t\tlocations  map[string]map[string][]search.Location\n\t\tfieldterms [][2]string\n\t}{\n\t\t{\n\t\t\tsearcher: phraseSearcher,\n\t\t\tresults: []*search.DocumentMatch{\n\t\t\t\t{\n\t\t\t\t\tIndexInternalID: index.IndexInternalID(\"2\"),\n\t\t\t\t\tScore:           1.0807601687084403,\n\t\t\t\t},\n\t\t\t},\n\t\t\tlocations:  map[string]map[string][]search.Location{\"desc\": {\"beer\": {{Pos: 2, Start: 6, End: 10}}, \"angst\": {{Pos: 1, Start: 0, End: 5}}}},\n\t\t\tfieldterms: [][2]string{{\"desc\", \"beer\"}, {\"desc\", \"angst\"}},\n\t\t},\n\t}\n\n\tfor testIndex, test := range tests {\n\t\tdefer func() {\n\t\t\terr := test.searcher.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}()\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := test.searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tnext.Complete(nil)\n\t\t\tif i < len(test.results) {\n\t\t\t\tif !next.IndexInternalID.Equals(test.results[i].IndexInternalID) {\n\t\t\t\t\tt.Errorf(\"expected result %d to have id %s got %s for test %d\\n\", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex)\n\t\t\t\t}\n\t\t\t\tif next.Score != test.results[i].Score {\n\t\t\t\t\tt.Errorf(\"expected result %d to have score %v got %v for test %d\\n\", i, test.results[i].Score, next.Score, testIndex)\n\t\t\t\t\tt.Logf(\"scoring explanation: %s\\n\", next.Expl)\n\t\t\t\t}\n\t\t\t\tfor _, ft := range test.fieldterms {\n\t\t\t\t\tlocs := next.Locations[ft[0]][ft[1]]\n\t\t\t\t\texplocs := test.locations[ft[0]][ft[1]]\n\t\t\t\t\tif len(explocs) != len(locs) {\n\t\t\t\t\t\tt.Fatalf(\"expected result %d to have %d Locations (%#v) but got %d (%#v) for test %d with field %q and term %q\\n\", i, len(explocs), explocs, len(locs), locs, testIndex, ft[0], ft[1])\n\t\t\t\t\t}\n\t\t\t\t\tfor ind, exploc := range explocs {\n\t\t\t\t\t\tif !reflect.DeepEqual(*locs[ind], exploc) {\n\t\t\t\t\t\t\tt.Errorf(\"expected result %d to have Location %v got %v for test %d\\n\", i, exploc, locs[ind], testIndex)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = test.searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, testIndex)\n\t\t}\n\t\tif len(test.results) != i {\n\t\t\tt.Errorf(\"expected %d results got %d for test %d\", len(test.results), i, testIndex)\n\t\t}\n\t}\n}\n\nfunc TestMultiPhraseSearch(t *testing.T) {\n\tsoptions := search.SearcherOptions{Explain: true, IncludeTermVectors: true}\n\n\ttests := []struct {\n\t\tphrase [][]string\n\t\tdocids [][]byte\n\t}{\n\t\t{\n\t\t\tphrase: [][]string{{\"angst\", \"what\"}, {\"beer\"}},\n\t\t\tdocids: [][]byte{[]byte(\"2\")},\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\n\t\treader, err := twoDocIndex.Reader()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tsearcher, err := NewMultiPhraseSearcher(context.TODO(), reader, test.phrase, 0, false, \"desc\", 1.0, soptions)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := searcher.Next(ctx)\n\t\tvar actualIds [][]byte\n\t\tfor err == nil && next != nil {\n\t\t\tactualIds = append(actualIds, next.IndexInternalID)\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = searcher.Next(ctx)\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, i)\n\t\t}\n\t\tif !reflect.DeepEqual(test.docids, actualIds) {\n\t\t\tt.Fatalf(\"expected ids: %v, got %v\", test.docids, actualIds)\n\t\t}\n\n\t\terr = searcher.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\n\t\terr = reader.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n}\n\nfunc TestFuzzyMultiPhraseSearch(t *testing.T) {\n\tsoptions := search.SearcherOptions{Explain: true, IncludeTermVectors: true}\n\n\ttests := []struct {\n\t\tmphrase   [][]string\n\t\tdocids    [][]byte\n\t\tfuzziness int\n\t\tprefix    int\n\t}{\n\t\t{\n\t\t\tmphrase:   [][]string{{\"pale\", \"anger\"}, {\"best\"}, {\"colon\", \"porch\"}},\n\t\t\tdocids:    [][]byte{[]byte(\"2\"), []byte(\"3\")},\n\t\t\tfuzziness: 2,\n\t\t},\n\t\t{\n\t\t\tmphrase:   [][]string{{\"pale\", \"anger\"}, {}, {\"colon\", \"porch\", \"could\"}},\n\t\t\tdocids:    nil,\n\t\t\tfuzziness: 1,\n\t\t},\n\t\t{\n\t\t\tmphrase:   [][]string{{\"app\"}, {\"best\"}, {\"volume\"}},\n\t\t\tdocids:    [][]byte{[]byte(\"3\")},\n\t\t\tfuzziness: 2,\n\t\t},\n\t\t{\n\t\t\tmphrase:   [][]string{{\"anger\", \"pale\", \"bar\"}, {\"beard\"}, {}, {}},\n\t\t\tdocids:    [][]byte{[]byte(\"1\"), []byte(\"2\"), []byte(\"3\"), []byte(\"4\")},\n\t\t\tfuzziness: 2,\n\t\t},\n\t\t{\n\t\t\tmphrase:   [][]string{{\"anger\", \"pale\", \"bar\"}, {}, {\"beard\"}, {}},\n\t\t\tdocids:    [][]byte{[]byte(\"1\"), []byte(\"4\")},\n\t\t\tfuzziness: 2,\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\n\t\treader, err := twoDocIndex.Reader()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tsearcher, err := NewMultiPhraseSearcher(context.TODO(), reader, test.mphrase, test.fuzziness, false, \"desc\", 1.0, soptions)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := searcher.Next(ctx)\n\t\tvar actualIds [][]byte\n\t\tfor err == nil && next != nil {\n\t\t\tactualIds = append(actualIds, next.IndexInternalID)\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = searcher.Next(ctx)\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, i)\n\t\t}\n\t\tif !reflect.DeepEqual(test.docids, actualIds) {\n\t\t\tt.Fatalf(\"expected ids: %v, got %v\", test.docids, actualIds)\n\t\t}\n\n\t\terr = searcher.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\n\t\terr = reader.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n}\n\nfunc TestFindPhrasePaths(t *testing.T) {\n\ttests := []struct {\n\t\tphrase [][]string\n\t\ttlm    search.TermLocationMap\n\t\tpaths  []phrasePath\n\t}{\n\t\t// simplest matching case\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\"}, {\"dog\"}},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 1,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 2}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// second term missing, no match\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\"}, {\"dog\"}},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 1,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpaths: nil,\n\t\t},\n\t\t// second term exists but in wrong position\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\"}, {\"dog\"}},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 1,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpaths: nil,\n\t\t},\n\t\t// matches multiple times\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\"}, {\"dog\"}},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 1,\n\t\t\t\t\t},\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 8,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 2,\n\t\t\t\t\t},\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 9,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 2}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 8}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 9}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// match over gaps\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\"}, {\"\"}, {\"dog\"}},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 1,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// match with leading \"\"\n\t\t{\n\t\t\tphrase: [][]string{{\"\"}, {\"cat\"}, {\"dog\"}},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// match with trailing \"\"\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\"}, {\"dog\"}, {\"\"}},\n\t\t\ttlm: search.TermLocationMap{\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{\n\t\t\t\t\t\tPos: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tactualPaths := findPhrasePaths(0, nil, test.phrase, test.tlm, nil, 0, nil)\n\t\tif !reflect.DeepEqual(actualPaths, test.paths) {\n\t\t\tt.Fatalf(\"expected: %v got %v for test %d\", test.paths, actualPaths, i)\n\t\t}\n\t}\n}\n\nfunc TestFindPhrasePathsSloppy(t *testing.T) {\n\ttlm := search.TermLocationMap{\n\t\t\"one\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 1,\n\t\t\t},\n\t\t},\n\t\t\"two\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 2,\n\t\t\t},\n\t\t},\n\t\t\"three\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 3,\n\t\t\t},\n\t\t},\n\t\t\"four\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 4,\n\t\t\t},\n\t\t},\n\t\t\"five\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 5,\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tphrase [][]string\n\t\tpaths  []phrasePath\n\t\tslop   int\n\t\ttlm    search.TermLocationMap\n\t}{\n\t\t// no match\n\t\t{\n\t\t\tphrase: [][]string{{\"one\"}, {\"five\"}},\n\t\t\tslop:   2,\n\t\t},\n\t\t// should match\n\t\t{\n\t\t\tphrase: [][]string{{\"one\"}, {\"five\"}},\n\t\t\tslop:   3,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"one\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"five\", &search.Location{Pos: 5}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// slop 0 finds exact match\n\t\t{\n\t\t\tphrase: [][]string{{\"four\"}, {\"five\"}},\n\t\t\tslop:   0,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"four\", &search.Location{Pos: 4}},\n\t\t\t\t\tphrasePart{\"five\", &search.Location{Pos: 5}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// slop 0 does not find exact match (reversed)\n\t\t{\n\t\t\tphrase: [][]string{{\"two\"}, {\"one\"}},\n\t\t\tslop:   0,\n\t\t},\n\t\t// slop 1 finds exact match\n\t\t{\n\t\t\tphrase: [][]string{{\"one\"}, {\"two\"}},\n\t\t\tslop:   1,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"one\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"two\", &search.Location{Pos: 2}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// slop 1 *still* does not find exact match (reversed) requires at least 2\n\t\t{\n\t\t\tphrase: [][]string{{\"two\"}, {\"one\"}},\n\t\t\tslop:   1,\n\t\t},\n\t\t// slop 2 does finds exact match reversed\n\t\t{\n\t\t\tphrase: [][]string{{\"two\"}, {\"one\"}},\n\t\t\tslop:   2,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"two\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"one\", &search.Location{Pos: 1}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// slop 2 not enough for this\n\t\t{\n\t\t\tphrase: [][]string{{\"three\"}, {\"one\"}},\n\t\t\tslop:   2,\n\t\t},\n\t\t// slop should be cumulative\n\t\t{\n\t\t\tphrase: [][]string{{\"one\"}, {\"three\"}, {\"five\"}},\n\t\t\tslop:   2,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"one\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"three\", &search.Location{Pos: 3}},\n\t\t\t\t\tphrasePart{\"five\", &search.Location{Pos: 5}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// should require 6\n\t\t{\n\t\t\tphrase: [][]string{{\"five\"}, {\"three\"}, {\"one\"}},\n\t\t\tslop:   5,\n\t\t},\n\t\t// so lets try 6\n\t\t{\n\t\t\tphrase: [][]string{{\"five\"}, {\"three\"}, {\"one\"}},\n\t\t\tslop:   6,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"five\", &search.Location{Pos: 5}},\n\t\t\t\t\tphrasePart{\"three\", &search.Location{Pos: 3}},\n\t\t\t\t\tphrasePart{\"one\", &search.Location{Pos: 1}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test an append() related edge case, where append()'s\n\t\t// current behavior needs to be called 3 times starting from a\n\t\t// nil slice before it grows to a slice with extra capacity --\n\t\t// hence, 3 initial terms of ark, bat, cat\n\t\t{\n\t\t\tphrase: [][]string{\n\t\t\t\t{\"ark\"}, {\"bat\"}, {\"cat\"}, {\"dog\"},\n\t\t\t},\n\t\t\tslop: 1,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"ark\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"bat\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 3}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 4}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"ark\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"bat\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 3}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 5}},\n\t\t\t\t},\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{ // ark bat cat dog dog\n\t\t\t\t\"ark\": search.Locations{\n\t\t\t\t\t&search.Location{Pos: 1},\n\t\t\t\t},\n\t\t\t\t\"bat\": search.Locations{\n\t\t\t\t\t&search.Location{Pos: 2},\n\t\t\t\t},\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{Pos: 3},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{Pos: 4},\n\t\t\t\t\t&search.Location{Pos: 5},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test that we don't see multiple hits from the same location\n\t\t{\n\t\t\tphrase: [][]string{\n\t\t\t\t{\"cat\"}, {\"dog\"}, {\"dog\"},\n\t\t\t},\n\t\t\tslop: 1,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{ // cat dog dog\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{Pos: 1},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{Pos: 2},\n\t\t\t\t\t&search.Location{Pos: 3},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test that we don't see multiple hits from the same location\n\t\t{\n\t\t\tphrase: [][]string{\n\t\t\t\t{\"cat\"}, {\"dog\"},\n\t\t\t},\n\t\t\tslop: 10,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 2}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 4}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 3}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 2}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 3}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 4}},\n\t\t\t\t},\n\t\t\t},\n\t\t\ttlm: search.TermLocationMap{ // cat dog cat dog\n\t\t\t\t\"cat\": search.Locations{\n\t\t\t\t\t&search.Location{Pos: 1},\n\t\t\t\t\t&search.Location{Pos: 3},\n\t\t\t\t},\n\t\t\t\t\"dog\": search.Locations{\n\t\t\t\t\t&search.Location{Pos: 2},\n\t\t\t\t\t&search.Location{Pos: 4},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\ttlmToUse := test.tlm\n\t\tif tlmToUse == nil {\n\t\t\ttlmToUse = tlm\n\t\t}\n\t\tactualPaths := findPhrasePaths(0, nil, test.phrase, tlmToUse, nil, test.slop, nil)\n\t\tif !reflect.DeepEqual(actualPaths, test.paths) {\n\t\t\tt.Fatalf(\"expected: %v got %v for test %d\", test.paths, actualPaths, i)\n\t\t}\n\t}\n}\n\nfunc TestFindPhrasePathsSloppyPalyndrome(t *testing.T) {\n\ttlm := search.TermLocationMap{\n\t\t\"one\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 1,\n\t\t\t},\n\t\t\t&search.Location{\n\t\t\t\tPos: 5,\n\t\t\t},\n\t\t},\n\t\t\"two\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 2,\n\t\t\t},\n\t\t\t&search.Location{\n\t\t\t\tPos: 4,\n\t\t\t},\n\t\t},\n\t\t\"three\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 3,\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tphrase [][]string\n\t\tpaths  []phrasePath\n\t\tslop   int\n\t}{\n\t\t// search non palyndrone, exact match\n\t\t{\n\t\t\tphrase: [][]string{{\"two\"}, {\"three\"}},\n\t\t\tslop:   0,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"two\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"three\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// same with slop 2 (not required) (find it twice)\n\t\t{\n\t\t\tphrase: [][]string{{\"two\"}, {\"three\"}},\n\t\t\tslop:   2,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"two\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"three\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"two\", &search.Location{Pos: 4}},\n\t\t\t\t\tphrasePart{\"three\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// palyndrone reversed\n\t\t{\n\t\t\tphrase: [][]string{{\"three\"}, {\"two\"}},\n\t\t\tslop:   2,\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"three\", &search.Location{Pos: 3}},\n\t\t\t\t\tphrasePart{\"two\", &search.Location{Pos: 2}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"three\", &search.Location{Pos: 3}},\n\t\t\t\t\tphrasePart{\"two\", &search.Location{Pos: 4}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tactualPaths := findPhrasePaths(0, nil, test.phrase, tlm, nil, test.slop, nil)\n\t\tif !reflect.DeepEqual(actualPaths, test.paths) {\n\t\t\tt.Fatalf(\"expected: %v got %v for test %d\", test.paths, actualPaths, i)\n\t\t}\n\t}\n}\n\nfunc TestFindMultiPhrasePaths(t *testing.T) {\n\ttlm := search.TermLocationMap{\n\t\t\"cat\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 1,\n\t\t\t},\n\t\t},\n\t\t\"dog\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 2,\n\t\t\t},\n\t\t},\n\t\t\"frog\": search.Locations{\n\t\t\t&search.Location{\n\t\t\t\tPos: 3,\n\t\t\t},\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tphrase [][]string\n\t\tpaths  []phrasePath\n\t}{\n\t\t// simplest, one of two possible terms matches\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\", \"rat\"}, {\"dog\"}},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 2}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// two possible terms, neither work\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\", \"rat\"}, {\"chicken\"}},\n\t\t},\n\t\t// two possible terms, one works, but out of position with next\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\", \"rat\"}, {\"frog\"}},\n\t\t},\n\t\t// matches multiple times, with different pairing\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\", \"dog\"}, {\"dog\", \"frog\"}},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 2}},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"dog\", &search.Location{Pos: 2}},\n\t\t\t\t\tphrasePart{\"frog\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// multi-match over a gap\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\", \"rat\"}, {\"\"}, {\"frog\"}},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"frog\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// multi-match over a gap (same as before, but with empty term list)\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\", \"rat\"}, {}, {\"frog\"}},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"frog\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// multi-match over a gap (same once again, but nil term list)\n\t\t{\n\t\t\tphrase: [][]string{{\"cat\", \"rat\"}, nil, {\"frog\"}},\n\t\t\tpaths: []phrasePath{\n\t\t\t\t{\n\t\t\t\t\tphrasePart{\"cat\", &search.Location{Pos: 1}},\n\t\t\t\t\tphrasePart{\"frog\", &search.Location{Pos: 3}},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tactualPaths := findPhrasePaths(0, nil, test.phrase, tlm, nil, 0, nil)\n\t\tif !reflect.DeepEqual(actualPaths, test.paths) {\n\t\t\tt.Fatalf(\"expected: %v got %v for test %d\", test.paths, actualPaths, i)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_regexp.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"regexp\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\n// The Regexp interface defines the subset of the regexp.Regexp API\n// methods that are used by bleve indexes, allowing callers to pass in\n// alternate implementations.\ntype Regexp interface {\n\tFindStringIndex(s string) (loc []int)\n\n\tLiteralPrefix() (prefix string, complete bool)\n\n\tString() string\n}\n\n// NewRegexpStringSearcher is similar to NewRegexpSearcher, but\n// additionally optimizes for index readers that handle regexp's.\nfunc NewRegexpStringSearcher(ctx context.Context, indexReader index.IndexReader, pattern string,\n\tfield string, boost float64, options search.SearcherOptions) (\n\tsearch.Searcher, error) {\n\tir, ok := indexReader.(index.IndexReaderRegexp)\n\tif !ok {\n\t\tr, err := regexp.Compile(pattern)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\treturn NewRegexpSearcher(ctx, indexReader, r, field, boost, options)\n\t}\n\n\tfieldDict, a, err := ir.FieldDictRegexpAutomaton(field, pattern)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := fieldDict.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tvar termSet = make(map[string]struct{})\n\tvar candidateTerms []string\n\n\ttfd, err := fieldDict.Next()\n\tfor err == nil && tfd != nil {\n\t\tif _, exists := termSet[tfd.Term]; !exists {\n\t\t\ttermSet[tfd.Term] = struct{}{}\n\t\t\tcandidateTerms = append(candidateTerms, tfd.Term)\n\t\t\ttfd, err = fieldDict.Next()\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif ctx != nil {\n\t\tif fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {\n\t\t\tif ts, exists := fts[field]; exists {\n\t\t\t\tfor term := range ts {\n\t\t\t\t\tif _, exists := termSet[term]; exists {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif a.MatchesRegex(term) {\n\t\t\t\t\t\ttermSet[term] = struct{}{}\n\t\t\t\t\t\tcandidateTerms = append(candidateTerms, term)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn NewMultiTermSearcher(ctx, indexReader, candidateTerms, field, boost,\n\t\toptions, true)\n}\n\n// NewRegexpSearcher creates a searcher which will match documents that\n// contain terms which match the pattern regexp.  The match must be EXACT\n// matching the entire term.  The provided regexp SHOULD NOT start with ^\n// or end with $ as this can interfere with the implementation.  Separately,\n// matches will be checked to ensure they match the entire term.\nfunc NewRegexpSearcher(ctx context.Context, indexReader index.IndexReader, pattern Regexp,\n\tfield string, boost float64, options search.SearcherOptions) (\n\tsearch.Searcher, error) {\n\tvar candidateTerms []string\n\tvar regexpCandidates *regexpCandidates\n\tprefixTerm, complete := pattern.LiteralPrefix()\n\tif complete {\n\t\t// there is no pattern\n\t\tcandidateTerms = []string{prefixTerm}\n\t} else {\n\t\tvar err error\n\t\tregexpCandidates, err = findRegexpCandidateTerms(indexReader, pattern, field,\n\t\t\tprefixTerm)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tvar dictBytesRead uint64\n\tif regexpCandidates != nil {\n\t\tcandidateTerms = regexpCandidates.candidates\n\t\tdictBytesRead = regexpCandidates.bytesRead\n\t}\n\n\tif ctx != nil {\n\t\treportIOStats(ctx, dictBytesRead)\n\t\tsearch.RecordSearchCost(ctx, search.AddM, dictBytesRead)\n\t}\n\n\treturn NewMultiTermSearcher(ctx, indexReader, candidateTerms, field, boost,\n\t\toptions, true)\n}\n\ntype regexpCandidates struct {\n\tcandidates []string\n\tbytesRead  uint64\n}\n\nfunc findRegexpCandidateTerms(indexReader index.IndexReader,\n\tpattern Regexp, field, prefixTerm string) (rv *regexpCandidates, err error) {\n\trv = &regexpCandidates{\n\t\tcandidates: make([]string, 0),\n\t}\n\tvar fieldDict index.FieldDict\n\tif len(prefixTerm) > 0 {\n\t\tfieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm))\n\t} else {\n\t\tfieldDict, err = indexReader.FieldDict(field)\n\t}\n\tdefer func() {\n\t\tif cerr := fieldDict.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\t// enumerate the terms and check against regexp\n\ttfd, err := fieldDict.Next()\n\tfor err == nil && tfd != nil {\n\t\tmatchPos := pattern.FindStringIndex(tfd.Term)\n\t\tif matchPos != nil && matchPos[0] == 0 && matchPos[1] == len(tfd.Term) {\n\t\t\trv.candidates = append(rv.candidates, tfd.Term)\n\t\t\tif tooManyClauses(len(rv.candidates)) {\n\t\t\t\treturn rv, tooManyClausesErr(field, len(rv.candidates))\n\t\t\t}\n\t\t}\n\t\ttfd, err = fieldDict.Next()\n\t}\n\trv.bytesRead = fieldDict.BytesRead()\n\treturn rv, err\n}\n"
  },
  {
    "path": "search/searcher/search_regexp_test.go",
    "content": "//  Copyright (c) 2015 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"os\"\n\t\"regexp\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestRegexpSearchUpsideDown(t *testing.T) {\n\ttwoDocIndex := initTwoDocUpsideDown()\n\ttestRegexpSearch(t, twoDocIndex, internalIDMakerUpsideDown, searcherMaker)\n\t_ = twoDocIndex.Close()\n}\n\nfunc TestRegexpStringSearchUpsideDown(t *testing.T) {\n\ttwoDocIndex := initTwoDocUpsideDown()\n\ttestRegexpSearch(t, twoDocIndex, internalIDMakerUpsideDown, searcherStringMaker)\n\t_ = twoDocIndex.Close()\n}\n\nfunc TestRegexpSearchScorch(t *testing.T) {\n\tdir, _ := os.MkdirTemp(\"\", \"scorchTwoDoc\")\n\tdefer func() {\n\t\t_ = os.RemoveAll(dir)\n\t}()\n\n\ttwoDocIndex := initTwoDocScorch(dir)\n\ttestRegexpSearch(t, twoDocIndex, internalIDMakerScorch, searcherMaker)\n\t_ = twoDocIndex.Close()\n}\n\nfunc TestRegexpStringSearchScorch(t *testing.T) {\n\tdir, _ := os.MkdirTemp(\"\", \"scorchTwoDoc\")\n\tdefer func() {\n\t\t_ = os.RemoveAll(dir)\n\t}()\n\n\ttwoDocIndex := initTwoDocScorch(dir)\n\ttestRegexpSearch(t, twoDocIndex, internalIDMakerScorch, searcherStringMaker)\n\t_ = twoDocIndex.Close()\n}\n\nfunc internalIDMakerUpsideDown(id int) index.IndexInternalID {\n\treturn index.IndexInternalID(fmt.Sprintf(\"%d\", id))\n}\n\nfunc internalIDMakerScorch(id int) index.IndexInternalID {\n\tbuf := make([]byte, 8)\n\tbinary.BigEndian.PutUint64(buf, uint64(id))\n\treturn index.IndexInternalID(buf)\n}\n\nfunc searcherMaker(t *testing.T, ir index.IndexReader, re, field string) search.Searcher {\n\tpattern, err := regexp.Compile(re)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tregexpSearcher, err := NewRegexpSearcher(context.TODO(), ir, pattern, field, 1.0,\n\t\tsearch.SearcherOptions{Explain: true})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn regexpSearcher\n}\n\nfunc searcherStringMaker(t *testing.T, ir index.IndexReader, re, field string) search.Searcher {\n\tregexpSearcher, err := NewRegexpStringSearcher(context.TODO(), ir, re, field, 1.0,\n\t\tsearch.SearcherOptions{Explain: true})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treturn regexpSearcher\n}\n\nfunc testRegexpSearch(t *testing.T, twoDocIndex index.Index,\n\tinternalIDMaker func(int) index.IndexInternalID,\n\tsearcherMaker func(t *testing.T, ir index.IndexReader, re, field string) search.Searcher,\n) {\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tregexpSearcher := searcherMaker(t, twoDocIndexReader, \"ma.*\", \"name\")\n\tregexpSearcherCo := searcherMaker(t, twoDocIndexReader, \"co.*\", \"desc\")\n\n\ttests := []struct {\n\t\tsearcher search.Searcher\n\t\tid2score map[string]float64\n\t}{\n\t\t{\n\t\t\tsearcher: regexpSearcher,\n\t\t\tid2score: map[string]float64{\n\t\t\t\t\"1\": 1.916290731874155,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tsearcher: regexpSearcherCo,\n\t\t\tid2score: map[string]float64{\n\t\t\t\t\"2\": 0.33875554280828685,\n\t\t\t\t\"3\": 0.33875554280828685,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor testIndex, test := range tests {\n\t\tdefer func() {\n\t\t\terr := test.searcher.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}()\n\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := test.searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\texID, _ := twoDocIndexReader.ExternalID(next.IndexInternalID)\n\t\t\tif _, ok := test.id2score[exID]; !ok {\n\t\t\t\tt.Errorf(\"test %d, found unexpected docID = %v, next = %v\", testIndex, exID, next)\n\t\t\t} else {\n\t\t\t\tscore := test.id2score[exID]\n\t\t\t\tif next.Score != score {\n\t\t\t\t\tt.Errorf(\"test %d, expected result %d to have score %v got %v,next: %#v\",\n\t\t\t\t\t\ttestIndex, i, score, next.Score, next)\n\t\t\t\t\tt.Logf(\"scoring explanation: %s\", next.Expl)\n\t\t\t\t}\n\t\t\t}\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = test.searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v for test %d\", err, testIndex)\n\t\t}\n\t\tif len(test.id2score) != i {\n\t\t\tt.Errorf(\"expected %d results got %d for test %d\", len(test.id2score), i, testIndex)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_term.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/scorer\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nvar reflectStaticSizeTermSearcher int\n\nfunc init() {\n\tvar ts TermSearcher\n\treflectStaticSizeTermSearcher = int(reflect.TypeOf(ts).Size())\n}\n\ntype TermSearcher struct {\n\tindexReader index.IndexReader\n\treader      index.TermFieldReader\n\tscorer      *scorer.TermQueryScorer\n\ttfd         index.TermFieldDoc\n}\n\nfunc NewTermSearcher(ctx context.Context, indexReader index.IndexReader,\n\tterm string, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {\n\tif isTermQuery(ctx) {\n\t\tctx = context.WithValue(ctx, search.QueryTypeKey, search.Term)\n\t}\n\treturn NewTermSearcherBytes(ctx, indexReader, []byte(term), field, boost, options)\n}\n\nfunc NewTermSearcherBytes(ctx context.Context, indexReader index.IndexReader,\n\tterm []byte, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {\n\tif ctx != nil {\n\t\tif fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {\n\t\t\tif ts, exists := fts[field]; exists {\n\t\t\t\tif s, found := ts[string(term)]; found {\n\t\t\t\t\treturn NewSynonymSearcher(ctx, indexReader, term, s, field, boost, options)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tneedFreqNorm := options.Score != \"none\"\n\treader, err := indexReader.TermFieldReader(ctx, term, field, needFreqNorm, needFreqNorm, options.IncludeTermVectors)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn newTermSearcherFromReader(ctx, indexReader, reader, term, field, boost, options)\n}\n\nfunc tfIDFScoreMetrics(indexReader index.IndexReader) (uint64, error) {\n\t// default tf-idf stats\n\tcount, err := indexReader.DocCount()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif count == 0 {\n\t\treturn 0, nil\n\t}\n\treturn count, nil\n}\n\nfunc bm25ScoreMetrics(ctx context.Context, field string,\n\tindexReader index.IndexReader) (uint64, float64, error) {\n\tvar count uint64\n\tvar fieldCardinality int\n\tvar err error\n\n\tbm25Stats, ok := ctx.Value(search.BM25StatsKey).(*search.BM25Stats)\n\tif !ok {\n\t\tcount, err = indexReader.DocCount()\n\t\tif err != nil {\n\t\t\treturn 0, 0, err\n\t\t}\n\t\tif bm25Reader, ok := indexReader.(index.BM25Reader); ok {\n\t\t\tfieldCardinality, err = bm25Reader.FieldCardinality(field)\n\t\t\tif err != nil {\n\t\t\t\treturn 0, 0, err\n\t\t\t}\n\t\t}\n\t} else {\n\t\tcount = uint64(bm25Stats.DocCount)\n\t\tfieldCardinality, ok = bm25Stats.FieldCardinality[field]\n\t\tif !ok {\n\t\t\treturn 0, 0, fmt.Errorf(\"field stat for bm25 not present %s\", field)\n\t\t}\n\t}\n\n\tif count == 0 && fieldCardinality == 0 {\n\t\treturn 0, 0, nil\n\t}\n\treturn count, math.Ceil(float64(fieldCardinality) / float64(count)), nil\n}\n\nfunc newTermSearcherFromReader(ctx context.Context, indexReader index.IndexReader,\n\treader index.TermFieldReader, term []byte, field string, boost float64,\n\toptions search.SearcherOptions) (*TermSearcher, error) {\n\tvar count uint64\n\tvar avgDocLength float64\n\tvar err error\n\tvar similarityModel string\n\n\t// as a fallback case we track certain stats for tf-idf scoring\n\tif ctx != nil {\n\t\tif similarityModelCallback, ok := ctx.Value(search.\n\t\t\tGetScoringModelCallbackKey).(search.GetScoringModelCallbackFn); ok {\n\t\t\tsimilarityModel = similarityModelCallback()\n\t\t}\n\t}\n\tswitch similarityModel {\n\tcase index.BM25Scoring:\n\t\tcount, avgDocLength, err = bm25ScoreMetrics(ctx, field, indexReader)\n\t\tif err != nil {\n\t\t\t_ = reader.Close()\n\t\t\treturn nil, err\n\t\t}\n\tcase index.TFIDFScoring:\n\t\tfallthrough\n\tdefault:\n\t\tcount, err = tfIDFScoreMetrics(indexReader)\n\t\tif err != nil {\n\t\t\t_ = reader.Close()\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tscorer := scorer.NewTermQueryScorer(term, field, boost, count, reader.Count(), avgDocLength, options)\n\treturn &TermSearcher{\n\t\tindexReader: indexReader,\n\t\treader:      reader,\n\t\tscorer:      scorer,\n\t}, nil\n}\n\nfunc NewSynonymSearcher(ctx context.Context, indexReader index.IndexReader, term []byte, synonyms []string, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {\n\tcreateTermSearcher := func(term []byte, boostVal float64) (search.Searcher, error) {\n\t\tneedFreqNorm := options.Score != \"none\"\n\t\treader, err := indexReader.TermFieldReader(ctx, term, field, needFreqNorm, needFreqNorm, options.IncludeTermVectors)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn newTermSearcherFromReader(ctx, indexReader, reader, term, field, boostVal, options)\n\t}\n\t// create a searcher for the term itself\n\ttermSearcher, err := createTermSearcher(term, boost)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// constituent searchers of the disjunction\n\tqsearchers := make([]search.Searcher, 0, len(synonyms)+1)\n\t// helper method to close all the searchers we've created\n\t// in case of an error\n\tqsearchersClose := func() {\n\t\tfor _, searcher := range qsearchers {\n\t\t\tif searcher != nil {\n\t\t\t\t_ = searcher.Close()\n\t\t\t}\n\t\t}\n\t}\n\tqsearchers = append(qsearchers, termSearcher)\n\t// create a searcher for each synonym\n\tfor _, synonym := range synonyms {\n\t\tsynonymSearcher, err := createTermSearcher([]byte(synonym), boost/2.0)\n\t\tif err != nil {\n\t\t\tqsearchersClose()\n\t\t\treturn nil, err\n\t\t}\n\t\tqsearchers = append(qsearchers, synonymSearcher)\n\t}\n\t// create a disjunction searcher\n\trv, err := NewDisjunctionSearcher(ctx, indexReader, qsearchers, 0, options)\n\tif err != nil {\n\t\tqsearchersClose()\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\nfunc (s *TermSearcher) Size() int {\n\treturn reflectStaticSizeTermSearcher + size.SizeOfPtr +\n\t\ts.reader.Size() +\n\t\ts.tfd.Size() +\n\t\ts.scorer.Size()\n}\n\nfunc (s *TermSearcher) Count() uint64 {\n\treturn s.reader.Count()\n}\n\nfunc (s *TermSearcher) Weight() float64 {\n\treturn s.scorer.Weight()\n}\n\nfunc (s *TermSearcher) SetQueryNorm(qnorm float64) {\n\ts.scorer.SetQueryNorm(qnorm)\n}\n\nfunc (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {\n\ttermMatch, err := s.reader.Next(s.tfd.Reset())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif termMatch == nil {\n\t\treturn nil, nil\n\t}\n\n\t// score match\n\tdocMatch := s.scorer.Score(ctx, termMatch)\n\t// return doc match\n\treturn docMatch, nil\n\n}\n\nfunc (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {\n\ttermMatch, err := s.reader.Advance(ID, s.tfd.Reset())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif termMatch == nil {\n\t\treturn nil, nil\n\t}\n\n\t// score match\n\tdocMatch := s.scorer.Score(ctx, termMatch)\n\n\t// return doc match\n\treturn docMatch, nil\n}\n\nfunc (s *TermSearcher) Close() error {\n\treturn s.reader.Close()\n}\n\nfunc (s *TermSearcher) Min() int {\n\treturn 0\n}\n\nfunc (s *TermSearcher) DocumentMatchPoolSize() int {\n\treturn 1\n}\n\nfunc (s *TermSearcher) Optimize(kind string, octx index.OptimizableContext) (\n\tindex.OptimizableContext, error) {\n\to, ok := s.reader.(index.Optimizable)\n\tif ok {\n\t\treturn o.Optimize(kind, octx)\n\t}\n\n\treturn nil, nil\n}\n\nfunc isTermQuery(ctx context.Context) bool {\n\tif ctx != nil {\n\t\t// if the ctx already has a value set for query type\n\t\t// it would've been done at a non term searcher level.\n\t\t_, ok := ctx.Value(search.QueryTypeKey).(string)\n\t\treturn !ok\n\t}\n\t// if the context is nil, then don't set the query type\n\treturn false\n}\n"
  },
  {
    "path": "search/searcher/search_term_prefix.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc NewTermPrefixSearcher(ctx context.Context, indexReader index.IndexReader, prefix string,\n\tfield string, boost float64, options search.SearcherOptions) (\n\tsearch.Searcher, error) {\n\t// find the terms with this prefix\n\tfieldDict, err := indexReader.FieldDictPrefix(field, []byte(prefix))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif cerr := fieldDict.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tvar terms []string\n\tvar termSet = make(map[string]struct{})\n\ttfd, err := fieldDict.Next()\n\tfor err == nil && tfd != nil {\n\t\tif _, exists := termSet[tfd.Term]; !exists {\n\t\t\ttermSet[tfd.Term] = struct{}{}\n\t\t\tterms = append(terms, tfd.Term)\n\t\t\tif tooManyClauses(len(terms)) {\n\t\t\t\treturn nil, tooManyClausesErr(field, len(terms))\n\t\t\t}\n\t\t\ttfd, err = fieldDict.Next()\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif ctx != nil {\n\t\treportIOStats(ctx, fieldDict.BytesRead())\n\t\tsearch.RecordSearchCost(ctx, search.AddM, fieldDict.BytesRead())\n\t}\n\n\tif ctx != nil {\n\t\tif fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {\n\t\t\tif ts, exists := fts[field]; exists {\n\t\t\t\tfor term := range ts {\n\t\t\t\t\tif _, exists := termSet[term]; exists {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif strings.HasPrefix(term, prefix) {\n\t\t\t\t\t\ttermSet[term] = struct{}{}\n\t\t\t\t\t\tterms = append(terms, term)\n\t\t\t\t\t\tif tooManyClauses(len(terms)) {\n\t\t\t\t\t\t\treturn nil, tooManyClausesErr(field, len(terms))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// check if the terms are empty or have one term which is the prefix itself\n\tif len(terms) == 0 || (len(terms) == 1 && terms[0] == prefix) {\n\t\treturn NewTermSearcher(ctx, indexReader, prefix, field, boost, options)\n\t}\n\n\treturn NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true)\n}\n"
  },
  {
    "path": "search/searcher/search_term_range.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc NewTermRangeSearcher(ctx context.Context, indexReader index.IndexReader,\n\tmin, max []byte, inclusiveMin, inclusiveMax *bool, field string,\n\tboost float64, options search.SearcherOptions) (search.Searcher, error) {\n\n\tif inclusiveMin == nil {\n\t\tdefaultInclusiveMin := true\n\t\tinclusiveMin = &defaultInclusiveMin\n\t}\n\tif inclusiveMax == nil {\n\t\tdefaultInclusiveMax := false\n\t\tinclusiveMax = &defaultInclusiveMax\n\t}\n\n\tif min == nil {\n\t\tmin = []byte{}\n\t}\n\n\trangeMax := max\n\tif rangeMax != nil {\n\t\t// the term dictionary range end has an unfortunate implementation\n\t\trangeMax = append(rangeMax, 0)\n\t}\n\n\t// find the terms with this prefix\n\tfieldDict, err := indexReader.FieldDictRange(field, min, rangeMax)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdefer func() {\n\t\tif cerr := fieldDict.Close(); cerr != nil && err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\tvar terms []string\n\ttfd, err := fieldDict.Next()\n\tfor err == nil && tfd != nil {\n\t\tterms = append(terms, tfd.Term)\n\t\ttfd, err = fieldDict.Next()\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(terms) < 1 {\n\t\treturn NewMatchNoneSearcher(indexReader)\n\t}\n\n\tif !*inclusiveMin && min != nil && string(min) == terms[0] {\n\t\tterms = terms[1:]\n\t\t// check again, as we might have removed only entry\n\t\tif len(terms) < 1 {\n\t\t\treturn NewMatchNoneSearcher(indexReader)\n\t\t}\n\t}\n\n\t// if our term list included the max, it would be the last item\n\tif !*inclusiveMax && max != nil && string(max) == terms[len(terms)-1] {\n\t\tterms = terms[:len(terms)-1]\n\t}\n\n\tif ctx != nil {\n\t\treportIOStats(ctx, fieldDict.BytesRead())\n\t\tsearch.RecordSearchCost(ctx, search.AddM, fieldDict.BytesRead())\n\t}\n\n\treturn NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true)\n}\n"
  },
  {
    "path": "search/searcher/search_term_range_test.go",
    "content": "//  Copyright (c) 2017 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\nfunc TestTermRangeSearch(t *testing.T) {\n\ttwoDocIndexReader, err := twoDocIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := twoDocIndexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\ttests := []struct {\n\t\tmin          []byte\n\t\tmax          []byte\n\t\tinclusiveMin bool\n\t\tinclusiveMax bool\n\t\tfield        string\n\t\twant         []string\n\t}{\n\t\t{\n\t\t\tmin:          []byte(\"marty\"),\n\t\t\tmax:          []byte(\"marty\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         []string{\"1\"},\n\t\t},\n\t\t{\n\t\t\tmin:          []byte(\"marty\"),\n\t\t\tmax:          []byte(\"ravi\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         []string{\"1\", \"4\"},\n\t\t},\n\t\t// inclusive max false should exclude ravi\n\t\t{\n\t\t\tmin:          []byte(\"marty\"),\n\t\t\tmax:          []byte(\"ravi\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: false,\n\t\t\twant:         []string{\"1\"},\n\t\t},\n\t\t// inclusive max false should remove last/only item\n\t\t{\n\t\t\tmin:          []byte(\"martz\"),\n\t\t\tmax:          []byte(\"ravi\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: false,\n\t\t\twant:         nil,\n\t\t},\n\t\t// inclusive min false should remove marty\n\t\t{\n\t\t\tmin:          []byte(\"marty\"),\n\t\t\tmax:          []byte(\"ravi\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: false,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         []string{\"4\"},\n\t\t},\n\t\t// inclusive min false should remove first/only item\n\t\t{\n\t\t\tmin:          []byte(\"marty\"),\n\t\t\tmax:          []byte(\"rav\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: false,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         nil,\n\t\t},\n\t\t// max nil sees everything after marty\n\t\t{\n\t\t\tmin:          []byte(\"marty\"),\n\t\t\tmax:          nil,\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         []string{\"1\", \"2\", \"4\"},\n\t\t},\n\t\t// min nil sees everything before ravi\n\t\t{\n\t\t\tmin:          nil,\n\t\t\tmax:          []byte(\"ravi\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         []string{\"1\", \"3\", \"4\", \"5\"},\n\t\t},\n\t\t// min and max nil sees everything\n\t\t{\n\t\t\tmin:          nil,\n\t\t\tmax:          nil,\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         []string{\"1\", \"2\", \"3\", \"4\", \"5\"},\n\t\t},\n\t\t// min and max nil sees everything, even with inclusiveMin false\n\t\t{\n\t\t\tmin:          nil,\n\t\t\tmax:          nil,\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: false,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         []string{\"1\", \"2\", \"3\", \"4\", \"5\"},\n\t\t},\n\t\t// min and max nil sees everything, even with inclusiveMax false\n\t\t{\n\t\t\tmin:          nil,\n\t\t\tmax:          nil,\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: false,\n\t\t\twant:         []string{\"1\", \"2\", \"3\", \"4\", \"5\"},\n\t\t},\n\t\t// min and max nil sees everything, even with both false\n\t\t{\n\t\t\tmin:          nil,\n\t\t\tmax:          nil,\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: false,\n\t\t\tinclusiveMax: false,\n\t\t\twant:         []string{\"1\", \"2\", \"3\", \"4\", \"5\"},\n\t\t},\n\t\t// min and max non-nil, but match 0 terms\n\t\t{\n\t\t\tmin:          []byte(\"martz\"),\n\t\t\tmax:          []byte(\"rav\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: true,\n\t\t\tinclusiveMax: true,\n\t\t\twant:         nil,\n\t\t},\n\t\t// min and max same (and term exists), both exclusive\n\t\t{\n\t\t\tmin:          []byte(\"marty\"),\n\t\t\tmax:          []byte(\"marty\"),\n\t\t\tfield:        \"name\",\n\t\t\tinclusiveMin: false,\n\t\t\tinclusiveMax: false,\n\t\t\twant:         nil,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\n\t\tsearcher, err := NewTermRangeSearcher(context.TODO(), twoDocIndexReader, test.min, test.max,\n\t\t\t&test.inclusiveMin, &test.inclusiveMax, test.field, 1.0, search.SearcherOptions{Explain: true})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tvar got []string\n\t\tctx := &search.SearchContext{\n\t\t\tDocumentMatchPool: search.NewDocumentMatchPool(\n\t\t\t\tsearcher.DocumentMatchPoolSize(), 0),\n\t\t}\n\t\tnext, err := searcher.Next(ctx)\n\t\ti := 0\n\t\tfor err == nil && next != nil {\n\t\t\tgot = append(got, string(next.IndexInternalID))\n\t\t\tctx.DocumentMatchPool.Put(next)\n\t\t\tnext, err = searcher.Next(ctx)\n\t\t\ti++\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error iterating searcher: %v\", err)\n\t\t}\n\t\tif !reflect.DeepEqual(got, test.want) {\n\t\t\tt.Errorf(\"expected: %v, got %v for test %#v\", test.want, got, test)\n\t\t}\n\n\t}\n}\n\nfunc TestTermRangeSearchTooManyTerms(t *testing.T) {\n\tdir, _ := os.MkdirTemp(\"\", \"scorchTwoDoc\")\n\tdefer func() {\n\t\t_ = os.RemoveAll(dir)\n\t}()\n\n\tscorchIndex := initTwoDocScorch(dir)\n\n\t// use lower limit for this test\n\torigLimit := DisjunctionMaxClauseCount\n\tDisjunctionMaxClauseCount = 2\n\tdefer func() {\n\t\tDisjunctionMaxClauseCount = origLimit\n\t}()\n\n\tscorchReader, err := scorchIndex.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := scorchReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\twant := []string{\"1\", \"3\", \"4\", \"5\"}\n\ttruth := true\n\tsearcher, err := NewTermRangeSearcher(context.TODO(), scorchReader, []byte(\"bobert\"), []byte(\"ravi\"),\n\t\t&truth, &truth, \"name\", 1.0, search.SearcherOptions{Score: \"none\", IncludeTermVectors: false})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar got []string\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(\n\t\t\tsearcher.DocumentMatchPoolSize(), 0),\n\t}\n\tnext, err := searcher.Next(ctx)\n\ti := 0\n\tfor err == nil && next != nil {\n\t\textId, err := scorchReader.ExternalID(next.IndexInternalID)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tgot = append(got, extId)\n\t\tctx.DocumentMatchPool.Put(next)\n\t\tnext, err = searcher.Next(ctx)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\n\t\ti++\n\t}\n\tif err != nil {\n\t\tt.Fatalf(\"error iterating searcher: %v\", err)\n\t}\n\terr = searcher.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// check that the expected number of term searchers were started\n\t// 6 = 4 original terms, 1 optimized after first round, then final searcher\n\t// from the last round\n\tstatsMap := scorchIndex.(*scorch.Scorch).StatsMap()\n\tif statsMap[\"term_searchers_started\"].(uint64) != 6 {\n\t\tt.Errorf(\"expected 6 term searchers started, got %d\", statsMap[\"term_searchers_started\"])\n\t}\n\t// check that all started searchers were closed\n\tif statsMap[\"term_searchers_started\"] != statsMap[\"term_searchers_finished\"] {\n\t\tt.Errorf(\"expected all term searchers closed, %d started %d closed\",\n\t\t\tstatsMap[\"term_searchers_started\"], statsMap[\"term_searchers_finished\"])\n\t}\n\n\tsort.Strings(got)\n\tif !reflect.DeepEqual(got, want) {\n\t\tt.Errorf(\"expected: %#v, got %#v\", want, got)\n\t}\n}\n"
  },
  {
    "path": "search/searcher/search_term_test.go",
    "content": "//  Copyright (c) 2013 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage searcher\n\nimport (\n\t\"context\"\n\t\"math\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestTermSearcher(t *testing.T) {\n\tqueryTerm := \"beer\"\n\tqueryField := \"desc\"\n\tqueryBoost := 3.0\n\tqueryExplain := search.SearcherOptions{Explain: true}\n\n\tanalysisQueue := index.NewAnalysisQueue(1)\n\ti, err := upsidedown.NewUpsideDownCouch(\n\t\tgtreap.Name,\n\t\tmap[string]interface{}{\n\t\t\t\"path\": \"\",\n\t\t},\n\t\tanalysisQueue)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = i.Open()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc := document.NewDocument(\"a\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"b\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"c\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"d\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"e\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"f\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"g\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"h\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"i\")\n\tdoc.AddField(document.NewTextField(\"desc\", []uint64{}, []byte(\"beer\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc = document.NewDocument(\"j\")\n\tdoc.AddField(document.NewTextField(\"title\", []uint64{}, []byte(\"cat\")))\n\terr = i.Update(doc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexReader, err := i.Reader()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tdefer func() {\n\t\terr := indexReader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tsearcher, err := NewTermSearcher(context.TODO(), indexReader, queryTerm, queryField, queryBoost, queryExplain)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := searcher.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tsearcher.SetQueryNorm(2.0)\n\tdocCount, err := indexReader.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tidf := 1.0 + math.Log(float64(docCount)/float64(searcher.Count()+1.0))\n\texpectedQueryWeight := 3 * idf * 3 * idf\n\tif expectedQueryWeight != searcher.Weight() {\n\t\tt.Errorf(\"expected weight %v got %v\", expectedQueryWeight, searcher.Weight())\n\t}\n\n\tif searcher.Count() != 9 {\n\t\tt.Errorf(\"expected count of 9, got %d\", searcher.Count())\n\t}\n\n\tctx := &search.SearchContext{\n\t\tDocumentMatchPool: search.NewDocumentMatchPool(1, 0),\n\t}\n\tdocMatch, err := searcher.Next(ctx)\n\tif err != nil {\n\t\tt.Errorf(\"expected result, got %v\", err)\n\t}\n\tif !docMatch.IndexInternalID.Equals(index.IndexInternalID(\"a\")) {\n\t\tt.Errorf(\"expected result ID to be 'a', got '%s\", docMatch.IndexInternalID)\n\t}\n\tctx.DocumentMatchPool.Put(docMatch)\n\tdocMatch, err = searcher.Advance(ctx, index.IndexInternalID(\"c\"))\n\tif err != nil {\n\t\tt.Errorf(\"expected result, got %v\", err)\n\t}\n\tif !docMatch.IndexInternalID.Equals(index.IndexInternalID(\"c\")) {\n\t\tt.Errorf(\"expected result ID to be 'c' got '%s'\", docMatch.IndexInternalID)\n\t}\n\n\t// try advancing past end\n\tctx.DocumentMatchPool.Put(docMatch)\n\tdocMatch, err = searcher.Advance(ctx, index.IndexInternalID(\"z\"))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docMatch != nil {\n\t\tt.Errorf(\"expected nil, got %v\", docMatch)\n\t}\n\n\t// try pushing next past end\n\tctx.DocumentMatchPool.Put(docMatch)\n\tdocMatch, err = searcher.Next(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif docMatch != nil {\n\t\tt.Errorf(\"expected nil, got %v\", docMatch)\n\t}\n}\n"
  },
  {
    "path": "search/sort.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\t\"unicode/utf8\"\n\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/numeric\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n)\n\nvar (\n\tHighTerm = strings.Repeat(string(utf8.MaxRune), 3)\n\tLowTerm  = string([]byte{0x00})\n)\n\ntype SearchSort interface {\n\tUpdateVisitor(field string, term []byte)\n\tValue(a *DocumentMatch) string\n\tDecodeValue(value string) string\n\tDescending() bool\n\n\tRequiresDocID() bool\n\tRequiresScoring() bool\n\tRequiresFields() []string\n\n\tReverse()\n\n\tCopy() SearchSort\n}\n\nfunc ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) {\n\tdescending, ok := input[\"desc\"].(bool)\n\tif !ok {\n\t\tdescending = false\n\t}\n\n\tby, ok := input[\"by\"].(string)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"search sort must specify by\")\n\t}\n\n\tswitch by {\n\tcase \"id\":\n\t\treturn &SortDocID{\n\t\t\tDesc: descending,\n\t\t}, nil\n\tcase \"score\":\n\t\treturn &SortScore{\n\t\t\tDesc: descending,\n\t\t}, nil\n\tcase \"geo_distance\":\n\t\tfield, ok := input[\"field\"].(string)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"search sort mode geo_distance must specify field\")\n\t\t}\n\t\tlon, lat, foundLocation := geo.ExtractGeoPoint(input[\"location\"])\n\t\tif !foundLocation {\n\t\t\treturn nil, fmt.Errorf(\"unable to parse geo_distance location\")\n\t\t}\n\t\trvd := &SortGeoDistance{\n\t\t\tField:    field,\n\t\t\tDesc:     descending,\n\t\t\tLon:      lon,\n\t\t\tLat:      lat,\n\t\t\tunitMult: 1.0,\n\t\t}\n\t\tif distUnit, ok := input[\"unit\"].(string); ok {\n\t\t\tvar err error\n\t\t\trvd.unitMult, err = geo.ParseDistanceUnit(distUnit)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\trvd.Unit = distUnit\n\t\t}\n\t\treturn rvd, nil\n\tcase \"field\":\n\t\tfield, ok := input[\"field\"].(string)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"search sort mode field must specify field\")\n\t\t}\n\t\trv := &SortField{\n\t\t\tField: field,\n\t\t\tDesc:  descending,\n\t\t}\n\t\ttyp, ok := input[\"type\"].(string)\n\t\tif ok {\n\t\t\tswitch typ {\n\t\t\tcase \"auto\":\n\t\t\t\trv.Type = SortFieldAuto\n\t\t\tcase \"string\":\n\t\t\t\trv.Type = SortFieldAsString\n\t\t\tcase \"number\":\n\t\t\t\trv.Type = SortFieldAsNumber\n\t\t\tcase \"date\":\n\t\t\t\trv.Type = SortFieldAsDate\n\t\t\tdefault:\n\t\t\t\treturn nil, fmt.Errorf(\"unknown sort field type: %s\", typ)\n\t\t\t}\n\t\t}\n\t\tmode, ok := input[\"mode\"].(string)\n\t\tif ok {\n\t\t\tswitch mode {\n\t\t\tcase \"default\":\n\t\t\t\trv.Mode = SortFieldDefault\n\t\t\tcase \"min\":\n\t\t\t\trv.Mode = SortFieldMin\n\t\t\tcase \"max\":\n\t\t\t\trv.Mode = SortFieldMax\n\t\t\tdefault:\n\t\t\t\treturn nil, fmt.Errorf(\"unknown sort field mode: %s\", mode)\n\t\t\t}\n\t\t}\n\t\tmissing, ok := input[\"missing\"].(string)\n\t\tif ok {\n\t\t\tswitch missing {\n\t\t\tcase \"first\":\n\t\t\t\trv.Missing = SortFieldMissingFirst\n\t\t\tcase \"last\":\n\t\t\t\trv.Missing = SortFieldMissingLast\n\t\t\tdefault:\n\t\t\t\treturn nil, fmt.Errorf(\"unknown sort field missing: %s\", missing)\n\t\t\t}\n\t\t}\n\t\treturn rv, nil\n\t}\n\n\treturn nil, fmt.Errorf(\"unknown search sort by: %s\", by)\n}\n\nfunc ParseSearchSortString(input string) SearchSort {\n\tdescending := false\n\tif strings.HasPrefix(input, \"-\") {\n\t\tdescending = true\n\t\tinput = input[1:]\n\t} else if strings.HasPrefix(input, \"+\") {\n\t\tinput = input[1:]\n\t}\n\n\tswitch input {\n\tcase \"_id\":\n\t\treturn &SortDocID{\n\t\t\tDesc: descending,\n\t\t}\n\tcase \"_score\":\n\t\treturn &SortScore{\n\t\t\tDesc: descending,\n\t\t}\n\t}\n\n\treturn &SortField{\n\t\tField: input,\n\t\tDesc:  descending,\n\t}\n}\n\nfunc ParseSearchSortJSON(input json.RawMessage) (SearchSort, error) {\n\t// first try to parse it as string\n\tvar sortString string\n\terr := util.UnmarshalJSON(input, &sortString)\n\tif err != nil {\n\t\tvar sortObj map[string]interface{}\n\t\terr = util.UnmarshalJSON(input, &sortObj)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn ParseSearchSortObj(sortObj)\n\t}\n\treturn ParseSearchSortString(sortString), nil\n}\n\nfunc ParseSortOrderStrings(in []string) SortOrder {\n\trv := make(SortOrder, 0, len(in))\n\tfor _, i := range in {\n\t\tss := ParseSearchSortString(i)\n\t\trv = append(rv, ss)\n\t}\n\treturn rv\n}\n\nfunc ParseSortOrderJSON(in []json.RawMessage) (SortOrder, error) {\n\trv := make(SortOrder, 0, len(in))\n\tfor _, i := range in {\n\t\tss, err := ParseSearchSortJSON(i)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trv = append(rv, ss)\n\t}\n\treturn rv, nil\n}\n\ntype SortOrder []SearchSort\n\nfunc (so SortOrder) Value(doc *DocumentMatch) {\n\tfor _, soi := range so {\n\t\tvalue := soi.Value(doc)\n\t\tdoc.Sort = append(doc.Sort, value)\n\t\tdoc.DecodedSort = append(doc.DecodedSort, soi.DecodeValue(value))\n\t}\n}\n\nfunc (so SortOrder) UpdateVisitor(field string, term []byte) {\n\tfor _, soi := range so {\n\t\tsoi.UpdateVisitor(field, term)\n\t}\n}\n\nfunc (so SortOrder) Copy() SortOrder {\n\trv := make(SortOrder, len(so))\n\tfor i, soi := range so {\n\t\trv[i] = soi.Copy()\n\t}\n\treturn rv\n}\n\n// Compare will compare two document matches using the specified sort order\n// if both are numbers, we avoid converting back to term\nfunc (so SortOrder) Compare(cachedScoring, cachedDesc []bool, i, j *DocumentMatch) int {\n\t// compare the documents on all search sorts until a differences is found\n\tfor x := range so {\n\t\tc := 0\n\t\tif cachedScoring[x] {\n\t\t\tif i.Score < j.Score {\n\t\t\t\tc = -1\n\t\t\t} else if i.Score > j.Score {\n\t\t\t\tc = 1\n\t\t\t}\n\t\t} else {\n\t\t\tiVal := i.Sort[x]\n\t\t\tjVal := j.Sort[x]\n\t\t\tif iVal < jVal {\n\t\t\t\tc = -1\n\t\t\t} else if iVal > jVal {\n\t\t\t\tc = 1\n\t\t\t}\n\t\t}\n\n\t\tif c == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tif cachedDesc[x] {\n\t\t\tc = -c\n\t\t}\n\t\treturn c\n\t}\n\t// if they are the same at this point, impose order based on index natural sort order\n\tif i.HitNumber == j.HitNumber {\n\t\treturn 0\n\t} else if i.HitNumber > j.HitNumber {\n\t\treturn 1\n\t}\n\treturn -1\n}\n\nfunc (so SortOrder) RequiresScore() bool {\n\tfor _, soi := range so {\n\t\tif soi.RequiresScoring() {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (so SortOrder) RequiresDocID() bool {\n\tfor _, soi := range so {\n\t\tif soi.RequiresDocID() {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (so SortOrder) RequiredFields() []string {\n\tvar rv []string\n\tfor _, soi := range so {\n\t\trv = append(rv, soi.RequiresFields()...)\n\t}\n\treturn rv\n}\n\nfunc (so SortOrder) CacheIsScore() []bool {\n\trv := make([]bool, 0, len(so))\n\tfor _, soi := range so {\n\t\trv = append(rv, soi.RequiresScoring())\n\t}\n\treturn rv\n}\n\nfunc (so SortOrder) CacheDescending() []bool {\n\trv := make([]bool, 0, len(so))\n\tfor _, soi := range so {\n\t\trv = append(rv, soi.Descending())\n\t}\n\treturn rv\n}\n\nfunc (so SortOrder) Reverse() {\n\tfor _, soi := range so {\n\t\tsoi.Reverse()\n\t}\n}\n\n// SortFieldType lets you control some internal sort behavior\n// normally leaving this to the zero-value of SortFieldAuto is fine\ntype SortFieldType int\n\nconst (\n\t// SortFieldAuto applies heuristics attempt to automatically sort correctly\n\tSortFieldAuto SortFieldType = iota\n\t// SortFieldAsString forces sort as string (no prefix coded terms removed)\n\tSortFieldAsString\n\t// SortFieldAsNumber forces sort as string (prefix coded terms with shift > 0 removed)\n\tSortFieldAsNumber\n\t// SortFieldAsDate forces sort as string (prefix coded terms with shift > 0 removed)\n\tSortFieldAsDate\n)\n\n// SortFieldMode describes the behavior if the field has multiple values\ntype SortFieldMode int\n\nconst (\n\t// SortFieldDefault uses the first (or only) value, this is the default zero-value\n\tSortFieldDefault SortFieldMode = iota // FIXME name is confusing\n\t// SortFieldMin uses the minimum value\n\tSortFieldMin\n\t// SortFieldMax uses the maximum value\n\tSortFieldMax\n)\n\n// SortFieldMissing controls where documents missing a field value should be sorted\ntype SortFieldMissing int\n\nconst (\n\t// SortFieldMissingLast sorts documents missing a field at the end\n\tSortFieldMissingLast SortFieldMissing = iota\n\n\t// SortFieldMissingFirst sorts documents missing a field at the beginning\n\tSortFieldMissingFirst\n)\n\n// SortField will sort results by the value of a stored field\n//\n//\tField is the name of the field\n//\tDescending reverse the sort order (default false)\n//\tType allows forcing of string/number/date behavior (default auto)\n//\tMode controls behavior for multi-values fields (default first)\n//\tMissing controls behavior of missing values (default last)\ntype SortField struct {\n\tField   string\n\tDesc    bool\n\tType    SortFieldType\n\tMode    SortFieldMode\n\tMissing SortFieldMissing\n\tvalues  [][]byte\n\ttmp     [][]byte\n}\n\n// UpdateVisitor notifies this sort field that in this document\n// this field has the specified term\nfunc (s *SortField) UpdateVisitor(field string, term []byte) {\n\tif field == s.Field {\n\t\ts.values = append(s.values, term)\n\t}\n}\n\n// Value returns the sort value of the DocumentMatch\n// it also resets the state of this SortField for\n// processing the next document\nfunc (s *SortField) Value(i *DocumentMatch) string {\n\tiTerms := s.filterTermsByType(s.values)\n\tiTerm := s.filterTermsByMode(iTerms)\n\ts.values = s.values[:0]\n\treturn iTerm\n}\n\nfunc (s *SortField) DecodeValue(value string) string {\n\tswitch s.Type {\n\tcase SortFieldAsNumber:\n\t\ti64, err := numeric.PrefixCoded(value).Int64()\n\t\tif err != nil {\n\t\t\treturn value\n\t\t}\n\t\treturn strconv.FormatFloat(numeric.Int64ToFloat64(i64), 'f', -1, 64)\n\tcase SortFieldAsDate:\n\t\ti64, err := numeric.PrefixCoded(value).Int64()\n\t\tif err != nil {\n\t\t\treturn value\n\t\t}\n\t\treturn time.Unix(0, i64).UTC().Format(time.RFC3339Nano)\n\tdefault:\n\t\treturn value\n\t}\n}\n\n// Descending determines the order of the sort\nfunc (s *SortField) Descending() bool {\n\treturn s.Desc\n}\n\nfunc (s *SortField) filterTermsByMode(terms [][]byte) string {\n\tif len(terms) == 1 || (len(terms) > 1 && s.Mode == SortFieldDefault) {\n\t\treturn string(terms[0])\n\t} else if len(terms) > 1 {\n\t\tswitch s.Mode {\n\t\tcase SortFieldMin:\n\t\t\tsort.Sort(BytesSlice(terms))\n\t\t\treturn string(terms[0])\n\t\tcase SortFieldMax:\n\t\t\tsort.Sort(BytesSlice(terms))\n\t\t\treturn string(terms[len(terms)-1])\n\t\t}\n\t}\n\n\t// handle missing terms\n\tif s.Missing == SortFieldMissingLast {\n\t\tif s.Desc {\n\t\t\treturn LowTerm\n\t\t}\n\t\treturn HighTerm\n\t}\n\tif s.Desc {\n\t\treturn HighTerm\n\t}\n\treturn LowTerm\n}\n\n// filterTermsByType attempts to make one pass on the terms\n// if we are in auto-mode AND all the terms look like prefix-coded numbers\n// return only the terms which had shift of 0\n// if we are in explicit number or date mode, return only valid\n// prefix coded numbers with shift of 0\nfunc (s *SortField) filterTermsByType(terms [][]byte) [][]byte {\n\tstype := s.Type\n\n\tswitch stype {\n\tcase SortFieldAuto:\n\t\tallTermsPrefixCoded := true\n\t\ttermsWithShiftZero := s.tmp[:0]\n\t\tfor _, term := range terms {\n\t\t\tvalid, shift := numeric.ValidPrefixCodedTermBytes(term)\n\t\t\tif valid && shift == 0 {\n\t\t\t\ttermsWithShiftZero = append(termsWithShiftZero, term)\n\t\t\t} else if !valid {\n\t\t\t\tallTermsPrefixCoded = false\n\t\t\t}\n\t\t}\n\t\t// reset the terms only when valid zero shift terms are found.\n\t\tif allTermsPrefixCoded && len(termsWithShiftZero) > 0 {\n\t\t\tterms = termsWithShiftZero\n\t\t\ts.tmp = termsWithShiftZero[:0]\n\t\t}\n\tcase SortFieldAsNumber, SortFieldAsDate:\n\t\ttermsWithShiftZero := s.tmp[:0]\n\t\tfor _, term := range terms {\n\t\t\tvalid, shift := numeric.ValidPrefixCodedTermBytes(term)\n\t\t\tif valid && shift == 0 {\n\t\t\t\ttermsWithShiftZero = append(termsWithShiftZero, term)\n\t\t\t}\n\t\t}\n\t\tterms = termsWithShiftZero\n\t\ts.tmp = termsWithShiftZero[:0]\n\t}\n\n\treturn terms\n}\n\n// RequiresDocID says this SearchSort does not require the DocID be loaded\nfunc (s *SortField) RequiresDocID() bool { return false }\n\n// RequiresScoring says this SearchStore does not require scoring\nfunc (s *SortField) RequiresScoring() bool { return false }\n\n// RequiresFields says this SearchStore requires the specified stored field\nfunc (s *SortField) RequiresFields() []string { return []string{s.Field} }\n\nfunc (s *SortField) MarshalJSON() ([]byte, error) {\n\t// see if simple format can be used\n\tif s.Missing == SortFieldMissingLast &&\n\t\ts.Mode == SortFieldDefault &&\n\t\ts.Type == SortFieldAuto {\n\t\tif s.Desc {\n\t\t\treturn json.Marshal(\"-\" + s.Field)\n\t\t}\n\t\treturn json.Marshal(s.Field)\n\t}\n\tsfm := map[string]interface{}{\n\t\t\"by\":    \"field\",\n\t\t\"field\": s.Field,\n\t}\n\tif s.Desc {\n\t\tsfm[\"desc\"] = true\n\t}\n\tif s.Missing > SortFieldMissingLast {\n\t\tswitch s.Missing {\n\t\tcase SortFieldMissingFirst:\n\t\t\tsfm[\"missing\"] = \"first\"\n\t\t}\n\t}\n\tif s.Mode > SortFieldDefault {\n\t\tswitch s.Mode {\n\t\tcase SortFieldMin:\n\t\t\tsfm[\"mode\"] = \"min\"\n\t\tcase SortFieldMax:\n\t\t\tsfm[\"mode\"] = \"max\"\n\t\t}\n\t}\n\tif s.Type > SortFieldAuto {\n\t\tswitch s.Type {\n\t\tcase SortFieldAsString:\n\t\t\tsfm[\"type\"] = \"string\"\n\t\tcase SortFieldAsNumber:\n\t\t\tsfm[\"type\"] = \"number\"\n\t\tcase SortFieldAsDate:\n\t\t\tsfm[\"type\"] = \"date\"\n\t\t}\n\t}\n\n\treturn json.Marshal(sfm)\n}\n\nfunc (s *SortField) Copy() SearchSort {\n\trv := *s\n\treturn &rv\n}\n\nfunc (s *SortField) Reverse() {\n\ts.Desc = !s.Desc\n\tif s.Missing == SortFieldMissingFirst {\n\t\ts.Missing = SortFieldMissingLast\n\t} else {\n\t\ts.Missing = SortFieldMissingFirst\n\t}\n}\n\n// SortDocID will sort results by the document identifier\ntype SortDocID struct {\n\tDesc bool\n}\n\n// UpdateVisitor is a no-op for SortDocID as it's value\n// is not dependent on any field terms\nfunc (s *SortDocID) UpdateVisitor(field string, term []byte) {\n}\n\n// Value returns the sort value of the DocumentMatch\nfunc (s *SortDocID) Value(i *DocumentMatch) string {\n\treturn i.ID\n}\n\nfunc (s *SortDocID) DecodeValue(value string) string {\n\treturn value\n}\n\n// Descending determines the order of the sort\nfunc (s *SortDocID) Descending() bool {\n\treturn s.Desc\n}\n\n// RequiresDocID says this SearchSort does require the DocID be loaded\nfunc (s *SortDocID) RequiresDocID() bool { return true }\n\n// RequiresScoring says this SearchStore does not require scoring\nfunc (s *SortDocID) RequiresScoring() bool { return false }\n\n// RequiresFields says this SearchStore does not require any stored fields\nfunc (s *SortDocID) RequiresFields() []string { return nil }\n\nfunc (s *SortDocID) MarshalJSON() ([]byte, error) {\n\tif s.Desc {\n\t\treturn json.Marshal(\"-_id\")\n\t}\n\treturn json.Marshal(\"_id\")\n}\n\nfunc (s *SortDocID) Copy() SearchSort {\n\trv := *s\n\treturn &rv\n}\n\nfunc (s *SortDocID) Reverse() {\n\ts.Desc = !s.Desc\n}\n\n// SortScore will sort results by the document match score\ntype SortScore struct {\n\tDesc bool\n}\n\n// UpdateVisitor is a no-op for SortScore as it's value\n// is not dependent on any field terms\nfunc (s *SortScore) UpdateVisitor(field string, term []byte) {\n}\n\n// Value returns the sort value of the DocumentMatch\nfunc (s *SortScore) Value(i *DocumentMatch) string {\n\treturn \"_score\"\n}\n\nfunc (s *SortScore) DecodeValue(value string) string {\n\treturn value\n}\n\n// Descending determines the order of the sort\nfunc (s *SortScore) Descending() bool {\n\treturn s.Desc\n}\n\n// RequiresDocID says this SearchSort does not require the DocID be loaded\nfunc (s *SortScore) RequiresDocID() bool { return false }\n\n// RequiresScoring says this SearchStore does require scoring\nfunc (s *SortScore) RequiresScoring() bool { return true }\n\n// RequiresFields says this SearchStore does not require any store fields\nfunc (s *SortScore) RequiresFields() []string { return nil }\n\nfunc (s *SortScore) MarshalJSON() ([]byte, error) {\n\tif s.Desc {\n\t\treturn json.Marshal(\"-_score\")\n\t}\n\treturn json.Marshal(\"_score\")\n}\n\nfunc (s *SortScore) Copy() SearchSort {\n\trv := *s\n\treturn &rv\n}\n\nfunc (s *SortScore) Reverse() {\n\ts.Desc = !s.Desc\n}\n\nvar maxDistance = string(numeric.MustNewPrefixCodedInt64(math.MaxInt64, 0))\n\n// NewSortGeoDistance creates SearchSort instance for sorting documents by\n// their distance from the specified point.\nfunc NewSortGeoDistance(field, unit string, lon, lat float64, desc bool) (\n\t*SortGeoDistance, error,\n) {\n\trv := &SortGeoDistance{\n\t\tField: field,\n\t\tDesc:  desc,\n\t\tUnit:  unit,\n\t\tLon:   lon,\n\t\tLat:   lat,\n\t}\n\tvar err error\n\trv.unitMult, err = geo.ParseDistanceUnit(unit)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn rv, nil\n}\n\n// SortGeoDistance will sort results by the distance of an\n// indexed geo point, from the provided location.\n//\n//\tField is the name of the field\n//\tDescending reverse the sort order (default false)\ntype SortGeoDistance struct {\n\tField    string\n\tDesc     bool\n\tUnit     string\n\tvalues   [][]byte\n\tLon      float64\n\tLat      float64\n\tunitMult float64\n\ttmp      []byte\n}\n\n// UpdateVisitor notifies this sort field that in this document\n// this field has the specified term\nfunc (s *SortGeoDistance) UpdateVisitor(field string, term []byte) {\n\tif field == s.Field {\n\t\ts.values = append(s.values, term)\n\t}\n}\n\n// Value returns the sort value of the DocumentMatch\n// it also resets the state of this SortGeoDistance for\n// processing the next document\nfunc (s *SortGeoDistance) Value(i *DocumentMatch) string {\n\tiTerm := s.findPrefixCodedNumericTerm(s.values)\n\ts.values = s.values[:0]\n\n\tif iTerm == nil {\n\t\treturn maxDistance\n\t}\n\n\ti64, err := numeric.PrefixCoded(iTerm).Int64()\n\tif err != nil {\n\t\treturn maxDistance\n\t}\n\tdocLon := geo.MortonUnhashLon(uint64(i64))\n\tdocLat := geo.MortonUnhashLat(uint64(i64))\n\n\tdist := geo.Haversin(s.Lon, s.Lat, docLon, docLat)\n\t// dist is returned in km, so convert to m\n\tdist *= 1000\n\tif s.unitMult != 0 {\n\t\tdist /= s.unitMult\n\t}\n\tdistInt64 := numeric.Float64ToInt64(dist)\n\ts.tmp = numeric.MustNewPrefixCodedInt64Prealloc(distInt64, 0, s.tmp)\n\treturn string(s.tmp)\n}\n\nfunc (s *SortGeoDistance) DecodeValue(value string) string {\n\tdistInt, err := numeric.PrefixCoded(value).Int64()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\treturn strconv.FormatFloat(numeric.Int64ToFloat64(distInt), 'f', -1, 64)\n}\n\n// Descending determines the order of the sort\nfunc (s *SortGeoDistance) Descending() bool {\n\treturn s.Desc\n}\n\n// findPrefixCodedNumericTerm looks through the provided terms\n// and returns the first valid prefix coded numeric term with shift of 0\nfunc (s *SortGeoDistance) findPrefixCodedNumericTerm(terms [][]byte) []byte {\n\tfor _, term := range terms {\n\t\tvalid, shift := numeric.ValidPrefixCodedTermBytes(term)\n\t\tif valid && shift == 0 {\n\t\t\treturn term\n\t\t}\n\t}\n\treturn nil\n}\n\n// RequiresDocID says this SearchSort does not require the DocID be loaded\nfunc (s *SortGeoDistance) RequiresDocID() bool { return false }\n\n// RequiresScoring says this SearchStore does not require scoring\nfunc (s *SortGeoDistance) RequiresScoring() bool { return false }\n\n// RequiresFields says this SearchStore requires the specified stored field\nfunc (s *SortGeoDistance) RequiresFields() []string { return []string{s.Field} }\n\nfunc (s *SortGeoDistance) MarshalJSON() ([]byte, error) {\n\tsfm := map[string]interface{}{\n\t\t\"by\":    \"geo_distance\",\n\t\t\"field\": s.Field,\n\t\t\"location\": map[string]interface{}{\n\t\t\t\"lon\": s.Lon,\n\t\t\t\"lat\": s.Lat,\n\t\t},\n\t}\n\tif s.Unit != \"\" {\n\t\tsfm[\"unit\"] = s.Unit\n\t}\n\tif s.Desc {\n\t\tsfm[\"desc\"] = true\n\t}\n\n\treturn json.Marshal(sfm)\n}\n\nfunc (s *SortGeoDistance) Copy() SearchSort {\n\trv := *s\n\treturn &rv\n}\n\nfunc (s *SortGeoDistance) Reverse() {\n\ts.Desc = !s.Desc\n}\n\ntype BytesSlice [][]byte\n\nfunc (p BytesSlice) Len() int           { return len(p) }\nfunc (p BytesSlice) Less(i, j int) bool { return bytes.Compare(p[i], p[j]) < 0 }\nfunc (p BytesSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }\n"
  },
  {
    "path": "search/sort_test.go",
    "content": "package search\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestParseSearchSortObj(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\tinput   map[string]interface{}\n\t\twant    SearchSort\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"sort by id\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":   \"id\",\n\t\t\t\t\"desc\": false,\n\t\t\t},\n\t\t\twant: &SortDocID{\n\t\t\t\tDesc: false,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"sort by id descending\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":   \"id\",\n\t\t\t\t\"desc\": true,\n\t\t\t},\n\t\t\twant: &SortDocID{\n\t\t\t\tDesc: true,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"sort by score\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":   \"score\",\n\t\t\t\t\"desc\": false,\n\t\t\t},\n\t\t\twant: &SortScore{\n\t\t\t\tDesc: false,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"sort by score descending\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":   \"score\",\n\t\t\t\t\"desc\": true,\n\t\t\t},\n\t\t\twant: &SortScore{\n\t\t\t\tDesc: true,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"sort by geo_distance\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":    \"geo_distance\",\n\t\t\t\t\"field\": \"location\",\n\t\t\t\t\"location\": map[string]interface{}{\n\t\t\t\t\t\"lon\": 1.0,\n\t\t\t\t\t\"lat\": 2.0,\n\t\t\t\t},\n\t\t\t\t\"unit\": \"km\",\n\t\t\t\t\"desc\": false,\n\t\t\t},\n\t\t\twant: &SortGeoDistance{\n\t\t\t\tField:    \"location\",\n\t\t\t\tDesc:     false,\n\t\t\t\tLon:      1.0,\n\t\t\t\tLat:      2.0,\n\t\t\t\tUnit:     \"km\",\n\t\t\t\tunitMult: 1000.0,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"sort by field\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":      \"field\",\n\t\t\t\t\"field\":   \"name\",\n\t\t\t\t\"desc\":    false,\n\t\t\t\t\"type\":    \"auto\",\n\t\t\t\t\"mode\":    \"default\",\n\t\t\t\t\"missing\": \"last\",\n\t\t\t},\n\t\t\twant: &SortField{\n\t\t\t\tField:   \"name\",\n\t\t\t\tDesc:    false,\n\t\t\t\tType:    SortFieldAuto,\n\t\t\t\tMode:    SortFieldDefault,\n\t\t\t\tMissing: SortFieldMissingLast,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"sort by field with missing\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":      \"field\",\n\t\t\t\t\"field\":   \"name\",\n\t\t\t\t\"desc\":    false,\n\t\t\t\t\"type\":    \"auto\",\n\t\t\t\t\"mode\":    \"default\",\n\t\t\t\t\"missing\": \"first\",\n\t\t\t},\n\t\t\twant: &SortField{\n\t\t\t\tField:   \"name\",\n\t\t\t\tDesc:    false,\n\t\t\t\tType:    SortFieldAuto,\n\t\t\t\tMode:    SortFieldDefault,\n\t\t\t\tMissing: SortFieldMissingFirst,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"sort by field descending\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":      \"field\",\n\t\t\t\t\"field\":   \"name\",\n\t\t\t\t\"desc\":    true,\n\t\t\t\t\"type\":    \"string\",\n\t\t\t\t\"mode\":    \"min\",\n\t\t\t\t\"missing\": \"first\",\n\t\t\t},\n\t\t\twant: &SortField{\n\t\t\t\tField:   \"name\",\n\t\t\t\tDesc:    true,\n\t\t\t\tType:    SortFieldAsString,\n\t\t\t\tMode:    SortFieldMin,\n\t\t\t\tMissing: SortFieldMissingFirst,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"missing by\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"desc\": true,\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"unknown by\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\": \"unknown\",\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"missing field for geo_distance\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\": \"geo_distance\",\n\t\t\t\t\"location\": map[string]interface{}{\n\t\t\t\t\t\"lon\": 1.0,\n\t\t\t\t\t\"lat\": 2.0,\n\t\t\t\t},\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"missing location for geo_distance\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":    \"geo_distance\",\n\t\t\t\t\"field\": \"location\",\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid unit for geo_distance\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":    \"geo_distance\",\n\t\t\t\t\"field\": \"location\",\n\t\t\t\t\"location\": map[string]interface{}{\n\t\t\t\t\t\"lon\": 1.0,\n\t\t\t\t\t\"lat\": 2.0,\n\t\t\t\t},\n\t\t\t\t\"unit\": \"invalid\",\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"missing field for field sort\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\": \"field\",\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"unknown type for field sort\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":    \"field\",\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"type\":  \"unknown\",\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"number type for field sort with desc\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":      \"field\",\n\t\t\t\t\"field\":   \"name\",\n\t\t\t\t\"type\":    \"number\",\n\t\t\t\t\"mode\":    \"default\",\n\t\t\t\t\"desc\":    true,\n\t\t\t\t\"missing\": \"last\",\n\t\t\t},\n\t\t\twant: &SortField{\n\t\t\t\tField:   \"name\",\n\t\t\t\tDesc:    true,\n\t\t\t\tType:    SortFieldAsNumber,\n\t\t\t\tMode:    SortFieldDefault,\n\t\t\t\tMissing: SortFieldMissingLast,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"date type for field sort with desc\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":      \"field\",\n\t\t\t\t\"field\":   \"name\",\n\t\t\t\t\"type\":    \"date\",\n\t\t\t\t\"mode\":    \"default\",\n\t\t\t\t\"desc\":    true,\n\t\t\t\t\"missing\": \"last\",\n\t\t\t},\n\t\t\twant: &SortField{\n\t\t\t\tField:   \"name\",\n\t\t\t\tDesc:    true,\n\t\t\t\tType:    SortFieldAsDate,\n\t\t\t\tMode:    SortFieldDefault,\n\t\t\t\tMissing: SortFieldMissingLast,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"unknown type for field sort with missing\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":      \"field\",\n\t\t\t\t\"field\":   \"name\",\n\t\t\t\t\"type\":    \"unknown\",\n\t\t\t\t\"mode\":    \"default\",\n\t\t\t\t\"missing\": \"last\",\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"unknown mode for field sort\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":    \"field\",\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"mode\":  \"unknown\",\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"default mode for field sort\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":    \"field\",\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"mode\":  \"default\",\n\t\t\t},\n\t\t\twant: &SortField{\n\t\t\t\tField:   \"name\",\n\t\t\t\tDesc:    false,\n\t\t\t\tType:    SortFieldAuto,\n\t\t\t\tMode:    SortFieldDefault,\n\t\t\t\tMissing: SortFieldMissingLast,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"max mode for field sort\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":    \"field\",\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"mode\":  \"max\",\n\t\t\t},\n\t\t\twant: &SortField{\n\t\t\t\tField:   \"name\",\n\t\t\t\tDesc:    false,\n\t\t\t\tType:    SortFieldAuto,\n\t\t\t\tMode:    SortFieldMax,\n\t\t\t\tMissing: SortFieldMissingLast,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"min mode for field sort\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":    \"field\",\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"mode\":  \"min\",\n\t\t\t},\n\t\t\twant: &SortField{\n\t\t\t\tField:   \"name\",\n\t\t\t\tDesc:    false,\n\t\t\t\tType:    SortFieldAuto,\n\t\t\t\tMode:    SortFieldMin,\n\t\t\t\tMissing: SortFieldMissingLast,\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"unknown missing for field sort\",\n\t\t\tinput: map[string]interface{}{\n\t\t\t\t\"by\":      \"field\",\n\t\t\t\t\"field\":   \"name\",\n\t\t\t\t\"missing\": \"unknown\",\n\t\t\t},\n\t\t\twant:    nil,\n\t\t\twantErr: true,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgot, err := ParseSearchSortObj(tt.input)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"ParseSearchSortObj() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(got, tt.want) {\n\t\t\t\tt.Errorf(\"ParseSearchSortObj() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "search/util.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"context\"\n\n\t\"github.com/blevesearch/geo/s2\"\n)\n\nfunc MergeLocations(locations []FieldTermLocationMap) FieldTermLocationMap {\n\trv := locations[0]\n\n\tfor i := 1; i < len(locations); i++ {\n\t\tnextLocations := locations[i]\n\t\tfor field, termLocationMap := range nextLocations {\n\t\t\trvTermLocationMap, rvHasField := rv[field]\n\t\t\tif rvHasField {\n\t\t\t\trv[field] = MergeTermLocationMaps(rvTermLocationMap, termLocationMap)\n\t\t\t} else {\n\t\t\t\trv[field] = termLocationMap\n\t\t\t}\n\t\t}\n\t}\n\n\treturn rv\n}\n\nfunc MergeTermLocationMaps(rv, other TermLocationMap) TermLocationMap {\n\tfor term, locationMap := range other {\n\t\t// for a given term/document there cannot be different locations\n\t\t// if they came back from different clauses, overwrite is ok\n\t\trv[term] = locationMap\n\t}\n\treturn rv\n}\n\nfunc MergeFieldTermLocations(dest []FieldTermLocation, matches []*DocumentMatch) []FieldTermLocation {\n\tn := len(dest)\n\tfor _, dm := range matches {\n\t\tif dm != nil {\n\t\t\tn += len(dm.FieldTermLocations)\n\t\t}\n\t}\n\tif cap(dest) < n {\n\t\tdest = append(make([]FieldTermLocation, 0, n), dest...)\n\t}\n\n\tfor _, dm := range matches {\n\t\tif dm != nil {\n\t\t\tdest = mergeFieldTermLocationFromMatch(dest, dm)\n\t\t}\n\t}\n\n\treturn dest\n}\n\n// MergeFieldTermLocationsFromMatch merges field term locations from a single DocumentMatch\n// into dest, returning the updated slice.\nfunc MergeFieldTermLocationsFromMatch(dest []FieldTermLocation, match *DocumentMatch) []FieldTermLocation {\n\tif match == nil {\n\t\treturn dest\n\t}\n\tn := len(dest) + len(match.FieldTermLocations)\n\tif cap(dest) < n {\n\t\tdest = append(make([]FieldTermLocation, 0, n), dest...)\n\t}\n\treturn mergeFieldTermLocationFromMatch(dest, match)\n}\n\n// mergeFieldTermLocationFromMatch appends field term locations from a DocumentMatch into dest.\n// Assumes dest has sufficient capacity.\nfunc mergeFieldTermLocationFromMatch(dest []FieldTermLocation, dm *DocumentMatch) []FieldTermLocation {\n\tfor _, ftl := range dm.FieldTermLocations {\n\t\tdest = append(dest, FieldTermLocation{\n\t\t\tField: ftl.Field,\n\t\t\tTerm:  ftl.Term,\n\t\t\tLocation: Location{\n\t\t\t\tPos:            ftl.Location.Pos,\n\t\t\t\tStart:          ftl.Location.Start,\n\t\t\t\tEnd:            ftl.Location.End,\n\t\t\t\tArrayPositions: append(ArrayPositions(nil), ftl.Location.ArrayPositions...),\n\t\t\t},\n\t\t})\n\t}\n\n\treturn dest\n}\n\ntype (\n\tSearchIncrementalCostCallbackMsg uint\n\tSearchQueryType                  uint\n)\n\nconst (\n\tTerm = SearchQueryType(1 << iota)\n\tGeo\n\tNumeric\n\tGenericCost\n)\n\nconst (\n\tAddM = SearchIncrementalCostCallbackMsg(1 << iota)\n\tAbortM\n\tDoneM\n)\n\n// ContextKey is used to identify the context key in the context.Context\ntype ContextKey string\n\nfunc (c ContextKey) String() string {\n\treturn string(c)\n}\n\nconst (\n\tSearchIncrementalCostKey ContextKey = \"_search_incremental_cost_key\"\n\tQueryTypeKey             ContextKey = \"_query_type_key\"\n\tFuzzyMatchPhraseKey      ContextKey = \"_fuzzy_match_phrase_key\"\n\tIncludeScoreBreakdownKey ContextKey = \"_include_score_breakdown_key\"\n\n\t// PreSearchKey indicates whether to perform a preliminary search to gather necessary\n\t// information which would be used in the actual search down the line.\n\tPreSearchKey ContextKey = \"_presearch_key\"\n\n\t// GetScoringModelCallbackKey is used to help the underlying searcher identify\n\t// which scoring mechanism to use based on index mapping.\n\tGetScoringModelCallbackKey ContextKey = \"_get_scoring_model\"\n\n\t// SearchIOStatsCallbackKey is used to help the underlying searcher identify\n\tSearchIOStatsCallbackKey ContextKey = \"_search_io_stats_callback_key\"\n\n\t// GeoBufferPoolCallbackKey ContextKey is used to help the underlying searcher\n\tGeoBufferPoolCallbackKey ContextKey = \"_geo_buffer_pool_callback_key\"\n\n\t// SearchTypeKey is used to identify type of the search being performed.\n\t//\n\t// for consistent scoring in cases an index is partitioned/sharded (using an\n\t// index alias), GlobalScoring helps in aggregating the necessary stats across\n\t// all the child bleve indexes (shards/partitions) first before the actual search\n\t// is performed, such that the scoring involved using these stats would be at a\n\t// global level.\n\tSearchTypeKey ContextKey = \"_search_type_key\"\n\n\t// The following keys are used to invoke the callbacks at the start and end stages\n\t// of optimizing the disjunction/conjunction searcher creation.\n\tSearcherStartCallbackKey ContextKey = \"_searcher_start_callback_key\"\n\tSearcherEndCallbackKey   ContextKey = \"_searcher_end_callback_key\"\n\n\t// FieldTermSynonymMapKey is used to store and transport the synonym definitions data\n\t// to the actual search phase which would use the synonyms to perform the search.\n\tFieldTermSynonymMapKey ContextKey = \"_field_term_synonym_map_key\"\n\n\t// BM25StatsKey is used to store and transport the BM25 Data\n\t// to the actual search phase which would use it to perform the search.\n\tBM25StatsKey ContextKey = \"_bm25_stats_key\"\n\n\t// ScoreFusionKey is used to communicate whether KNN hits need to be preserved for\n\t// hybrid search algorithms (like RRF)\n\tScoreFusionKey ContextKey = \"_fusion_rescoring_key\"\n\n\t// NestedSearchKey is used to communicate whether the search is performed\n\t// in an index with nested documents\n\tNestedSearchKey ContextKey = \"_nested_search_key\"\n)\n\nfunc RecordSearchCost(ctx context.Context,\n\tmsg SearchIncrementalCostCallbackMsg, bytes uint64,\n) {\n\tif ctx != nil {\n\t\tqueryType, ok := ctx.Value(QueryTypeKey).(SearchQueryType)\n\t\tif !ok {\n\t\t\t// for the cost of the non query type specific factors such as\n\t\t\t// doc values and stored fields section.\n\t\t\tqueryType = GenericCost\n\t\t}\n\n\t\taggCallbackFn := ctx.Value(SearchIncrementalCostKey)\n\t\tif aggCallbackFn != nil {\n\t\t\taggCallbackFn.(SearchIncrementalCostCallbackFn)(msg, queryType, bytes)\n\t\t}\n\t}\n}\n\n// Assigning the size of the largest buffer in the pool to 24KB and\n// the smallest buffer to 24 bytes. The pools are used to read a\n// sequence of vertices which are always 24 bytes each.\nconst (\n\tMaxGeoBufPoolSize = 24 * 1024\n\tMinGeoBufPoolSize = 24\n)\n\n// PreSearchDataKey are used to store the data gathered during the presearch phase\n// which would be use in the actual search phase.\nconst (\n\tKnnPreSearchDataKey     = \"_knn_pre_search_data_key\"\n\tSynonymPreSearchDataKey = \"_synonym_pre_search_data_key\"\n\tBM25PreSearchDataKey    = \"_bm25_pre_search_data_key\"\n)\n\nconst GlobalScoring = \"_global_scoring\"\n\ntype (\n\t// SearcherStartCallbackFn is a callback function type used to signal the start of\n\t// searcher creation phase.\n\tSearcherStartCallbackFn func(size uint64) error\n\t// SearcherEndCallbackFn is a callback function type used to signal the end of\n\t// a searcher creation phase.\n\tSearcherEndCallbackFn func(size uint64) error\n\t// GetScoringModelCallbackFn is a callback function type used to get the scoring model\n\t// to be used for scoring documents during search.\n\tGetScoringModelCallbackFn func() string\n\t// HybridMergeCallbackFn is a callback function type used to merge a KNN document match\n\t// into a full text search document match, of the same docID as part of hybrid search.\n\tHybridMergeCallbackFn func(ftsMatch *DocumentMatch, knnMatch *DocumentMatch)\n\t// DescendantAdderCallback is a callback function type used to customize how a descendant\n\t// DocumentMatch is merged into its parent. This allows different descendant addition strategies for\n\t// different use cases (e.g., TopN vs KNN collection).\n\tDescendantAdderCallbackFn func(parent *DocumentMatch, descendant *DocumentMatch) error\n\t// GeoBufferPoolCallbackFunc is a callback function type used to get the geo buffer pool\n\t// to be used during geo searches.\n\tGeoBufferPoolCallbackFunc func() *s2.GeoBufferPool\n\t// SearchIOStatsCallbackFunc is a callback function type used to report search IO stats\n\t// during search.\n\tSearchIOStatsCallbackFunc func(uint64)\n\t// Implementation of SearchIncrementalCostCallbackFn should handle the following messages\n\t//   - add: increment the cost of a search operation\n\t//     (which can be specific to a query type as well)\n\t//   - abort: query was aborted due to a cancel of search's context (for eg),\n\t//     which can be handled differently as well\n\t//   - done: indicates that a search was complete and the tracked cost can be\n\t//     handled safely by the implementation.\n\tSearchIncrementalCostCallbackFn func(SearchIncrementalCostCallbackMsg,\n\t\tSearchQueryType, uint64)\n)\n\n// field -> term -> synonyms\ntype FieldTermSynonymMap map[string]map[string][]string\n\nfunc (f FieldTermSynonymMap) MergeWith(fts FieldTermSynonymMap) {\n\tfor field, termSynonymMap := range fts {\n\t\t// Ensure the field exists in the receiver\n\t\tif _, exists := f[field]; !exists {\n\t\t\tf[field] = make(map[string][]string)\n\t\t}\n\t\tfor term, synonyms := range termSynonymMap {\n\t\t\t// Append synonyms\n\t\t\tf[field][term] = append(f[field][term], synonyms...)\n\t\t}\n\t}\n}\n\n// BM25 specific multipliers which control the scoring of a document.\n//\n// BM25_b - controls the extent to which doc's field length normalize term frequency part of score\n// BM25_k1 - controls the saturation of the score due to term frequency\n// the default values are as per elastic search's implementation\n//   - https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-similarity.html#bm25\n//   - https://www.elastic.co/blog/practical-bm25-part-3-considerations-for-picking-b-and-k1-in-elasticsearch\nvar (\n\tBM25_k1 float64 = 1.2\n\tBM25_b  float64 = 0.75\n)\n\ntype BM25Stats struct {\n\tDocCount         float64        `json:\"doc_count\"`\n\tFieldCardinality map[string]int `json:\"field_cardinality\"`\n}\n\n// FieldSet represents a set of queried fields.\ntype FieldSet map[string]struct{}\n\n// NewFieldSet creates a new FieldSet.\nfunc NewFieldSet() FieldSet {\n\treturn make(map[string]struct{})\n}\n\n// Add adds a field to the set.\nfunc (fs FieldSet) AddField(field string) {\n\tfs[field] = struct{}{}\n}\n\n// HasID returns true if the field set contains the \"_id\" field.\nfunc (fs FieldSet) HasID() bool {\n\t_, ok := fs[\"_id\"]\n\treturn ok\n}\n\n// HasAll returns true if the field set contains the \"_all\" field.\nfunc (fs FieldSet) HasAll() bool {\n\t_, ok := fs[\"_all\"]\n\treturn ok\n}\n"
  },
  {
    "path": "search/util_test.go",
    "content": "//  Copyright (c) 2013 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage search\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestMergeLocations(t *testing.T) {\n\tflm1 := FieldTermLocationMap{\n\t\t\"marty\": TermLocationMap{\n\t\t\t\"name\": {\n\t\t\t\t&Location{\n\t\t\t\t\tPos:   1,\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tflm2 := FieldTermLocationMap{\n\t\t\"marty\": TermLocationMap{\n\t\t\t\"description\": {\n\t\t\t\t&Location{\n\t\t\t\t\tPos:   5,\n\t\t\t\t\tStart: 20,\n\t\t\t\t\tEnd:   25,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tflm3 := FieldTermLocationMap{\n\t\t\"josh\": TermLocationMap{\n\t\t\t\"description\": {\n\t\t\t\t&Location{\n\t\t\t\t\tPos:   5,\n\t\t\t\t\tStart: 20,\n\t\t\t\t\tEnd:   25,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\texpectedMerge := FieldTermLocationMap{\n\t\t\"marty\": TermLocationMap{\n\t\t\t\"description\": {\n\t\t\t\t&Location{\n\t\t\t\t\tPos:   5,\n\t\t\t\t\tStart: 20,\n\t\t\t\t\tEnd:   25,\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"name\": {\n\t\t\t\t&Location{\n\t\t\t\t\tPos:   1,\n\t\t\t\t\tStart: 0,\n\t\t\t\t\tEnd:   5,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"josh\": TermLocationMap{\n\t\t\t\"description\": {\n\t\t\t\t&Location{\n\t\t\t\t\tPos:   5,\n\t\t\t\t\tStart: 20,\n\t\t\t\t\tEnd:   25,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tmergedLocations := MergeLocations([]FieldTermLocationMap{flm1, flm2, flm3})\n\tif !reflect.DeepEqual(expectedMerge, mergedLocations) {\n\t\tt.Errorf(\"expected %v, got %v\", expectedMerge, mergedLocations)\n\t}\n}\n"
  },
  {
    "path": "search.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/optional\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/registry\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/collector\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\t\"github.com/blevesearch/bleve/v2/size\"\n\t\"github.com/blevesearch/bleve/v2/util\"\n)\n\nvar (\n\treflectStaticSizeSearchResult int\n\treflectStaticSizeSearchStatus int\n)\n\nfunc init() {\n\tvar sr SearchResult\n\treflectStaticSizeSearchResult = int(reflect.TypeOf(sr).Size())\n\tvar ss SearchStatus\n\treflectStaticSizeSearchStatus = int(reflect.TypeOf(ss).Size())\n}\n\nvar cache = registry.NewCache()\n\nconst defaultDateTimeParser = optional.Name\n\nconst (\n\tScoreDefault = \"\"\n\tScoreNone    = \"none\"\n\tScoreRRF     = \"rrf\"\n\tScoreRSF     = \"rsf\"\n)\n\nvar AllowedFusionSort = search.SortOrder{&search.SortScore{Desc: true}}\n\ntype dateTimeRange struct {\n\tName           string    `json:\"name,omitempty\"`\n\tStart          time.Time `json:\"start,omitempty\"`\n\tEnd            time.Time `json:\"end,omitempty\"`\n\tDateTimeParser string    `json:\"datetime_parser,omitempty\"`\n\tstartString    *string\n\tendString      *string\n}\n\nfunc (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time, err error) {\n\tstart = dr.Start\n\tif dr.Start.IsZero() && dr.startString != nil {\n\t\ts, _, parseError := dateTimeParser.ParseDateTime(*dr.startString)\n\t\tif parseError != nil {\n\t\t\treturn start, end, fmt.Errorf(\"error parsing start date '%s' for date range name '%s': %v\", *dr.startString, dr.Name, parseError)\n\t\t}\n\t\tstart = s\n\t}\n\tend = dr.End\n\tif dr.End.IsZero() && dr.endString != nil {\n\t\te, _, parseError := dateTimeParser.ParseDateTime(*dr.endString)\n\t\tif parseError != nil {\n\t\t\treturn start, end, fmt.Errorf(\"error parsing end date '%s' for date range name '%s': %v\", *dr.endString, dr.Name, parseError)\n\t\t}\n\t\tend = e\n\t}\n\treturn start, end, err\n}\n\nfunc (dr *dateTimeRange) UnmarshalJSON(input []byte) error {\n\tvar temp struct {\n\t\tName           string  `json:\"name,omitempty\"`\n\t\tStart          *string `json:\"start,omitempty\"`\n\t\tEnd            *string `json:\"end,omitempty\"`\n\t\tDateTimeParser string  `json:\"datetime_parser,omitempty\"`\n\t}\n\n\tif err := util.UnmarshalJSON(input, &temp); err != nil {\n\t\treturn err\n\t}\n\n\tdr.Name = temp.Name\n\tif temp.Start != nil {\n\t\tdr.startString = temp.Start\n\t}\n\tif temp.End != nil {\n\t\tdr.endString = temp.End\n\t}\n\tif temp.DateTimeParser != \"\" {\n\t\tdr.DateTimeParser = temp.DateTimeParser\n\t}\n\n\treturn nil\n}\n\nfunc (dr *dateTimeRange) MarshalJSON() ([]byte, error) {\n\trv := map[string]interface{}{\n\t\t\"name\": dr.Name,\n\t}\n\n\tif !dr.Start.IsZero() {\n\t\trv[\"start\"] = dr.Start\n\t} else if dr.startString != nil {\n\t\trv[\"start\"] = dr.startString\n\t}\n\n\tif !dr.End.IsZero() {\n\t\trv[\"end\"] = dr.End\n\t} else if dr.endString != nil {\n\t\trv[\"end\"] = dr.endString\n\t}\n\n\tif dr.DateTimeParser != \"\" {\n\t\trv[\"datetime_parser\"] = dr.DateTimeParser\n\t}\n\treturn util.MarshalJSON(rv)\n}\n\ntype numericRange struct {\n\tName string   `json:\"name,omitempty\"`\n\tMin  *float64 `json:\"min,omitempty\"`\n\tMax  *float64 `json:\"max,omitempty\"`\n}\n\n// A FacetRequest describes a facet or aggregation\n// of the result document set you would like to be\n// built.\ntype FacetRequest struct {\n\tSize           int              `json:\"size\"`\n\tField          string           `json:\"field\"`\n\tTermPrefix     string           `json:\"term_prefix,omitempty\"`\n\tTermPattern    string           `json:\"term_pattern,omitempty\"`\n\tNumericRanges  []*numericRange  `json:\"numeric_ranges,omitempty\"`\n\tDateTimeRanges []*dateTimeRange `json:\"date_ranges,omitempty\"`\n\n\t// Compiled regex pattern (cached during validation)\n\tcompiledPattern *regexp.Regexp\n}\n\n// NewFacetRequest creates a facet on the specified\n// field that limits the number of entries to the\n// specified size.\nfunc NewFacetRequest(field string, size int) *FacetRequest {\n\treturn &FacetRequest{\n\t\tField: field,\n\t\tSize:  size,\n\t}\n}\n\n// SetPrefixFilter sets the prefix filter for term facets.\nfunc (fr *FacetRequest) SetPrefixFilter(prefix string) {\n\tfr.TermPrefix = prefix\n}\n\n// SetRegexFilter sets the regex pattern filter for term facets.\nfunc (fr *FacetRequest) SetRegexFilter(pattern string) {\n\tfr.TermPattern = pattern\n}\n\nfunc (fr *FacetRequest) Validate() error {\n\t// Validate regex pattern if provided and cache the compiled regex\n\tif fr.TermPattern != \"\" {\n\t\tcompiled, err := regexp.Compile(fr.TermPattern)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"invalid term pattern: %v\", err)\n\t\t}\n\t\tfr.compiledPattern = compiled\n\t}\n\n\tnrCount := len(fr.NumericRanges)\n\tdrCount := len(fr.DateTimeRanges)\n\tif nrCount > 0 && drCount > 0 {\n\t\treturn fmt.Errorf(\"facet can only contain numeric ranges or date ranges, not both\")\n\t}\n\n\tif nrCount > 0 {\n\t\tnrNames := map[string]interface{}{}\n\t\tfor _, nr := range fr.NumericRanges {\n\t\t\tif _, ok := nrNames[nr.Name]; ok {\n\t\t\t\treturn fmt.Errorf(\"numeric ranges contains duplicate name '%s'\", nr.Name)\n\t\t\t}\n\t\t\tnrNames[nr.Name] = struct{}{}\n\t\t\tif nr.Min == nil && nr.Max == nil {\n\t\t\t\treturn fmt.Errorf(\"numeric range query must specify either min, max or both for range name '%s'\", nr.Name)\n\t\t\t}\n\t\t}\n\n\t} else {\n\t\tdateTimeParser, err := cache.DateTimeParserNamed(defaultDateTimeParser)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdrNames := map[string]interface{}{}\n\t\tfor _, dr := range fr.DateTimeRanges {\n\t\t\tif _, ok := drNames[dr.Name]; ok {\n\t\t\t\treturn fmt.Errorf(\"date ranges contains duplicate name '%s'\", dr.Name)\n\t\t\t}\n\t\t\tdrNames[dr.Name] = struct{}{}\n\t\t\tif dr.DateTimeParser == \"\" {\n\t\t\t\t// cannot parse the date range dates as the defaultDateTimeParser is overridden\n\t\t\t\t// so perform this validation at query time\n\t\t\t\tstart, end, err := dr.ParseDates(dateTimeParser)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"ParseDates err: %v, using date time parser named %s\", err, defaultDateTimeParser)\n\t\t\t\t}\n\t\t\t\tif start.IsZero() && end.IsZero() {\n\t\t\t\t\treturn fmt.Errorf(\"date range query must specify either start, end or both for range name '%s'\", dr.Name)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// AddDateTimeRange adds a bucket to a field\n// containing date values.  Documents with a\n// date value falling into this range are tabulated\n// as part of this bucket/range.\nfunc (fr *FacetRequest) AddDateTimeRange(name string, start, end time.Time) {\n\tif fr.DateTimeRanges == nil {\n\t\tfr.DateTimeRanges = make([]*dateTimeRange, 0, 1)\n\t}\n\tfr.DateTimeRanges = append(fr.DateTimeRanges, &dateTimeRange{Name: name, Start: start, End: end})\n}\n\n// AddDateTimeRangeString adds a bucket to a field\n// containing date values. Uses defaultDateTimeParser to parse the date strings.\nfunc (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) {\n\tif fr.DateTimeRanges == nil {\n\t\tfr.DateTimeRanges = make([]*dateTimeRange, 0, 1)\n\t}\n\tfr.DateTimeRanges = append(fr.DateTimeRanges,\n\t\t&dateTimeRange{Name: name, startString: start, endString: end})\n}\n\n// AddDateTimeRangeString adds a bucket to a field\n// containing date values. Uses the specified parser to parse the date strings.\n// provided the parser is registered in the index mapping.\nfunc (fr *FacetRequest) AddDateTimeRangeStringWithParser(name string, start, end *string, parser string) {\n\tif fr.DateTimeRanges == nil {\n\t\tfr.DateTimeRanges = make([]*dateTimeRange, 0, 1)\n\t}\n\tfr.DateTimeRanges = append(fr.DateTimeRanges,\n\t\t&dateTimeRange{Name: name, startString: start, endString: end, DateTimeParser: parser})\n}\n\n// AddNumericRange adds a bucket to a field\n// containing numeric values.  Documents with a\n// numeric value falling into this range are\n// tabulated as part of this bucket/range.\nfunc (fr *FacetRequest) AddNumericRange(name string, min, max *float64) {\n\tif fr.NumericRanges == nil {\n\t\tfr.NumericRanges = make([]*numericRange, 0, 1)\n\t}\n\tfr.NumericRanges = append(fr.NumericRanges, &numericRange{Name: name, Min: min, Max: max})\n}\n\n// FacetsRequest groups together all the\n// FacetRequest objects for a single query.\ntype FacetsRequest map[string]*FacetRequest\n\nfunc (fr FacetsRequest) Validate() error {\n\tfor _, v := range fr {\n\t\tif err := v.Validate(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// HighlightRequest describes how field matches\n// should be highlighted.\ntype HighlightRequest struct {\n\tStyle  *string  `json:\"style\"`\n\tFields []string `json:\"fields\"`\n}\n\n// NewHighlight creates a default\n// HighlightRequest.\nfunc NewHighlight() *HighlightRequest {\n\treturn &HighlightRequest{}\n}\n\n// NewHighlightWithStyle creates a HighlightRequest\n// with an alternate style.\nfunc NewHighlightWithStyle(style string) *HighlightRequest {\n\treturn &HighlightRequest{\n\t\tStyle: &style,\n\t}\n}\n\nfunc (h *HighlightRequest) AddField(field string) {\n\tif h.Fields == nil {\n\t\th.Fields = make([]string, 0, 1)\n\t}\n\th.Fields = append(h.Fields, field)\n}\n\nfunc (r *SearchRequest) Validate() error {\n\tif srq, ok := r.Query.(query.ValidatableQuery); ok {\n\t\terr := srq.Validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif r.SearchAfter != nil && r.SearchBefore != nil {\n\t\treturn fmt.Errorf(\"cannot use search after and search before together\")\n\t}\n\n\tif r.SearchAfter != nil {\n\t\tif r.From != 0 {\n\t\t\treturn fmt.Errorf(\"cannot use search after with from !=0\")\n\t\t}\n\t\tif len(r.SearchAfter) != len(r.Sort) {\n\t\t\treturn fmt.Errorf(\"search after must have same size as sort order\")\n\t\t}\n\t}\n\tif r.SearchBefore != nil {\n\t\tif r.From != 0 {\n\t\t\treturn fmt.Errorf(\"cannot use search before with from !=0\")\n\t\t}\n\t\tif len(r.SearchBefore) != len(r.Sort) {\n\t\t\treturn fmt.Errorf(\"search before must have same size as sort order\")\n\t\t}\n\t}\n\n\terr := r.validatePagination()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif IsScoreFusionRequested(r) {\n\t\tif r.SearchAfter != nil || r.SearchBefore != nil {\n\t\t\treturn fmt.Errorf(\"cannot use search after or search before with score fusion\")\n\t\t}\n\n\t\tif r.Sort != nil {\n\t\t\tif !reflect.DeepEqual(r.Sort, AllowedFusionSort) {\n\t\t\t\treturn fmt.Errorf(\"sort must be empty or descending order of score for score fusion\")\n\t\t\t}\n\t\t}\n\t}\n\n\terr = validateKNN(r)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn r.Facets.Validate()\n}\n\n// Validates SearchAfter/SearchBefore\nfunc (r *SearchRequest) validatePagination() error {\n\tvar pagination []string\n\tvar afterOrBefore string\n\n\tif r.SearchAfter != nil {\n\t\tpagination = r.SearchAfter\n\t\tafterOrBefore = \"search after\"\n\t} else if r.SearchBefore != nil {\n\t\tpagination = r.SearchBefore\n\t\tafterOrBefore = \"search before\"\n\t} else {\n\t\treturn nil\n\t}\n\n\tfor i := range pagination {\n\t\tswitch ss := r.Sort[i].(type) {\n\t\tcase *search.SortGeoDistance:\n\t\t\t_, err := strconv.ParseFloat(pagination[i], 64)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"invalid %s value for sort field '%s': '%s'. %s\", afterOrBefore, ss.Field, pagination[i], err)\n\t\t\t}\n\t\tcase *search.SortField:\n\t\t\tswitch ss.Type {\n\t\t\tcase search.SortFieldAsNumber:\n\t\t\t\t_, err := strconv.ParseFloat(pagination[i], 64)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"invalid %s value for sort field '%s': '%s'. %s\", afterOrBefore, ss.Field, pagination[i], err)\n\t\t\t\t}\n\t\t\tcase search.SortFieldAsDate:\n\t\t\t\t_, err := time.Parse(time.RFC3339Nano, pagination[i])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"invalid %s value for sort field '%s': '%s'. %s\", afterOrBefore, ss.Field, pagination[i], err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// AddFacet adds a FacetRequest to this SearchRequest\nfunc (r *SearchRequest) AddFacet(facetName string, f *FacetRequest) {\n\tif r.Facets == nil {\n\t\tr.Facets = make(FacetsRequest, 1)\n\t}\n\tr.Facets[facetName] = f\n}\n\n// SortBy changes the request to use the requested sort order\n// this form uses the simplified syntax with an array of strings\n// each string can either be a field name\n// or the magic value _id and _score which refer to the doc id and search score\n// any of these values can optionally be prefixed with - to reverse the order\nfunc (r *SearchRequest) SortBy(order []string) {\n\tso := search.ParseSortOrderStrings(order)\n\tr.Sort = so\n}\n\n// SortByCustom changes the request to use the requested sort order\nfunc (r *SearchRequest) SortByCustom(order search.SortOrder) {\n\tr.Sort = order\n}\n\n// SetSearchAfter sets the request to skip over hits with a sort\n// value less than the provided sort after key\nfunc (r *SearchRequest) SetSearchAfter(after []string) {\n\tr.SearchAfter = after\n}\n\n// SetSearchBefore sets the request to skip over hits with a sort\n// value greater than the provided sort before key\nfunc (r *SearchRequest) SetSearchBefore(before []string) {\n\tr.SearchBefore = before\n}\n\n// AddParams adds a RequestParams field to the search request\nfunc (r *SearchRequest) AddParams(params RequestParams) {\n\tr.Params = &params\n}\n\n// NewSearchRequest creates a new SearchRequest\n// for the Query, using default values for all\n// other search parameters.\nfunc NewSearchRequest(q query.Query) *SearchRequest {\n\treturn NewSearchRequestOptions(q, 10, 0, false)\n}\n\n// NewSearchRequestOptions creates a new SearchRequest\n// for the Query, with the requested size, from\n// and explanation search parameters.\n// By default results are ordered by score, descending.\nfunc NewSearchRequestOptions(q query.Query, size, from int, explain bool) *SearchRequest {\n\treturn &SearchRequest{\n\t\tQuery:   q,\n\t\tSize:    size,\n\t\tFrom:    from,\n\t\tExplain: explain,\n\t\tSort:    search.SortOrder{&search.SortScore{Desc: true}},\n\t}\n}\n\n// IndexErrMap tracks errors with the name of the index where it occurred\ntype IndexErrMap map[string]error\n\n// MarshalJSON serializes the error into a string for JSON consumption\nfunc (iem IndexErrMap) MarshalJSON() ([]byte, error) {\n\ttmp := make(map[string]string, len(iem))\n\tfor k, v := range iem {\n\t\ttmp[k] = v.Error()\n\t}\n\treturn util.MarshalJSON(tmp)\n}\n\nfunc (iem IndexErrMap) UnmarshalJSON(data []byte) error {\n\tvar tmp map[string]string\n\terr := util.UnmarshalJSON(data, &tmp)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor k, v := range tmp {\n\t\tiem[k] = fmt.Errorf(\"%s\", v)\n\t}\n\treturn nil\n}\n\n// SearchStatus is a section in the SearchResult reporting how many\n// underlying indexes were queried, how many were successful/failed\n// and a map of any errors that were encountered\ntype SearchStatus struct {\n\tTotal      int         `json:\"total\"`\n\tFailed     int         `json:\"failed\"`\n\tSuccessful int         `json:\"successful\"`\n\tErrors     IndexErrMap `json:\"errors,omitempty\"`\n}\n\n// Merge will merge together multiple SearchStatuses during a MultiSearch\nfunc (ss *SearchStatus) Merge(other *SearchStatus) {\n\tss.Total += other.Total\n\tss.Failed += other.Failed\n\tss.Successful += other.Successful\n\tif len(other.Errors) > 0 {\n\t\tif ss.Errors == nil {\n\t\t\tss.Errors = make(map[string]error)\n\t\t}\n\t\tfor otherIndex, otherError := range other.Errors {\n\t\t\tss.Errors[otherIndex] = otherError\n\t\t}\n\t}\n}\n\n// A SearchResult describes the results of executing\n// a SearchRequest.\n//\n// Status - Whether the search was executed on the underlying indexes successfully\n// or failed, and the corresponding errors.\n// Request - The SearchRequest that was executed.\n// Hits - The list of documents that matched the query and their corresponding\n// scores, score explanation, location info and so on.\n// Total - The total number of documents that matched the query.\n// Cost - indicates how expensive was the query with respect to bytes read\n// from the mapped index files.\n// MaxScore - The maximum score seen across all document hits seen for this query.\n// Took - The time taken to execute the search.\n// Facets - The facet results for the search.\ntype SearchResult struct {\n\tStatus   *SearchStatus                  `json:\"status\"`\n\tRequest  *SearchRequest                 `json:\"request,omitempty\"`\n\tHits     search.DocumentMatchCollection `json:\"hits\"`\n\tTotal    uint64                         `json:\"total_hits\"`\n\tCost     uint64                         `json:\"cost\"`\n\tMaxScore float64                        `json:\"max_score\"`\n\tTook     time.Duration                  `json:\"took\"`\n\tFacets   search.FacetResults            `json:\"facets\"`\n\t// special fields that are applicable only for search\n\t// results that are obtained from a presearch\n\tSynonymResult search.FieldTermSynonymMap `json:\"synonym_result,omitempty\"`\n\n\t// The following fields are applicable to BM25 preSearch\n\tBM25Stats *search.BM25Stats `json:\"bm25_stats,omitempty\"`\n}\n\nfunc (sr *SearchResult) Size() int {\n\tsizeInBytes := reflectStaticSizeSearchResult + size.SizeOfPtr +\n\t\treflectStaticSizeSearchStatus\n\n\tfor _, entry := range sr.Hits {\n\t\tif entry != nil {\n\t\t\tsizeInBytes += entry.Size()\n\t\t}\n\t}\n\n\tfor k, v := range sr.Facets {\n\t\tsizeInBytes += size.SizeOfString + len(k) +\n\t\t\tv.Size()\n\t}\n\n\treturn sizeInBytes\n}\n\nfunc (sr *SearchResult) String() string {\n\trv := &strings.Builder{}\n\tif sr.Total > 0 {\n\t\tswitch {\n\t\tcase sr.Request != nil && sr.Request.Size > 0:\n\t\t\tstart := sr.Request.From + 1\n\t\t\tend := sr.Request.From + len(sr.Hits)\n\t\t\tfmt.Fprintf(rv, \"%d matches, showing %d through %d, took %s\\n\", sr.Total, start, end, sr.Took)\n\t\t\tfor i, hit := range sr.Hits {\n\t\t\t\trv = formatHit(rv, hit, start+i)\n\t\t\t}\n\t\tcase sr.Request == nil:\n\t\t\tfmt.Fprintf(rv, \"%d matches, took %s\\n\", sr.Total, sr.Took)\n\t\t\tfor i, hit := range sr.Hits {\n\t\t\t\trv = formatHit(rv, hit, i+1)\n\t\t\t}\n\t\tdefault:\n\t\t\tfmt.Fprintf(rv, \"%d matches, took %s\\n\", sr.Total, sr.Took)\n\t\t}\n\t} else {\n\t\tfmt.Fprintf(rv, \"No matches\\n\")\n\t}\n\tif len(sr.Facets) > 0 {\n\t\tfmt.Fprintf(rv, \"Facets:\\n\")\n\t\tfor fn, f := range sr.Facets {\n\t\t\tfmt.Fprintf(rv, \"%s(%d)\\n\", fn, f.Total)\n\t\t\tfor _, t := range f.Terms.Terms() {\n\t\t\t\tfmt.Fprintf(rv, \"\\t%s(%d)\\n\", t.Term, t.Count)\n\t\t\t}\n\t\t\tfor _, n := range f.NumericRanges {\n\t\t\t\tfmt.Fprintf(rv, \"\\t%s(%d)\\n\", n.Name, n.Count)\n\t\t\t}\n\t\t\tfor _, d := range f.DateRanges {\n\t\t\t\tfmt.Fprintf(rv, \"\\t%s(%d)\\n\", d.Name, d.Count)\n\t\t\t}\n\t\t\tif f.Other != 0 {\n\t\t\t\tfmt.Fprintf(rv, \"\\tOther(%d)\\n\", f.Other)\n\t\t\t}\n\t\t}\n\t}\n\treturn rv.String()\n}\n\n// formatHit is a helper function to format a single hit in the search result for\n// the String() method of SearchResult\nfunc formatHit(rv *strings.Builder, hit *search.DocumentMatch, hitNumber int) *strings.Builder {\n\tfmt.Fprintf(rv, \"%5d. %s (%f)\\n\", hitNumber, hit.ID, hit.Score)\n\tfor fragmentField, fragments := range hit.Fragments {\n\t\tfmt.Fprintf(rv, \"\\t%s\\n\", fragmentField)\n\t\tfor _, fragment := range fragments {\n\t\t\tfmt.Fprintf(rv, \"\\t\\t%s\\n\", fragment)\n\t\t}\n\t}\n\tfor otherFieldName, otherFieldValue := range hit.Fields {\n\t\tif otherFieldName == NestedDocumentKey {\n\t\t\tcontinue\n\t\t}\n\t\tif _, ok := hit.Fragments[otherFieldName]; !ok {\n\t\t\tfmt.Fprintf(rv, \"\\t%s\\n\", otherFieldName)\n\t\t\tfmt.Fprintf(rv, \"\\t\\t%v\\n\", otherFieldValue)\n\t\t}\n\t}\n\t// nested documents\n\tif nested, ok := hit.Fields[NestedDocumentKey]; ok {\n\t\tif list, ok := nested.([]*search.NestedDocumentMatch); ok {\n\t\t\tfmt.Fprintf(rv, \"\\t%s (%d nested documents)\\n\", NestedDocumentKey, len(list))\n\t\t\tfor ni, nd := range list {\n\t\t\t\tfmt.Fprintf(rv, \"\\t\\tNested #%d:\\n\", ni+1)\n\t\t\t\tfor f, frags := range nd.Fragments {\n\t\t\t\t\tfmt.Fprintf(rv, \"\\t\\t\\t%s\\n\", f)\n\t\t\t\t\tfor _, frag := range frags {\n\t\t\t\t\t\tfmt.Fprintf(rv, \"\\t\\t\\t\\t%s\\n\", frag)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfor f, v := range nd.Fields {\n\t\t\t\t\tif _, ok := nd.Fragments[f]; !ok {\n\t\t\t\t\t\tfmt.Fprintf(rv, \"\\t\\t\\t%s\\n\", f)\n\t\t\t\t\t\tfmt.Fprintf(rv, \"\\t\\t\\t\\t%v\\n\", v)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif len(hit.DecodedSort) > 0 {\n\t\tfmt.Fprintf(rv, \"\\t_sort: [\")\n\t\tfor k, v := range hit.DecodedSort {\n\t\t\tif k > 0 {\n\t\t\t\tfmt.Fprintf(rv, \", \")\n\t\t\t}\n\t\t\tfmt.Fprintf(rv, \"%v\", v)\n\t\t}\n\t\tfmt.Fprintf(rv, \"]\\n\")\n\t}\n\treturn rv\n}\n\n// Merge will merge together multiple SearchResults during a MultiSearch\nfunc (sr *SearchResult) Merge(other *SearchResult) {\n\tsr.Status.Merge(other.Status)\n\tsr.Hits = append(sr.Hits, other.Hits...)\n\tsr.Total += other.Total\n\tsr.Cost += other.Cost\n\tif other.MaxScore > sr.MaxScore {\n\t\tsr.MaxScore = other.MaxScore\n\t}\n\tif sr.Facets == nil && len(other.Facets) != 0 {\n\t\tsr.Facets = other.Facets\n\t\treturn\n\t}\n\n\tsr.Facets.Merge(other.Facets)\n}\n\n// MemoryNeededForSearchResult is an exported helper function to determine the RAM\n// needed to accommodate the results for a given search request.\nfunc MemoryNeededForSearchResult(req *SearchRequest) uint64 {\n\tif req == nil {\n\t\treturn 0\n\t}\n\n\tnumDocMatches := req.Size + req.From\n\tif req.Size+req.From > collector.PreAllocSizeSkipCap {\n\t\tnumDocMatches = collector.PreAllocSizeSkipCap\n\t}\n\n\testimate := 0\n\n\t// overhead from the SearchResult structure\n\tvar sr SearchResult\n\testimate += sr.Size()\n\n\tvar dm search.DocumentMatch\n\tsizeOfDocumentMatch := dm.Size()\n\n\t// overhead from results\n\testimate += numDocMatches * sizeOfDocumentMatch\n\n\t// overhead from facet results\n\tif req.Facets != nil {\n\t\tvar fr search.FacetResult\n\t\testimate += len(req.Facets) * fr.Size()\n\t}\n\n\t// overhead from fields, highlighting\n\tvar d document.Document\n\tif len(req.Fields) > 0 || req.Highlight != nil {\n\t\tnumDocsApplicable := req.Size\n\t\tif numDocsApplicable > collector.PreAllocSizeSkipCap {\n\t\t\tnumDocsApplicable = collector.PreAllocSizeSkipCap\n\t\t}\n\t\testimate += numDocsApplicable * d.Size()\n\t}\n\n\treturn uint64(estimate)\n}\n\n// SetSortFunc sets the sort implementation to use when sorting hits.\n//\n// SearchRequests can specify a custom sort implementation to meet\n// their needs. For instance, by specifying a parallel sort\n// that uses all available cores.\nfunc (r *SearchRequest) SetSortFunc(s func(sort.Interface)) {\n\tr.sortFunc = s\n}\n\n// SortFunc returns the sort implementation to use when sorting hits.\n// Defaults to sort.Sort.\nfunc (r *SearchRequest) SortFunc() func(data sort.Interface) {\n\tif r.sortFunc != nil {\n\t\treturn r.sortFunc\n\t}\n\n\treturn sort.Sort\n}\n\nfunc isMatchNoneQuery(q query.Query) bool {\n\t_, ok := q.(*query.MatchNoneQuery)\n\treturn ok\n}\n\nfunc isMatchAllQuery(q query.Query) bool {\n\t_, ok := q.(*query.MatchAllQuery)\n\treturn ok\n}\n\n// Checks if the request is hybrid search. Currently supports: RRF, RSF.\nfunc IsScoreFusionRequested(req *SearchRequest) bool {\n\tswitch req.Score {\n\tcase ScoreRRF, ScoreRSF:\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n\n// Additional parameters in the search request. Currently only being\n// used for score fusion parameters.\ntype RequestParams struct {\n\tScoreRankConstant int `json:\"score_rank_constant,omitempty\"`\n\tScoreWindowSize   int `json:\"score_window_size,omitempty\"`\n}\n\nfunc NewDefaultParams(from, size int) *RequestParams {\n\treturn &RequestParams{\n\t\tScoreRankConstant: DefaultScoreRankConstant,\n\t\tScoreWindowSize:   from + size,\n\t}\n}\n\nfunc (p *RequestParams) UnmarshalJSON(input []byte) error {\n\tvar temp struct {\n\t\tScoreRankConstant *int `json:\"score_rank_constant,omitempty\"`\n\t\tScoreWindowSize   *int `json:\"score_window_size,omitempty\"`\n\t}\n\n\tif err := util.UnmarshalJSON(input, &temp); err != nil {\n\t\treturn err\n\t}\n\n\tif temp.ScoreRankConstant != nil {\n\t\tp.ScoreRankConstant = *temp.ScoreRankConstant\n\t}\n\n\tif temp.ScoreWindowSize != nil {\n\t\tp.ScoreWindowSize = *temp.ScoreWindowSize\n\t}\n\n\treturn nil\n}\n\nfunc (p *RequestParams) Validate(size int) error {\n\tif p.ScoreWindowSize < 1 {\n\t\treturn fmt.Errorf(\"score window size must be greater than 0\")\n\t} else if p.ScoreWindowSize < size {\n\t\treturn fmt.Errorf(\"score window size must be greater than or equal to Size (%d)\", size)\n\t}\n\n\treturn nil\n}\n\nfunc ParseParams(r *SearchRequest, input []byte) (*RequestParams, error) {\n\tparams := NewDefaultParams(r.From, r.Size)\n\tif len(input) == 0 {\n\t\treturn params, nil\n\t}\n\n\terr := util.UnmarshalJSON(input, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// validate params\n\terr = params.Validate(r.Size)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn params, nil\n}\n"
  },
  {
    "path": "search_knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/collector\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nconst supportForVectorSearch = true\n\ntype knnOperator string\n\n// Must be updated only at init\nvar BleveMaxK = int64(10000)\n\ntype SearchRequest struct {\n\tClientContextID  string            `json:\"client_context_id,omitempty\"`\n\tQuery            query.Query       `json:\"query\"`\n\tSize             int               `json:\"size\"`\n\tFrom             int               `json:\"from\"`\n\tHighlight        *HighlightRequest `json:\"highlight\"`\n\tFields           []string          `json:\"fields\"`\n\tFacets           FacetsRequest     `json:\"facets\"`\n\tExplain          bool              `json:\"explain\"`\n\tSort             search.SortOrder  `json:\"sort\"`\n\tIncludeLocations bool              `json:\"includeLocations\"`\n\tScore            string            `json:\"score,omitempty\"`\n\tSearchAfter      []string          `json:\"search_after\"`\n\tSearchBefore     []string          `json:\"search_before\"`\n\n\tKNN         []*KNNRequest `json:\"knn\"`\n\tKNNOperator knnOperator   `json:\"knn_operator\"`\n\n\t// PreSearchData will be a  map that will be used\n\t// in the second phase of any 2-phase search, to provide additional\n\t// context to the second phase. This is useful in the case of index\n\t// aliases where the first phase will gather the PreSearchData from all\n\t// the indexes in the alias, and the second phase will use that\n\t// PreSearchData to perform the actual search.\n\t// The currently accepted map configuration is:\n\t//\n\t// \"_knn_pre_search_data_key\": []*search.DocumentMatch\n\n\tPreSearchData map[string]interface{} `json:\"pre_search_data,omitempty\"`\n\n\tParams *RequestParams `json:\"params,omitempty\"`\n\n\tsortFunc func(sort.Interface)\n}\n\n// Vector takes precedence over vectorBase64 in case both fields are given\ntype KNNRequest struct {\n\tField        string       `json:\"field\"`\n\tVector       []float32    `json:\"vector\"`\n\tVectorBase64 string       `json:\"vector_base64\"`\n\tK            int64        `json:\"k\"`\n\tBoost        *query.Boost `json:\"boost,omitempty\"`\n\n\t// Search parameters for the field's vector index part of the segment.\n\t// Value of it depends on the field's backing vector index implementation.\n\t//\n\t// For Faiss IVF index, supported search params are:\n\t//  - ivf_nprobe_pct    : int  // percentage of total clusters to search\n\t//  - ivf_max_codes_pct : float // percentage of total vectors to visit to do a query (across all clusters)\n\t//\n\t// Consult go-faiss to know all supported search params\n\tParams json.RawMessage `json:\"params\"`\n\n\t// Filter query to use with kNN pre-filtering.\n\t// Supports pre-filtering with all existing types of query clauses.\n\tFilterQuery query.Query `json:\"filter,omitempty\"`\n}\n\nfunc (r *SearchRequest) AddKNN(field string, vector []float32, k int64, boost float64) {\n\tb := query.Boost(boost)\n\tr.KNN = append(r.KNN, &KNNRequest{\n\t\tField:  field,\n\t\tVector: vector,\n\t\tK:      k,\n\t\tBoost:  &b,\n\t})\n}\n\nfunc (r *SearchRequest) AddKNNWithFilter(field string, vector []float32, k int64,\n\tboost float64, filterQuery query.Query) {\n\tb := query.Boost(boost)\n\tr.KNN = append(r.KNN, &KNNRequest{\n\t\tField:       field,\n\t\tVector:      vector,\n\t\tK:           k,\n\t\tBoost:       &b,\n\t\tFilterQuery: filterQuery,\n\t})\n}\n\nfunc (r *SearchRequest) AddKNNOperator(operator knnOperator) {\n\tr.KNNOperator = operator\n}\n\n// UnmarshalJSON deserializes a JSON representation of\n// a SearchRequest\nfunc (r *SearchRequest) UnmarshalJSON(input []byte) error {\n\ttype tempKNNReq struct {\n\t\tField        string          `json:\"field\"`\n\t\tVector       []float32       `json:\"vector\"`\n\t\tVectorBase64 string          `json:\"vector_base64\"`\n\t\tK            int64           `json:\"k\"`\n\t\tBoost        *query.Boost    `json:\"boost,omitempty\"`\n\t\tParams       json.RawMessage `json:\"params\"`\n\t\tFilterQuery  json.RawMessage `json:\"filter,omitempty\"`\n\t}\n\n\tvar temp struct {\n\t\tQ                json.RawMessage   `json:\"query\"`\n\t\tSize             *int              `json:\"size\"`\n\t\tFrom             int               `json:\"from\"`\n\t\tHighlight        *HighlightRequest `json:\"highlight\"`\n\t\tFields           []string          `json:\"fields\"`\n\t\tFacets           FacetsRequest     `json:\"facets\"`\n\t\tExplain          bool              `json:\"explain\"`\n\t\tSort             []json.RawMessage `json:\"sort\"`\n\t\tIncludeLocations bool              `json:\"includeLocations\"`\n\t\tScore            string            `json:\"score\"`\n\t\tSearchAfter      []string          `json:\"search_after\"`\n\t\tSearchBefore     []string          `json:\"search_before\"`\n\t\tKNN              []*tempKNNReq     `json:\"knn\"`\n\t\tKNNOperator      knnOperator       `json:\"knn_operator\"`\n\t\tPreSearchData    json.RawMessage   `json:\"pre_search_data\"`\n\t\tParams           json.RawMessage   `json:\"params\"`\n\t}\n\n\terr := json.Unmarshal(input, &temp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif temp.Size == nil {\n\t\tr.Size = 10\n\t} else {\n\t\tr.Size = *temp.Size\n\t}\n\tif temp.Sort == nil {\n\t\tr.Sort = search.SortOrder{&search.SortScore{Desc: true}}\n\t} else {\n\t\tr.Sort, err = search.ParseSortOrderJSON(temp.Sort)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tr.From = temp.From\n\tr.Explain = temp.Explain\n\tr.Highlight = temp.Highlight\n\tr.Fields = temp.Fields\n\tr.Facets = temp.Facets\n\tr.IncludeLocations = temp.IncludeLocations\n\tr.Score = temp.Score\n\tr.SearchAfter = temp.SearchAfter\n\tr.SearchBefore = temp.SearchBefore\n\tr.Query, err = query.ParseQuery(temp.Q)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif r.Size < 0 {\n\t\tr.Size = 10\n\t}\n\tif r.From < 0 {\n\t\tr.From = 0\n\t}\n\n\tif IsScoreFusionRequested(r) {\n\t\tif temp.Params == nil {\n\t\t\t// If params is not present and it is requires rescoring, assign\n\t\t\t// default values\n\t\t\tr.Params = NewDefaultParams(r.From, r.Size)\n\t\t} else {\n\t\t\t// if it is a request that requires rescoring, parse the rescoring\n\t\t\t// parameters.\n\t\t\tparams, err := ParseParams(r, temp.Params)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tr.Params = params\n\t\t}\n\t}\n\n\tr.KNN = make([]*KNNRequest, len(temp.KNN))\n\tfor i, knnReq := range temp.KNN {\n\t\tr.KNN[i] = &KNNRequest{}\n\t\tr.KNN[i].Field = temp.KNN[i].Field\n\t\tr.KNN[i].Vector = temp.KNN[i].Vector\n\t\tr.KNN[i].VectorBase64 = temp.KNN[i].VectorBase64\n\t\tr.KNN[i].K = temp.KNN[i].K\n\t\tr.KNN[i].Boost = temp.KNN[i].Boost\n\t\tr.KNN[i].Params = temp.KNN[i].Params\n\t\tif len(knnReq.FilterQuery) == 0 {\n\t\t\t// Setting this to nil to avoid ParseQuery() setting it to a match none\n\t\t\tr.KNN[i].FilterQuery = nil\n\t\t} else {\n\t\t\tr.KNN[i].FilterQuery, err = query.ParseQuery(knnReq.FilterQuery)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\tr.KNNOperator = temp.KNNOperator\n\tif r.KNNOperator == \"\" {\n\t\tr.KNNOperator = knnOperatorOr\n\t}\n\n\tif temp.PreSearchData != nil {\n\t\tr.PreSearchData, err = query.ParsePreSearchData(temp.PreSearchData)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n\n}\n\n// -----------------------------------------------------------------------------\n\nfunc copySearchRequest(req *SearchRequest, preSearchData map[string]interface{}) *SearchRequest {\n\trv := SearchRequest{\n\t\tQuery:            req.Query,\n\t\tSize:             req.Size + req.From,\n\t\tFrom:             0,\n\t\tHighlight:        req.Highlight,\n\t\tFields:           req.Fields,\n\t\tFacets:           req.Facets,\n\t\tExplain:          req.Explain,\n\t\tSort:             req.Sort.Copy(),\n\t\tIncludeLocations: req.IncludeLocations,\n\t\tScore:            req.Score,\n\t\tSearchAfter:      req.SearchAfter,\n\t\tSearchBefore:     req.SearchBefore,\n\t\tKNN:              req.KNN,\n\t\tKNNOperator:      req.KNNOperator,\n\t\tPreSearchData:    preSearchData,\n\t\tParams:           req.Params,\n\t}\n\treturn &rv\n\n}\n\nvar (\n\tknnOperatorAnd = knnOperator(\"and\")\n\tknnOperatorOr  = knnOperator(\"or\")\n)\n\nfunc createKNNQuery(req *SearchRequest, knnFilterResults map[int]index.EligibleDocumentSelector) (\n\tquery.Query, []int64, int64, error) {\n\tif requestHasKNN(req) {\n\t\t// first perform validation\n\t\terr := validateKNN(req)\n\t\tif err != nil {\n\t\t\treturn nil, nil, 0, err\n\t\t}\n\t\tvar subQueries []query.Query\n\t\tkArray := make([]int64, 0, len(req.KNN))\n\t\tsumOfK := int64(0)\n\t\tfor i, knn := range req.KNN {\n\t\t\t// If it's a filtered kNN but has no eligible filter hits, then\n\t\t\t// do not run the kNN query.\n\t\t\tif selector, exists := knnFilterResults[i]; exists && selector == nil {\n\t\t\t\t// if the kNN query is filtered and has no eligible filter hits, then\n\t\t\t\t// do not run the kNN query, so we add a match_none query to the subQueries.\n\t\t\t\t// this will ensure that the score breakdown is set to 0 for this kNN query.\n\t\t\t\tsubQueries = append(subQueries, NewMatchNoneQuery())\n\t\t\t\tkArray = append(kArray, 0)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tknnQuery := query.NewKNNQuery(knn.Vector)\n\t\t\tknnQuery.SetField(knn.Field)\n\t\t\tknnQuery.SetK(knn.K)\n\t\t\tknnQuery.SetBoost(knn.Boost.Value())\n\t\t\tknnQuery.SetParams(knn.Params)\n\t\t\tif selector, exists := knnFilterResults[i]; exists {\n\t\t\t\tknnQuery.SetEligibleSelector(selector)\n\t\t\t}\n\t\t\tsubQueries = append(subQueries, knnQuery)\n\t\t\tkArray = append(kArray, knn.K)\n\t\t\tsumOfK += knn.K\n\t\t}\n\t\trv := query.NewDisjunctionQuery(subQueries)\n\t\trv.RetrieveScoreBreakdown(true)\n\t\treturn rv, kArray, sumOfK, nil\n\t}\n\treturn nil, nil, 0, nil\n}\n\nfunc validateKNN(req *SearchRequest) error {\n\tfor _, q := range req.KNN {\n\t\tif q == nil {\n\t\t\treturn fmt.Errorf(\"knn query cannot be nil\")\n\t\t}\n\t\tif len(q.Vector) == 0 && q.VectorBase64 != \"\" {\n\t\t\t// consider vector_base64 only if vector is not provided\n\t\t\tdecodedVector, err := document.DecodeVector(q.VectorBase64)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tq.Vector = decodedVector\n\t\t}\n\t\tif q.K <= 0 || len(q.Vector) == 0 {\n\t\t\treturn fmt.Errorf(\"k must be greater than 0 and vector must be non-empty\")\n\t\t}\n\t\tif q.K > BleveMaxK {\n\t\t\treturn fmt.Errorf(\"k must be less than %d\", BleveMaxK)\n\t\t}\n\t\t// since the DefaultField is not applicable for knn,\n\t\t// the field must be specified.\n\t\tif q.Field == \"\" {\n\t\t\treturn fmt.Errorf(\"knn query field must be non-empty\")\n\t\t}\n\t\tif vfq, ok := q.FilterQuery.(query.ValidatableQuery); ok {\n\t\t\terr := vfq.Validate()\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"knn filter query is invalid: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\tswitch req.KNNOperator {\n\tcase knnOperatorAnd, knnOperatorOr, \"\":\n\t\t// Valid cases, do nothing\n\tdefault:\n\t\treturn fmt.Errorf(\"knn_operator must be either 'and' / 'or'\")\n\t}\n\n\treturn nil\n}\n\nfunc addSortAndFieldsToKNNHits(req *SearchRequest, knnHits []*search.DocumentMatch, reader index.IndexReader, name string) (err error) {\n\trequiredSortFields := req.Sort.RequiredFields()\n\tvar dvReader index.DocValueReader\n\tvar updateFieldVisitor index.DocValueVisitor\n\tif len(requiredSortFields) > 0 {\n\t\tdvReader, err = reader.DocValueReader(requiredSortFields)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tupdateFieldVisitor = func(field string, term []byte) {\n\t\t\treq.Sort.UpdateVisitor(field, term)\n\t\t}\n\t}\n\tfor _, hit := range knnHits {\n\t\tif len(requiredSortFields) > 0 {\n\t\t\terr = dvReader.VisitDocValues(hit.IndexInternalID, updateFieldVisitor)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treq.Sort.Value(hit)\n\t\terr, _ = LoadAndHighlightAllFields(hit, req, \"\", reader, nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\thit.Index = name\n\t}\n\treturn nil\n}\n\nfunc (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader, preSearch bool) (knnHits []*search.DocumentMatch, err error) {\n\t// Maps the index of a KNN query in the request to its pre-filter result:\n\t// - If the KNN query is **not filtered**, the value will be `nil`.\n\t// - If the KNN query **is filtered**, the value will be an eligible document selector\n\t//   that can be used to retrieve eligible documents.\n\t// - If there is an **empty entry** for a KNN query, it means no documents match\n\t//   the filter query, and the KNN query can be skipped.\n\tknnFilterResults := make(map[int]index.EligibleDocumentSelector)\n\tfor idx, knnReq := range req.KNN {\n\t\tfilterQ := knnReq.FilterQuery\n\t\tif filterQ == nil || isMatchAllQuery(filterQ) {\n\t\t\t// When there is no filter query or the filter query is match_all,\n\t\t\t// all documents are eligible, and can be treated as unfiltered query.\n\t\t\tcontinue\n\t\t} else if isMatchNoneQuery(filterQ) {\n\t\t\t// If the filter query is match_none, then no documents match the filter query.\n\t\t\tknnFilterResults[idx] = nil\n\t\t\tcontinue\n\t\t}\n\t\t// Applies to all supported types of queries.\n\t\tfilterSearcher, err := filterQ.Searcher(ctx, reader, i.m, search.SearcherOptions{\n\t\t\tScore: \"none\", // just want eligible hits --> don't compute scores if not needed\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// Using the index doc count to determine collector size since we do not\n\t\t// have an estimate of the number of eligible docs in the index yet.\n\t\tindexDocCount, err := i.DocCount()\n\t\tif err != nil {\n\t\t\t// close the searcher before returning\n\t\t\tfilterSearcher.Close()\n\t\t\treturn nil, err\n\t\t}\n\t\tfilterColl := collector.NewEligibleCollector(int(indexDocCount))\n\t\terr = filterColl.Collect(ctx, filterSearcher, reader)\n\t\tif err != nil {\n\t\t\t// close the searcher before returning\n\t\t\tfilterSearcher.Close()\n\t\t\treturn nil, err\n\t\t}\n\t\tknnFilterResults[idx] = filterColl.EligibleSelector()\n\t\t// Close the filter searcher, as we are done with it.\n\t\terr = filterSearcher.Close()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// Add the filter hits when creating the kNN query\n\tKNNQuery, kArray, sumOfK, err := createKNNQuery(req, knnFilterResults)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tknnSearcher, err := KNNQuery.Searcher(ctx, reader, i.m, search.SearcherOptions{\n\t\tExplain: req.Explain,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer func() {\n\t\tif serr := knnSearcher.Close(); err == nil && serr != nil {\n\t\t\terr = serr\n\t\t}\n\t}()\n\tknnCollector := collector.NewKNNCollector(kArray, sumOfK)\n\terr = knnCollector.Collect(ctx, knnSearcher, reader)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tknnHits = knnCollector.Results()\n\tif !preSearch {\n\t\tknnHits = finalizeKNNResults(req, knnHits)\n\t}\n\t// at this point, irrespective of whether it is a preSearch or not,\n\t// the knn hits are populated with Sort and Fields.\n\t// it must be ensured downstream that the Sort and Fields are not\n\t// re-evaluated, for these hits.\n\t// also add the index names to the hits, so that when early\n\t// exit takes place after the first phase, the hits will have\n\t// a valid value for Index.\n\terr = addSortAndFieldsToKNNHits(req, knnHits, reader, i.name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn knnHits, nil\n}\n\nfunc setKnnHitsInCollector(knnHits []*search.DocumentMatch, coll *collector.TopNCollector) {\n\tif len(knnHits) > 0 {\n\t\tmergeFn := func(ftsMatch *search.DocumentMatch, knnMatch *search.DocumentMatch) {\n\t\t\t// Boost the FTS score using the KNN score\n\t\t\tftsMatch.Score += knnMatch.Score\n\t\t\t// Combine the FTS explanation with the KNN explanation, if present\n\t\t\tftsMatch.Expl.MergeWith(knnMatch.Expl)\n\t\t}\n\t\tcoll.SetKNNHits(knnHits, search.HybridMergeCallbackFn(mergeFn))\n\t}\n}\n\nfunc finalizeKNNResults(req *SearchRequest, knnHits []*search.DocumentMatch) []*search.DocumentMatch {\n\t// if the KNN operator is AND, then we need to filter out the hits that\n\t// do not have match the KNN queries.\n\tif req.KNNOperator == knnOperatorAnd {\n\t\tidx := 0\n\t\tfor _, hit := range knnHits {\n\t\t\tif len(hit.ScoreBreakdown) == len(req.KNN) {\n\t\t\t\tknnHits[idx] = hit\n\t\t\t\tidx++\n\t\t\t}\n\t\t}\n\t\tknnHits = knnHits[:idx]\n\t}\n\n\t// if score fusion required, return early because\n\t// score breakdown is retained\n\tif IsScoreFusionRequested(req) {\n\t\treturn knnHits\n\t}\n\t// fix the score using score breakdown now\n\t// if the score is none, then we need to set the score to 0.0\n\t// if req.Explain is true, then we need to use the expl breakdown to\n\t// finalize the correct explanation.\n\tfor _, hit := range knnHits {\n\t\thit.Score = 0.0\n\t\tif req.Score != \"none\" {\n\t\t\tfor _, score := range hit.ScoreBreakdown {\n\t\t\t\thit.Score += score\n\t\t\t}\n\t\t}\n\t\tif req.Explain {\n\t\t\tchildrenExpl := make([]*search.Explanation, 0, len(hit.ScoreBreakdown))\n\t\t\tfor i := range hit.ScoreBreakdown {\n\t\t\t\tchildrenExpl = append(childrenExpl, hit.Expl.Children[i])\n\t\t\t}\n\t\t\thit.Expl = &search.Explanation{Value: hit.Score, Message: \"sum of:\", Children: childrenExpl}\n\t\t}\n\t\t// we don't need the score breakdown anymore\n\t\t// so we can set it to nil\n\t\thit.ScoreBreakdown = nil\n\t}\n\treturn knnHits\n}\n\n// when we are setting KNN hits in the preSearchData, we need to make sure that\n// the KNN hit goes to the right index. This is because the KNN hits are\n// collected from all the indexes in the alias, but the preSearchData is\n// specific to each index. If alias A1 contains indexes I1 and I2 and\n// the KNN hits collected from both I1 and I2, and merged to get top K\n// hits, then the top K hits need to be distributed to I1 and I2,\n// so that the preSearchData for I1 contains the top K hits from I1 and\n// the preSearchData for I2 contains the top K hits from I2.\nfunc validateAndDistributeKNNHits(knnHits []*search.DocumentMatch, indexes []Index) (map[string][]*search.DocumentMatch, error) {\n\t// create a set of all the index names of this alias\n\tindexNames := make(map[string]struct{}, len(indexes))\n\tfor _, index := range indexes {\n\t\tindexNames[index.Name()] = struct{}{}\n\t}\n\tsegregatedKnnHits := make(map[string][]*search.DocumentMatch)\n\tfor _, hit := range knnHits {\n\t\t// for each hit, we need to perform a validation check to ensure that the stack\n\t\t// is still valid.\n\t\t//\n\t\t// if the stack is empty, then we have an inconsistency/abnormality\n\t\t// since any hit with an empty stack is supposed to land on a leaf index,\n\t\t// and not an alias. This cannot happen in normal circumstances. But\n\t\t// performing this check to be safe. Since we extract the stack top\n\t\t// in the following steps.\n\t\tif len(hit.IndexNames) == 0 {\n\t\t\treturn nil, ErrorTwoPhaseSearchInconsistency\n\t\t}\n\t\t// since the stack is not empty, we need to check if the top of the stack\n\t\t// is a valid index name, of an index that is part of this alias. If not,\n\t\t// then we have an inconsistency that could be caused due to a topology\n\t\t// change.\n\t\tstackTopIdx := len(hit.IndexNames) - 1\n\t\ttop := hit.IndexNames[stackTopIdx]\n\t\tif _, exists := indexNames[top]; !exists {\n\t\t\treturn nil, ErrorTwoPhaseSearchInconsistency\n\t\t}\n\t\tif stackTopIdx == 0 {\n\t\t\t// if the stack consists of only one index, then popping the top\n\t\t\t// would result in an empty slice, and handle this case by setting\n\t\t\t// indexNames to nil. So that the final search results will not\n\t\t\t// contain the indexNames field.\n\t\t\thit.IndexNames = nil\n\t\t} else {\n\t\t\thit.IndexNames = hit.IndexNames[:stackTopIdx]\n\t\t}\n\t\tsegregatedKnnHits[top] = append(segregatedKnnHits[top], hit)\n\t}\n\treturn segregatedKnnHits, nil\n}\n\nfunc requestHasKNN(req *SearchRequest) bool {\n\treturn len(req.KNN) > 0\n}\n\nfunc numKNNQueries(req *SearchRequest) int {\n\treturn len(req.KNN)\n}\n\n// returns true if the search request contains a KNN request that can be\n// satisfied by just performing a preSearch, completely bypassing the\n// actual search.\nfunc isKNNrequestSatisfiedByPreSearch(req *SearchRequest) bool {\n\t// if req.Query is not match_none => then we need to go to phase 2\n\t// to perform the actual query.\n\tif !isMatchNoneQuery(req.Query) {\n\t\treturn false\n\t}\n\t// req.Query is a match_none query\n\t//\n\t// if request contains facets, we need to perform phase 2 to calculate\n\t// the facet result. Since documents were removed as part of the\n\t// merging process after phase 1, if the facet results were to be calculated\n\t// during phase 1, then they will be now be incorrect, since merging would\n\t// remove some documents.\n\tif req.Facets != nil {\n\t\treturn false\n\t}\n\t// the request is a match_none query and does not contain any facets\n\t// so we can satisfy the request using just the preSearch result.\n\treturn true\n}\n\nfunc constructKnnPreSearchData(mergedOut map[string]map[string]interface{}, preSearchResult *SearchResult,\n\tindexes []Index) (map[string]map[string]interface{}, error) {\n\n\tdistributedHits, err := validateAndDistributeKNNHits([]*search.DocumentMatch(preSearchResult.Hits), indexes)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, index := range indexes {\n\t\tmergedOut[index.Name()][search.KnnPreSearchDataKey] = distributedHits[index.Name()]\n\t}\n\treturn mergedOut, nil\n}\n\nfunc addKnnToDummyRequest(dummyReq *SearchRequest, realReq *SearchRequest) {\n\tdummyReq.KNN = realReq.KNN\n\tdummyReq.KNNOperator = knnOperatorOr\n\tdummyReq.Explain = realReq.Explain\n\tdummyReq.Fields = realReq.Fields\n\tdummyReq.Sort = realReq.Sort\n}\n\nfunc newKnnPreSearchResultProcessor(req *SearchRequest) *knnPreSearchResultProcessor {\n\tkArray := make([]int64, len(req.KNN))\n\tfor i, knnReq := range req.KNN {\n\t\tkArray[i] = knnReq.K\n\t}\n\tknnStore := collector.GetNewKNNCollectorStore(kArray)\n\treturn &knnPreSearchResultProcessor{\n\t\taddFn: func(sr *SearchResult, indexName string) {\n\t\t\tfor _, hit := range sr.Hits {\n\t\t\t\t// tag the hit with the index name, so that when the\n\t\t\t\t// final search result is constructed, the hit will have\n\t\t\t\t// a valid path to follow along the alias tree to reach\n\t\t\t\t// the index.\n\t\t\t\thit.IndexNames = append(hit.IndexNames, indexName)\n\t\t\t\tknnStore.AddDocument(hit)\n\t\t\t}\n\t\t},\n\t\tfinalizeFn: func(sr *SearchResult) {\n\t\t\t// passing nil as the document fixup function, because we don't need to\n\t\t\t// fixup the document, since this was already done in the first phase,\n\t\t\t// hence error is always nil.\n\t\t\t// the merged knn hits are finalized and set in the search result.\n\t\t\tsr.Hits, _ = knnStore.Final(nil)\n\t\t},\n\t}\n}\n\n// Replace knn boost values for fusion rescoring queries\nfunc (r *rescorer) prepareKnnRequest() {\n\tfor i := range r.req.KNN {\n\t\tb := r.req.KNN[i].Boost\n\t\tif b != nil {\n\t\t\tr.origBoosts[i+1] = b.Value()\n\t\t\tnewB := query.Boost(1.0)\n\t\t\tr.req.KNN[i].Boost = &newB\n\t\t} else {\n\t\t\tr.origBoosts[i+1] = 1.0\n\t\t}\n\t}\n}\n\n// Restore knn boost values for fusion rescoring queries\nfunc (r *rescorer) restoreKnnRequest() {\n\tfor i := range r.req.KNN {\n\t\tb := query.Boost(r.origBoosts[i+1])\n\t\tr.req.KNN[i].Boost = &b\n\t}\n}\n"
  },
  {
    "path": "search_knn_test.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build vectors\n// +build vectors\n\npackage bleve\n\nimport (\n\t\"archive/zip\"\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword\"\n\t\"github.com/blevesearch/bleve/v2/analysis/lang/en\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nconst testInputCompressedFile = \"test/knn/knn_dataset_queries.zip\"\nconst testDatasetFileName = \"knn_dataset.json\"\nconst testQueryFileName = \"knn_queries.json\"\n\nconst testDatasetDims = 384\n\nvar knnOperators []knnOperator = []knnOperator{knnOperatorAnd, knnOperatorOr}\n\nfunc TestSimilaritySearchPartitionedIndex(t *testing.T) {\n\tdataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocuments := makeDatasetIntoDocuments(dataset)\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Analyzer = en.AnalyzerName\n\n\tvecFieldMappingL2 := mapping.NewVectorFieldMapping()\n\tvecFieldMappingL2.Dims = testDatasetDims\n\tvecFieldMappingL2.Similarity = index.EuclideanDistance\n\n\tindexMappingL2Norm := NewIndexMapping()\n\tindexMappingL2Norm.DefaultMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tindexMappingL2Norm.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFieldMappingL2)\n\n\tvecFieldMappingDot := mapping.NewVectorFieldMapping()\n\tvecFieldMappingDot.Dims = testDatasetDims\n\tvecFieldMappingDot.Similarity = index.InnerProduct\n\n\tindexMappingDotProduct := NewIndexMapping()\n\tindexMappingDotProduct.DefaultMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tindexMappingDotProduct.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFieldMappingDot)\n\n\tvecFieldMappingCosine := mapping.NewVectorFieldMapping()\n\tvecFieldMappingCosine.Dims = testDatasetDims\n\tvecFieldMappingCosine.Similarity = index.CosineSimilarity\n\n\tindexMappingCosine := NewIndexMapping()\n\tindexMappingCosine.DefaultMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tindexMappingCosine.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFieldMappingCosine)\n\n\ttype testCase struct {\n\t\ttestType           string\n\t\tqueryIndex         int\n\t\tnumIndexPartitions int\n\t\tmapping            mapping.IndexMapping\n\t}\n\n\ttestCases := []testCase{\n\t\t// l2 norm similarity\n\t\t{\n\t\t\ttestType:           \"multi_partition:match_none:oneKNNreq:k=3\",\n\t\t\tqueryIndex:         0,\n\t\t\tnumIndexPartitions: 4,\n\t\t\tmapping:            indexMappingL2Norm,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:match_none:oneKNNreq:k=2\",\n\t\t\tqueryIndex:         0,\n\t\t\tnumIndexPartitions: 10,\n\t\t\tmapping:            indexMappingL2Norm,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:match:oneKNNreq:k=2\",\n\t\t\tqueryIndex:         1,\n\t\t\tnumIndexPartitions: 5,\n\t\t\tmapping:            indexMappingL2Norm,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:disjunction:twoKNNreq:k=2,2\",\n\t\t\tqueryIndex:         2,\n\t\t\tnumIndexPartitions: 4,\n\t\t\tmapping:            indexMappingL2Norm,\n\t\t},\n\t\t// dot product similarity\n\t\t{\n\t\t\ttestType:           \"multi_partition:match_none:oneKNNreq:k=3\",\n\t\t\tqueryIndex:         0,\n\t\t\tnumIndexPartitions: 4,\n\t\t\tmapping:            indexMappingDotProduct,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:match_none:oneKNNreq:k=2\",\n\t\t\tqueryIndex:         0,\n\t\t\tnumIndexPartitions: 10,\n\t\t\tmapping:            indexMappingDotProduct,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:match:oneKNNreq:k=2\",\n\t\t\tqueryIndex:         1,\n\t\t\tnumIndexPartitions: 5,\n\t\t\tmapping:            indexMappingDotProduct,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:disjunction:twoKNNreq:k=2,2\",\n\t\t\tqueryIndex:         2,\n\t\t\tnumIndexPartitions: 4,\n\t\t\tmapping:            indexMappingDotProduct,\n\t\t},\n\t\t// cosine similarity\n\t\t{\n\t\t\ttestType:           \"multi_partition:match_none:oneKNNreq:k=3\",\n\t\t\tqueryIndex:         0,\n\t\t\tnumIndexPartitions: 7,\n\t\t\tmapping:            indexMappingCosine,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:match_none:oneKNNreq:k=2\",\n\t\t\tqueryIndex:         0,\n\t\t\tnumIndexPartitions: 5,\n\t\t\tmapping:            indexMappingCosine,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:match:oneKNNreq:k=2\",\n\t\t\tqueryIndex:         1,\n\t\t\tnumIndexPartitions: 3,\n\t\t\tmapping:            indexMappingCosine,\n\t\t},\n\t\t{\n\t\t\ttestType:           \"multi_partition:disjunction:twoKNNreq:k=2,2\",\n\t\t\tqueryIndex:         2,\n\t\t\tnumIndexPartitions: 9,\n\t\t\tmapping:            indexMappingCosine,\n\t\t},\n\t}\n\n\tindex := NewIndexAlias()\n\tvar reqSort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: \"content\"}}\n\tfor testCaseNum, testCase := range testCases {\n\t\toriginalRequest := searchRequests[testCase.queryIndex]\n\t\tfor _, operator := range knnOperators {\n\n\t\t\tindex.indexes = make([]Index, 0)\n\t\t\tquery := copySearchRequest(originalRequest, nil)\n\t\t\tquery.AddKNNOperator(operator)\n\t\t\tquery.Sort = reqSort.Copy()\n\t\t\tquery.Explain = true\n\n\t\t\tnameToIndex := createPartitionedIndex(documents, index, 1, testCase.mapping, t, false)\n\t\t\tcontrolResult, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(controlResult.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected control result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\t\t\tcleanUp(t, nameToIndex)\n\n\t\t\tindex.indexes = make([]Index, 0)\n\t\t\tquery = copySearchRequest(originalRequest, nil)\n\t\t\tquery.AddKNNOperator(operator)\n\t\t\tquery.Sort = reqSort.Copy()\n\t\t\tquery.Explain = true\n\n\t\t\tnameToIndex = createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, false)\n\t\t\texperimentalResult, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(experimentalResult.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected experimental Result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\t\t\tverifyResult(t, controlResult, experimentalResult, testCaseNum, true)\n\t\t\tcleanUp(t, nameToIndex)\n\n\t\t\tindex.indexes = make([]Index, 0)\n\t\t\tquery = copySearchRequest(originalRequest, nil)\n\t\t\tquery.AddKNNOperator(operator)\n\t\t\tquery.Sort = reqSort.Copy()\n\t\t\tquery.Explain = true\n\n\t\t\tnameToIndex = createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, true)\n\t\t\tmultiLevelIndexResult, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(multiLevelIndexResult.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected experimental Result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\t\t\tverifyResult(t, multiLevelIndexResult, experimentalResult, testCaseNum, false)\n\t\t\tcleanUp(t, nameToIndex)\n\n\t\t}\n\t}\n\n\tvar facets = map[string]*FacetRequest{\n\t\t\"content\": {\n\t\t\tField: \"content\",\n\t\t\tSize:  10,\n\t\t},\n\t}\n\n\tindex = NewIndexAlias()\n\tfor testCaseNum, testCase := range testCases {\n\t\tindex.indexes = make([]Index, 0)\n\t\tnameToIndex := createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, false)\n\t\toriginalRequest := searchRequests[testCase.queryIndex]\n\t\tfor _, operator := range knnOperators {\n\t\t\tfrom, size := originalRequest.From, originalRequest.Size\n\t\t\tquery := copySearchRequest(originalRequest, nil)\n\t\t\tquery.AddKNNOperator(operator)\n\t\t\tquery.Explain = true\n\t\t\tquery.From = from\n\t\t\tquery.Size = size\n\n\t\t\t// Three types of queries to run wrt sort and facet fields that require fields.\n\t\t\t// 1. Sort And Facet are there\n\t\t\t// 2. Sort is there, Facet is not there\n\t\t\t// 3. Sort is not there, Facet is there\n\t\t\t// The case where both sort and facet are not there is already covered in the previous tests.\n\n\t\t\t// 1. Sort And Facet are there\n\t\t\tquery.Facets = facets\n\t\t\tquery.Sort = reqSort.Copy()\n\n\t\t\tres1, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(res1.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected experimental Result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\n\t\t\tfacetRes1 := res1.Facets\n\t\t\tfacetRes1Str, err := json.Marshal(facetRes1)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// 2. Sort is there, Facet is not there\n\t\t\tquery.Facets = nil\n\t\t\tquery.Sort = reqSort.Copy()\n\n\t\t\tres2, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(res2.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected experimental Result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\n\t\t\t// 3. Sort is not there, Facet is there\n\t\t\tquery.Facets = facets\n\t\t\tquery.Sort = nil\n\t\t\tres3, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(res3.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected experimental Result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\n\t\t\tfacetRes3 := res3.Facets\n\t\t\tfacetRes3Str, err := json.Marshal(facetRes3)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// Verify the facet results\n\t\t\tif string(facetRes1Str) != string(facetRes3Str) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected facet results to be equal\", testCaseNum)\n\t\t\t}\n\n\t\t\t// Verify the results\n\t\t\tverifyResult(t, res1, res2, testCaseNum, false)\n\t\t\tverifyResult(t, res2, res3, testCaseNum, true)\n\n\t\t\t// Test early exit fail case -> matchNone + facetRequest\n\t\t\tquery.Query = NewMatchNoneQuery()\n\t\t\tquery.Sort = reqSort.Copy()\n\t\t\t// control case\n\t\t\tquery.Facets = nil\n\t\t\tres4Ctrl, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(res4Ctrl.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected control Result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\n\t\t\t// experimental case\n\t\t\tquery.Facets = facets\n\t\t\tres4Exp, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(res4Exp.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected experimental Result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\n\t\t\tif !(operator == knnOperatorAnd && res4Ctrl.Total == 0 && res4Exp.Total == 0) {\n\t\t\t\t// catch case where no hits are returned\n\t\t\t\t// due to matchNone query with a KNN request with operator AND\n\t\t\t\t// where no hits are part of the intersection in multi knn request\n\t\t\t\tverifyResult(t, res4Ctrl, res4Exp, testCaseNum, false)\n\t\t\t}\n\t\t}\n\t\tcleanUp(t, nameToIndex)\n\t}\n\n\t// Test Pagination with multi partitioned index\n\tindex = NewIndexAlias()\n\tindex.indexes = make([]Index, 0)\n\tnameToIndex := createPartitionedIndex(documents, index, 8, indexMappingL2Norm, t, true)\n\n\t// Test From + Size pagination for Hybrid Search (2-Phase)\n\tquery := copySearchRequest(searchRequests[4], nil)\n\tquery.Sort = reqSort.Copy()\n\tquery.Facets = facets\n\tquery.Explain = true\n\n\ttestFromSizePagination(t, query, index, nameToIndex)\n\n\t// Test From + Size pagination for Early Exit Hybrid Search (1-Phase)\n\tquery = copySearchRequest(searchRequests[4], nil)\n\tquery.Query = NewMatchNoneQuery()\n\tquery.Sort = reqSort.Copy()\n\tquery.Facets = nil\n\tquery.Explain = true\n\n\ttestFromSizePagination(t, query, index, nameToIndex)\n\n\tcleanUp(t, nameToIndex)\n}\n\nfunc testFromSizePagination(t *testing.T, query *SearchRequest, index Index, nameToIndex map[string]Index) {\n\tquery.From = 0\n\tquery.Size = 30\n\n\tresCtrl, err := index.Search(query)\n\tif err != nil {\n\t\tcleanUp(t, nameToIndex)\n\t\tt.Fatal(err)\n\t}\n\n\tctrlHitIds := make([]string, len(resCtrl.Hits))\n\tfor i, doc := range resCtrl.Hits {\n\t\tctrlHitIds[i] = doc.ID\n\t}\n\t// experimental case\n\n\tfromValues := []int{0, 5, 10, 15, 20, 25}\n\tsize := 5\n\tfor fromIdx := 0; fromIdx < len(fromValues); fromIdx++ {\n\t\tfrom := fromValues[fromIdx]\n\t\tquery.From = from\n\t\tquery.Size = size\n\t\tresExp, err := index.Search(query)\n\t\tif err != nil {\n\t\t\tcleanUp(t, nameToIndex)\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif from >= len(ctrlHitIds) {\n\t\t\tif len(resExp.Hits) != 0 {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"expected 0 hits, got %d\", len(resExp.Hits))\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tnumHitsExp := len(resExp.Hits)\n\t\tnumHitsCtrl := min(len(ctrlHitIds)-from, size)\n\t\tif numHitsExp != numHitsCtrl {\n\t\t\tcleanUp(t, nameToIndex)\n\t\t\tt.Fatalf(\"expected %d hits, got %d\", numHitsCtrl, numHitsExp)\n\t\t}\n\t\tfor i := 0; i < numHitsExp; i++ {\n\t\t\tdoc := resExp.Hits[i]\n\t\t\tstartOffset := from + i\n\t\t\tif doc.ID != ctrlHitIds[startOffset] {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"expected %s at index %d, got %s\", ctrlHitIds[startOffset], i, doc.ID)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestVectorBase64Index(t *testing.T) {\n\tdataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocuments := makeDatasetIntoDocuments(dataset)\n\n\t_, searchRequestsCopy, err := readDatasetAndQueries(testInputCompressedFile)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, doc := range documents {\n\t\tvec, ok := doc[\"vector\"].([]float32)\n\t\tif !ok {\n\t\t\tt.Fatal(\"Typecasting vector to float array failed\")\n\t\t}\n\n\t\tbuf := new(bytes.Buffer)\n\t\tfor _, v := range vec {\n\t\t\terr := binary.Write(buf, binary.LittleEndian, v)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\n\t\tdoc[\"vectorEncoded\"] = base64.StdEncoding.EncodeToString(buf.Bytes())\n\t}\n\n\tfor _, sr := range searchRequestsCopy {\n\t\tfor _, kr := range sr.KNN {\n\t\t\tkr.Field = \"vectorEncoded\"\n\t\t}\n\t}\n\n\tcontentFM := NewTextFieldMapping()\n\tcontentFM.Analyzer = en.AnalyzerName\n\n\tvecFML2 := mapping.NewVectorFieldMapping()\n\tvecFML2.Dims = testDatasetDims\n\tvecFML2.Similarity = index.EuclideanDistance\n\n\tvecBFML2 := mapping.NewVectorBase64FieldMapping()\n\tvecBFML2.Dims = testDatasetDims\n\tvecBFML2.Similarity = index.EuclideanDistance\n\n\tvecFMDot := mapping.NewVectorFieldMapping()\n\tvecFMDot.Dims = testDatasetDims\n\tvecFMDot.Similarity = index.InnerProduct\n\n\tvecBFMDot := mapping.NewVectorBase64FieldMapping()\n\tvecBFMDot.Dims = testDatasetDims\n\tvecBFMDot.Similarity = index.InnerProduct\n\n\tindexMappingL2 := NewIndexMapping()\n\tindexMappingL2.DefaultMapping.AddFieldMappingsAt(\"content\", contentFM)\n\tindexMappingL2.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFML2)\n\tindexMappingL2.DefaultMapping.AddFieldMappingsAt(\"vectorEncoded\", vecBFML2)\n\n\tindexMappingDot := NewIndexMapping()\n\tindexMappingDot.DefaultMapping.AddFieldMappingsAt(\"content\", contentFM)\n\tindexMappingDot.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFMDot)\n\tindexMappingDot.DefaultMapping.AddFieldMappingsAt(\"vectorEncoded\", vecBFMDot)\n\n\ttmpIndexPathL2 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPathL2)\n\n\ttmpIndexPathDot := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPathDot)\n\n\tindexL2, err := New(tmpIndexPathL2, indexMappingL2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := indexL2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tindexDot, err := New(tmpIndexPathDot, indexMappingDot)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := indexDot.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatchL2 := indexL2.NewBatch()\n\tbatchDot := indexDot.NewBatch()\n\n\tfor _, doc := range documents {\n\t\terr = batchL2.Index(doc[\"id\"].(string), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = batchDot.Index(doc[\"id\"].(string), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = indexL2.Batch(batchL2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = indexDot.Batch(batchDot)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := range searchRequests {\n\t\tfor _, operator := range knnOperators {\n\t\t\tcontrolQuery := searchRequests[i]\n\t\t\ttestQuery := searchRequestsCopy[i]\n\n\t\t\tcontrolQuery.AddKNNOperator(operator)\n\t\t\ttestQuery.AddKNNOperator(operator)\n\n\t\t\tcontrolResultL2, err := indexL2.Search(controlQuery)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\ttestResultL2, err := indexL2.Search(testQuery)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif controlResultL2 != nil && testResultL2 != nil {\n\t\t\t\tif len(controlResultL2.Hits) == len(testResultL2.Hits) {\n\t\t\t\t\tfor j := range controlResultL2.Hits {\n\t\t\t\t\t\tif controlResultL2.Hits[j].ID != testResultL2.Hits[j].ID {\n\t\t\t\t\t\t\tt.Fatalf(\"testcase %d failed: expected hit id %s, got hit id %s\", i, controlResultL2.Hits[j].ID, testResultL2.Hits[j].ID)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if (controlResultL2 == nil && testResultL2 != nil) ||\n\t\t\t\t(controlResultL2 != nil && testResultL2 == nil) {\n\t\t\t\tt.Fatalf(\"testcase %d failed: expected result %s, got result %s\", i, controlResultL2, testResultL2)\n\t\t\t}\n\n\t\t\tcontrolResultDot, err := indexDot.Search(controlQuery)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\ttestResultDot, err := indexDot.Search(testQuery)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif controlResultDot != nil && testResultDot != nil {\n\t\t\t\tif len(controlResultDot.Hits) == len(testResultDot.Hits) {\n\t\t\t\t\tfor j := range controlResultDot.Hits {\n\t\t\t\t\t\tif controlResultDot.Hits[j].ID != testResultDot.Hits[j].ID {\n\t\t\t\t\t\t\tt.Fatalf(\"testcase %d failed: expected hit id %s, got hit id %s\", i, controlResultDot.Hits[j].ID, testResultDot.Hits[j].ID)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if (controlResultDot == nil && testResultDot != nil) ||\n\t\t\t\t(controlResultDot != nil && testResultDot == nil) {\n\t\t\t\tt.Fatalf(\"testcase %d failed: expected result %s, got result %s\", i, controlResultDot, testResultDot)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Test to verify that the bivf-flat indexes with vector base64 field mapping returns the\n// same results as the non-optimized vector field mapping for L2, Dot Product and Cosine similarities.\n// Also test to see no differences in results for any distance metric\nfunc TestVectorBivfIndexes(t *testing.T) {\n\toptimizations := []string{index.IndexBIVFWithBackingSQ8, index.IndexBIVFWithBackingFlat}\n\tfor _, optimization := range optimizations {\n\t\ttestVectorBivfIndex(t, optimization)\n\t}\n}\n\nfunc testVectorBivfIndex(t *testing.T, optimization string) {\n\n\tdataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocuments := makeDatasetIntoDocuments(dataset)\n\n\t_, searchRequestsCopy, err := readDatasetAndQueries(testInputCompressedFile)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, doc := range documents {\n\t\tvec, ok := doc[\"vector\"].([]float32)\n\t\tif !ok {\n\t\t\tt.Fatal(\"Typecasting vector to float array failed\")\n\t\t}\n\n\t\tbuf := new(bytes.Buffer)\n\t\tfor _, v := range vec {\n\t\t\terr := binary.Write(buf, binary.LittleEndian, v)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\n\t\tdoc[\"vectorEncoded\"] = base64.StdEncoding.EncodeToString(buf.Bytes())\n\t}\n\n\tfor _, sr := range searchRequestsCopy {\n\t\tfor _, kr := range sr.KNN {\n\t\t\tkr.Field = \"vectorEncoded\"\n\t\t}\n\t}\n\n\tcontentFM := NewTextFieldMapping()\n\tcontentFM.Analyzer = en.AnalyzerName\n\n\tvecFML2 := mapping.NewVectorFieldMapping()\n\tvecFML2.Dims = testDatasetDims\n\tvecFML2.Similarity = index.EuclideanDistance\n\tvecFML2.VectorIndexOptimizedFor = optimization\n\n\tvecBFML2 := mapping.NewVectorBase64FieldMapping()\n\tvecBFML2.Dims = testDatasetDims\n\tvecBFML2.Similarity = index.EuclideanDistance\n\tvecBFML2.VectorIndexOptimizedFor = optimization\n\n\tvecFMDot := mapping.NewVectorFieldMapping()\n\tvecFMDot.Dims = testDatasetDims\n\tvecFMDot.Similarity = index.InnerProduct\n\tvecFMDot.VectorIndexOptimizedFor = optimization\n\n\tvecBFMDot := mapping.NewVectorBase64FieldMapping()\n\tvecBFMDot.Dims = testDatasetDims\n\tvecBFMDot.Similarity = index.InnerProduct\n\tvecBFMDot.VectorIndexOptimizedFor = optimization\n\n\tvecFMCosine := mapping.NewVectorFieldMapping()\n\tvecFMCosine.Dims = testDatasetDims\n\tvecFMCosine.Similarity = index.CosineSimilarity\n\n\tvecBFMCosine := mapping.NewVectorBase64FieldMapping()\n\tvecBFMCosine.Dims = testDatasetDims\n\tvecBFMCosine.Similarity = index.CosineSimilarity\n\tvecBFMCosine.VectorIndexOptimizedFor = optimization\n\n\tindexMappingL2 := NewIndexMapping()\n\tindexMappingL2.DefaultMapping.AddFieldMappingsAt(\"content\", contentFM)\n\tindexMappingL2.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFML2)\n\tindexMappingL2.DefaultMapping.AddFieldMappingsAt(\"vectorEncoded\", vecBFML2)\n\n\tindexMappingDot := NewIndexMapping()\n\tindexMappingDot.DefaultMapping.AddFieldMappingsAt(\"content\", contentFM)\n\tindexMappingDot.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFMDot)\n\tindexMappingDot.DefaultMapping.AddFieldMappingsAt(\"vectorEncoded\", vecBFMDot)\n\n\tindexMappingCosine := NewIndexMapping()\n\tindexMappingCosine.DefaultMapping.AddFieldMappingsAt(\"content\", contentFM)\n\tindexMappingCosine.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFMCosine)\n\tindexMappingCosine.DefaultMapping.AddFieldMappingsAt(\"vectorEncoded\", vecBFMCosine)\n\n\ttmpIndexPathL2 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPathL2)\n\n\ttmpIndexPathDot := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPathDot)\n\n\ttmpIndexPathCosine := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPathCosine)\n\n\tindexL2, err := New(tmpIndexPathL2, indexMappingL2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := indexL2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tindexDot, err := New(tmpIndexPathDot, indexMappingDot)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := indexDot.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tindexCosine, err := New(tmpIndexPathCosine, indexMappingCosine)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := indexCosine.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatchL2 := indexL2.NewBatch()\n\tbatchDot := indexDot.NewBatch()\n\tbatchCosine := indexCosine.NewBatch()\n\n\tfor _, doc := range documents {\n\t\terr = batchL2.Index(doc[\"id\"].(string), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = batchDot.Index(doc[\"id\"].(string), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = batchCosine.Index(doc[\"id\"].(string), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = indexL2.Batch(batchL2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = indexDot.Batch(batchDot)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = indexCosine.Batch(batchCosine)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := range searchRequests {\n\t\tfor _, operator := range knnOperators {\n\t\t\tnormQuery := searchRequests[i]\n\t\t\tbase64Query := searchRequestsCopy[i]\n\n\t\t\tnormQuery.AddKNNOperator(operator)\n\t\t\tbase64Query.AddKNNOperator(operator)\n\n\t\t\tnormResultL2, err := indexL2.Search(normQuery)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tbase64ResultL2, err := indexL2.Search(base64Query)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif normResultL2 != nil && base64ResultL2 != nil {\n\t\t\t\tif len(normResultL2.Hits) == len(base64ResultL2.Hits) {\n\t\t\t\t\tfor j := range normResultL2.Hits {\n\t\t\t\t\t\tif normResultL2.Hits[j].ID != base64ResultL2.Hits[j].ID {\n\t\t\t\t\t\t\tt.Fatalf(\"testcase %d failed: expected hit id %s, got hit id %s\", i, normResultL2.Hits[j].ID, base64ResultL2.Hits[j].ID)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if (normResultL2 == nil && base64ResultL2 != nil) ||\n\t\t\t\t(normResultL2 != nil && base64ResultL2 == nil) {\n\t\t\t\tt.Fatalf(\"testcase %d failed: expected result %s, got result %s\", i, normResultL2, base64ResultL2)\n\t\t\t}\n\n\t\t\tnormResultDot, err := indexDot.Search(normQuery)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tbase64ResultDot, err := indexDot.Search(base64Query)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif normResultDot != nil && base64ResultDot != nil {\n\t\t\t\tif len(normResultDot.Hits) == len(base64ResultDot.Hits) {\n\t\t\t\t\tfor j := range normResultDot.Hits {\n\t\t\t\t\t\tif normResultDot.Hits[j].ID != base64ResultDot.Hits[j].ID {\n\t\t\t\t\t\t\tt.Fatalf(\"testcase %d failed: expected hit id %s, got hit id %s\", i, normResultDot.Hits[j].ID, base64ResultDot.Hits[j].ID)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if (normResultDot == nil && base64ResultDot != nil) ||\n\t\t\t\t(normResultDot != nil && base64ResultDot == nil) {\n\t\t\t\tt.Fatalf(\"testcase %d failed: expected result %s, got result %s\", i, normResultDot, base64ResultDot)\n\t\t\t}\n\n\t\t\tnormResultCosine, err := indexCosine.Search(normQuery)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tbase64ResultCosine, err := indexCosine.Search(base64Query)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif normResultCosine != nil && base64ResultCosine != nil {\n\t\t\t\tif len(normResultCosine.Hits) == len(base64ResultCosine.Hits) {\n\t\t\t\t\tfor j := range normResultCosine.Hits {\n\t\t\t\t\t\tif normResultCosine.Hits[j].ID != base64ResultCosine.Hits[j].ID {\n\t\t\t\t\t\t\tt.Fatalf(\"testcase %d failed: expected hit id %s, got hit id %s\", i, normResultCosine.Hits[j].ID, base64ResultCosine.Hits[j].ID)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if (normResultCosine == nil && base64ResultCosine != nil) ||\n\t\t\t\t(normResultCosine != nil && base64ResultCosine == nil) {\n\t\t\t\tt.Fatalf(\"testcase %d failed: expected result %s, got result %s\", i, normResultCosine, base64ResultCosine)\n\t\t\t}\n\n\t\t\tif normResultCosine != nil && normResultL2 != nil {\n\t\t\t\tif len(normResultCosine.Hits) == len(normResultL2.Hits) {\n\t\t\t\t\tfor j := range normResultCosine.Hits {\n\t\t\t\t\t\tif normResultCosine.Hits[j].ID != normResultL2.Hits[j].ID {\n\t\t\t\t\t\t\tif normResultCosine.Hits[j].Score != normResultL2.Hits[j].Score {\n\t\t\t\t\t\t\t\tt.Fatalf(\"testcase %d failed: expected hit id %s, got hit id %s\", i, normResultCosine.Hits[j].ID, normResultL2.Hits[j].ID)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if (normResultCosine == nil && normResultL2 != nil) ||\n\t\t\t\t(normResultCosine != nil && normResultL2 == nil) {\n\t\t\t\tt.Fatalf(\"testcase %d failed: expected result %s, got result %s\", i, normResultCosine, normResultL2)\n\t\t\t}\n\n\t\t\tif normResultCosine != nil && normResultDot != nil {\n\t\t\t\tif len(normResultCosine.Hits) == len(normResultDot.Hits) {\n\t\t\t\t\tfor j := range normResultCosine.Hits {\n\t\t\t\t\t\tif normResultCosine.Hits[j].ID != normResultDot.Hits[j].ID {\n\t\t\t\t\t\t\tif normResultCosine.Hits[j].Score != normResultDot.Hits[j].Score {\n\t\t\t\t\t\t\t\tt.Fatalf(\"testcase %d failed: expected hit id %s, got hit id %s\", i, normResultCosine.Hits[j].ID, normResultDot.Hits[j].ID)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if (normResultCosine == nil && normResultDot != nil) ||\n\t\t\t\t(normResultCosine != nil && normResultDot == nil) {\n\t\t\t\tt.Fatalf(\"testcase %d failed: expected result %s, got result %s\", i, normResultCosine, normResultDot)\n\t\t\t}\n\t\t}\n\t}\n}\n\ntype testDocument struct {\n\tID      string    `json:\"id\"`\n\tContent string    `json:\"content\"`\n\tVector  []float32 `json:\"vector\"`\n}\n\nfunc readDatasetAndQueries(fileName string) ([]testDocument, []*SearchRequest, error) {\n\t// Open the zip archive for reading\n\tr, err := zip.OpenReader(fileName)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tvar dataset []testDocument\n\tvar queries []*SearchRequest\n\n\tdefer r.Close()\n\tfor _, f := range r.File {\n\t\tjsonFile, err := f.Open()\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tdefer jsonFile.Close()\n\t\tif f.Name == testDatasetFileName {\n\t\t\terr = json.NewDecoder(jsonFile).Decode(&dataset)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t} else if f.Name == testQueryFileName {\n\t\t\terr = json.NewDecoder(jsonFile).Decode(&queries)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t}\n\t}\n\treturn dataset, queries, nil\n}\n\nfunc makeDatasetIntoDocuments(dataset []testDocument) []map[string]interface{} {\n\tdocuments := make([]map[string]interface{}, len(dataset))\n\tfor i := 0; i < len(dataset); i++ {\n\t\tdocument := make(map[string]interface{})\n\t\tdocument[\"id\"] = dataset[i].ID\n\t\tdocument[\"content\"] = dataset[i].Content\n\t\tdocument[\"vector\"] = dataset[i].Vector\n\t\tdocuments[i] = document\n\t}\n\treturn documents\n}\n\nfunc cleanUp(t *testing.T, nameToIndex map[string]Index) {\n\tfor path, childIndex := range nameToIndex {\n\t\terr := childIndex.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tcleanupTmpIndexPath(t, path)\n\t}\n}\n\nfunc createChildIndex(docs []map[string]interface{}, mapping mapping.IndexMapping, t *testing.T, nameToIndex map[string]Index) Index {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tindex, err := New(tmpIndexPath, mapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnameToIndex[index.Name()] = index\n\tbatch := index.NewBatch()\n\tfor _, doc := range docs {\n\t\terr := batch.Index(doc[\"id\"].(string), doc)\n\t\tif err != nil {\n\t\t\tcleanUp(t, nameToIndex)\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tcleanUp(t, nameToIndex)\n\t\tt.Fatal(err)\n\t}\n\treturn index\n}\n\nfunc createPartitionedIndex(documents []map[string]interface{}, index *indexAliasImpl, numPartitions int,\n\tmapping mapping.IndexMapping, t *testing.T, multiLevel bool) map[string]Index {\n\n\tpartitionSize := len(documents) / numPartitions\n\textraDocs := len(documents) % numPartitions\n\tnumDocsPerPartition := make([]int, numPartitions)\n\tfor i := 0; i < numPartitions; i++ {\n\t\tnumDocsPerPartition[i] = partitionSize\n\t\tif extraDocs > 0 {\n\t\t\tnumDocsPerPartition[i]++\n\t\t\textraDocs--\n\t\t}\n\t}\n\tdocsPerPartition := make([][]map[string]interface{}, numPartitions)\n\tprevCutoff := 0\n\tfor i := 0; i < numPartitions; i++ {\n\t\tdocsPerPartition[i] = make([]map[string]interface{}, numDocsPerPartition[i])\n\t\tfor j := 0; j < numDocsPerPartition[i]; j++ {\n\t\t\tdocsPerPartition[i][j] = documents[prevCutoff+j]\n\t\t}\n\t\tprevCutoff += numDocsPerPartition[i]\n\t}\n\n\trv := make(map[string]Index)\n\tif !multiLevel {\n\t\t// all indexes are at the same level\n\t\tfor i := 0; i < numPartitions; i++ {\n\t\t\tindex.Add(createChildIndex(docsPerPartition[i], mapping, t, rv))\n\t\t}\n\t} else {\n\t\t// alias tree\n\t\tindexes := make([]Index, numPartitions)\n\t\tfor i := 0; i < numPartitions; i++ {\n\t\t\tindexes[i] = createChildIndex(docsPerPartition[i], mapping, t, rv)\n\t\t}\n\t\tnumAlias := int(math.Ceil(float64(numPartitions) / 2.0))\n\t\taliases := make([]IndexAlias, numAlias)\n\t\tfor i := 0; i < numAlias; i++ {\n\t\t\taliases[i] = NewIndexAlias()\n\t\t\taliases[i].SetName(fmt.Sprintf(\"alias%d\", i))\n\t\t\tfor j := 0; j < 2; j++ {\n\t\t\t\tif i*2+j < numPartitions {\n\t\t\t\t\taliases[i].Add(indexes[i*2+j])\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor i := 0; i < numAlias; i++ {\n\t\t\tindex.Add(aliases[i])\n\t\t}\n\t}\n\treturn rv\n}\n\nfunc createMultipleSegmentsIndex(documents []map[string]interface{}, index Index, numSegments int) error {\n\t// create multiple batches to simulate more than one segment\n\tnumBatches := numSegments\n\n\tbatches := make([]*Batch, numBatches)\n\tnumDocsPerBatch := len(documents) / numBatches\n\textraDocs := len(documents) % numBatches\n\n\tdocsPerBatch := make([]int, numBatches)\n\tfor i := 0; i < numBatches; i++ {\n\t\tdocsPerBatch[i] = numDocsPerBatch\n\t\tif extraDocs > 0 {\n\t\t\tdocsPerBatch[i]++\n\t\t\textraDocs--\n\t\t}\n\t}\n\tprevCutoff := 0\n\tfor i := 0; i < numBatches; i++ {\n\t\tbatches[i] = index.NewBatch()\n\t\tfor j := prevCutoff; j < prevCutoff+docsPerBatch[i]; j++ {\n\t\t\tdoc := documents[j]\n\t\t\terr := batches[i].Index(doc[\"id\"].(string), doc)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tprevCutoff += docsPerBatch[i]\n\t}\n\terrMutex := sync.Mutex{}\n\tvar errors []error\n\twg := sync.WaitGroup{}\n\twg.Add(len(batches))\n\tfor i, batch := range batches {\n\t\tgo func(ix int, batchx *Batch) {\n\t\t\tdefer wg.Done()\n\t\t\terr := index.Batch(batchx)\n\t\t\tif err != nil {\n\t\t\t\terrMutex.Lock()\n\t\t\t\terrors = append(errors, err)\n\t\t\t\terrMutex.Unlock()\n\t\t\t}\n\t\t}(i, batch)\n\t}\n\twg.Wait()\n\tif len(errors) > 0 {\n\t\treturn errors[0]\n\t}\n\treturn nil\n}\n\nfunc truncateScore(score float64) float64 {\n\tepsilon := 1e-4\n\ttruncated := float64(int(score*1e6)) / 1e6\n\tif math.Abs(truncated-1.0) <= epsilon {\n\t\treturn 1.0\n\t}\n\treturn truncated\n}\n\n// Function to compare two Explanation structs recursively\nfunc compareExplanation(a, b *search.Explanation) bool {\n\tif a == nil && b == nil {\n\t\treturn true\n\t}\n\tif a == nil || b == nil {\n\t\treturn false\n\t}\n\n\tif truncateScore(a.Value) != truncateScore(b.Value) || len(a.Children) != len(b.Children) {\n\t\treturn false\n\t}\n\n\t// Sort the children slices before comparison\n\tsortChildren(a.Children)\n\tsortChildren(b.Children)\n\n\tfor i := range a.Children {\n\t\tif !compareExplanation(a.Children[i], b.Children[i]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Function to sort the children slices\nfunc sortChildren(children []*search.Explanation) {\n\tsort.Slice(children, func(i, j int) bool {\n\t\treturn children[i].Value < children[j].Value\n\t})\n}\n\n// All hits from a hybrid search/knn search should not have\n// index names or score breakdown.\nfunc finalHitsOmitKNNMetadata(hits []*search.DocumentMatch) bool {\n\tfor _, hit := range hits {\n\t\tif hit.IndexNames != nil || hit.ScoreBreakdown != nil {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc finalHitsHaveValidIndex(hits []*search.DocumentMatch, indexes map[string]Index) bool {\n\tfor _, hit := range hits {\n\t\tif hit.Index == \"\" {\n\t\t\treturn false\n\t\t}\n\t\tvar idx Index\n\t\tvar ok bool\n\t\tif idx, ok = indexes[hit.Index]; !ok {\n\t\t\treturn false\n\t\t}\n\t\tif idx == nil {\n\t\t\treturn false\n\t\t}\n\t\tvar doc index.Document\n\t\tdoc, err = idx.Document(hit.ID)\n\t\tif err != nil {\n\t\t\treturn false\n\t\t}\n\t\tif doc == nil {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc verifyResult(t *testing.T, controlResult *SearchResult, experimentalResult *SearchResult, testCaseNum int, verifyOnlyDocIDs bool) {\n\tif controlResult.Hits.Len() == 0 || experimentalResult.Hits.Len() == 0 {\n\t\tt.Fatalf(\"test case #%d failed: 0 hits returned\", testCaseNum)\n\t}\n\tif len(controlResult.Hits) != len(experimentalResult.Hits) {\n\t\tt.Fatalf(\"test case #%d failed: expected %d results, got %d\", testCaseNum, len(controlResult.Hits), len(experimentalResult.Hits))\n\t}\n\tif controlResult.Total != experimentalResult.Total {\n\t\tt.Fatalf(\"test case #%d failed: expected total hits to be %d, got %d\", testCaseNum, controlResult.Total, experimentalResult.Total)\n\t}\n\t// KNN Metadata -> Score Breakdown and IndexNames MUST be omitted from the final hits\n\tif !finalHitsOmitKNNMetadata(controlResult.Hits) || !finalHitsOmitKNNMetadata(experimentalResult.Hits) {\n\t\tt.Fatalf(\"test case #%d failed: expected no KNN metadata in hits\", testCaseNum)\n\t}\n\tif controlResult.Took == 0 || experimentalResult.Took == 0 {\n\t\tt.Fatalf(\"test case #%d failed: expected non-zero took time\", testCaseNum)\n\t}\n\tif controlResult.Request == nil || experimentalResult.Request == nil {\n\t\tt.Fatalf(\"test case #%d failed: expected non-nil request\", testCaseNum)\n\t}\n\tif verifyOnlyDocIDs {\n\t\t// in multi partitioned index, we cannot be sure of the score or the ordering of the hits as the tf-idf scores are localized to each partition\n\t\t// so we only check the ids\n\t\tcontrolMap := make(map[string]struct{})\n\t\texperimentalMap := make(map[string]struct{})\n\t\tfor _, hit := range controlResult.Hits {\n\t\t\tcontrolMap[hit.ID] = struct{}{}\n\t\t}\n\t\tfor _, hit := range experimentalResult.Hits {\n\t\t\texperimentalMap[hit.ID] = struct{}{}\n\t\t}\n\t\tif len(controlMap) != len(experimentalMap) {\n\t\t\tt.Fatalf(\"test case #%d failed: expected %d results, got %d\", testCaseNum, len(controlMap), len(experimentalMap))\n\t\t}\n\t\tfor id := range controlMap {\n\t\t\tif _, ok := experimentalMap[id]; !ok {\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected id %s to be in experimental result\", testCaseNum, id)\n\t\t\t}\n\t\t}\n\t\treturn\n\t}\n\tfor i := 0; i < len(controlResult.Hits); i++ {\n\t\tif controlResult.Hits[i].ID != experimentalResult.Hits[i].ID {\n\t\t\tt.Fatalf(\"test case #%d failed: expected hit %d to have id %s, got %s\", testCaseNum, i, controlResult.Hits[i].ID, experimentalResult.Hits[i].ID)\n\t\t}\n\t\t// Truncate to 6 decimal places\n\t\tactualScore := truncateScore(experimentalResult.Hits[i].Score)\n\t\texpectScore := truncateScore(controlResult.Hits[i].Score)\n\t\tif expectScore != actualScore {\n\t\t\tt.Fatalf(\"test case #%d failed: expected hit %d to have score %f, got %f\", testCaseNum, i, expectScore, actualScore)\n\t\t}\n\t\tif !compareExplanation(controlResult.Hits[i].Expl, experimentalResult.Hits[i].Expl) {\n\t\t\tt.Fatalf(\"test case #%d failed: expected hit %d to have explanation %v, got %v\", testCaseNum, i, controlResult.Hits[i].Expl, experimentalResult.Hits[i].Expl)\n\t\t}\n\t}\n\tif truncateScore(controlResult.MaxScore) != truncateScore(experimentalResult.MaxScore) {\n\t\tt.Fatalf(\"test case #%d: expected maxScore to be %f, got %f\", testCaseNum, controlResult.MaxScore, experimentalResult.MaxScore)\n\t}\n}\n\nfunc TestSimilaritySearchMultipleSegments(t *testing.T) {\n\t// using scorch options to prevent merges during the course of this test\n\t// so that the knnCollector can be accurately tested\n\tscorch.DefaultMemoryPressurePauseThreshold = 0\n\tscorch.DefaultMinSegmentsForInMemoryMerge = math.MaxInt\n\tdataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocuments := makeDatasetIntoDocuments(dataset)\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Analyzer = en.AnalyzerName\n\n\tvecFieldMappingL2 := mapping.NewVectorFieldMapping()\n\tvecFieldMappingL2.Dims = testDatasetDims\n\tvecFieldMappingL2.Similarity = index.EuclideanDistance\n\n\tvecFieldMappingDot := mapping.NewVectorFieldMapping()\n\tvecFieldMappingDot.Dims = testDatasetDims\n\tvecFieldMappingDot.Similarity = index.InnerProduct\n\n\tvecFieldMappingCosine := mapping.NewVectorFieldMapping()\n\tvecFieldMappingCosine.Dims = testDatasetDims\n\tvecFieldMappingCosine.Similarity = index.CosineSimilarity\n\n\tindexMappingL2Norm := NewIndexMapping()\n\tindexMappingL2Norm.DefaultMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tindexMappingL2Norm.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFieldMappingL2)\n\n\tindexMappingDotProduct := NewIndexMapping()\n\tindexMappingDotProduct.DefaultMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tindexMappingDotProduct.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFieldMappingDot)\n\n\tindexMappingCosine := NewIndexMapping()\n\tindexMappingCosine.DefaultMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tindexMappingCosine.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFieldMappingCosine)\n\n\tvar reqSort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: \"content\"}}\n\n\ttestCases := []struct {\n\t\tnumSegments int\n\t\tqueryIndex  int\n\t\tmapping     mapping.IndexMapping\n\t\tscoreValue  string\n\t}{\n\t\t// L2 norm similarity\n\t\t{\n\t\t\tnumSegments: 6,\n\t\t\tqueryIndex:  0,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 7,\n\t\t\tqueryIndex:  1,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 8,\n\t\t\tqueryIndex:  2,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 9,\n\t\t\tqueryIndex:  3,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 10,\n\t\t\tqueryIndex:  4,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 11,\n\t\t\tqueryIndex:  5,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t},\n\t\t// dot_product similarity\n\t\t{\n\t\t\tnumSegments: 6,\n\t\t\tqueryIndex:  0,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 7,\n\t\t\tqueryIndex:  1,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 8,\n\t\t\tqueryIndex:  2,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 9,\n\t\t\tqueryIndex:  3,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 10,\n\t\t\tqueryIndex:  4,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 11,\n\t\t\tqueryIndex:  5,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t},\n\t\t// cosine similarity\n\t\t{\n\t\t\tnumSegments: 9,\n\t\t\tqueryIndex:  0,\n\t\t\tmapping:     indexMappingCosine,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 5,\n\t\t\tqueryIndex:  1,\n\t\t\tmapping:     indexMappingCosine,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 4,\n\t\t\tqueryIndex:  2,\n\t\t\tmapping:     indexMappingCosine,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 12,\n\t\t\tqueryIndex:  3,\n\t\t\tmapping:     indexMappingCosine,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 7,\n\t\t\tqueryIndex:  4,\n\t\t\tmapping:     indexMappingCosine,\n\t\t},\n\t\t{\n\t\t\tnumSegments: 11,\n\t\t\tqueryIndex:  5,\n\t\t\tmapping:     indexMappingCosine,\n\t\t},\n\t\t// score none test\n\t\t{\n\t\t\tnumSegments: 3,\n\t\t\tqueryIndex:  0,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t\t{\n\t\t\tnumSegments: 7,\n\t\t\tqueryIndex:  1,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t\t{\n\t\t\tnumSegments: 8,\n\t\t\tqueryIndex:  2,\n\t\t\tmapping:     indexMappingL2Norm,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t\t{\n\t\t\tnumSegments: 3,\n\t\t\tqueryIndex:  0,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t\t{\n\t\t\tnumSegments: 7,\n\t\t\tqueryIndex:  1,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t\t{\n\t\t\tnumSegments: 8,\n\t\t\tqueryIndex:  2,\n\t\t\tmapping:     indexMappingDotProduct,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t\t{\n\t\t\tnumSegments: 3,\n\t\t\tqueryIndex:  0,\n\t\t\tmapping:     indexMappingCosine,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t\t{\n\t\t\tnumSegments: 7,\n\t\t\tqueryIndex:  1,\n\t\t\tmapping:     indexMappingCosine,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t\t{\n\t\t\tnumSegments: 8,\n\t\t\tqueryIndex:  2,\n\t\t\tmapping:     indexMappingCosine,\n\t\t\tscoreValue:  \"none\",\n\t\t},\n\t}\n\tfor testCaseNum, testCase := range testCases {\n\t\toriginalRequest := searchRequests[testCase.queryIndex]\n\t\tfor _, operator := range knnOperators {\n\t\t\t// run single segment test first\n\t\t\ttmpIndexPath := createTmpIndexPath(t)\n\t\t\tindex, err := New(tmpIndexPath, testCase.mapping)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tquery := copySearchRequest(originalRequest, nil)\n\t\t\tquery.Sort = reqSort.Copy()\n\t\t\tquery.AddKNNOperator(operator)\n\t\t\tquery.Explain = true\n\n\t\t\tnameToIndex := make(map[string]Index)\n\t\t\tnameToIndex[index.Name()] = index\n\n\t\t\terr = createMultipleSegmentsIndex(documents, index, 1)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tcontrolResult, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(controlResult.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected control result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\t\t\tif testCase.scoreValue == \"none\" {\n\n\t\t\t\tquery := copySearchRequest(originalRequest, nil)\n\t\t\t\tquery.Sort = reqSort.Copy()\n\t\t\t\tquery.AddKNNOperator(operator)\n\t\t\t\tquery.Explain = true\n\t\t\t\tquery.Score = testCase.scoreValue\n\n\t\t\t\texpectedResultScoreNone, err := index.Search(query)\n\t\t\t\tif err != nil {\n\t\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t\tif !finalHitsHaveValidIndex(expectedResultScoreNone.Hits, nameToIndex) {\n\t\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\t\tt.Fatalf(\"test case #%d failed: expected score none hits to have valid `Index`\", testCaseNum)\n\t\t\t\t}\n\t\t\t\tverifyResult(t, controlResult, expectedResultScoreNone, testCaseNum, true)\n\t\t\t}\n\t\t\tcleanUp(t, nameToIndex)\n\n\t\t\t// run multiple segments test\n\t\t\ttmpIndexPath = createTmpIndexPath(t)\n\t\t\tindex, err = New(tmpIndexPath, testCase.mapping)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tnameToIndex = make(map[string]Index)\n\t\t\tnameToIndex[index.Name()] = index\n\t\t\terr = createMultipleSegmentsIndex(documents, index, testCase.numSegments)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tquery = copySearchRequest(originalRequest, nil)\n\t\t\tquery.Sort = reqSort.Copy()\n\t\t\tquery.AddKNNOperator(operator)\n\t\t\tquery.Explain = true\n\n\t\t\texperimentalResult, err := index.Search(query)\n\t\t\tif err != nil {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif !finalHitsHaveValidIndex(experimentalResult.Hits, nameToIndex) {\n\t\t\t\tcleanUp(t, nameToIndex)\n\t\t\t\tt.Fatalf(\"test case #%d failed: expected experimental result hits to have valid `Index`\", testCaseNum)\n\t\t\t}\n\t\t\tverifyResult(t, controlResult, experimentalResult, testCaseNum, false)\n\t\t\tcleanUp(t, nameToIndex)\n\t\t}\n\t}\n}\n\n// Test to determine the impact of boost on kNN queries.\nfunc TestKNNScoreBoosting(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tconst dims = 5\n\tgetRandomVector := func() []float32 {\n\t\tvec := make([]float32, dims)\n\t\tfor i := 0; i < dims; i++ {\n\t\t\tvec[i] = rand.Float32()\n\t\t}\n\t\treturn vec\n\t}\n\n\tdataset := make([]map[string]interface{}, 10)\n\n\t// Indexing just a few docs to populate index.\n\tfor i := 0; i < 100; i++ {\n\t\tdataset = append(dataset, map[string]interface{}{\n\t\t\t\"type\":    \"vectorStuff\",\n\t\t\t\"content\": strconv.Itoa(i),\n\t\t\t\"vector\":  getRandomVector(),\n\t\t})\n\t}\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tdocumentMapping := NewDocumentMapping()\n\tindexMapping.AddDocumentMapping(\"vectorStuff\", documentMapping)\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Index = true\n\tcontentFieldMapping.Store = true\n\tdocumentMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\n\tvecFieldMapping := mapping.NewVectorFieldMapping()\n\tvecFieldMapping.Index = true\n\tvecFieldMapping.Dims = 5\n\tvecFieldMapping.Similarity = \"dot_product\"\n\tdocumentMapping.AddFieldMappingsAt(\"vector\", vecFieldMapping)\n\n\tindex, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch := index.NewBatch()\n\tfor i := 0; i < len(dataset); i++ {\n\t\terr = batch.Index(strconv.Itoa(i), dataset[i])\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tqueryVec := getRandomVector()\n\tsearchRequest := NewSearchRequest(NewMatchNoneQuery())\n\tsearchRequest.AddKNN(\"vector\", queryVec, 3, 1.0)\n\tsearchRequest.Fields = []string{\"content\", \"vector\"}\n\n\thits, err := index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\thitsMap := make(map[string]float64, 0)\n\tfor _, hit := range hits.Hits {\n\t\thitsMap[hit.ID] = (hit.Score)\n\t}\n\n\tsearchRequest = NewSearchRequest(NewMatchNoneQuery())\n\tsearchRequest.AddKNN(\"vector\", queryVec, 3, 10.0)\n\tsearchRequest.Fields = []string{\"content\", \"vector\"}\n\n\thits, err = index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\thitsMap2 := make(map[string]float64, 0)\n\tfor _, hit := range hits.Hits {\n\t\thitsMap2[hit.ID] = (hit.Score)\n\t}\n\n\tfor _, hit := range hits.Hits {\n\t\tif hitsMap[hit.ID] != hitsMap2[hit.ID]/10 {\n\t\t\tt.Errorf(\"boosting not working: %v %v \\n\", hitsMap[hit.ID], hitsMap2[hit.ID])\n\t\t}\n\t}\n}\n\n// Test to see if KNN Operators get added right to the query.\nfunc TestKNNOperator(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tconst dims = 5\n\tgetRandomVector := func() []float32 {\n\t\tvec := make([]float32, dims)\n\t\tfor i := 0; i < dims; i++ {\n\t\t\tvec[i] = rand.Float32()\n\t\t}\n\t\treturn vec\n\t}\n\n\tdataset := make([]map[string]interface{}, 10)\n\n\t// Indexing just a few docs to populate index.\n\tfor i := 0; i < 10; i++ {\n\t\tdataset = append(dataset, map[string]interface{}{\n\t\t\t\"type\":    \"vectorStuff\",\n\t\t\t\"content\": strconv.Itoa(i),\n\t\t\t\"vector\":  getRandomVector(),\n\t\t})\n\t}\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tdocumentMapping := NewDocumentMapping()\n\tindexMapping.AddDocumentMapping(\"vectorStuff\", documentMapping)\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Index = true\n\tcontentFieldMapping.Store = true\n\tdocumentMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\n\tvecFieldMapping := mapping.NewVectorFieldMapping()\n\tvecFieldMapping.Index = true\n\tvecFieldMapping.Dims = 5\n\tvecFieldMapping.Similarity = \"dot_product\"\n\tdocumentMapping.AddFieldMappingsAt(\"vector\", vecFieldMapping)\n\n\tindex, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch := index.NewBatch()\n\tfor i := 0; i < len(dataset); i++ {\n\t\terr = batch.Index(strconv.Itoa(i), dataset[i])\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttermQuery := query.NewTermQuery(\"2\")\n\n\tsearchRequest := NewSearchRequest(termQuery)\n\tsearchRequest.AddKNN(\"vector\", getRandomVector(), 3, 2.0)\n\tsearchRequest.AddKNN(\"vector\", getRandomVector(), 2, 1.5)\n\tsearchRequest.Fields = []string{\"content\", \"vector\"}\n\n\t// Conjunction\n\tsearchRequest.AddKNNOperator(knnOperatorAnd)\n\tconjunction, _, _, err := createKNNQuery(searchRequest, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error for AND knn operator\")\n\t}\n\n\tconj, ok := conjunction.(*query.DisjunctionQuery)\n\tif !ok {\n\t\tt.Fatalf(\"expected disjunction query\")\n\t}\n\n\tif len(conj.Disjuncts) != 2 {\n\t\tt.Fatalf(\"expected 2 disjuncts\")\n\t}\n\n\t// Disjunction\n\tsearchRequest.AddKNNOperator(knnOperatorOr)\n\tdisjunction, _, _, err := createKNNQuery(searchRequest, nil)\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error for OR knn operator\")\n\t}\n\n\tdisj, ok := disjunction.(*query.DisjunctionQuery)\n\tif !ok {\n\t\tt.Fatalf(\"expected disjunction query\")\n\t}\n\n\tif len(disj.Disjuncts) != 2 {\n\t\tt.Fatalf(\"expected 2 disjuncts\")\n\t}\n\n\t// Incorrect operator.\n\tsearchRequest.AddKNNOperator(\"bs_op\")\n\tsearchRequest.Query, _, _, err = createKNNQuery(searchRequest, nil)\n\tif err == nil {\n\t\tt.Fatalf(\"expected error for incorrect knn operator\")\n\t}\n}\n\nfunc TestKNNFiltering(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tconst dims = 5\n\tgetRandomVector := func() []float32 {\n\t\tvec := make([]float32, dims)\n\t\tfor i := 0; i < dims; i++ {\n\t\t\tvec[i] = rand.Float32()\n\t\t}\n\t\treturn vec\n\t}\n\n\tdataset := make([]map[string]interface{}, 0)\n\n\t// Indexing just a few docs to populate index.\n\tfor i := 0; i < 10; i++ {\n\t\tdataset = append(dataset, map[string]interface{}{\n\t\t\t\"type\":    \"vectorStuff\",\n\t\t\t\"content\": strconv.Itoa(i + 1000),\n\t\t\t\"vector\":  getRandomVector(),\n\t\t})\n\t}\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tdocumentMapping := NewDocumentMapping()\n\tindexMapping.AddDocumentMapping(\"vectorStuff\", documentMapping)\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Index = true\n\tcontentFieldMapping.Store = true\n\tdocumentMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\n\tvecFieldMapping := mapping.NewVectorFieldMapping()\n\tvecFieldMapping.Index = true\n\tvecFieldMapping.Dims = 5\n\tvecFieldMapping.Similarity = \"dot_product\"\n\tdocumentMapping.AddFieldMappingsAt(\"vector\", vecFieldMapping)\n\n\tindex, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch := index.NewBatch()\n\tfor i := 0; i < len(dataset); i++ {\n\t\t// the id of term \"i\" is (i-1000)\n\t\terr = batch.Index(strconv.Itoa(i), dataset[i])\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttermQuery := query.NewTermQuery(\"1004\")\n\tfilterRequest := NewSearchRequest(termQuery)\n\tfilteredHits, err := index.Search(filterRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfilteredDocIDs := make(map[string]struct{})\n\tfor _, match := range filteredHits.Hits {\n\t\tfilteredDocIDs[match.ID] = struct{}{}\n\t}\n\n\tsearchRequest := NewSearchRequest(NewMatchNoneQuery())\n\tsearchRequest.AddKNNWithFilter(\"vector\", getRandomVector(), 3, 2.0, termQuery)\n\tsearchRequest.Fields = []string{\"content\", \"vector\"}\n\n\tres, err := index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// check if any of the returned results are not part of the filtered hits.\n\tfor _, match := range res.Hits {\n\t\tif _, exists := filteredDocIDs[match.ID]; !exists {\n\t\t\tt.Errorf(\"returned result not present in filtered hits\")\n\t\t}\n\t}\n\n\t// No results should be returned with a match_none filter.\n\tsearchRequest = NewSearchRequest(NewMatchNoneQuery())\n\tsearchRequest.AddKNNWithFilter(\"vector\", getRandomVector(), 3, 2.0,\n\t\tNewMatchNoneQuery())\n\tres, err = index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res.Hits) != 0 {\n\t\tt.Errorf(\"match none filter should return no hits\")\n\t}\n\n\t// Testing with a disjunction query.\n\n\ttermQuery = query.NewTermQuery(\"1003\")\n\ttermQuery2 := query.NewTermQuery(\"1005\")\n\tdisjQuery := query.NewDisjunctionQuery([]query.Query{termQuery, termQuery2})\n\tfilterRequest = NewSearchRequest(disjQuery)\n\tfilteredHits, err = index.Search(filterRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfilteredDocIDs = make(map[string]struct{})\n\tfor _, match := range filteredHits.Hits {\n\t\tfilteredDocIDs[match.ID] = struct{}{}\n\t}\n\n\tsearchRequest = NewSearchRequest(NewMatchNoneQuery())\n\tsearchRequest.AddKNNWithFilter(\"vector\", getRandomVector(), 3, 2.0, disjQuery)\n\tsearchRequest.Fields = []string{\"content\", \"vector\"}\n\n\tres, err = index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, match := range res.Hits {\n\t\tif _, exists := filteredDocIDs[match.ID]; !exists {\n\t\t\tt.Errorf(\"returned result not present in filtered hits\")\n\t\t}\n\t}\n}\n\n// -----------------------------------------------------------------------------\n// Test nested vectors\n\nfunc TestNestedVectors(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tconst dims = 3\n\tconst k = 1 // one nearest neighbor\n\tconst vecFieldName = \"vecData\"\n\n\tdataset := map[string]map[string]interface{}{ // docID -> Doc\n\t\t\"doc1\": {\n\t\t\tvecFieldName: []float32{100, 100, 100},\n\t\t},\n\t\t\"doc2\": {\n\t\t\tvecFieldName: [][]float32{{0, 0, 0}, {1000, 1000, 1000}},\n\t\t},\n\t}\n\n\t// Index mapping\n\tindexMapping := NewIndexMapping()\n\tvm := mapping.NewVectorFieldMapping()\n\tvm.Dims = dims\n\tvm.Similarity = \"l2_norm\"\n\tindexMapping.DefaultMapping.AddFieldMappingsAt(vecFieldName, vm)\n\n\t// Create index and upload documents\n\tindex, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch := index.NewBatch()\n\tfor docID, doc := range dataset {\n\t\terr = batch.Index(docID, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Run searches\n\n\ttests := []struct {\n\t\tqueryVec      []float32\n\t\texpectedDocID string\n\t}{\n\t\t{\n\t\t\tqueryVec:      []float32{100, 100, 100},\n\t\t\texpectedDocID: \"doc1\",\n\t\t},\n\t\t{\n\t\t\tqueryVec:      []float32{0, 0, 0},\n\t\t\texpectedDocID: \"doc2\",\n\t\t},\n\t\t{\n\t\t\tqueryVec:      []float32{1000, 1000, 1000},\n\t\t\texpectedDocID: \"doc2\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tsearchReq := NewSearchRequest(query.NewMatchNoneQuery())\n\t\tsearchReq.AddKNNWithFilter(vecFieldName, test.queryVec, k, 1000,\n\t\t\tNewMatchAllQuery())\n\n\t\tres, err := index.Search(searchReq)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif len(res.Hits) != 1 {\n\t\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t\t}\n\n\t\tif res.Hits[0].ID != test.expectedDocID {\n\t\t\tt.Fatalf(\"expected docID %s, got %s\", test.expectedDocID,\n\t\t\t\tres.Hits[0].ID)\n\t\t}\n\t}\n}\n\n// -----------------------------------------------------------------------------\n// TestMultiVector tests the KNN functionality which handles duplicate\n// vectors being matched within the same document. When a document has multiple vectors\n// (via [[]] array of vectors or [{}] array of objects with vectors), the KNN\n// searcher must pick the best scoring vector match for that document. This test covers these scenarios:\n// - Single vector field (baseline)\n// - [[]] style: array of vectors (same doc appears multiple times)\n// - [{}] style: array of objects with vector field (chunks pattern)\nfunc TestMultiVector(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\t// JSON documents covering merger scenarios:\n\t// - Single vector (baseline)\n\t// - [[]] style: array of vectors (same doc appears multiple times)\n\t// - [{}] style: array of objects with vector field (chunks pattern)\n\tdocs := map[string]string{\n\t\t// Single vector - baseline\n\t\t\"doc1\": `{\n\t\t\t\"vec\": [10, 10, 10],\n\t\t\t\"vecB\": [100, 100, 100]\n\t\t}`,\n\t\t// [[]] style - array of 2 vectors\n\t\t\"doc2\": `{\n\t\t\t\"vec\": [[0, 0, 0], [500, 500, 500]],\n\t\t\t\"vecB\": [[900, 900, 900], [950, 950, 950], [975, 975, 975], [990, 990, 990]]\n\t\t}`,\n\t\t// [[]] style - array of 3 vectors\n\t\t\"doc3\": `{ \n\t\t\t\"vec\": [[50, 50, 50], [200, 200, 200], [400, 400, 400]],\n\t\t\t\"vecB\": [[800, 800, 800], [850, 850, 850]]\n\t\t}`,\n\t\t// Single vector - baseline\n\t\t\"doc4\": `{\n\t\t\t\"vec\": [1000, 1000, 1000],\n\t\t\t\"vecB\": [1, 1, 1]\n\t\t}`,\n\t\t// [{}] style - array of objects with vector field (chunks pattern)\n\t\t\"doc5\": `{\n\t\t\t\"chunks\": [\n\t\t\t\t{\"vec\": [10, 10, 10], \"text\": \"chunk1\"},\n\t\t\t\t{\"vec\": [20, 20, 20], \"text\": \"chunk2\"},\n\t\t\t\t{\"vec\": [30, 30, 30], \"text\": \"chunk3\"},\n\t\t\t\t{\"vec\": [40, 40, 40], \"text\": \"chunk4\"}\n\t\t\t]\n\t\t}`,\n\t\t\"doc6\": `{\n\t\t\t\"chunks\": [\n\t\t\t\t{\"vec\": [[10, 10, 10],[20, 20, 20]], \"text\": \"chunk1\"},\n\t\t\t\t{\"vec\": [[30, 30, 30],[40, 40, 40]], \"text\": \"chunk2\"}\n\t\t\t]\n\t\t}`,\n\t}\n\n\t// Parse JSON documents\n\tdataset := make(map[string]map[string]interface{})\n\tfor docID, jsonStr := range docs {\n\t\tvar doc map[string]interface{}\n\t\tif err := json.Unmarshal([]byte(jsonStr), &doc); err != nil {\n\t\t\tt.Fatalf(\"failed to unmarshal %s: %v\", docID, err)\n\t\t}\n\t\tdataset[docID] = doc\n\t}\n\n\t// Index mapping\n\tindexMapping := NewIndexMapping()\n\n\tvecMapping := mapping.NewVectorFieldMapping()\n\tvecMapping.Dims = 3\n\tvecMapping.Similarity = index.InnerProduct\n\tindexMapping.DefaultMapping.AddFieldMappingsAt(\"vec\", vecMapping)\n\tindexMapping.DefaultMapping.AddFieldMappingsAt(\"vecB\", vecMapping)\n\n\t// Nested chunks mapping for [{}] style\n\tchunksMapping := mapping.NewDocumentMapping()\n\tchunksMapping.AddFieldMappingsAt(\"vec\", vecMapping)\n\tindexMapping.DefaultMapping.AddSubDocumentMapping(\"chunks\", chunksMapping)\n\n\t// Create and populate index\n\tidx, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\tif err := idx.Close(); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch := idx.NewBatch()\n\tfor docID, doc := range dataset {\n\t\tif err := batch.Index(docID, doc); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tif err := idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Test: Single KNN query - basic functionality\n\tt.Run(\"VecFieldSingle\", func(t *testing.T) {\n\t\tsearchReq := NewSearchRequest(query.NewMatchNoneQuery())\n\t\tsearchReq.AddKNN(\"vec\", []float32{1, 1, 1}, 20, 1.0)\n\t\tres, err := idx.Search(searchReq)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\t// Inner product: score = sum(query_i * doc_i)\n\t\t// doc1 vec=[10,10,10]: 1*10*3 = 30\n\t\t// doc2 vec best is [500,500,500]: 1*500*3 = 1500\n\t\t// doc3 vec best is [400,400,400]: 1*400*3 = 1200\n\t\t// doc4 vec=[1000,1000,1000]: 1*1000*3 = 3000\n\t\texpectedResult := []struct {\n\t\t\tdocID         string\n\t\t\texpectedScore float64\n\t\t}{\n\t\t\t{docID: \"doc4\", expectedScore: 3000},\n\t\t\t{docID: \"doc2\", expectedScore: 1500},\n\t\t\t{docID: \"doc3\", expectedScore: 1200},\n\t\t\t{docID: \"doc1\", expectedScore: 30},\n\t\t}\n\n\t\tif len(res.Hits) != len(expectedResult) {\n\t\t\tt.Fatalf(\"expected %d hits, got %d\", len(expectedResult), len(res.Hits))\n\t\t}\n\n\t\tfor i, expected := range expectedResult {\n\t\t\tif res.Hits[i].ID != expected.docID {\n\t\t\t\tt.Fatalf(\"at rank %d, expected docID %s, got %s\", i+1, expected.docID, res.Hits[i].ID)\n\t\t\t}\n\t\t\tif res.Hits[i].Score != expected.expectedScore {\n\t\t\t\tt.Fatalf(\"at rank %d, expected score %v, got %v\", i+1, expected.expectedScore, res.Hits[i].Score)\n\t\t\t}\n\t\t}\n\t})\n\n\t// Test: Single KNN query on vecB field\n\tt.Run(\"VecBFieldSingle\", func(t *testing.T) {\n\t\tsearchReq := NewSearchRequest(query.NewMatchNoneQuery())\n\t\tsearchReq.AddKNN(\"vecB\", []float32{1000, 1000, 1000}, 20, 1.0)\n\t\tres, err := idx.Search(searchReq)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\t// Inner product: score = sum(query_i * doc_i) for each dimension\n\t\t// doc1: vecB=[100,100,100] -> 1000*100*3 = 300,000\n\t\t// doc2: vecB best is [990,990,990] -> 1000*990*3 = 2,970,000\n\t\t// doc3: vecB best is [850,850,850] -> 1000*850*3 = 2,550,000\n\t\t// doc4: vecB=[1,1,1] -> 1000*1*3 = 3,000\n\t\texpectedResult := []struct {\n\t\t\tdocID         string\n\t\t\texpectedScore float64\n\t\t}{\n\t\t\t{docID: \"doc2\", expectedScore: 2970000},\n\t\t\t{docID: \"doc3\", expectedScore: 2550000},\n\t\t\t{docID: \"doc1\", expectedScore: 300000},\n\t\t\t{docID: \"doc4\", expectedScore: 3000},\n\t\t}\n\n\t\tif len(res.Hits) != len(expectedResult) {\n\t\t\tt.Fatalf(\"expected %d hits, got %d\", len(expectedResult), len(res.Hits))\n\t\t}\n\n\t\tfor i, expected := range expectedResult {\n\t\t\tif res.Hits[i].ID != expected.docID {\n\t\t\t\tt.Fatalf(\"at rank %d, expected docID %s, got %s\", i+1, expected.docID, res.Hits[i].ID)\n\t\t\t}\n\t\t\tif res.Hits[i].Score != expected.expectedScore {\n\t\t\t\tt.Fatalf(\"at rank %d, expected score %v, got %v\", i+1, expected.expectedScore, res.Hits[i].Score)\n\t\t\t}\n\t\t}\n\t})\n\n\t// Test: Single KNN query on nested chunks.vec field\n\tt.Run(\"ChunksVecFieldSingle\", func(t *testing.T) {\n\t\tsearchReq := NewSearchRequest(query.NewMatchNoneQuery())\n\t\tsearchReq.AddKNN(\"chunks.vec\", []float32{1, 1, 1}, 20, 1.0)\n\t\tsearchReq.SortBy([]string{\"_score\", \"docID\"})\n\t\tres, err := idx.Search(searchReq)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\t// Only doc5 and doc6 have chunks.vec\n\t\t// doc5 chunks: [10,10,10], [20,20,20], [30,30,30], [40,40,40]\n\t\t//   Best score: 1*40*3 = 120\n\t\t// doc6 chunks: [[10,10,10],[20,20,20]], [[30,30,30],[40,40,40]]\n\t\t//   Best score: 1*40*3 = 120\n\t\tif len(res.Hits) != 2 {\n\t\t\tt.Fatalf(\"expected 2 hits, got %d\", len(res.Hits))\n\t\t}\n\n\t\t// Both should have score 120\n\t\tfor _, hit := range res.Hits {\n\t\t\tif hit.ID != \"doc5\" && hit.ID != \"doc6\" {\n\t\t\t\tt.Fatalf(\"unexpected docID %s, expected doc5 or doc6\", hit.ID)\n\t\t\t}\n\t\t\tif hit.Score != 120 {\n\t\t\t\tt.Fatalf(\"for %s, expected score 120, got %v\", hit.ID, hit.Score)\n\t\t\t}\n\t\t}\n\t})\n}\n\n// TestMultiVectorCosineNormalization verifies that multi-vector fields are\n// normalized correctly with cosine similarity. Each sub-vector in a multi-vector\n// should be independently normalized, producing correct similarity scores.\nfunc TestMultiVectorCosineNormalization(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tconst dims = 3\n\n\t// Create index with cosine similarity\n\tindexMapping := NewIndexMapping()\n\tvecFieldMapping := mapping.NewVectorFieldMapping()\n\tvecFieldMapping.Dims = dims\n\tvecFieldMapping.Similarity = index.CosineSimilarity\n\n\t// Single-vector field\n\tindexMapping.DefaultMapping.AddFieldMappingsAt(\"vec\", vecFieldMapping)\n\t// Multi-vector field\n\tindexMapping.DefaultMapping.AddFieldMappingsAt(\"multi_vec\", vecFieldMapping)\n\n\tidx, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocsString := []string{\n\t\t`{\"vec\": [3, 0, 0]}`,\n\t\t`{\"vec\": [0, 4, 0]}`,\n\t\t`{\"multi_vec\": [[3, 0, 0], [0, 4, 0]]}`,\n\t}\n\n\tfor i, docStr := range docsString {\n\t\tvar doc map[string]interface{}\n\t\terr = json.Unmarshal([]byte(docStr), &doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = idx.Index(fmt.Sprintf(\"doc%d\", i+1), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\t// Query for X direction [1,0,0]\n\tsearchReq := NewSearchRequest(query.NewMatchNoneQuery())\n\tsearchReq.AddKNN(\"vec\", []float32{1, 0, 0}, 3, 1.0)\n\tres, err := idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res.Hits) != 2 {\n\t\tt.Fatalf(\"expected 2 hits, got %d\", len(res.Hits))\n\t}\n\t// Hit 1 should be doc1 with score 1.0 (perfect match)\n\tif res.Hits[0].ID != \"doc1\" {\n\t\tt.Fatalf(\"expected doc1 as first hit, got %s\", res.Hits[0].ID)\n\t}\n\tif math.Abs(float64(res.Hits[0].Score-1.0)) > 1e-6 {\n\t\tt.Fatalf(\"expected score 1.0, got %f\", res.Hits[0].Score)\n\t}\n\t// Hit 2 should be doc2 with a score of 0.0 (orthogonal)\n\tif res.Hits[1].ID != \"doc2\" {\n\t\tt.Fatalf(\"expected doc2 as second hit, got %s\", res.Hits[1].ID)\n\t}\n\tif math.Abs(float64(res.Hits[1].Score-0.0)) > 1e-6 {\n\t\tt.Fatalf(\"expected score 0.0, got %f\", res.Hits[1].Score)\n\t}\n\n\t// Query for Y direction [0,1,0]\n\tsearchReq = NewSearchRequest(query.NewMatchNoneQuery())\n\tsearchReq.AddKNN(\"vec\", []float32{0, 1, 0}, 3, 1.0)\n\tres, err = idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res.Hits) != 2 {\n\t\tt.Fatalf(\"expected 2 hits, got %d\", len(res.Hits))\n\t}\n\t// Hit 1 should be doc2 with score 1.0 (perfect match)\n\tif res.Hits[0].ID != \"doc2\" {\n\t\tt.Fatalf(\"expected doc2 as first hit, got %s\", res.Hits[0].ID)\n\t}\n\tif math.Abs(float64(res.Hits[0].Score-1.0)) > 1e-6 {\n\t\tt.Fatalf(\"expected score 1.0, got %f\", res.Hits[0].Score)\n\t}\n\t// Hit 2 should be doc1 with a score of 0.0 (orthogonal)\n\tif res.Hits[1].ID != \"doc1\" {\n\t\tt.Fatalf(\"expected doc1 as second hit, got %s\", res.Hits[1].ID)\n\t}\n\tif math.Abs(float64(res.Hits[1].Score-0.0)) > 1e-6 {\n\t\tt.Fatalf(\"expected score 0.0, got %f\", res.Hits[1].Score)\n\t}\n\n\t// Now test querying the nested multi-vector field\n\tsearchReq = NewSearchRequest(query.NewMatchNoneQuery())\n\tsearchReq.AddKNN(\"multi_vec\", []float32{1, 0, 0}, 3, 1.0)\n\tres, err = idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\t// Hit should be doc3 with score 1.0 (perfect match on first sub-vector)\n\tif res.Hits[0].ID != \"doc3\" {\n\t\tt.Fatalf(\"expected doc3 as first hit, got %s\", res.Hits[0].ID)\n\t}\n\tif math.Abs(float64(res.Hits[0].Score-1.0)) > 1e-6 {\n\t\tt.Fatalf(\"expected score 1.0, got %f\", res.Hits[0].Score)\n\t}\n\t// Query for Y direction [0,1,0] on nested field\n\tsearchReq = NewSearchRequest(query.NewMatchNoneQuery())\n\tsearchReq.AddKNN(\"multi_vec\", []float32{0, 1, 0}, 3, 1.0)\n\tres, err = idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\t// Hit should be doc3 with score 1.0 (perfect match on second sub-vector)\n\tif res.Hits[0].ID != \"doc3\" {\n\t\tt.Fatalf(\"expected doc3 as first hit, got %s\", res.Hits[0].ID)\n\t}\n\tif math.Abs(float64(res.Hits[0].Score-1.0)) > 1e-6 {\n\t\tt.Fatalf(\"expected score 1.0, got %f\", res.Hits[0].Score)\n\t}\n}\n\nfunc TestNumVecsStat(t *testing.T) {\n\n\tdataset, _, err := readDatasetAndQueries(testInputCompressedFile)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdocuments := makeDatasetIntoDocuments(dataset)\n\n\tindexMapping := NewIndexMapping()\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Analyzer = en.AnalyzerName\n\tindexMapping.DefaultMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\n\tvecFieldMapping1 := mapping.NewVectorFieldMapping()\n\tvecFieldMapping1.Dims = testDatasetDims\n\tvecFieldMapping1.Similarity = index.EuclideanDistance\n\tindexMapping.DefaultMapping.AddFieldMappingsAt(\"vector\", vecFieldMapping1)\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tindex, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor i := 0; i < 10; i++ {\n\t\tbatch := index.NewBatch()\n\t\tfor j := 0; j < 3; j++ {\n\t\t\tfor k := 0; k < 10; k++ {\n\t\t\t\terr := batch.Index(fmt.Sprintf(\"%d\", i*30+j*10+k), documents[j*10+k])\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\terr = index.Batch(batch)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tstatsMap := index.StatsMap()\n\n\tif indexStats, exists := statsMap[\"index\"]; exists {\n\t\tif indexStatsMap, ok := indexStats.(map[string]interface{}); ok {\n\t\t\tv1, ok := indexStatsMap[\"field:vector:num_vectors\"].(uint64)\n\t\t\tif !ok || v1 != uint64(300) {\n\t\t\t\tt.Fatalf(\"mismatch in the number of vectors, expected 300, got %d\", indexStatsMap[\"field:vector:num_vectors\"])\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestIndexUpdateVector(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindexMappingBefore := mapping.NewIndexMapping()\n\tindexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{}\n\tindexMappingBefore.DefaultMapping = &mapping.DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: false,\n\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\"a\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:                    \"vector\",\n\t\t\t\t\t\tIndex:                   true,\n\t\t\t\t\t\tDims:                    4,\n\t\t\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"b\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:                    \"vector\",\n\t\t\t\t\t\tIndex:                   true,\n\t\t\t\t\t\tDims:                    4,\n\t\t\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"c\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:                    \"vector_base64\",\n\t\t\t\t\t\tIndex:                   true,\n\t\t\t\t\t\tDims:                    4,\n\t\t\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"d\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:                    \"vector_base64\",\n\t\t\t\t\t\tIndex:                   true,\n\t\t\t\t\t\tDims:                    4,\n\t\t\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tFields: []*mapping.FieldMapping{},\n\t}\n\tindexMappingBefore.IndexDynamic = false\n\tindexMappingBefore.StoreDynamic = false\n\tindexMappingBefore.DocValuesDynamic = false\n\n\tindex, err := New(tmpIndexPath, indexMappingBefore)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdoc1 := map[string]interface{}{\"a\": []float32{0.32894259691238403, 0.6973215341567993, 0.6835201978683472, 0.38296082615852356}, \"b\": []float32{0.32894259691238403, 0.6973215341567993, 0.6835201978683472, 0.38296082615852356}, \"c\": \"L5MOPw7NID5SQMU9pHUoPw==\", \"d\": \"L5MOPw7NID5SQMU9pHUoPw==\"}\n\tdoc2 := map[string]interface{}{\"a\": []float32{0.0018692062003538013, 0.41076546907424927, 0.5675257444381714, 0.45832985639572144}, \"b\": []float32{0.0018692062003538013, 0.41076546907424927, 0.5675257444381714, 0.45832985639572144}, \"c\": \"czloP94ZCD71ldY+GbAOPw==\", \"d\": \"czloP94ZCD71ldY+GbAOPw==\"}\n\tdoc3 := map[string]interface{}{\"a\": []float32{0.7853356599807739, 0.6904757618904114, 0.5643226504325867, 0.682637631893158}, \"b\": []float32{0.7853356599807739, 0.6904757618904114, 0.5643226504325867, 0.682637631893158}, \"c\": \"Chh6P2lOqT47mjg/0odlPg==\", \"d\": \"Chh6P2lOqT47mjg/0odlPg==\"}\n\tbatch := index.NewBatch()\n\terr = batch.Index(\"001\", doc1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = batch.Index(\"002\", doc2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = batch.Index(\"003\", doc3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = index.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tindexMappingAfter := mapping.NewIndexMapping()\n\tindexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{}\n\tindexMappingAfter.DefaultMapping = &mapping.DocumentMapping{\n\t\tEnabled: true,\n\t\tDynamic: false,\n\t\tProperties: map[string]*mapping.DocumentMapping{\n\t\t\t\"a\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:                    \"vector\",\n\t\t\t\t\t\tIndex:                   true,\n\t\t\t\t\t\tDims:                    4,\n\t\t\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"c\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:                    \"vector_base64\",\n\t\t\t\t\t\tIndex:                   true,\n\t\t\t\t\t\tDims:                    4,\n\t\t\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"d\": {\n\t\t\t\tEnabled:    true,\n\t\t\t\tDynamic:    false,\n\t\t\t\tProperties: map[string]*mapping.DocumentMapping{},\n\t\t\t\tFields: []*mapping.FieldMapping{\n\t\t\t\t\t{\n\t\t\t\t\t\tType:                    \"vector_base64\",\n\t\t\t\t\t\tIndex:                   false,\n\t\t\t\t\t\tDims:                    4,\n\t\t\t\t\t\tSimilarity:              \"l2_norm\",\n\t\t\t\t\t\tVectorIndexOptimizedFor: \"latency\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tFields: []*mapping.FieldMapping{},\n\t}\n\tindexMappingAfter.IndexDynamic = false\n\tindexMappingAfter.StoreDynamic = false\n\tindexMappingAfter.DocValuesDynamic = false\n\n\tmappingString, err := json.Marshal(indexMappingAfter)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tconfig := map[string]interface{}{\n\t\t\"updated_mapping\": string(mappingString),\n\t}\n\n\tindex, err = OpenUsing(tmpIndexPath, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tq1 := NewSearchRequest(NewMatchNoneQuery())\n\tq1.AddKNN(\"a\", []float32{1, 2, 3, 4}, 3, 1.0)\n\tres1, err := index.Search(q1)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res1.Hits) != 3 {\n\t\tt.Fatalf(\"Expected 3 hits, got %d\", len(res1.Hits))\n\t}\n\tq2 := NewSearchRequest(NewMatchNoneQuery())\n\tq2.AddKNN(\"b\", []float32{1, 2, 3, 4}, 3, 1.0)\n\tres2, err := index.Search(q2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res2.Hits) != 0 {\n\t\tt.Fatalf(\"Expected 0 hits, got %d\", len(res2.Hits))\n\t}\n\tq3 := NewSearchRequest(NewMatchNoneQuery())\n\tq3.AddKNN(\"c\", []float32{1, 2, 3, 4}, 3, 1.0)\n\tres3, err := index.Search(q3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res3.Hits) != 3 {\n\t\tt.Fatalf(\"Expected 3 hits, got %d\", len(res3.Hits))\n\t}\n\tq4 := NewSearchRequest(NewMatchNoneQuery())\n\tq4.AddKNN(\"d\", []float32{1, 2, 3, 4}, 3, 1.0)\n\tres4, err := index.Search(q4)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res4.Hits) != 0 {\n\t\tt.Fatalf(\"Expected 0 hits, got %d\", len(res4.Hits))\n\t}\n}\n\nfunc TestIndexInsightsTermFrequencies(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tmp := mapping.NewIndexMapping()\n\ttextMapping := mapping.NewTextFieldMapping()\n\ttextMapping.Analyzer = \"en\"\n\tmp.DefaultMapping.AddFieldMappingsAt(\"text\", textMapping)\n\n\tidx, err := New(tmpIndexPath, mp)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdata := []map[string]string{\n\t\t{\n\t\t\t\"id\":   \"one\",\n\t\t\t\"text\": \"She sells sea shells by the sea shore\",\n\t\t},\n\t\t{\n\t\t\t\"id\":   \"two\",\n\t\t\t\"text\": \"The quick brown fox jumps over the lazy dog\",\n\t\t},\n\t\t{\n\t\t\t\"id\":   \"three\",\n\t\t\t\"text\": \"She sold sea shells to the person with the dog\",\n\t\t},\n\t\t{\n\t\t\t\"id\":   \"four\",\n\t\t\t\"text\": \"But there are a lot of dogs on the beach\",\n\t\t},\n\t\t{\n\t\t\t\"id\":   \"five\",\n\t\t\t\"text\": \"To hell with the foxes\",\n\t\t},\n\t\t{\n\t\t\t\"id\":   \"six\",\n\t\t\t\"text\": \"What about the dogs\",\n\t\t},\n\t\t{\n\t\t\t\"id\":   \"seven\",\n\t\t\t\"text\": \"Dogs are OK, foxes are not\",\n\t\t},\n\t}\n\n\texpectTermFreqs := []index.TermFreq{\n\t\t{Term: \"dog\", Frequency: 5},\n\t\t{Term: \"fox\", Frequency: 3},\n\t\t{Term: \"sea\", Frequency: 2},\n\t\t{Term: \"shell\", Frequency: 2},\n\t\t{Term: \"beach\", Frequency: 1},\n\t}\n\n\tfor _, d := range data {\n\t\terr = idx.Index(d[\"id\"], d)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Error updating index: %v\", err)\n\t\t}\n\t}\n\n\tinsightsIdx, ok := idx.(InsightsIndex)\n\tif !ok {\n\t\tt.Fatal(\"index does not support insights\")\n\t}\n\n\ttermFreqs, err := insightsIdx.TermFrequencies(\"text\", 5, true)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif !reflect.DeepEqual(termFreqs, expectTermFreqs) {\n\t\tt.Fatalf(\"term freqs do not match: got: %v, expected: %v\", termFreqs, expectTermFreqs)\n\t}\n}\n\nfunc TestIndexInsightsCentroidCardinalities(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tvectorDims := 5\n\n\tmp := mapping.NewIndexMapping()\n\tvecFieldMapping := mapping.NewVectorFieldMapping()\n\tvecFieldMapping.Dims = vectorDims\n\tvecFieldMapping.Similarity = index.CosineSimilarity\n\tmp.DefaultMapping.AddFieldMappingsAt(\"vec\", vecFieldMapping)\n\n\tidx, err := New(tmpIndexPath, mp)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\trand.Seed(time.Now().UnixNano())\n\tmin, max := float32(-10.0), float32(10.0)\n\tgenRandomVector := func() []float32 {\n\t\tvec := make([]float32, vectorDims)\n\t\tfor i := range vec {\n\t\t\tvec[i] = min + rand.Float32()*(max-min)\n\t\t}\n\t\treturn vec\n\t}\n\n\tbatch := idx.NewBatch()\n\tfor i := 1; i <= 50000; i++ {\n\t\tif err = batch.Index(fmt.Sprintf(\"doc-%d\", i), map[string]interface{}{\n\t\t\t\"vec\": genRandomVector(),\n\t\t}); err != nil {\n\t\t\tt.Fatalf(\"error indexing doc: %v\", err)\n\t\t}\n\n\t\tif i%200 == 0 {\n\t\t\terr = idx.Batch(batch)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Error adding batch to index: %v\", err)\n\t\t\t}\n\t\t\tbatch = idx.NewBatch()\n\t\t}\n\t}\n\n\tif batch.Size() > 0 {\n\t\t// In case doc count is not a multiple of 200, we need to add the final batch\n\t\terr = idx.Batch(batch)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Error adding final batch to index: %v\", err)\n\t\t}\n\t}\n\n\tinsightsIdx, ok := idx.(InsightsIndex)\n\tif !ok {\n\t\tt.Fatal(\"index does not support insights\")\n\t}\n\n\tcentroids, err := insightsIdx.CentroidCardinalities(\"vec\", 5, true)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(centroids) != 5 {\n\t\tt.Fatalf(\"expected 5 centroids, got %d\", len(centroids))\n\t}\n\n\tfor _, entry := range centroids {\n\t\tif len(entry.Index) == 0 {\n\t\t\tt.Fatal(\"expected index name for each centroid\")\n\t\t}\n\t}\n}\n\nfunc TestHierarchicalNestedVectorSearch(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tdataset := `\n\t[\n\t\t{\n\t\t\t\"id\": \"doc1\",\n\t\t\t\"items\": [\n\t\t\t\t{\n\t\t\t\t\t\"description\": \"I like trains\",\n\t\t\t\t\t\"embedding_vector\": [\n\t\t\t\t\t\t1,\n\t\t\t\t\t\t0,\n\t\t\t\t\t\t0\n\t\t\t\t\t],\n\t\t\t\t\t\"type\": \"transport\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"description\": \"I love pizza\",\n\t\t\t\t\t\"embedding_vector\": [\n\t\t\t\t\t\t0,\n\t\t\t\t\t\t1,\n\t\t\t\t\t\t0\n\t\t\t\t\t],\n\t\t\t\t\t\"type\": \"food\"\n\t\t\t\t}\n\t\t\t]\n\t\t},\n\t\t{\n\t\t\t\"id\": \"doc2\",\n\t\t\t\"items\": [\n\t\t\t\t{\n\t\t\t\t\t\"description\": \"I go to school by bus\",\n\t\t\t\t\t\"embedding_vector\": [\n\t\t\t\t\t\t0.9,\n\t\t\t\t\t\t0.1,\n\t\t\t\t\t\t0\n\t\t\t\t\t],\n\t\t\t\t\t\"type\": \"transport\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"description\": \"Sushi is delicious\",\n\t\t\t\t\t\"embedding_vector\": [\n\t\t\t\t\t\t0,\n\t\t\t\t\t\t1,\n\t\t\t\t\t\t0\n\t\t\t\t\t],\n\t\t\t\t\t\"type\": \"food\"\n\t\t\t\t}\n\t\t\t]\n\t\t},\n\t\t{\n\t\t\t\"id\": \"doc3\",\n\t\t\t\"items\": [\n\t\t\t\t{\n\t\t\t\t\t\"description\": \"Hamburgers are tasty\",\n\t\t\t\t\t\"embedding_vector\": [\n\t\t\t\t\t\t0,\n\t\t\t\t\t\t0.8,\n\t\t\t\t\t\t0.2\n\t\t\t\t\t],\n\t\t\t\t\t\"type\": \"food\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"description\": \"I enjoy biking\",\n\t\t\t\t\t\"embedding_vector\": [\n\t\t\t\t\t\t0.7,\n\t\t\t\t\t\t0,\n\t\t\t\t\t\t0.3\n\t\t\t\t\t],\n\t\t\t\t\t\"type\": \"transport\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t]`\n\tvar documents []map[string]interface{}\n\terr := json.Unmarshal([]byte(dataset), &documents)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to unmarshal dataset: %v\", err)\n\t}\n\tindexMapping := NewIndexMapping()\n\tvecFieldMapping := mapping.NewVectorFieldMapping()\n\tvecFieldMapping.Dims = 3\n\tvecFieldMapping.Similarity = index.CosineSimilarity\n\n\ttypeMapping := mapping.NewTextFieldMapping()\n\ttypeMapping.Analyzer = keyword.Name\n\n\tdescMapping := mapping.NewTextFieldMapping()\n\tdescMapping.Analyzer = en.AnalyzerName\n\n\t// items is NOT nested\n\titemsMapping := mapping.NewDocumentMapping()\n\titemsMapping.AddFieldMappingsAt(\"embedding_vector\", vecFieldMapping)\n\titemsMapping.AddFieldMappingsAt(\"type\", typeMapping)\n\titemsMapping.AddFieldMappingsAt(\"description\", descMapping)\n\n\tindexMapping.DefaultMapping.AddSubDocumentMapping(\"items\", itemsMapping)\n\tidx, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to create index: %v\", err)\n\t}\n\tdefer func() {\n\t\tif err := idx.Close(); err != nil {\n\t\t\tt.Fatalf(\"failed to close index: %v\", err)\n\t\t}\n\t}()\n\n\tbatch := idx.NewBatch()\n\tfor _, doc := range documents {\n\t\terr := batch.Index(doc[\"id\"].(string), doc)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to index document %s: %v\", doc[\"id\"], err)\n\t\t}\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to batch index documents: %v\", err)\n\t}\n\n\t// Plain vector search\n\tsearchReq := NewSearchRequest(query.NewMatchNoneQuery())\n\tsearchReq.AddKNN(\"items.embedding_vector\", []float32{0, 1, 0}, 5, 1.0)\n\tsearchReq.SortBy([]string{\"-_score\", \"_id\"})\n\n\tres, err := idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to execute search: %v\", err)\n\t}\n\n\texpectedOrder := []string{\"doc1\", \"doc2\", \"doc3\"}\n\texpectedScores := []float64{1.0, 1.0, 0.970}\n\tif len(res.Hits) != len(expectedOrder) {\n\t\tt.Fatalf(\"expected %d hits, got %d\", len(expectedOrder), len(res.Hits))\n\t}\n\tfor i, expectedID := range expectedOrder {\n\t\tif res.Hits[i].ID != expectedID {\n\t\t\tt.Fatalf(\"at rank %d, expected docID %s, got %s\", i+1, expectedID, res.Hits[i].ID)\n\t\t}\n\t\tif math.Abs(res.Hits[i].Score-expectedScores[i]) > 0.01 {\n\t\t\tt.Fatalf(\"at rank %d, expected score %.3f, got %.3f\", i+1, expectedScores[i], res.Hits[i].Score)\n\t\t}\n\t}\n\n\t// Filtered vector search - should match output of plain vector search in non-nested case\n\tfilterQuery := NewTermQuery(\"transport\")\n\tfilterQuery.SetField(\"items.type\")\n\tsearchReq = NewSearchRequest(query.NewMatchNoneQuery())\n\tsearchReq.AddKNNWithFilter(\"items.embedding_vector\", []float32{0, 1, 0}, 5, 1.0, filterQuery)\n\tsearchReq.SortBy([]string{\"-_score\", \"_id\"})\n\tres, err = idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to execute filtered search: %v\", err)\n\t}\n\tif len(res.Hits) != len(expectedOrder) {\n\t\tt.Fatalf(\"expected %d hits, got %d\", len(expectedOrder), len(res.Hits))\n\t}\n\tfor i, expectedID := range expectedOrder {\n\t\tif res.Hits[i].ID != expectedID {\n\t\t\tt.Fatalf(\"at rank %d, expected docID %s, got %s\", i+1, expectedID, res.Hits[i].ID)\n\t\t}\n\t\tif math.Abs(res.Hits[i].Score-expectedScores[i]) > 0.01 {\n\t\t\tt.Fatalf(\"at rank %d, expected score %.3f, got %.3f\", i+1, expectedScores[i], res.Hits[i].Score)\n\t\t}\n\t}\n\n\t// items IS nested\n\tnestedItemsMapping := mapping.NewNestedDocumentMapping()\n\tnestedItemsMapping.AddFieldMappingsAt(\"embedding_vector\", vecFieldMapping)\n\tnestedItemsMapping.AddFieldMappingsAt(\"type\", typeMapping)\n\tnestedItemsMapping.AddFieldMappingsAt(\"description\", descMapping)\n\n\tindexMappingNested := NewIndexMapping()\n\tindexMappingNested.DefaultMapping.AddSubDocumentMapping(\"items\", nestedItemsMapping)\n\tidxNested, err := New(tmpIndexPath+\"_nested\", indexMappingNested)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to create nested index: %v\", err)\n\t}\n\tdefer func() {\n\t\tif err := idxNested.Close(); err != nil {\n\t\t\tt.Fatalf(\"failed to close nested index: %v\", err)\n\t\t}\n\t}()\n\n\tbatch = idxNested.NewBatch()\n\tfor _, doc := range documents {\n\t\terr := batch.Index(doc[\"id\"].(string), doc)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to index document %s in nested index: %v\", doc[\"id\"], err)\n\t\t}\n\t}\n\terr = idxNested.Batch(batch)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to batch index documents in nested index: %v\", err)\n\t}\n\t// Plain vector search on nested index\n\tsearchReq = NewSearchRequest(query.NewMatchNoneQuery())\n\tsearchReq.AddKNN(\"items.embedding_vector\", []float32{0, 1, 0}, 5, 1.0)\n\tsearchReq.SortBy([]string{\"-_score\", \"_id\"})\n\n\tres, err = idxNested.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to execute search on nested index: %v\", err)\n\t}\n\t// Exact same behavior as non-nested in this case\n\tif len(res.Hits) != len(expectedOrder) {\n\t\tt.Fatalf(\"expected %d hits, got %d\", len(expectedOrder), len(res.Hits))\n\t}\n\tfor i, expectedID := range expectedOrder {\n\t\tif res.Hits[i].ID != expectedID {\n\t\t\tt.Fatalf(\"at rank %d, expected docID %s, got %s\", i+1, expectedID, res.Hits[i].ID)\n\t\t}\n\t\tif math.Abs(res.Hits[i].Score-expectedScores[i]) > 0.01 {\n\t\t\tt.Fatalf(\"at rank %d, expected score %.3f, got %.3f\", i+1, expectedScores[i], res.Hits[i].Score)\n\t\t}\n\t}\n\n\t// Filtered vector search on nested index - should NOT match output of plain vector search in nested case\n\tfilterQuery = NewTermQuery(\"transport\")\n\tfilterQuery.SetField(\"items.type\")\n\tsearchReq = NewSearchRequest(query.NewMatchNoneQuery())\n\tsearchReq.AddKNNWithFilter(\"items.embedding_vector\", []float32{0, 1, 0}, 5, 1.0, filterQuery)\n\tsearchReq.SortBy([]string{\"-_score\", \"_id\"})\n\tres, err = idxNested.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to execute filtered search on nested index: %v\", err)\n\t}\n\texpectedNestedOrder := []string{\"doc2\", \"doc1\", \"doc3\"}\n\texpectedNestedScores := []float64{0.110, 0, 0}\n\tif len(res.Hits) != len(expectedNestedOrder) {\n\t\tt.Fatalf(\"expected %d hits, got %d\", len(expectedNestedOrder), len(res.Hits))\n\t}\n\tfor i, expectedID := range expectedNestedOrder {\n\t\tif res.Hits[i].ID != expectedID {\n\t\t\tt.Fatalf(\"at rank %d, expected docID %s, got %s\", i+1, expectedID, res.Hits[i].ID)\n\t\t}\n\t\tif math.Abs(res.Hits[i].Score-expectedNestedScores[i]) > 0.01 {\n\t\t\tt.Fatalf(\"at rank %d, expected score %.3f, got %.3f\", i+1, expectedNestedScores[i], res.Hits[i].Score)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "search_nested_test.go",
    "content": "//  Copyright (c) 2026 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight/highlighter/ansi\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n)\n\nfunc createNestedIndexMapping() mapping.IndexMapping {\n\t/*\n\t\tcompany\n\t\t├── id\n\t\t├── name\n\t\t├── departments[] (nested)\n\t\t│     ├── name\n\t\t│     ├── budget\n\t\t│     ├── employees[] (nested)\n\t\t│     │     ├── name\n\t\t│     │     ├── role\n\t\t│     └── projects[] (nested)\n\t\t│           ├── title\n\t\t│           ├── status\n\t\t└── locations[] (nested)\n\t\t\t\t├── city\n\t\t\t\t├── country\n\t*/\n\n\t// Create the index mapping\n\timap := mapping.NewIndexMapping()\n\n\t// Create company mapping\n\tcompanyMapping := mapping.NewDocumentMapping()\n\n\t// Company ID field\n\tcompanyIDField := mapping.NewTextFieldMapping()\n\tcompanyMapping.AddFieldMappingsAt(\"id\", companyIDField)\n\n\t// Company name field\n\tcompanyNameField := mapping.NewTextFieldMapping()\n\tcompanyMapping.AddFieldMappingsAt(\"name\", companyNameField)\n\n\t// Departments mapping\n\tdepartmentsMapping := mapping.NewNestedDocumentMapping()\n\n\t// Department name field\n\tdeptNameField := mapping.NewTextFieldMapping()\n\tdepartmentsMapping.AddFieldMappingsAt(\"name\", deptNameField)\n\n\t// Department budget field\n\tdeptBudgetField := mapping.NewNumericFieldMapping()\n\tdepartmentsMapping.AddFieldMappingsAt(\"budget\", deptBudgetField)\n\n\t// Employees mapping\n\temployeesMapping := mapping.NewNestedDocumentMapping()\n\n\t// Employee name field\n\tempNameField := mapping.NewTextFieldMapping()\n\temployeesMapping.AddFieldMappingsAt(\"name\", empNameField)\n\n\t// Employee role field\n\tempRoleField := mapping.NewTextFieldMapping()\n\temployeesMapping.AddFieldMappingsAt(\"role\", empRoleField)\n\n\tdepartmentsMapping.AddSubDocumentMapping(\"employees\", employeesMapping)\n\n\t// Projects mapping\n\tprojectsMapping := mapping.NewNestedDocumentMapping()\n\n\t// Project title field\n\tprojTitleField := mapping.NewTextFieldMapping()\n\tprojectsMapping.AddFieldMappingsAt(\"title\", projTitleField)\n\n\t// Project status field\n\tprojStatusField := mapping.NewTextFieldMapping()\n\tprojectsMapping.AddFieldMappingsAt(\"status\", projStatusField)\n\n\tdepartmentsMapping.AddSubDocumentMapping(\"projects\", projectsMapping)\n\n\tcompanyMapping.AddSubDocumentMapping(\"departments\", departmentsMapping)\n\n\t// Locations mapping\n\tlocationsMapping := mapping.NewNestedDocumentMapping()\n\n\t// Location city field\n\tcityField := mapping.NewTextFieldMapping()\n\tlocationsMapping.AddFieldMappingsAt(\"city\", cityField)\n\n\t// Location country field\n\tcountryField := mapping.NewTextFieldMapping()\n\tlocationsMapping.AddFieldMappingsAt(\"country\", countryField)\n\n\tcompanyMapping.AddSubDocumentMapping(\"locations\", locationsMapping)\n\n\t// Add company to type mapping\n\timap.DefaultMapping.AddSubDocumentMapping(\"company\", companyMapping)\n\n\treturn imap\n}\n\nfunc TestNestedPrefixes(t *testing.T) {\n\timap := createNestedIndexMapping()\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\tif err := idx.Close(); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tnmap, ok := imap.(mapping.NestedMapping)\n\tif !ok {\n\t\tt.Fatal(\"index mapping is not a NestedMapping\")\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 1: Employee Role AND Employee Name\n\t// ----------------------------------------------------------------------\n\tfs := search.NewFieldSet()\n\tfs.AddField(\"company.departments.employees.role\")\n\tfs.AddField(\"company.departments.employees.name\")\n\n\texpectedCommon := 2\n\texpectedMax := 2\n\n\tcommon, max := nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test1: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 2: Employee Role AND Employee Name AND Department Name\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\tfs.AddField(\"company.departments.employees.role\")\n\tfs.AddField(\"company.departments.employees.name\")\n\tfs.AddField(\"company.departments.name\")\n\n\texpectedCommon = 1\n\texpectedMax = 2 // employees nested deeper\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test2: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 3: Employee Role AND Location City\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\tfs.AddField(\"company.departments.employees.role\")\n\tfs.AddField(\"company.locations.city\")\n\n\texpectedCommon = 0\n\texpectedMax = 2 // employees deeper than locations (1)\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test3: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 4: Company Name AND Location Country\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\tfs.AddField(\"company.name\")\n\tfs.AddField(\"company.locations.country\")\n\tfs.AddField(\"company.locations.city\")\n\n\texpectedCommon = 0\n\texpectedMax = 1 // locations.country and locations.city share depth 1\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test4: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 5: Department Budget AND Project Status AND Employee Name\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\tfs.AddField(\"company.departments.budget\")\n\tfs.AddField(\"company.departments.projects.status\")\n\tfs.AddField(\"company.departments.employees.name\")\n\n\texpectedCommon = 1\n\texpectedMax = 2 // employees + projects go deeper\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test5: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 6: Single Field\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\tfs.AddField(\"company.id\")\n\n\texpectedCommon = 0\n\texpectedMax = 0\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test6: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 7: No Fields\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\n\texpectedCommon = 0\n\texpectedMax = 0\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test7: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 8: All Fields\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\tfs.AddField(\"company.id\")\n\tfs.AddField(\"company.name\")\n\tfs.AddField(\"company.departments.name\")\n\tfs.AddField(\"company.departments.budget\")\n\tfs.AddField(\"company.departments.employees.name\")\n\tfs.AddField(\"company.departments.employees.role\")\n\tfs.AddField(\"company.departments.projects.title\")\n\tfs.AddField(\"company.departments.projects.status\")\n\tfs.AddField(\"company.locations.city\")\n\tfs.AddField(\"company.locations.country\")\n\n\texpectedCommon = 0 // spans different contexts\n\texpectedMax = 2\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test8: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 9: Project Title AND Project Status\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\tfs.AddField(\"company.departments.projects.title\")\n\tfs.AddField(\"company.departments.projects.status\")\n\n\texpectedCommon = 2\n\texpectedMax = 2\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test9: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n\n\t// ----------------------------------------------------------------------\n\t// Test 10: Department Name AND Location Country\n\t// ----------------------------------------------------------------------\n\tfs = search.NewFieldSet()\n\tfs.AddField(\"company.departments.name\")\n\tfs.AddField(\"company.locations.country\")\n\tfs.AddField(\"company.locations.city\")\n\n\texpectedCommon = 0\n\texpectedMax = 1 // locations share depth 1\n\n\tcommon, max = nmap.NestedDepth(fs)\n\tif common != expectedCommon || max != expectedMax {\n\t\tt.Fatalf(\"Test10: expected (common=%d, max=%d), got (common=%d, max=%d)\",\n\t\t\texpectedCommon, expectedMax, common, max)\n\t}\n}\n\nfunc TestNestedConjunctionQuery(t *testing.T) {\n\timap := createNestedIndexMapping()\n\terr := imap.Validate()\n\tif err != nil {\n\t\tt.Fatalf(\"expected valid nested index mapping, got error: %v\", err)\n\t}\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\t// Index 3 sample documents\n\tdocs := []struct {\n\t\tid   string\n\t\tdata string\n\t}{\n\t\t{\n\t\t\tid: \"doc1\",\n\t\t\tdata: `{\n\t\t\t\t\"company\": {\n\t\t\t\t\t\"id\": \"c1\",\n\t\t\t\t\t\"name\": \"TechCorp\",\n\t\t\t\t\t\"departments\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"Engineering\",\n\t\t\t\t\t\t\t\"budget\": 2000000,\n\t\t\t\t\t\t\t\"employees\": [\n\t\t\t\t\t\t\t\t{\"name\": \"Alice\", \"role\": \"Engineer\"},\n\t\t\t\t\t\t\t\t{\"name\": \"Bob\", \"role\": \"Manager\"}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"projects\": [\n\t\t\t\t\t\t\t\t{\"title\": \"Project X\", \"status\": \"ongoing\"},\n\t\t\t\t\t\t\t\t{\"title\": \"Project Y\", \"status\": \"completed\"}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"Sales\",\n\t\t\t\t\t\t\t\"budget\": 300000,\n\t\t\t\t\t\t\t\"employees\": [\n\t\t\t\t\t\t\t\t{\"name\": \"Eve\", \"role\": \"Salesperson\"},\n\t\t\t\t\t\t\t\t{\"name\": \"Mallory\", \"role\": \"Manager\"}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"projects\": [\n\t\t\t\t\t\t\t\t{\"title\": \"Project A\", \"status\": \"completed\"},\n\t\t\t\t\t\t\t\t{\"title\": \"Project B\", \"status\": \"ongoing\"}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t}\t\n\t\t\t\t\t],\n\t\t\t\t\t\"locations\": [\n\t\t\t\t\t\t{\"city\": \"Athens\", \"country\": \"Greece\"},\n\t\t\t\t\t\t{\"city\": \"Berlin\", \"country\": \"USA\"}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}`,\n\t\t},\n\t\t{\n\t\t\tid: \"doc2\",\n\t\t\tdata: `{\n\t\t\t\t\"company\" : {\n\t\t\t\t\t\"id\": \"c2\",\n\t\t\t\t\t\"name\": \"BizInc\",\n\t\t\t\t\t\"departments\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"Marketing\",\n\t\t\t\t\t\t\t\"budget\": 800000,\n\t\t\t\t\t\t\t\"employees\": [\n\t\t\t\t\t\t\t\t{\"name\": \"Eve\", \"role\": \"Marketer\"},\n\t\t\t\t\t\t\t\t{\"name\": \"David\", \"role\": \"Manager\"}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"projects\": [\n\t\t\t\t\t\t\t\t{\"title\": \"Project Z\", \"status\": \"ongoing\"},\n\t\t\t\t\t\t\t\t{\"title\": \"Project W\", \"status\": \"planned\"}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"Engineering\",\n\t\t\t\t\t\t\t\"budget\": 800000,\n\t\t\t\t\t\t\t\"employees\": [\n\t\t\t\t\t\t\t\t{\"name\": \"Frank\", \"role\": \"Manager\"},\n\t\t\t\t\t\t\t\t{\"name\": \"Grace\", \"role\": \"Engineer\"}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"projects\": [\n\t\t\t\t\t\t\t\t{\"title\": \"Project Alpha\", \"status\": \"completed\"},\n\t\t\t\t\t\t\t\t{\"title\": \"Project Beta\", \"status\": \"ongoing\"}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t}\t\n\t\t\t\t\t],\n\t\t\t\t\t\"locations\": [\n\t\t\t\t\t\t{\"city\": \"Athens\", \"country\": \"USA\"},\n\t\t\t\t\t\t{\"city\": \"London\", \"country\": \"UK\"}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}`,\n\t\t},\n\t\t{\n\t\t\tid: \"doc3\",\n\t\t\tdata: `{\n\t\t\t\t\"company\": {\n\t\t\t\t\t\"id\": \"c3\",\n\t\t\t\t\t\"name\": \"WebSolutions\",\n\t\t\t\t\t\"departments\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"HR\",\n\t\t\t\t\t\t\t\"budget\": 800000,\n\t\t\t\t\t\t\t\"employees\": [\n\t\t\t\t\t\t\t\t{\"name\": \"Eve\", \"role\": \"Manager\"},\n\t\t\t\t\t\t\t\t{\"name\": \"Frank\", \"role\": \"HR\"}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"projects\": [\n\t\t\t\t\t\t\t\t{\"title\": \"Project Beta\", \"status\": \"completed\"},\n\t\t\t\t\t\t\t\t{\"title\": \"Project B\", \"status\": \"ongoing\"}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"Engineering\",\n\t\t\t\t\t\t\t\"budget\": 200000,\n\t\t\t\t\t\t\t\"employees\": [\n\t\t\t\t\t\t\t\t{\"name\": \"Heidi\", \"role\": \"Support Engineer\"},\n\t\t\t\t\t\t\t\t{\"name\": \"Ivan\", \"role\": \"Manager\"}\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\"projects\": [\n\t\t\t\t\t\t\t\t{\"title\": \"Project Helpdesk\", \"status\": \"ongoing\"},\n\t\t\t\t\t\t\t\t{\"title\": \"Project FAQ\", \"status\": \"completed\"}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t}\n\t\t\t\t\t],\n\t\t\t\t\t\"locations\": [\n\t\t\t\t\t\t{\"city\": \"Edinburgh\", \"country\": \"UK\"},\n\t\t\t\t\t\t{\"city\": \"London\", \"country\": \"Canada\"}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}`,\n\t\t},\n\t}\n\n\tfor _, doc := range docs {\n\t\tvar dataMap map[string]interface{}\n\t\terr := json.Unmarshal([]byte(doc.data), &dataMap)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to unmarshal document %s: %v\", doc.id, err)\n\t\t}\n\t\terr = idx.Index(doc.id, dataMap)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to index document %s: %v\", doc.id, err)\n\t\t}\n\t}\n\n\tvar buildReq = func(subQueries []query.Query) *SearchRequest {\n\t\trv := NewSearchRequest(query.NewConjunctionQuery(subQueries))\n\t\trv.SortBy([]string{\"_id\"})\n\t\trv.Fields = []string{\"*\"}\n\t\trv.Highlight = NewHighlightWithStyle(ansi.Name)\n\t\treturn rv\n\t}\n\n\tvar (\n\t\treq             *SearchRequest\n\t\tres             *SearchResult\n\t\tdeptNameQuery   *query.MatchQuery\n\t\tdeptBudgetQuery *query.NumericRangeQuery\n\t\tempNameQuery    *query.MatchQuery\n\t\tempRoleQuery    *query.MatchQuery\n\t\tprojTitleQuery  *query.MatchPhraseQuery\n\t\tprojStatusQuery *query.MatchQuery\n\t\tcountryQuery    *query.MatchQuery\n\t\tcityQuery       *query.MatchQuery\n\t)\n\n\t// Test 1: Find companies with a department named \"Engineering\" AND budget over 900000\n\tdeptNameQuery = query.NewMatchQuery(\"Engineering\")\n\tdeptNameQuery.SetField(\"company.departments.name\")\n\n\tmin := float64(800000)\n\tdeptBudgetQuery = query.NewNumericRangeQuery(&min, nil)\n\tdeptBudgetQuery.SetField(\"company.departments.budget\")\n\n\treq = buildReq([]query.Query{deptNameQuery, deptBudgetQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 2 {\n\t\tt.Fatalf(\"expected 2 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc1\" || res.Hits[1].ID != \"doc2\" {\n\t\tt.Fatalf(\"unexpected hit IDs: %v, %v\", res.Hits[0].ID, res.Hits[1].ID)\n\t}\n\n\t// Test 2: Find companies with an employee named \"Eve\" AND project status \"completed\"\n\tempNameQuery = query.NewMatchQuery(\"Eve\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tprojStatusQuery = query.NewMatchQuery(\"completed\")\n\tprojStatusQuery.SetField(\"company.departments.projects.status\")\n\n\treq = buildReq([]query.Query{empNameQuery, projStatusQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 2 {\n\t\tt.Fatalf(\"expected 2 hits, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc1\" || res.Hits[1].ID != \"doc3\" {\n\t\tt.Fatalf(\"unexpected hit IDs: %v, %v\", res.Hits[0].ID, res.Hits[1].ID)\n\t}\n\n\t// Test 3: Find companies located in \"Athens, USA\" AND with an Engineering department\n\tcountryQuery = query.NewMatchQuery(\"USA\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"Athens\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery := query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\tdeptNameQuery = query.NewMatchQuery(\"Engineering\")\n\tdeptNameQuery.SetField(\"company.departments.name\")\n\n\treq = buildReq([]query.Query{locQuery, deptNameQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc2\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\t// Test 4a: Find companies located in \"Athens, USA\" AND with an Engineering department with a budget over 1M\n\tcountryQuery = query.NewMatchQuery(\"USA\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"Athens\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\tdeptNameQuery = query.NewMatchQuery(\"Engineering\")\n\tdeptNameQuery.SetField(\"company.departments.name\")\n\n\tmin = float64(1000000)\n\tdeptBudgetQuery = query.NewNumericRangeQuery(&min, nil)\n\tdeptBudgetQuery.SetField(\"company.departments.budget\")\n\n\tdeptQuery := query.NewConjunctionQuery([]query.Query{deptNameQuery, deptBudgetQuery})\n\n\treq = buildReq([]query.Query{locQuery, deptQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hits, got %d\", len(res.Hits))\n\t}\n\n\t// Test 4b: Find companies located in \"Athens, Greece\" AND with an Engineering department with a budget over 1M\n\tcountryQuery = query.NewMatchQuery(\"Greece\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"Athens\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\tdeptNameQuery = query.NewMatchQuery(\"Engineering\")\n\tdeptNameQuery.SetField(\"company.departments.name\")\n\n\tmin = float64(1000000)\n\tdeptBudgetQuery = query.NewNumericRangeQuery(&min, nil)\n\tdeptBudgetQuery.SetField(\"company.departments.budget\")\n\n\tdeptQuery = query.NewConjunctionQuery([]query.Query{deptNameQuery, deptBudgetQuery})\n\n\treq = buildReq([]query.Query{locQuery, deptQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hits, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc1\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\t// Test 5a: Find companies with an employee named \"Frank\" AND role \"Manager\" whose department is\n\t// handling a project titled \"Project Beta\" which is marked as \"completed\"\n\tempNameQuery = query.NewMatchQuery(\"Frank\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery := query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tprojTitleQuery = query.NewMatchPhraseQuery(\"Project Beta\")\n\tprojTitleQuery.SetField(\"company.departments.projects.title\")\n\n\tprojStatusQuery = query.NewMatchQuery(\"completed\")\n\tprojStatusQuery.SetField(\"company.departments.projects.status\")\n\n\tprojQuery := query.NewConjunctionQuery([]query.Query{projTitleQuery, projStatusQuery})\n\n\treq = buildReq([]query.Query{empQuery, projQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hit, got %d\", len(res.Hits))\n\t}\n\n\t// Test 5b: Find companies with an employee named \"Frank\" AND role \"Manager\" whose department is\n\t// handling a project titled \"Project Beta\" which is marked as \"ongoing\"\n\tempNameQuery = query.NewMatchQuery(\"Frank\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tprojTitleQuery = query.NewMatchPhraseQuery(\"Project Beta\")\n\tprojTitleQuery.SetField(\"company.departments.projects.title\")\n\n\tprojStatusQuery = query.NewMatchQuery(\"ongoing\")\n\tprojStatusQuery.SetField(\"company.departments.projects.status\")\n\n\tprojQuery = query.NewConjunctionQuery([]query.Query{projTitleQuery, projStatusQuery})\n\n\treq = buildReq([]query.Query{empQuery, projQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc2\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\t// Test 6a: Find companies with an employee named \"Eve\" AND role \"Manager\"\n\t// who is working in a department located in \"London, UK\"\n\tempNameQuery = query.NewMatchQuery(\"Eve\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tcountryQuery = query.NewMatchQuery(\"UK\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"London\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\treq = buildReq([]query.Query{empQuery, locQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hit, got %d\", len(res.Hits))\n\t}\n\n\t// Test 6b: Find companies with an employee named \"Eve\" AND role \"Manager\"\n\t// who is working in a department located in \"London, Canada\"\n\tempNameQuery = query.NewMatchQuery(\"Eve\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tcountryQuery = query.NewMatchQuery(\"Canada\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"London\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\treq = buildReq([]query.Query{empQuery, locQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc3\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\t// Test 7a: Find companies where Ivan the Manager works London, UK\n\n\tempNameQuery = query.NewMatchQuery(\"Ivan\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tcountryQuery = query.NewMatchQuery(\"UK\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"London\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\treq = buildReq([]query.Query{empQuery, locQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hit, got %d\", len(res.Hits))\n\t}\n\n\t// Test 7b: Find companies where Ivan the Manager works London, Canada\n\n\tempNameQuery = query.NewMatchQuery(\"Ivan\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tcountryQuery = query.NewMatchQuery(\"Canada\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"London\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\treq = buildReq([]query.Query{empQuery, locQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc3\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\t// Test 8: Find companies where Frank the Manager works in Engineering department located in London, UK\n\tempNameQuery = query.NewMatchQuery(\"Frank\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tdeptNameQuery = query.NewMatchQuery(\"Engineering\")\n\tdeptNameQuery.SetField(\"company.departments.name\")\n\n\tdeptQuery = query.NewConjunctionQuery([]query.Query{empQuery, deptNameQuery})\n\n\tcountryQuery = query.NewMatchQuery(\"UK\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"London\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\treq = buildReq([]query.Query{deptQuery, locQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc2\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\t// Test 9: Match_All query must return only top-level documents\n\tmatchAllQuery := query.NewMatchAllQuery()\n\treq = buildReq([]query.Query{matchAllQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 3 {\n\t\tt.Fatalf(\"expected 3 hits, got %d\", len(res.Hits))\n\t}\n\n\t// Test 10: DocID query must return only top-level documents\n\tdocIDQuery := query.NewDocIDQuery([]string{\"doc1\", \"doc2\", \"doc3\", \"doc2_$company.locations_$0\", \"doc3_$company.departments_$0_$company.departments.employees_$0\"})\n\treq = buildReq([]query.Query{docIDQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 3 {\n\t\tt.Fatalf(\"expected 3 hits, got %d\", len(res.Hits))\n\t}\n\n\t// Test 11: Boolean query in Filter-only mode must return correct top-level documents\n\tempNameQuery = query.NewMatchQuery(\"Frank\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tboolQuery := query.NewBooleanQuery(nil, nil, nil)\n\tboolQuery.AddFilter(empNameQuery)\n\n\treq = buildReq([]query.Query{boolQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 2 {\n\t\tt.Fatalf(\"expected 2 hits, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc2\" || res.Hits[1].ID != \"doc3\" {\n\t\tt.Fatalf(\"unexpected hit IDs: %v, %v\", res.Hits[0].ID, res.Hits[1].ID)\n\t}\n\n\t// Test 12: Boolean query Must clause should work in nested context\n\tempNameQuery = query.NewMatchQuery(\"Ivan\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tcountryQuery = query.NewMatchQuery(\"Canada\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"London\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\tboolQuery = query.NewBooleanQuery([]query.Query{empQuery, locQuery}, nil, nil)\n\treq = buildReq([]query.Query{boolQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc3\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\t// Test 13: Queries targetting _all field should:\n\t// - match only top-level fields when no specific field is set\n\t// - not match nested fields when no specific field is set\n\t// - work correctly when combined with nested field queries,\n\t//   returning only top-level documents where both conditions are met\n\tallRootFieldQuery := query.NewMatchQuery(\"TechCorp\")\n\n\treq = buildReq([]query.Query{allRootFieldQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc1\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\tallNestedFieldQuery := query.NewMatchQuery(\"Alice\")\n\treq = buildReq([]query.Query{allNestedFieldQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hits, got %d\", len(res.Hits))\n\t}\n\n\tallMixedQuery := buildReq([]query.Query{allRootFieldQuery, allNestedFieldQuery})\n\tres, err = idx.Search(allMixedQuery)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hits, got %d\", len(res.Hits))\n\t}\n\n\tnestedFieldQuery := query.NewMatchQuery(\"Alice\")\n\tnestedFieldQuery.SetField(\"company.departments.employees.name\")\n\n\tallMixedQueryWithNested := buildReq([]query.Query{allRootFieldQuery, nestedFieldQuery})\n\tres, err = idx.Search(allMixedQueryWithNested)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc1\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\tempNameQuery = query.NewMatchQuery(\"Frank\")\n\tempNameQuery.SetField(\"company.departments.employees.name\")\n\n\tempRoleQuery = query.NewMatchQuery(\"Manager\")\n\tempRoleQuery.SetField(\"company.departments.employees.role\")\n\n\tempQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery})\n\n\tdeptNameQuery = query.NewMatchQuery(\"Engineering\")\n\tdeptNameQuery.SetField(\"company.departments.name\")\n\n\tdeptQuery = query.NewConjunctionQuery([]query.Query{empQuery, deptNameQuery})\n\n\tcountryQuery = query.NewMatchQuery(\"UK\")\n\tcountryQuery.SetField(\"company.locations.country\")\n\n\tcityQuery = query.NewMatchQuery(\"London\")\n\tcityQuery.SetField(\"company.locations.city\")\n\n\tlocQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery})\n\n\t// mixed queries with _all field and _id field should match at root level always\n\tcompanyNameAllQuery := query.NewMatchQuery(\"BizInc\")\n\tmatchAllQuery = query.NewMatchAllQuery()\n\treq = buildReq([]query.Query{deptQuery, locQuery, companyNameAllQuery, matchAllQuery})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(res.Hits))\n\t}\n\tif res.Hits[0].ID != \"doc2\" {\n\t\tt.Fatalf(\"unexpected hit ID: %v\", res.Hits[0].ID)\n\t}\n\n\tcompanyNameAllQueryNoMatch := query.NewMatchQuery(\"WebSolutions\")\n\treq = buildReq([]query.Query{deptQuery, locQuery, companyNameAllQueryNoMatch})\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hits, got %d\", len(res.Hits))\n\t}\n}\n\nfunc TestNestedArrayConjunctionQuery(t *testing.T) {\n\timap := NewIndexMapping()\n\tgroupsMapping := mapping.NewNestedDocumentMapping()\n\n\tnameField := mapping.NewTextFieldMapping()\n\tgroupsMapping.AddFieldMappingsAt(\"first_name\", nameField)\n\tgroupsMapping.AddFieldMappingsAt(\"last_name\", nameField)\n\n\timap.DefaultMapping.AddSubDocumentMapping(\"groups\", groupsMapping)\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocs := []string{\n\t\t`{\n\t\t\t\"groups\": [\n\t\t\t\t[\n\t\t\t\t\t{\n\t\t\t\t\t\t\"first_name\": \"Alice\",\n\t\t\t\t\t\t\"last_name\": \"Smith\"\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t\"first_name\": \"Bob\",\n\t\t\t\t\t\t\"last_name\": \"Johnson\"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t[\n\t\t\t\t\t{\n\t\t\t\t\t\t\"first_name\": \"Charlie\",\n\t\t\t\t\t\t\"last_name\": \"Williams\"\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t\"first_name\": \"Diana\",\n\t\t\t\t\t\t\"last_name\": \"Brown\"\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t]\n\t\t}`,\n\t\t`{\n\t\t\t\"groups\": [\n\t\t\t\t{\n\t\t\t\t\t\"first_name\": \"Alice\",\n\t\t\t\t\t\"last_name\": \"Smith\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"first_name\": \"Bob\",\n\t\t\t\t\t\"last_name\": \"Johnson\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"first_name\": \"Charlie\",\n\t\t\t\t\t\"last_name\": \"Williams\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"first_name\": \"Diana\",\n\t\t\t\t\t\"last_name\": \"Brown\"\n\t\t\t\t}\n\t\t\t]\n\t\t}`,\n\t}\n\n\tfor i, doc := range docs {\n\t\tvar dataMap map[string]interface{}\n\t\terr := json.Unmarshal([]byte(doc), &dataMap)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to unmarshal document %d: %v\", i, err)\n\t\t}\n\t\terr = idx.Index(fmt.Sprintf(\"%d\", i+1), dataMap)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to index document %d: %v\", i, err)\n\t\t}\n\t}\n\n\tvar (\n\t\tfirstNameQuery *query.MatchQuery\n\t\tlastNameQuery  *query.MatchQuery\n\t\tconjQuery      *query.ConjunctionQuery\n\t\tsearchReq      *SearchRequest\n\t\tres            *SearchResult\n\t)\n\n\t// Search for documents where first_name is \"Alice\" AND last_name is \"Johnson\"\n\tfirstNameQuery = query.NewMatchQuery(\"Alice\")\n\tfirstNameQuery.SetField(\"groups.first_name\")\n\n\tlastNameQuery = query.NewMatchQuery(\"Johnson\")\n\tlastNameQuery.SetField(\"groups.last_name\")\n\n\tconjQuery = query.NewConjunctionQuery([]query.Query{firstNameQuery, lastNameQuery})\n\n\tsearchReq = NewSearchRequest(conjQuery)\n\tsearchReq.SortBy([]string{\"_id\"})\n\n\tres, err = idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hits, got %d\", len(res.Hits))\n\t}\n\n\t// Search for documents where first_name is \"Bob\" AND last_name is \"Johnson\"\n\tfirstNameQuery = query.NewMatchQuery(\"Bob\")\n\tfirstNameQuery.SetField(\"groups.first_name\")\n\n\tlastNameQuery = query.NewMatchQuery(\"Johnson\")\n\tlastNameQuery.SetField(\"groups.last_name\")\n\n\tconjQuery = query.NewConjunctionQuery([]query.Query{firstNameQuery, lastNameQuery})\n\n\tsearchReq = NewSearchRequest(conjQuery)\n\tsearchReq.SortBy([]string{\"_id\"})\n\n\tres, err = idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\n\tif len(res.Hits) != 2 {\n\t\tt.Fatalf(\"expected 2 hits, got %d\", len(res.Hits))\n\t}\n\n\tif res.Hits[0].ID != \"1\" || res.Hits[1].ID != \"2\" {\n\t\tt.Fatalf(\"unexpected hit IDs: %v, %v\", res.Hits[0].ID, res.Hits[1].ID)\n\t}\n\n\t// Search for documents where first_name is \"Alice\" AND last_name is \"Williams\"\n\tfirstNameQuery = query.NewMatchQuery(\"Alice\")\n\tfirstNameQuery.SetField(\"groups.first_name\")\n\n\tlastNameQuery = query.NewMatchQuery(\"Williams\")\n\tlastNameQuery.SetField(\"groups.last_name\")\n\n\tconjQuery = query.NewConjunctionQuery([]query.Query{firstNameQuery, lastNameQuery})\n\n\tsearchReq = NewSearchRequest(conjQuery)\n\tsearchReq.SortBy([]string{\"_id\"})\n\n\tres, err = idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\n\tif len(res.Hits) != 0 {\n\t\tt.Fatalf(\"expected 0 hits, got %d\", len(res.Hits))\n\t}\n\n\t// Search for documents where first_name is \"Diana\" AND last_name is \"Brown\"\n\tfirstNameQuery = query.NewMatchQuery(\"Diana\")\n\tfirstNameQuery.SetField(\"groups.first_name\")\n\n\tlastNameQuery = query.NewMatchQuery(\"Brown\")\n\tlastNameQuery.SetField(\"groups.last_name\")\n\n\tconjQuery = query.NewConjunctionQuery([]query.Query{firstNameQuery, lastNameQuery})\n\n\tsearchReq = NewSearchRequest(conjQuery)\n\tsearchReq.SortBy([]string{\"_id\"})\n\n\tres, err = idx.Search(searchReq)\n\tif err != nil {\n\t\tt.Fatalf(\"search failed: %v\", err)\n\t}\n\n\tif len(res.Hits) != 2 {\n\t\tt.Fatalf(\"expected 2 hits, got %d\", len(res.Hits))\n\t}\n\n\tif res.Hits[0].ID != \"1\" || res.Hits[1].ID != \"2\" {\n\t\tt.Fatalf(\"unexpected hit IDs: %v, %v\", res.Hits[0].ID, res.Hits[1].ID)\n\t}\n}\n\nfunc TestValidNestedMapping(t *testing.T) {\n\t// ensure that top-level mappings - DefaultMapping and any type mappings - cannot be nested mappings\n\timap := mapping.NewIndexMapping()\n\tnestedMapping := mapping.NewNestedDocumentMapping()\n\timap.DefaultMapping = nestedMapping\n\terr := imap.Validate()\n\tif err == nil {\n\t\tt.Fatalf(\"expected error for nested DefaultMapping, got nil\")\n\t}\n\t// invalid nested type mapping\n\timap = mapping.NewIndexMapping()\n\timap.AddDocumentMapping(\"type1\", nestedMapping)\n\terr = imap.Validate()\n\tif err == nil {\n\t\tt.Fatalf(\"expected error for nested type mapping, got nil\")\n\t}\n\t// valid nested mappings within DefaultMapping\n\timap = mapping.NewIndexMapping()\n\tdocMapping := mapping.NewDocumentMapping()\n\tnestedMapping = mapping.NewNestedDocumentMapping()\n\tfieldMapping := mapping.NewTextFieldMapping()\n\tnestedMapping.AddFieldMappingsAt(\"field1\", fieldMapping)\n\tdocMapping.AddSubDocumentMapping(\"nestedField\", nestedMapping)\n\timap.DefaultMapping = docMapping\n\terr = imap.Validate()\n\tif err != nil {\n\t\tt.Fatalf(\"expected valid nested mapping, got error: %v\", err)\n\t}\n\t// valid nested mappings within type mapping\n\timap = mapping.NewIndexMapping()\n\tdocMapping = mapping.NewDocumentMapping()\n\tnestedMapping = mapping.NewNestedDocumentMapping()\n\tfieldMapping = mapping.NewTextFieldMapping()\n\tnestedMapping.AddFieldMappingsAt(\"field1\", fieldMapping)\n\tdocMapping.AddSubDocumentMapping(\"nestedField\", nestedMapping)\n\timap.AddDocumentMapping(\"type1\", docMapping)\n\terr = imap.Validate()\n\tif err != nil {\n\t\tt.Fatalf(\"expected valid nested mapping, got error: %v\", err)\n\t}\n\t// some nested type mappings\n\timap = mapping.NewIndexMapping()\n\tnestedMapping = mapping.NewNestedDocumentMapping()\n\tregularMapping := mapping.NewDocumentMapping()\n\timap.AddDocumentMapping(\"non_nested1\", regularMapping)\n\timap.AddDocumentMapping(\"non_nested2\", regularMapping)\n\timap.AddDocumentMapping(\"nested1\", nestedMapping)\n\timap.AddDocumentMapping(\"nested2\", nestedMapping)\n\terr = imap.Validate()\n\tif err == nil {\n\t\tt.Fatalf(\"expected error for nested type mappings, got nil\")\n\t}\n}\n"
  },
  {
    "path": "search_no_knn.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !vectors\n// +build !vectors\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"sort\"\n\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/collector\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nconst supportForVectorSearch = false\n\n// A SearchRequest describes all the parameters\n// needed to search the index.\n// Query is required.\n// Size/From describe how much and which part of the\n// result set to return.\n// Highlight describes optional search result\n// highlighting.\n// Fields describes a list of field values which\n// should be retrieved for result documents, provided they\n// were stored while indexing.\n// Facets describe the set of facets to be computed.\n// Explain triggers inclusion of additional search\n// result score explanations.\n// Sort describes the desired order for the results to be returned.\n// Score controls the kind of scoring performed\n// SearchAfter supports deep paging by providing a minimum sort key\n// SearchBefore supports deep paging by providing a maximum sort key\n// sortFunc specifies the sort implementation to use for sorting results.\n//\n// A special field named \"*\" can be used to return all fields.\ntype SearchRequest struct {\n\tClientContextID  string            `json:\"client_context_id,omitempty\"`\n\tQuery            query.Query       `json:\"query\"`\n\tSize             int               `json:\"size\"`\n\tFrom             int               `json:\"from\"`\n\tHighlight        *HighlightRequest `json:\"highlight\"`\n\tFields           []string          `json:\"fields\"`\n\tFacets           FacetsRequest     `json:\"facets\"`\n\tExplain          bool              `json:\"explain\"`\n\tSort             search.SortOrder  `json:\"sort\"`\n\tIncludeLocations bool              `json:\"includeLocations\"`\n\tScore            string            `json:\"score,omitempty\"`\n\tSearchAfter      []string          `json:\"search_after\"`\n\tSearchBefore     []string          `json:\"search_before\"`\n\n\t// PreSearchData will be a  map that will be used\n\t// in the second phase of any 2-phase search, to provide additional\n\t// context to the second phase. This is useful in the case of index\n\t// aliases where the first phase will gather the PreSearchData from all\n\t// the indexes in the alias, and the second phase will use that\n\t// PreSearchData to perform the actual search.\n\t// The currently accepted map configuration is:\n\t//\n\t// \"_knn_pre_search_data_key\": []*search.DocumentMatch\n\n\tPreSearchData map[string]interface{} `json:\"pre_search_data,omitempty\"`\n\n\tParams *RequestParams `json:\"params,omitempty\"`\n\n\tsortFunc func(sort.Interface)\n}\n\n// UnmarshalJSON deserializes a JSON representation of\n// a SearchRequest\nfunc (r *SearchRequest) UnmarshalJSON(input []byte) error {\n\tvar temp struct {\n\t\tQ                json.RawMessage   `json:\"query\"`\n\t\tSize             *int              `json:\"size\"`\n\t\tFrom             int               `json:\"from\"`\n\t\tHighlight        *HighlightRequest `json:\"highlight\"`\n\t\tFields           []string          `json:\"fields\"`\n\t\tFacets           FacetsRequest     `json:\"facets\"`\n\t\tExplain          bool              `json:\"explain\"`\n\t\tSort             []json.RawMessage `json:\"sort\"`\n\t\tIncludeLocations bool              `json:\"includeLocations\"`\n\t\tScore            string            `json:\"score\"`\n\t\tSearchAfter      []string          `json:\"search_after\"`\n\t\tSearchBefore     []string          `json:\"search_before\"`\n\t\tPreSearchData    json.RawMessage   `json:\"pre_search_data\"`\n\t\tParams           json.RawMessage   `json:\"params\"`\n\t}\n\n\terr := json.Unmarshal(input, &temp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif temp.Size == nil {\n\t\tr.Size = 10\n\t} else {\n\t\tr.Size = *temp.Size\n\t}\n\tif temp.Sort == nil {\n\t\tr.Sort = search.SortOrder{&search.SortScore{Desc: true}}\n\t} else {\n\t\tr.Sort, err = search.ParseSortOrderJSON(temp.Sort)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tr.From = temp.From\n\tr.Explain = temp.Explain\n\tr.Highlight = temp.Highlight\n\tr.Fields = temp.Fields\n\tr.Facets = temp.Facets\n\tr.IncludeLocations = temp.IncludeLocations\n\tr.Score = temp.Score\n\tr.SearchAfter = temp.SearchAfter\n\tr.SearchBefore = temp.SearchBefore\n\tr.Query, err = query.ParseQuery(temp.Q)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif r.Size < 0 {\n\t\tr.Size = 10\n\t}\n\tif r.From < 0 {\n\t\tr.From = 0\n\t}\n\n\tif IsScoreFusionRequested(r) {\n\t\tif temp.Params == nil {\n\t\t\t// If params is not present and it is requires rescoring, assign\n\t\t\t// default values\n\t\t\tr.Params = NewDefaultParams(r.From, r.Size)\n\t\t} else {\n\t\t\t// if it is a request that requires rescoring, parse the rescoring\n\t\t\t// parameters.\n\t\t\tparams, err := ParseParams(r, temp.Params)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tr.Params = params\n\t\t}\n\t}\n\n\tif temp.PreSearchData != nil {\n\t\tr.PreSearchData, err = query.ParsePreSearchData(temp.PreSearchData)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n\n}\n\n// -----------------------------------------------------------------------------\n\nfunc copySearchRequest(req *SearchRequest, preSearchData map[string]interface{}) *SearchRequest {\n\trv := SearchRequest{\n\t\tQuery:            req.Query,\n\t\tSize:             req.Size + req.From,\n\t\tFrom:             0,\n\t\tHighlight:        req.Highlight,\n\t\tFields:           req.Fields,\n\t\tFacets:           req.Facets,\n\t\tExplain:          req.Explain,\n\t\tSort:             req.Sort.Copy(),\n\t\tIncludeLocations: req.IncludeLocations,\n\t\tScore:            req.Score,\n\t\tSearchAfter:      req.SearchAfter,\n\t\tSearchBefore:     req.SearchBefore,\n\t\tPreSearchData:    preSearchData,\n\t}\n\treturn &rv\n}\n\nfunc validateKNN(req *SearchRequest) error {\n\treturn nil\n}\n\nfunc (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader, preSearch bool) ([]*search.DocumentMatch, error) {\n\treturn nil, nil\n}\n\nfunc setKnnHitsInCollector(knnHits []*search.DocumentMatch, coll *collector.TopNCollector) {\n}\n\nfunc requestHasKNN(req *SearchRequest) bool {\n\treturn false\n}\n\nfunc numKNNQueries(req *SearchRequest) int {\n\treturn 0\n}\n\nfunc addKnnToDummyRequest(dummyReq *SearchRequest, realReq *SearchRequest) {\n}\n\nfunc validateAndDistributeKNNHits(knnHits []*search.DocumentMatch, indexes []Index) (map[string][]*search.DocumentMatch, error) {\n\treturn nil, nil\n}\n\nfunc isKNNrequestSatisfiedByPreSearch(req *SearchRequest) bool {\n\treturn false\n}\n\nfunc constructKnnPreSearchData(mergedOut map[string]map[string]interface{}, preSearchResult *SearchResult,\n\tindexes []Index) (map[string]map[string]interface{}, error) {\n\treturn mergedOut, nil\n}\n\nfunc finalizeKNNResults(req *SearchRequest, knnHits []*search.DocumentMatch) []*search.DocumentMatch {\n\treturn knnHits\n}\n\nfunc newKnnPreSearchResultProcessor(req *SearchRequest) *knnPreSearchResultProcessor {\n\treturn &knnPreSearchResultProcessor{} // equivalent to nil\n}\n\nfunc (r *rescorer) prepareKnnRequest() {\n}\n\nfunc (r *rescorer) restoreKnnRequest() {\n}\n"
  },
  {
    "path": "search_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bleve\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2/analysis\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/custom\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/simple\"\n\t\"github.com/blevesearch/bleve/v2/analysis/analyzer/standard\"\n\thtml_char_filter \"github.com/blevesearch/bleve/v2/analysis/char/html\"\n\tregexp_char_filter \"github.com/blevesearch/bleve/v2/analysis/char/regexp\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/flexible\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/iso\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/percent\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/sanitized\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds\"\n\t\"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds\"\n\t\"github.com/blevesearch/bleve/v2/analysis/lang/en\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/length\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/lowercase\"\n\t\"github.com/blevesearch/bleve/v2/analysis/token/shingle\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/single\"\n\t\"github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/geo\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight/highlighter/ansi\"\n\t\"github.com/blevesearch/bleve/v2/search/highlight/highlighter/html\"\n\t\"github.com/blevesearch/bleve/v2/search/query\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestSortedFacetedQuery(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tdocumentMapping := NewDocumentMapping()\n\tindexMapping.AddDocumentMapping(\"hotel\", documentMapping)\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Index = true\n\tcontentFieldMapping.DocValues = true\n\tdocumentMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\tdocumentMapping.AddFieldMappingsAt(\"country\", contentFieldMapping)\n\n\tindex, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tif err := index.Index(\"1\", map[string]interface{}{\n\t\t\"country\": \"india\",\n\t\t\"content\": \"k\",\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := index.Index(\"2\", map[string]interface{}{\n\t\t\"country\": \"india\",\n\t\t\"content\": \"l\",\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := index.Index(\"3\", map[string]interface{}{\n\t\t\"country\": \"india\",\n\t\t\"content\": \"k\",\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\td, err := index.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif d != 3 {\n\t\tt.Errorf(\"expected 3, got %d\", d)\n\t}\n\n\tquery := NewMatchPhraseQuery(\"india\")\n\tquery.SetField(\"country\")\n\tsearchRequest := NewSearchRequest(query)\n\tsearchRequest.SortBy([]string{\"content\"})\n\tfr := NewFacetRequest(\"content\", 100)\n\tsearchRequest.AddFacet(\"content_facet\", fr)\n\n\tsearchResults, err := index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedResults := map[string]int{\"k\": 2, \"l\": 1}\n\n\tfor _, v := range searchResults.Facets {\n\t\tfor _, v1 := range v.Terms.Terms() {\n\t\t\tif v1.Count != expectedResults[v1.Term] {\n\t\t\t\tt.Errorf(\"expected %d, got %d\", expectedResults[v1.Term], v1.Count)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestMatchAllScorer(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tindexMapping := NewIndexMapping()\n\tindexMapping.TypeField = \"type\"\n\tindexMapping.DefaultAnalyzer = \"en\"\n\tdocumentMapping := NewDocumentMapping()\n\n\tcontentFieldMapping := NewTextFieldMapping()\n\tcontentFieldMapping.Index = true\n\tcontentFieldMapping.Store = true\n\tdocumentMapping.AddFieldMappingsAt(\"content\", contentFieldMapping)\n\n\tindex, err := New(tmpIndexPath, indexMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tif err := index.Index(\"1\", map[string]interface{}{\n\t\t\"country\": \"india\",\n\t\t\"content\": \"k\",\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := index.Index(\"2\", map[string]interface{}{\n\t\t\"country\": \"india\",\n\t\t\"content\": \"l\",\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := index.Index(\"3\", map[string]interface{}{\n\t\t\"country\": \"india\",\n\t\t\"content\": \"k\",\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\td, err := index.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif d != 3 {\n\t\tt.Errorf(\"expected 3, got %d\", d)\n\t}\n\n\tsearchRequest := NewSearchRequest(NewMatchAllQuery())\n\tsearchRequest.Score = \"none\"\n\tsearchResults, err := index.Search(searchRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif searchResults.Total != 3 {\n\t\tt.Fatalf(\"expected all the 3 docs in the index, got %v\", searchResults.Total)\n\t}\n\n\tfor _, hit := range searchResults.Hits {\n\t\tif hit.Score != 0.0 {\n\t\t\tt.Fatalf(\"expected 0 score since score = none, got %v\", hit.Score)\n\t\t}\n\t}\n}\n\nfunc TestSearchResultString(t *testing.T) {\n\ttests := []struct {\n\t\tresult *SearchResult\n\t\tstr    string\n\t}{\n\t\t{\n\t\t\tresult: &SearchResult{\n\t\t\t\tRequest: &SearchRequest{\n\t\t\t\t\tSize: 10,\n\t\t\t\t},\n\t\t\t\tTotal: 5,\n\t\t\t\tTook:  1 * time.Second,\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t&search.DocumentMatch{},\n\t\t\t\t\t&search.DocumentMatch{},\n\t\t\t\t\t&search.DocumentMatch{},\n\t\t\t\t\t&search.DocumentMatch{},\n\t\t\t\t\t&search.DocumentMatch{},\n\t\t\t\t},\n\t\t\t},\n\t\t\tstr: \"5 matches, showing 1 through 5, took 1s\",\n\t\t},\n\t\t{\n\t\t\tresult: &SearchResult{\n\t\t\t\tRequest: &SearchRequest{\n\t\t\t\t\tSize: 0,\n\t\t\t\t},\n\t\t\t\tTotal: 5,\n\t\t\t\tHits:  search.DocumentMatchCollection{},\n\t\t\t},\n\t\t\tstr: \"5 matches\",\n\t\t},\n\t\t{\n\t\t\tresult: &SearchResult{\n\t\t\t\tRequest: &SearchRequest{\n\t\t\t\t\tSize: 10,\n\t\t\t\t},\n\t\t\t\tTotal: 0,\n\t\t\t\tHits:  search.DocumentMatchCollection{},\n\t\t\t},\n\t\t\tstr: \"No matches\",\n\t\t},\n\t\t// no search request\n\t\t{\n\t\t\tresult: &SearchResult{\n\t\t\t\tTotal: 3,\n\t\t\t\tTook:  500 * time.Millisecond,\n\t\t\t\tHits: search.DocumentMatchCollection{\n\t\t\t\t\t&search.DocumentMatch{},\n\t\t\t\t\t&search.DocumentMatch{},\n\t\t\t\t\t&search.DocumentMatch{},\n\t\t\t\t},\n\t\t\t},\n\t\t\tstr: \"3 matches, took 500ms\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tsrstring := test.result.String()\n\t\tif !strings.HasPrefix(srstring, test.str) {\n\t\t\tt.Errorf(\"expected to start %s, got %s\", test.str, srstring)\n\t\t}\n\t}\n}\n\nfunc TestSearchResultMerge(t *testing.T) {\n\tl := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal:    1,\n\t\tMaxScore: 1,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t&search.DocumentMatch{\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1,\n\t\t\t},\n\t\t},\n\t}\n\n\tr := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal:    1,\n\t\tMaxScore: 2,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t&search.DocumentMatch{\n\t\t\t\tID:    \"b\",\n\t\t\t\tScore: 2,\n\t\t\t},\n\t\t},\n\t}\n\n\texpected := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      2,\n\t\t\tSuccessful: 2,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal:    2,\n\t\tMaxScore: 2,\n\t\tHits: search.DocumentMatchCollection{\n\t\t\t&search.DocumentMatch{\n\t\t\t\tID:    \"a\",\n\t\t\t\tScore: 1,\n\t\t\t},\n\t\t\t&search.DocumentMatch{\n\t\t\t\tID:    \"b\",\n\t\t\t\tScore: 2,\n\t\t\t},\n\t\t},\n\t}\n\n\tl.Merge(r)\n\n\tif !reflect.DeepEqual(l, expected) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expected, l)\n\t}\n}\n\nfunc TestUnmarshalingSearchResult(t *testing.T) {\n\tsearchResponse := []byte(`{\n    \"status\":{\n      \"total\":1,\n      \"failed\":1,\n      \"successful\":0,\n      \"errors\":{\n        \"default_index_362ce020b3d62b13_348f5c3c\":\"context deadline exceeded\"\n      }\n    },\n    \"request\":{\n      \"query\":{\n        \"match\":\"emp\",\n        \"field\":\"type\",\n        \"boost\":1,\n        \"prefix_length\":0,\n        \"fuzziness\":0\n      },\n    \"size\":10000000,\n    \"from\":0,\n    \"highlight\":null,\n    \"fields\":[],\n    \"facets\":null,\n    \"explain\":false\n  },\n  \"hits\":null,\n  \"total_hits\":0,\n  \"max_score\":0,\n  \"took\":0,\n  \"facets\":null\n}`)\n\n\trv := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tErrors: make(map[string]error),\n\t\t},\n\t}\n\terr = json.Unmarshal(searchResponse, rv)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif len(rv.Status.Errors) != 1 {\n\t\tt.Errorf(\"expected 1 error, got %d\", len(rv.Status.Errors))\n\t}\n}\n\nfunc TestFacetNumericDateRangeRequests(t *testing.T) {\n\tdrMissingErr := fmt.Errorf(\"date range query must specify either start, end or both for range name 'testName'\")\n\tnrMissingErr := fmt.Errorf(\"numeric range query must specify either min, max or both for range name 'testName'\")\n\tdrNrErr := fmt.Errorf(\"facet can only contain numeric ranges or date ranges, not both\")\n\tdrNameDupErr := fmt.Errorf(\"date ranges contains duplicate name 'testName'\")\n\tnrNameDupErr := fmt.Errorf(\"numeric ranges contains duplicate name 'testName'\")\n\tvalue := float64(5)\n\n\ttests := []struct {\n\t\tfacet  *FacetRequest\n\t\tresult error\n\t}{\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Date_Range_Success_With_StartEnd\",\n\t\t\t\tSize:  1,\n\t\t\t\tDateTimeRanges: []*dateTimeRange{\n\t\t\t\t\t{Name: \"testName\", Start: time.Unix(0, 0), End: time.Now()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: nil,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Date_Range_Success_With_Start\",\n\t\t\t\tSize:  1,\n\t\t\t\tDateTimeRanges: []*dateTimeRange{\n\t\t\t\t\t{Name: \"testName\", Start: time.Unix(0, 0)},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: nil,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Date_Range_Success_With_End\",\n\t\t\t\tSize:  1,\n\t\t\t\tDateTimeRanges: []*dateTimeRange{\n\t\t\t\t\t{Name: \"testName\", End: time.Now()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: nil,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Numeric_Range_Success_With_MinMax\",\n\t\t\t\tSize:  1,\n\t\t\t\tNumericRanges: []*numericRange{\n\t\t\t\t\t{Name: \"testName\", Min: &value, Max: &value},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: nil,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Numeric_Range_Success_With_Min\",\n\t\t\t\tSize:  1,\n\t\t\t\tNumericRanges: []*numericRange{\n\t\t\t\t\t{Name: \"testName\", Min: &value},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: nil,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Numeric_Range_Success_With_Max\",\n\t\t\t\tSize:  1,\n\t\t\t\tNumericRanges: []*numericRange{\n\t\t\t\t\t{Name: \"testName\", Max: &value},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: nil,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Date_Range_Missing_Failure\",\n\t\t\t\tSize:  1,\n\t\t\t\tDateTimeRanges: []*dateTimeRange{\n\t\t\t\t\t{Name: \"testName2\", Start: time.Unix(0, 0)},\n\t\t\t\t\t{Name: \"testName1\", End: time.Now()},\n\t\t\t\t\t{Name: \"testName\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: drMissingErr,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Numeric_Range_Missing_Failure\",\n\t\t\t\tSize:  1,\n\t\t\t\tNumericRanges: []*numericRange{\n\t\t\t\t\t{Name: \"testName2\", Min: &value},\n\t\t\t\t\t{Name: \"testName1\", Max: &value},\n\t\t\t\t\t{Name: \"testName\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: nrMissingErr,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Numeric_And_DateRanges_Failure\",\n\t\t\t\tSize:  1,\n\t\t\t\tNumericRanges: []*numericRange{\n\t\t\t\t\t{Name: \"testName\", Max: &value},\n\t\t\t\t},\n\t\t\t\tDateTimeRanges: []*dateTimeRange{\n\t\t\t\t\t{Name: \"testName\", End: time.Now()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: drNrErr,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Numeric_Range_Name_Repeat_Failure\",\n\t\t\t\tSize:  1,\n\t\t\t\tNumericRanges: []*numericRange{\n\t\t\t\t\t{Name: \"testName\", Min: &value},\n\t\t\t\t\t{Name: \"testName\", Max: &value},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: nrNameDupErr,\n\t\t},\n\t\t{\n\t\t\tfacet: &FacetRequest{\n\t\t\t\tField: \"Date_Range_Name_Repeat_Failure\",\n\t\t\t\tSize:  1,\n\t\t\t\tDateTimeRanges: []*dateTimeRange{\n\t\t\t\t\t{Name: \"testName\", Start: time.Unix(0, 0)},\n\t\t\t\t\t{Name: \"testName\", End: time.Now()},\n\t\t\t\t},\n\t\t\t},\n\t\t\tresult: drNameDupErr,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tresult := test.facet.Validate()\n\t\tif !reflect.DeepEqual(result, test.result) {\n\t\t\tt.Errorf(\"expected %#v, got %#v\", test.result, result)\n\t\t}\n\t}\n}\n\nfunc TestSearchResultFacetsMerge(t *testing.T) {\n\tlowmed := \"2010-01-01\"\n\tmedhi := \"2011-01-01\"\n\thihigher := \"2012-01-01\"\n\n\tfr := &search.FacetResult{\n\t\tField:   \"birthday\",\n\t\tTotal:   100,\n\t\tMissing: 25,\n\t\tOther:   25,\n\t\tDateRanges: []*search.DateRangeFacet{\n\t\t\t{\n\t\t\t\tName:  \"low\",\n\t\t\t\tEnd:   &lowmed,\n\t\t\t\tCount: 25,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"med\",\n\t\t\t\tCount: 24,\n\t\t\t\tStart: &lowmed,\n\t\t\t\tEnd:   &medhi,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:  \"hi\",\n\t\t\t\tCount: 1,\n\t\t\t\tStart: &medhi,\n\t\t\t\tEnd:   &hihigher,\n\t\t\t},\n\t\t},\n\t}\n\tfrs := search.FacetResults{\n\t\t\"birthdays\": fr,\n\t}\n\n\tl := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      10,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal:    10,\n\t\tMaxScore: 1,\n\t}\n\n\tr := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      1,\n\t\t\tSuccessful: 1,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal:    1,\n\t\tMaxScore: 2,\n\t\tFacets:   frs,\n\t}\n\n\texpected := &SearchResult{\n\t\tStatus: &SearchStatus{\n\t\t\tTotal:      11,\n\t\t\tSuccessful: 2,\n\t\t\tErrors:     make(map[string]error),\n\t\t},\n\t\tTotal:    11,\n\t\tMaxScore: 2,\n\t\tFacets:   frs,\n\t}\n\n\tl.Merge(r)\n\n\tif !reflect.DeepEqual(l, expected) {\n\t\tt.Errorf(\"expected %#v, got %#v\", expected, l)\n\t}\n}\n\nfunc TestMemoryNeededForSearchResult(t *testing.T) {\n\tquery := NewTermQuery(\"blah\")\n\treq := NewSearchRequest(query)\n\n\tvar sr SearchResult\n\texpect := sr.Size()\n\tvar dm search.DocumentMatch\n\texpect += 10 * dm.Size()\n\n\testimate := MemoryNeededForSearchResult(req)\n\tif estimate != uint64(expect) {\n\t\tt.Errorf(\"estimate not what is expected: %v != %v\", estimate, expect)\n\t}\n}\n\n// https://github.com/blevesearch/bleve/issues/954\nfunc TestNestedBooleanSearchers(t *testing.T) {\n\t// create an index with a custom analyzer\n\tidxMapping := NewIndexMapping()\n\tif err := idxMapping.AddCustomAnalyzer(\"3xbla\", map[string]interface{}{\n\t\t\"type\":          custom.Name,\n\t\t\"tokenizer\":     whitespace.Name,\n\t\t\"token_filters\": []interface{}{lowercase.Name, \"stop_en\"},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidxMapping.DefaultAnalyzer = \"3xbla\"\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// create and insert documents as a batch\n\tbatch := idx.NewBatch()\n\tmatches := 0\n\tfor i := 0; i < 100; i++ {\n\t\thostname := fmt.Sprintf(\"planner_hostname_%d\", i%5)\n\t\tmetadata := map[string]string{\"region\": fmt.Sprintf(\"planner_us-east-%d\", i%5)}\n\n\t\t// Expected matches\n\t\tif (hostname == \"planner_hostname_1\" || hostname == \"planner_hostname_2\") &&\n\t\t\tmetadata[\"region\"] == \"planner_us-east-1\" {\n\t\t\tmatches++\n\t\t}\n\n\t\tdoc := document.NewDocument(strconv.Itoa(i))\n\t\tdoc.Fields = []document.Field{\n\t\t\tdocument.NewTextFieldCustom(\"hostname\", []uint64{}, []byte(hostname),\n\t\t\t\tindex.IndexField,\n\t\t\t\t&analysis.DefaultAnalyzer{\n\t\t\t\t\tTokenizer: single.NewSingleTokenTokenizer(),\n\t\t\t\t\tTokenFilters: []analysis.TokenFilter{\n\t\t\t\t\t\tlowercase.NewLowerCaseFilter(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t),\n\t\t}\n\t\tfor k, v := range metadata {\n\t\t\tdoc.AddField(document.NewTextFieldWithIndexingOptions(\n\t\t\t\tfmt.Sprintf(\"metadata.%s\", k), []uint64{}, []byte(v), index.IndexField))\n\t\t}\n\t\tdoc.CompositeFields = []*document.CompositeField{\n\t\t\tdocument.NewCompositeFieldWithIndexingOptions(\n\t\t\t\t\"_all\", true, []string{\"text\"}, []string{},\n\t\t\t\tindex.IndexField|index.IncludeTermVectors),\n\t\t}\n\n\t\tif err = batch.IndexAdvanced(doc); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tque, err := query.ParseQuery([]byte(\n\t\t`{\n\t\t\t\"conjuncts\": [\n\t\t\t{\n\t\t\t\t\"must\": {\n\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t{\n\t\t\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"match\": \"planner_hostname_1\",\n\t\t\t\t\t\t\t\"field\": \"hostname\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"match\": \"planner_hostname_2\",\n\t\t\t\t\t\t\t\"field\": \"hostname\"\n\t\t\t\t\t\t}\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t},\n\t\t\t{\n\t\t\t\t\"must\": {\n\t\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t{\n\t\t\t\t\t\t\"match\": \"planner_us-east-1\",\n\t\t\t\t\t\t\"field\": \"metadata.region\"\n\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t\t]\n\t\t}`,\n\t))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treq := NewSearchRequest(que)\n\treq.Size = 100\n\treq.Fields = []string{\"hostname\", \"metadata.region\"}\n\tsearchResults, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif matches != len(searchResults.Hits) {\n\t\tt.Fatalf(\"Unexpected result set, %v != %v\", matches, len(searchResults.Hits))\n\t}\n}\n\nfunc TestNestedBooleanMustNotSearcherUpsidedown(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\t// create an index with default settings\n\tidxMapping := NewIndexMapping()\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// create and insert documents as a batch\n\tbatch := idx.NewBatch()\n\n\tdocs := []struct {\n\t\tid              string\n\t\thasRole         bool\n\t\tinvestigationId string\n\t}{\n\t\t{\n\t\t\tid:              \"1@1\",\n\t\t\thasRole:         true,\n\t\t\tinvestigationId: \"1\",\n\t\t},\n\t\t{\n\t\t\tid:              \"1@2\",\n\t\t\thasRole:         false,\n\t\t\tinvestigationId: \"2\",\n\t\t},\n\t\t{\n\t\t\tid:              \"2@1\",\n\t\t\thasRole:         true,\n\t\t\tinvestigationId: \"1\",\n\t\t},\n\t\t{\n\t\t\tid:              \"2@2\",\n\t\t\thasRole:         false,\n\t\t\tinvestigationId: \"2\",\n\t\t},\n\t\t{\n\t\t\tid:              \"3@1\",\n\t\t\thasRole:         true,\n\t\t\tinvestigationId: \"1\",\n\t\t},\n\t\t{\n\t\t\tid:              \"3@2\",\n\t\t\thasRole:         false,\n\t\t\tinvestigationId: \"2\",\n\t\t},\n\t\t{\n\t\t\tid:              \"4@1\",\n\t\t\thasRole:         true,\n\t\t\tinvestigationId: \"1\",\n\t\t},\n\t\t{\n\t\t\tid:              \"5@1\",\n\t\t\thasRole:         true,\n\t\t\tinvestigationId: \"1\",\n\t\t},\n\t\t{\n\t\t\tid:              \"6@1\",\n\t\t\thasRole:         true,\n\t\t\tinvestigationId: \"1\",\n\t\t},\n\t\t{\n\t\t\tid:              \"7@1\",\n\t\t\thasRole:         true,\n\t\t\tinvestigationId: \"1\",\n\t\t},\n\t}\n\n\tfor i := 0; i < len(docs); i++ {\n\t\tdoc := document.NewDocument(docs[i].id)\n\t\tdoc.Fields = []document.Field{\n\t\t\tdocument.NewTextField(\"id\", []uint64{}, []byte(docs[i].id)),\n\t\t\tdocument.NewBooleanField(\"hasRole\", []uint64{}, docs[i].hasRole),\n\t\t\tdocument.NewTextField(\"investigationId\", []uint64{}, []byte(docs[i].investigationId)),\n\t\t}\n\n\t\tdoc.CompositeFields = []*document.CompositeField{\n\t\t\tdocument.NewCompositeFieldWithIndexingOptions(\n\t\t\t\t\"_all\", true, []string{\"text\"}, []string{},\n\t\t\t\tindex.IndexField|index.IncludeTermVectors),\n\t\t}\n\n\t\tif err = batch.IndexAdvanced(doc); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttq := NewTermQuery(\"1\")\n\ttq.SetField(\"investigationId\")\n\t// using must not, for cases that the field did not exists at all\n\thasRole := NewBoolFieldQuery(true)\n\thasRole.SetField(\"hasRole\")\n\tnoRole := NewBooleanQuery()\n\tnoRole.AddMustNot(hasRole)\n\toneRolesOrNoRoles := NewBooleanQuery()\n\toneRolesOrNoRoles.AddShould(noRole)\n\toneRolesOrNoRoles.SetMinShould(1)\n\tq := NewConjunctionQuery(tq, oneRolesOrNoRoles)\n\n\tsr := NewSearchRequestOptions(q, 100, 0, false)\n\tsr.Fields = []string{\"hasRole\"}\n\tsr.Highlight = NewHighlight()\n\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif res.Total != 0 {\n\t\tt.Fatalf(\"Unexpected result, %v != 0\", res.Total)\n\t}\n}\n\nfunc TestSearchScorchOverEmptyKeyword(t *testing.T) {\n\tdefaultIndexType := Config.DefaultIndexType\n\tConfig.DefaultIndexType = scorch.Name\n\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.DefaultAnalyzer = standard.Name\n\n\tfm := mapping.NewTextFieldMapping()\n\tfm.Analyzer = keyword.Name\n\n\tfm1 := mapping.NewTextFieldMapping()\n\tfm1.Analyzer = standard.Name\n\n\tdmap.AddFieldMappingsAt(\"id\", fm)\n\tdmap.AddFieldMappingsAt(\"name\", fm1)\n\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping = dmap\n\timap.DefaultAnalyzer = standard.Name\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tConfig.DefaultIndexType = defaultIndexType\n\t}()\n\n\tfor i := 0; i < 10; i++ {\n\t\terr = idx.Index(fmt.Sprint(i), map[string]string{\"name\": fmt.Sprintf(\"test%d\", i), \"id\": \"\"})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tcount, err := idx.DocCount()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif count != 10 {\n\t\tt.Fatalf(\"Unexpected doc count: %v, expected 10\", count)\n\t}\n\n\tq := query.NewWildcardQuery(\"test*\")\n\tsr := NewSearchRequestOptions(q, 40, 0, false)\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif res.Total != 10 {\n\t\tt.Fatalf(\"Unexpected search hits: %v, expected 10\", res.Total)\n\t}\n}\n\nfunc TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) {\n\tdefaultIndexType := Config.DefaultIndexType\n\tConfig.DefaultIndexType = scorch.Name\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\t// create an index with default settings\n\tidxMapping := NewIndexMapping()\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tConfig.DefaultIndexType = defaultIndexType\n\t}()\n\n\t// create and insert documents as a batch\n\tbatch := idx.NewBatch()\n\n\tdoc := document.NewDocument(\"1-child-0\")\n\tdoc.Fields = []document.Field{\n\t\tdocument.NewTextField(\"id\", []uint64{}, []byte(\"1-child-0\")),\n\t\tdocument.NewBooleanField(\"hasRole\", []uint64{}, false),\n\t\tdocument.NewTextField(\"roles\", []uint64{}, []byte(\"R1\")),\n\t\tdocument.NewNumericField(\"type\", []uint64{}, 0),\n\t}\n\tdoc.CompositeFields = []*document.CompositeField{\n\t\tdocument.NewCompositeFieldWithIndexingOptions(\n\t\t\t\"_all\", true, []string{\"text\"}, []string{},\n\t\t\tindex.IndexField|index.IncludeTermVectors),\n\t}\n\n\tif err = batch.IndexAdvanced(doc); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdocs := []struct {\n\t\tid      string\n\t\thasRole bool\n\t\ttyp     int\n\t}{\n\t\t{\n\t\t\tid:      \"16d6fa37-48fd-4dea-8b3d-a52bddf73951\",\n\t\t\thasRole: false,\n\t\t\ttyp:     9,\n\t\t},\n\t\t{\n\t\t\tid:      \"18fa9eb2-8b1f-46f0-8b56-b4c551213f78\",\n\t\t\thasRole: false,\n\t\t\ttyp:     9,\n\t\t},\n\t\t{\n\t\t\tid:      \"3085855b-d74b-474a-86c3-9bf3e4504382\",\n\t\t\thasRole: false,\n\t\t\ttyp:     9,\n\t\t},\n\t\t{\n\t\t\tid:      \"38ef5d28-0f85-4fb0-8a94-dd20751c3364\",\n\t\t\thasRole: false,\n\t\t\ttyp:     9,\n\t\t},\n\t}\n\n\tfor i := 0; i < len(docs); i++ {\n\t\tdoc := document.NewDocument(docs[i].id)\n\t\tdoc.Fields = []document.Field{\n\t\t\tdocument.NewTextField(\"id\", []uint64{}, []byte(docs[i].id)),\n\t\t\tdocument.NewBooleanField(\"hasRole\", []uint64{}, docs[i].hasRole),\n\t\t\tdocument.NewNumericField(\"type\", []uint64{}, float64(docs[i].typ)),\n\t\t}\n\n\t\tdoc.CompositeFields = []*document.CompositeField{\n\t\t\tdocument.NewCompositeFieldWithIndexingOptions(\n\t\t\t\t\"_all\", true, []string{\"text\"}, []string{},\n\t\t\t\tindex.IndexField|index.IncludeTermVectors),\n\t\t}\n\n\t\tif err = batch.IndexAdvanced(doc); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tbatch = idx.NewBatch()\n\n\t// Update 1st doc\n\tdoc = document.NewDocument(\"1-child-0\")\n\tdoc.Fields = []document.Field{\n\t\tdocument.NewTextField(\"id\", []uint64{}, []byte(\"1-child-0\")),\n\t\tdocument.NewBooleanField(\"hasRole\", []uint64{}, false),\n\t\tdocument.NewNumericField(\"type\", []uint64{}, 0),\n\t}\n\tdoc.CompositeFields = []*document.CompositeField{\n\t\tdocument.NewCompositeFieldWithIndexingOptions(\n\t\t\t\"_all\", true, []string{\"text\"}, []string{},\n\t\t\tindex.IndexField|index.IncludeTermVectors),\n\t}\n\n\tif err = batch.IndexAdvanced(doc); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tinclusive := true\n\tval := float64(9)\n\tq := query.NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive)\n\tq.SetField(\"type\")\n\tinitialQuery := query.NewBooleanQuery(nil, nil, []query.Query{q})\n\n\t// using must not, for cases that the field did not exists at all\n\thasRole := NewBoolFieldQuery(true)\n\thasRole.SetField(\"hasRole\")\n\tnoRole := NewBooleanQuery()\n\tnoRole.AddMustNot(hasRole)\n\n\trq := query.NewBooleanQuery([]query.Query{initialQuery, noRole}, nil, nil)\n\n\tsr := NewSearchRequestOptions(rq, 100, 0, false)\n\tsr.Fields = []string{\"id\", \"hasRole\", \"type\"}\n\tsr.Highlight = NewHighlight()\n\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 1 {\n\t\tt.Fatalf(\"Unexpected result, %v != 1\", res.Total)\n\t}\n}\n\nfunc testBooleanMustNotSearcher(t *testing.T, indexName string) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tim := NewIndexMapping()\n\tidx, err := NewUsing(tmpIndexPath, im, indexName, Config.DefaultKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocs := []struct {\n\t\tName    string\n\t\tHasRole bool\n\t}{\n\t\t{\n\t\t\tName: \"13900\",\n\t\t},\n\t\t{\n\t\t\tName: \"13901\",\n\t\t},\n\t\t{\n\t\t\tName: \"13965\",\n\t\t},\n\t\t{\n\t\t\tName:    \"13966\",\n\t\t\tHasRole: true,\n\t\t},\n\t\t{\n\t\t\tName:    \"13967\",\n\t\t\tHasRole: true,\n\t\t},\n\t}\n\n\tfor _, doc := range docs {\n\t\terr := idx.Index(doc.Name, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tlhs := NewDocIDQuery([]string{\"13965\", \"13966\", \"13967\"})\n\thasRole := NewBoolFieldQuery(true)\n\thasRole.SetField(\"HasRole\")\n\trhs := NewBooleanQuery()\n\trhs.AddMustNot(hasRole)\n\n\tcompareLeftRightAndConjunction := func(idx Index, left, right query.Query) error {\n\t\t// left\n\t\tlr := NewSearchRequestOptions(left, 100, 0, false)\n\t\tlres, err := idx.Search(lr)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error left: %v\", err)\n\t\t}\n\t\tlresIds := map[string]struct{}{}\n\t\tfor i := range lres.Hits {\n\t\t\tlresIds[lres.Hits[i].ID] = struct{}{}\n\t\t}\n\t\t// right\n\t\trr := NewSearchRequestOptions(right, 100, 0, false)\n\t\trres, err := idx.Search(rr)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error right: %v\", err)\n\t\t}\n\t\trresIds := map[string]struct{}{}\n\t\tfor i := range rres.Hits {\n\t\t\trresIds[rres.Hits[i].ID] = struct{}{}\n\t\t}\n\t\t// conjunction\n\t\tcr := NewSearchRequestOptions(NewConjunctionQuery(left, right), 100, 0, false)\n\t\tcres, err := idx.Search(cr)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error conjunction: %v\", err)\n\t\t}\n\t\tfor i := range cres.Hits {\n\t\t\tif _, ok := lresIds[cres.Hits[i].ID]; ok {\n\t\t\t\tif _, ok := rresIds[cres.Hits[i].ID]; !ok {\n\t\t\t\t\treturn fmt.Errorf(\"error id %s missing from right\", cres.Hits[i].ID)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn fmt.Errorf(\"error id %s missing from left\", cres.Hits[i].ID)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\n\terr = compareLeftRightAndConjunction(idx, lhs, rhs)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestBooleanMustNotSearcherUpsidedown(t *testing.T) {\n\ttestBooleanMustNotSearcher(t, upsidedown.Name)\n}\n\nfunc TestBooleanMustNotSearcherScorch(t *testing.T) {\n\ttestBooleanMustNotSearcher(t, scorch.Name)\n}\n\nfunc TestQueryStringEmptyConjunctionSearcher(t *testing.T) {\n\tmapping := NewIndexMapping()\n\tmapping.DefaultAnalyzer = keyword.Name\n\tindex, err := NewMemOnly(mapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\t_ = index.Close()\n\t}()\n\n\tquery := NewQueryStringQuery(\"foo:bar +baz:\\\"\\\"\")\n\tsearchReq := NewSearchRequest(query)\n\n\t_, _ = index.Search(searchReq)\n}\n\nfunc TestDisjunctionQueryIncorrectMin(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\t// create an index with default settings\n\tidxMapping := NewIndexMapping()\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// create and insert documents as a batch\n\tbatch := idx.NewBatch()\n\tdocs := []struct {\n\t\tfield1 string\n\t\tfield2 int\n\t}{\n\t\t{\n\t\t\tfield1: \"one\",\n\t\t\tfield2: 1,\n\t\t},\n\t\t{\n\t\t\tfield1: \"two\",\n\t\t\tfield2: 2,\n\t\t},\n\t}\n\n\tfor i := 0; i < len(docs); i++ {\n\t\tdoc := document.NewDocument(strconv.Itoa(docs[i].field2))\n\t\tdoc.Fields = []document.Field{\n\t\t\tdocument.NewTextField(\"field1\", []uint64{}, []byte(docs[i].field1)),\n\t\t\tdocument.NewNumericField(\"field2\", []uint64{}, float64(docs[i].field2)),\n\t\t}\n\t\tdoc.CompositeFields = []*document.CompositeField{\n\t\t\tdocument.NewCompositeFieldWithIndexingOptions(\n\t\t\t\t\"_all\", true, []string{\"text\"}, []string{},\n\t\t\t\tindex.IndexField|index.IncludeTermVectors),\n\t\t}\n\t\tif err = batch.IndexAdvanced(doc); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttq := NewTermQuery(\"one\")\n\tdq := NewDisjunctionQuery(tq)\n\tdq.SetMin(2)\n\tsr := NewSearchRequestOptions(dq, 1, 0, false)\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total > 0 {\n\t\tt.Fatalf(\"Expected 0 matches as disjunction query contains a single clause\"+\n\t\t\t\" but got: %v\", res.Total)\n\t}\n}\n\nfunc TestMatchQueryPartialMatch(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\tidx, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\tdoc1 := map[string]interface{}{\n\t\t\"description\": \"Patrick is first name Stewart is last name\",\n\t}\n\tdoc2 := map[string]interface{}{\n\t\t\"description\": \"Manager given name is Patrick\",\n\t}\n\tbatch := idx.NewBatch()\n\tif err = batch.Index(\"doc1\", doc1); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err = batch.Index(\"doc2\", doc2); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// Test 1 - Both Docs hit, doc 1 = Full Match and doc 2 = Partial Match\n\tmq1 := NewMatchQuery(\"patrick stewart\")\n\tmq1.SetField(\"description\")\n\n\tsr := NewSearchRequest(mq1)\n\tsr.Explain = true\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif res.Total != 2 {\n\t\tt.Errorf(\"Expected 2 results, but got: %v\", res.Total)\n\t}\n\tfor _, hit := range res.Hits {\n\t\tswitch hit.ID {\n\t\tcase \"doc1\":\n\t\t\tif hit.Expl.PartialMatch {\n\t\t\t\tt.Errorf(\"Expected doc1 to be a full match\")\n\t\t\t}\n\t\tcase \"doc2\":\n\t\t\tif !hit.Expl.PartialMatch {\n\t\t\t\tt.Errorf(\"Expected doc2 to be a partial match\")\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Errorf(\"Unexpected document ID: %s\", hit.ID)\n\t\t}\n\t}\n\n\t// Test 2 - Both Docs hit, doc 1 = Partial Match and doc 2 = Full Match\n\tmq2 := NewMatchQuery(\"paltric manner\")\n\tmq2.SetField(\"description\")\n\tmq2.SetFuzziness(2)\n\n\tsr = NewSearchRequest(mq2)\n\tsr.Explain = true\n\tres, err = idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif res.Total != 2 {\n\t\tt.Errorf(\"Expected 2 results, but got: %v\", res.Total)\n\t}\n\tfor _, hit := range res.Hits {\n\t\tswitch hit.ID {\n\t\tcase \"doc1\":\n\t\t\tif !hit.Expl.PartialMatch {\n\t\t\t\tt.Errorf(\"Expected doc1 to be a partial match\")\n\t\t\t}\n\t\tcase \"doc2\":\n\t\t\tif hit.Expl.PartialMatch {\n\t\t\t\tt.Errorf(\"Expected doc2 to be a full match\")\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Errorf(\"Unexpected document ID: %s\", hit.ID)\n\t\t}\n\t}\n\t// Test 3 - Two Docs hits, both full match\n\tmq3 := NewMatchQuery(\"patrick\")\n\tmq3.SetField(\"description\")\n\n\tsr = NewSearchRequest(mq3)\n\tsr.Explain = true\n\tres, err = idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif res.Total != 2 {\n\t\tt.Errorf(\"Expected 2 results, but got: %v\", res.Total)\n\t}\n\tfor _, hit := range res.Hits {\n\t\tswitch hit.ID {\n\t\tcase \"doc1\":\n\t\t\tif hit.Expl.PartialMatch {\n\t\t\t\tt.Errorf(\"Expected doc1 to be a full match\")\n\t\t\t}\n\t\tcase \"doc2\":\n\t\t\tif hit.Expl.PartialMatch {\n\t\t\t\tt.Errorf(\"Expected doc2 to be a full match\")\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Errorf(\"Unexpected document ID: %s\", hit.ID)\n\t\t}\n\t}\n\t// Test 4 - Two Docs hits, both partial match\n\tmq4 := NewMatchQuery(\"patrick stewart manager\")\n\tmq4.SetField(\"description\")\n\n\tsr = NewSearchRequest(mq4)\n\tsr.Explain = true\n\tres, err = idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif res.Total != 2 {\n\t\tt.Errorf(\"Expected 2 results, but got: %v\", res.Total)\n\t}\n\tfor _, hit := range res.Hits {\n\t\tswitch hit.ID {\n\t\tcase \"doc1\":\n\t\t\tif !hit.Expl.PartialMatch {\n\t\t\t\tt.Errorf(\"Expected doc1 to be a full match\")\n\t\t\t}\n\t\tcase \"doc2\":\n\t\t\tif !hit.Expl.PartialMatch {\n\t\t\t\tt.Errorf(\"Expected doc2 to be a full match\")\n\t\t\t}\n\t\tdefault:\n\t\t\tt.Errorf(\"Unexpected document ID: %s\", hit.ID)\n\t\t}\n\t}\n\n\t// Test 5 - Match Query AND operator always results in full match\n\tmq5 := NewMatchQuery(\"patrick stewart\")\n\tmq5.SetField(\"description\")\n\tmq5.SetOperator(1)\n\n\tsr = NewSearchRequest(mq5)\n\tsr.Explain = true\n\tres, err = idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif res.Total != 1 {\n\t\tt.Errorf(\"Expected 1 result, but got: %v\", res.Total)\n\t}\n\thit := res.Hits[0]\n\tif hit.ID != \"doc1\" || hit.Expl.PartialMatch {\n\t\tt.Errorf(\"Expected doc1 to be a full match\")\n\t}\n}\n\nfunc TestBooleanShouldMinPropagation(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc1 := map[string]interface{}{\n\t\t\"dept\": \"queen\",\n\t\t\"name\": \"cersei lannister\",\n\t}\n\n\tdoc2 := map[string]interface{}{\n\t\t\"dept\": \"kings guard\",\n\t\t\"name\": \"jaime lannister\",\n\t}\n\n\tbatch := idx.NewBatch()\n\n\tif err = batch.Index(\"doc1\", doc1); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = batch.Index(\"doc2\", doc2); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// term dictionaries in the index for field..\n\t//  dept: queen kings guard\n\t//  name: cersei jaime lannister\n\n\t// the following match query would match doc2\n\tmq1 := NewMatchQuery(\"kings guard\")\n\tmq1.SetField(\"dept\")\n\n\t// the following match query would match both doc1 and doc2,\n\t// as both docs share common lastname\n\tmq2 := NewMatchQuery(\"jaime lannister\")\n\tmq2.SetField(\"name\")\n\n\tbq := NewBooleanQuery()\n\tbq.AddShould(mq1)\n\tbq.AddMust(mq2)\n\n\tsr := NewSearchRequest(bq)\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 2 {\n\t\tt.Errorf(\"Expected 2 results, but got: %v\", res.Total)\n\t}\n}\n\nfunc TestDisjunctionMinPropagation(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, NewIndexMapping())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc1 := map[string]interface{}{\n\t\t\"dept\": \"finance\",\n\t\t\"name\": \"xyz\",\n\t}\n\n\tdoc2 := map[string]interface{}{\n\t\t\"dept\": \"marketing\",\n\t\t\"name\": \"xyz\",\n\t}\n\n\tdoc3 := map[string]interface{}{\n\t\t\"dept\": \"engineering\",\n\t\t\"name\": \"abc\",\n\t}\n\n\tbatch := idx.NewBatch()\n\n\tif err = batch.Index(\"doc1\", doc1); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = batch.Index(\"doc2\", doc2); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = batch.Index(\"doc3\", doc3); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmq1 := NewMatchQuery(\"finance\")\n\tmq2 := NewMatchQuery(\"marketing\")\n\tdq := NewDisjunctionQuery(mq1, mq2)\n\tdq.SetMin(3)\n\n\tdq2 := NewDisjunctionQuery(dq)\n\tdq2.SetMin(1)\n\n\tsr := NewSearchRequest(dq2)\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 0 {\n\t\tt.Fatalf(\"Expect 0 results, but got: %v\", res.Total)\n\t}\n}\n\nfunc TestDuplicateLocationsIssue1168(t *testing.T) {\n\tfm1 := NewTextFieldMapping()\n\tfm1.Analyzer = keyword.Name\n\tfm1.Name = \"name1\"\n\n\tdm := NewDocumentStaticMapping()\n\tdm.AddFieldMappingsAt(\"name\", fm1)\n\n\tm := NewIndexMapping()\n\tm.DefaultMapping = dm\n\n\tidx, err := NewMemOnly(m)\n\tif err != nil {\n\t\tt.Fatalf(\"bleve new err: %v\", err)\n\t}\n\n\terr = idx.Index(\"x\", map[string]interface{}{\n\t\t\"name\": \"marty\",\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"bleve index err: %v\", err)\n\t}\n\n\tq1 := NewTermQuery(\"marty\")\n\tq2 := NewTermQuery(\"marty\")\n\tdq := NewDisjunctionQuery(q1, q2)\n\n\tsreq := NewSearchRequest(dq)\n\tsreq.Fields = []string{\"*\"}\n\tsreq.Highlight = NewHighlightWithStyle(html.Name)\n\n\tsres, err := idx.Search(sreq)\n\tif err != nil {\n\t\tt.Fatalf(\"bleve search err: %v\", err)\n\t}\n\tif len(sres.Hits[0].Locations[\"name1\"][\"marty\"]) != 1 {\n\t\tt.Fatalf(\"duplicate marty\")\n\t}\n}\n\nfunc TestBooleanMustSingleMatchNone(t *testing.T) {\n\tidxMapping := NewIndexMapping()\n\tif err := idxMapping.AddCustomTokenFilter(length.Name, map[string]interface{}{\n\t\t\"min\":  3.0,\n\t\t\"max\":  5.0,\n\t\t\"type\": length.Name,\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idxMapping.AddCustomAnalyzer(\"custom1\", map[string]interface{}{\n\t\t\"type\":          \"custom\",\n\t\t\"tokenizer\":     \"single\",\n\t\t\"token_filters\": []interface{}{length.Name},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidxMapping.DefaultAnalyzer = \"custom1\"\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := map[string]interface{}{\n\t\t\"languages_known\": \"Dutch\",\n\t\t\"dept\":            \"Sales\",\n\t}\n\n\tbatch := idx.NewBatch()\n\tif err = batch.Index(\"doc\", doc); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// this is a successful match\n\tmatchSales := NewMatchQuery(\"Sales\")\n\tmatchSales.SetField(\"dept\")\n\n\t// this would spin off a MatchNoneSearcher as the\n\t// token filter rules out the word \"French\"\n\tmatchFrench := NewMatchQuery(\"French\")\n\tmatchFrench.SetField(\"languages_known\")\n\n\tbq := NewBooleanQuery()\n\tbq.AddShould(matchSales)\n\tbq.AddMust(matchFrench)\n\n\tsr := NewSearchRequest(bq)\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 0 {\n\t\tt.Fatalf(\"Expected 0 results but got: %v\", res.Total)\n\t}\n}\n\nfunc TestBooleanMustNotSingleMatchNone(t *testing.T) {\n\tidxMapping := NewIndexMapping()\n\tif err := idxMapping.AddCustomTokenFilter(shingle.Name, map[string]interface{}{\n\t\t\"min\":  3.0,\n\t\t\"max\":  5.0,\n\t\t\"type\": shingle.Name,\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idxMapping.AddCustomAnalyzer(\"custom1\", map[string]interface{}{\n\t\t\"type\":          \"custom\",\n\t\t\"tokenizer\":     \"unicode\",\n\t\t\"token_filters\": []interface{}{shingle.Name},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidxMapping.DefaultAnalyzer = \"custom1\"\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := map[string]interface{}{\n\t\t\"languages_known\": \"Dutch\",\n\t\t\"dept\":            \"Sales\",\n\t}\n\n\tbatch := idx.NewBatch()\n\tif err = batch.Index(\"doc\", doc); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// this is a successful match\n\tmatchSales := NewMatchQuery(\"Sales\")\n\tmatchSales.SetField(\"dept\")\n\n\t// this would spin off a MatchNoneSearcher as the\n\t// token filter rules out the word \"Dutch\"\n\tmatchDutch := NewMatchQuery(\"Dutch\")\n\tmatchDutch.SetField(\"languages_known\")\n\n\tmatchEngineering := NewMatchQuery(\"Engineering\")\n\tmatchEngineering.SetField(\"dept\")\n\n\tbq := NewBooleanQuery()\n\tbq.AddShould(matchSales)\n\tbq.AddMustNot(matchDutch, matchEngineering)\n\n\tsr := NewSearchRequest(bq)\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 0 {\n\t\tt.Fatalf(\"Expected 0 results but got: %v\", res.Total)\n\t}\n}\n\nfunc TestBooleanSearchBug1185(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tof := NewTextFieldMapping()\n\tof.Analyzer = keyword.Name\n\tof.Name = \"owner\"\n\n\tdm := NewDocumentMapping()\n\tdm.AddFieldMappingsAt(\"owner\", of)\n\n\tm := NewIndexMapping()\n\tm.DefaultMapping = dm\n\n\tidx, err := NewUsing(tmpIndexPath, m, \"scorch\", \"scorch\", nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\terr = idx.Index(\"17112\", map[string]interface{}{\n\t\t\"owner\": \"marty\",\n\t\t\"type\":  \"A Demo Type\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Index(\"17139\", map[string]interface{}{\n\t\t\"type\": \"A Demo Type\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Index(\"177777\", map[string]interface{}{\n\t\t\"type\": \"x\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Index(\"177778\", map[string]interface{}{\n\t\t\"type\": \"A Demo Type\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Index(\"17140\", map[string]interface{}{\n\t\t\"type\": \"A Demo Type\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Index(\"17000\", map[string]interface{}{\n\t\t\"owner\": \"marty\",\n\t\t\"type\":  \"x\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Index(\"17141\", map[string]interface{}{\n\t\t\"type\": \"A Demo Type\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Index(\"17428\", map[string]interface{}{\n\t\t\"owner\": \"marty\",\n\t\t\"type\":  \"A Demo Type\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idx.Index(\"17113\", map[string]interface{}{\n\t\t\"owner\": \"marty\",\n\t\t\"type\":  \"x\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmatchTypeQ := NewMatchPhraseQuery(\"A Demo Type\")\n\tmatchTypeQ.SetField(\"type\")\n\n\tmatchAnyOwnerRegQ := NewRegexpQuery(\".+\")\n\tmatchAnyOwnerRegQ.SetField(\"owner\")\n\n\tmatchNoOwner := NewBooleanQuery()\n\tmatchNoOwner.AddMustNot(matchAnyOwnerRegQ)\n\n\tnotNoOwner := NewBooleanQuery()\n\tnotNoOwner.AddMustNot(matchNoOwner)\n\n\tmatchTypeAndNoOwner := NewConjunctionQuery()\n\tmatchTypeAndNoOwner.AddQuery(matchTypeQ)\n\tmatchTypeAndNoOwner.AddQuery(notNoOwner)\n\n\treq := NewSearchRequest(matchTypeAndNoOwner)\n\tres, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// query 2\n\tmatchTypeAndNoOwnerBoolean := NewBooleanQuery()\n\tmatchTypeAndNoOwnerBoolean.AddMust(matchTypeQ)\n\tmatchTypeAndNoOwnerBoolean.AddMustNot(matchNoOwner)\n\n\treq2 := NewSearchRequest(matchTypeAndNoOwnerBoolean)\n\tres2, err := idx.Search(req2)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(res.Hits) != len(res2.Hits) {\n\t\tt.Fatalf(\"expected same number of hits, got: %d and %d\", len(res.Hits), len(res2.Hits))\n\t}\n}\n\nfunc TestSearchScoreNone(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := NewUsing(tmpIndexPath, NewIndexMapping(), scorch.Name, Config.DefaultKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := map[string]interface{}{\n\t\t\"field1\": \"asd fgh jkl\",\n\t\t\"field2\": \"more content blah blah\",\n\t\t\"id\":     \"doc\",\n\t}\n\n\tif err = idx.Index(\"doc\", doc); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tq := NewQueryStringQuery(\"content\")\n\tsr := NewSearchRequest(q)\n\tsr.IncludeLocations = true\n\tsr.Score = \"none\"\n\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(res.Hits) != 1 {\n\t\tt.Fatal(\"unexpected number of hits\")\n\t}\n\n\tif len(res.Hits[0].Locations) != 1 {\n\t\tt.Fatal(\"unexpected locations for the hit\")\n\t}\n\n\tif res.Hits[0].Score != 0 {\n\t\tt.Fatal(\"unexpected score for the hit\")\n\t}\n}\n\nfunc TestGeoDistanceIssue1301(t *testing.T) {\n\tshopMapping := NewDocumentMapping()\n\tshopMapping.AddFieldMappingsAt(\"GEO\", NewGeoPointFieldMapping())\n\tshopIndexMapping := NewIndexMapping()\n\tshopIndexMapping.DefaultMapping = shopMapping\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := NewUsing(tmpIndexPath, shopIndexMapping, scorch.Name, Config.DefaultKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tfor i, g := range []string{\"wecpkbeddsmf\", \"wecpk8tne453\", \"wecpkb80s09t\"} {\n\t\tif err = idx.Index(strconv.Itoa(i), map[string]interface{}{\n\t\t\t\"ID\":  i,\n\t\t\t\"GEO\": g,\n\t\t}); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\t// Not setting \"Field\" for the following query, targets it against the _all\n\t// field and this is returning inconsistent results, when there's another\n\t// field indexed along with the geopoint which is numeric.\n\t// As reported in: https://github.com/blevesearch/bleve/issues/1301\n\tlat, lon := 22.371154, 114.112603\n\tq := NewGeoDistanceQuery(lon, lat, \"1km\")\n\n\treq := NewSearchRequest(q)\n\tsr, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif sr.Total != 3 {\n\t\tt.Fatalf(\"Size expected: 3, actual %d\\n\", sr.Total)\n\t}\n}\n\nfunc TestSearchHighlightingWithRegexpReplacement(t *testing.T) {\n\tidxMapping := NewIndexMapping()\n\tif err := idxMapping.AddCustomCharFilter(regexp_char_filter.Name, map[string]interface{}{\n\t\t\"regexp\":  `([a-z])\\s+(\\d)`,\n\t\t\"replace\": \"ooooo$1-$2\",\n\t\t\"type\":    regexp_char_filter.Name,\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idxMapping.AddCustomAnalyzer(\"regexp_replace\", map[string]interface{}{\n\t\t\"type\":      custom.Name,\n\t\t\"tokenizer\": \"unicode\",\n\t\t\"char_filters\": []string{\n\t\t\tregexp_char_filter.Name,\n\t\t},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidxMapping.DefaultAnalyzer = \"regexp_replace\"\n\tidxMapping.StoreDynamic = true\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := NewUsing(tmpIndexPath, idxMapping, scorch.Name, Config.DefaultKVStore, nil)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := map[string]interface{}{\n\t\t\"status\": \"fool 10\",\n\t}\n\n\tbatch := idx.NewBatch()\n\tif err = batch.Index(\"doc\", doc); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tquery := NewMatchQuery(\"fool 10\")\n\tsreq := NewSearchRequest(query)\n\tsreq.Fields = []string{\"*\"}\n\tsreq.Highlight = NewHighlightWithStyle(ansi.Name)\n\n\tsres, err := idx.Search(sreq)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif sres.Total != 1 {\n\t\tt.Fatalf(\"Expected 1 hit, got: %v\", sres.Total)\n\t}\n}\n\nfunc TestAnalyzerInheritance(t *testing.T) {\n\ttests := []struct {\n\t\tname       string\n\t\tmappingStr string\n\t\tdoc        map[string]interface{}\n\t\tqueryField string\n\t\tqueryTerm  string\n\t}{\n\t\t{\n\t\t\t/*\n\t\t\t\tindex_mapping: keyword\n\t\t\t\tdefault_mapping: \"\"\n\t\t\t\t\t-> child field (should inherit keyword)\n\t\t\t*/\n\t\t\tname: \"Child field to inherit index mapping's default analyzer\",\n\t\t\tmappingStr: `{\"default_mapping\":{\"enabled\":true,\"dynamic\":false,\"properties\":` +\n\t\t\t\t`{\"city\":{\"enabled\":true,\"dynamic\":false,\"fields\":[{\"name\":\"city\",\"type\":\"text\",` +\n\t\t\t\t`\"store\":false,\"index\":true}]}}},\"default_analyzer\":\"keyword\"}`,\n\t\t\tdoc:        map[string]interface{}{\"city\": \"San Francisco\"},\n\t\t\tqueryField: \"city\",\n\t\t\tqueryTerm:  \"San Francisco\",\n\t\t},\n\t\t{\n\t\t\t/*\n\t\t\t\tindex_mapping: standard\n\t\t\t\tdefault_mapping: keyword\n\t\t\t\t    -> child field (should inherit keyword)\n\t\t\t*/\n\t\t\tname: \"Child field to inherit default mapping's default analyzer\",\n\t\t\tmappingStr: `{\"default_mapping\":{\"enabled\":true,\"dynamic\":false,\"properties\":` +\n\t\t\t\t`{\"city\":{\"enabled\":true,\"dynamic\":false,\"fields\":[{\"name\":\"city\",\"type\":\"text\",` +\n\t\t\t\t`\"index\":true}]}},\"default_analyzer\":\"keyword\"},\"default_analyzer\":\"standard\"}`,\n\t\t\tdoc:        map[string]interface{}{\"city\": \"San Francisco\"},\n\t\t\tqueryField: \"city\",\n\t\t\tqueryTerm:  \"San Francisco\",\n\t\t},\n\t\t{\n\t\t\t/*\n\t\t\t\tindex_mapping: standard\n\t\t\t\tdefault_mapping: keyword (dynamic)\n\t\t\t\t    -> search over field to (should inherit keyword)\n\t\t\t*/\n\t\t\tname: \"Child field to inherit default mapping's default analyzer\",\n\t\t\tmappingStr: `{\"default_mapping\":{\"enabled\":true,\"dynamic\":true,\"default_analyzer\":\"keyword\"}` +\n\t\t\t\t`,\"default_analyzer\":\"standard\"}`,\n\t\t\tdoc:        map[string]interface{}{\"city\": \"San Francisco\"},\n\t\t\tqueryField: \"city\",\n\t\t\tqueryTerm:  \"San Francisco\",\n\t\t},\n\t\t{\n\t\t\t/*\n\t\t\t\tindex_mapping: standard\n\t\t\t\tdefault_mapping: keyword\n\t\t\t\t    -> child mapping: \"\"\n\t\t\t\t\t    -> child field: (should inherit keyword)\n\t\t\t*/\n\t\t\tname: \"Nested child field to inherit default mapping's default analyzer\",\n\t\t\tmappingStr: `{\"default_mapping\":{\"enabled\":true,\"dynamic\":false,\"default_analyzer\":` +\n\t\t\t\t`\"keyword\",\"properties\":{\"address\":{\"enabled\":true,\"dynamic\":false,\"properties\":` +\n\t\t\t\t`{\"city\":{\"enabled\":true,\"dynamic\":false,\"fields\":[{\"name\":\"city\",\"type\":\"text\",` +\n\t\t\t\t`\"index\":true}]}}}}},\"default_analyzer\":\"standard\"}`,\n\t\t\tdoc: map[string]interface{}{\n\t\t\t\t\"address\": map[string]interface{}{\"city\": \"San Francisco\"},\n\t\t\t},\n\t\t\tqueryField: \"address.city\",\n\t\t\tqueryTerm:  \"San Francisco\",\n\t\t},\n\t\t{\n\t\t\t/*\n\t\t\t\tindex_mapping: standard\n\t\t\t\tdefault_mapping: \"\"\n\t\t\t\t    -> child mapping: \"keyword\"\n\t\t\t\t\t    -> child mapping: \"\"\n\t\t\t\t\t\t    -> child field: (should inherit keyword)\n\t\t\t*/\n\t\t\tname: \"Nested child field to inherit first child mapping's default analyzer\",\n\t\t\tmappingStr: `{\"default_mapping\":{\"enabled\":true,\"dynamic\":false,\"properties\":` +\n\t\t\t\t`{\"address\":{\"enabled\":true,\"dynamic\":false,\"default_analyzer\":\"keyword\",` +\n\t\t\t\t`\"properties\":{\"state\":{\"enabled\":true,\"dynamic\":false,\"properties\":{\"city\":` +\n\t\t\t\t`{\"enabled\":true,\"dynamic\":false,\"fields\":[{\"name\":\"city\",\"type\":\"text\",` +\n\t\t\t\t`\"store\":false,\"index\":true}]}}}}}}},\"default_analyer\":\"standard\"}`,\n\t\t\tdoc: map[string]interface{}{\n\t\t\t\t\"address\": map[string]interface{}{\n\t\t\t\t\t\"state\": map[string]interface{}{\"city\": \"San Francisco\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tqueryField: \"address.state.city\",\n\t\t\tqueryTerm:  \"San Francisco\",\n\t\t},\n\t}\n\n\tfor i := range tests {\n\t\tt.Run(tests[i].name, func(t *testing.T) {\n\t\t\tidxMapping := NewIndexMapping()\n\t\t\tif err := idxMapping.UnmarshalJSON([]byte(tests[i].mappingStr)); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\ttmpIndexPath := createTmpIndexPath(t)\n\t\t\tidx, err := New(tmpIndexPath, idxMapping)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tdefer func() {\n\t\t\t\tif err := idx.Close(); err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\tif err = idx.Index(\"doc\", tests[i].doc); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tq := NewTermQuery(tests[i].queryTerm)\n\t\t\tq.SetField(tests[i].queryField)\n\n\t\t\tres, err := idx.Search(NewSearchRequest(q))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif len(res.Hits) != 1 {\n\t\t\t\tt.Errorf(\"Unexpected number of hits: %v\", len(res.Hits))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestHightlightingWithHTMLCharacterFilter(t *testing.T) {\n\tidxMapping := NewIndexMapping()\n\tif err := idxMapping.AddCustomAnalyzer(\"custom-html\", map[string]interface{}{\n\t\t\"type\":         custom.Name,\n\t\t\"tokenizer\":    \"unicode\",\n\t\t\"char_filters\": []interface{}{html_char_filter.Name},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfm := mapping.NewTextFieldMapping()\n\tfm.Analyzer = \"custom-html\"\n\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"content\", fm)\n\n\tidxMapping.DefaultMapping = dmap\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tcontent := \"<div> Welcome to blevesearch. </div>\"\n\tif err = idx.Index(\"doc\", map[string]string{\n\t\t\"content\": content,\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tsearchStr := \"blevesearch\"\n\tq := query.NewMatchQuery(searchStr)\n\tq.SetField(\"content\")\n\tsr := NewSearchRequest(q)\n\tsr.IncludeLocations = true\n\tsr.Fields = []string{\"*\"}\n\tsr.Highlight = NewHighlightWithStyle(html.Name)\n\tsearchResults, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(searchResults.Hits) != 1 ||\n\t\tlen(searchResults.Hits[0].Locations[\"content\"][searchStr]) != 1 {\n\t\tt.Fatalf(\"Expected 1 hit with 1 location\")\n\t}\n\n\texpectedLocation := &search.Location{\n\t\tPos:   3,\n\t\tStart: uint64(strings.Index(content, searchStr)),\n\t\tEnd:   uint64(strings.Index(content, searchStr) + len(searchStr)),\n\t}\n\texpectedFragment := \"&lt;div&gt; Welcome to <mark>blevesearch</mark>. &lt;/div&gt;\"\n\n\tgotLocation := searchResults.Hits[0].Locations[\"content\"][\"blevesearch\"][0]\n\tgotFragment := searchResults.Hits[0].Fragments[\"content\"][0]\n\n\tif !reflect.DeepEqual(expectedLocation, gotLocation) {\n\t\tt.Fatalf(\"Mismatch in locations, got: %v, expected: %v\",\n\t\t\tgotLocation, expectedLocation)\n\t}\n\n\tif expectedFragment != gotFragment {\n\t\tt.Fatalf(\"Mismatch in fragment, got: %v, expected: %v\",\n\t\t\tgotFragment, expectedFragment)\n\t}\n}\n\nfunc TestIPRangeQuery(t *testing.T) {\n\tidxMapping := NewIndexMapping()\n\tim := NewIPFieldMapping()\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"ip_content\", im)\n\tidxMapping.DefaultMapping = dmap\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tipContent := \"192.168.10.11\"\n\tif err = idx.Index(\"doc\", map[string]string{\n\t\t\"ip_content\": ipContent,\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tq := query.NewIPRangeQuery(\"192.168.10.0/24\")\n\tq.SetField(\"ip_content\")\n\tsr := NewSearchRequest(q)\n\n\tsearchResults, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(searchResults.Hits) != 1 ||\n\t\tsearchResults.Hits[0].ID != \"doc\" {\n\t\tt.Fatal(\"Expected the 1 result - doc\")\n\t}\n}\n\nfunc TestGeoShapePolygonContainsPoint(t *testing.T) {\n\tfm := mapping.NewGeoShapeFieldMapping()\n\tdmap := mapping.NewDocumentMapping()\n\tdmap.AddFieldMappingsAt(\"geometry\", fm)\n\n\tidxMapping := NewIndexMapping()\n\tidxMapping.DefaultMapping = dmap\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// Polygon coordinates to be ordered in counter-clock-wise order\n\t// for the outer loop, and holes to follow clock-wise order.\n\t// See: https://www.rfc-editor.org/rfc/rfc7946.html#section-3.1.6\n\n\tone := []byte(`{\n\t\t\"geometry\":{\n\t\t\t\"type\":\"Polygon\",\n\t\t\t\"coordinates\":[[\n\t\t\t\t[4.8089,46.9307],\n\t\t\t\t[4.8223,46.8915],\n\t\t\t\t[4.8149,46.886],\n\t\t\t\t[4.8252,46.8647],\n\t\t\t\t[4.8305,46.8531],\n\t\t\t\t[4.8506,46.8509],\n\t\t\t\t[4.8574,46.8621],\n\t\t\t\t[4.8576,46.8769],\n\t\t\t\t[4.8753,46.8774],\n\t\t\t\t[4.8909,46.8519],\n\t\t\t\t[4.8837,46.8485],\n\t\t\t\t[4.9014,46.8318],\n\t\t\t\t[4.9067,46.8179],\n\t\t\t\t[4.8986,46.8122],\n\t\t\t\t[4.9081,46.7969],\n\t\t\t\t[4.9535,46.8254],\n\t\t\t\t[4.9577,46.8053],\n\t\t\t\t[5.0201,46.821],\n\t\t\t\t[5.0357,46.8207],\n\t\t\t\t[5.0656,46.8434],\n\t\t\t\t[5.0955,46.8411],\n\t\t\t\t[5.1149,46.8435],\n\t\t\t\t[5.1259,46.8395],\n\t\t\t\t[5.1433,46.8463],\n\t\t\t\t[5.1415,46.8589],\n\t\t\t\t[5.1533,46.873],\n\t\t\t\t[5.138,46.8843],\n\t\t\t\t[5.1525,46.9012],\n\t\t\t\t[5.1485,46.9165],\n\t\t\t\t[5.1582,46.926],\n\t\t\t\t[5.1882,46.9251],\n\t\t\t\t[5.2039,46.9129],\n\t\t\t\t[5.2223,46.9175],\n\t\t\t\t[5.2168,46.926],\n\t\t\t\t[5.2338,46.9316],\n\t\t\t\t[5.228,46.9505],\n\t\t\t\t[5.2078,46.9722],\n\t\t\t\t[5.2117,46.98],\n\t\t\t\t[5.1961,46.9783],\n\t\t\t\t[5.1663,46.9638],\n\t\t\t\t[5.1213,46.9634],\n\t\t\t\t[5.1086,46.9596],\n\t\t\t\t[5.0729,46.9604],\n\t\t\t\t[5.0731,46.9668],\n\t\t\t\t[5.0493,46.9817],\n\t\t\t\t[5.0034,46.9722],\n\t\t\t\t[4.9852,46.9585],\n\t\t\t\t[4.9479,46.9664],\n\t\t\t\t[4.8943,46.9663],\n\t\t\t\t[4.8937,46.951],\n\t\t\t\t[4.8534,46.9458],\n\t\t\t\t[4.8089,46.9307]\n\t\t\t]]\n\t\t}\n\t}`)\n\n\ttwo := []byte(`{\n\t\t\"geometry\":{\n\t\t\t\"type\":\"Polygon\",\n\t\t\t\"coordinates\":[[\n\t\t\t\t[2.2266,48.7816],\n\t\t\t\t[2.2266,48.7761],\n\t\t\t\t[2.2288,48.7745],\n\t\t\t\t[2.2717,48.7905],\n\t\t\t\t[2.2799,48.8109],\n\t\t\t\t[2.3013,48.8251],\n\t\t\t\t[2.2894,48.8283],\n\t\t\t\t[2.2726,48.8144],\n\t\t\t\t[2.2518,48.8164],\n\t\t\t\t[2.255,48.8101],\n\t\t\t\t[2.2348,48.7954],\n\t\t\t\t[2.2266,48.7816]\n\t\t\t]]\n\t\t}\n\t}`)\n\n\tvar doc1, doc2 map[string]interface{}\n\n\tif err = json.Unmarshal(one, &doc1); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err = idx.Index(\"doc1\", doc1); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = json.Unmarshal(two, &doc2); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err = idx.Index(\"doc2\", doc2); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor testi, test := range []struct {\n\t\tcoordinates []float64\n\t\texpectHits  []string\n\t}{\n\t\t{\n\t\t\tcoordinates: []float64{5, 46.9},\n\t\t\texpectHits:  []string{\"doc1\"},\n\t\t},\n\t\t{\n\t\t\tcoordinates: []float64{1.5, 48.2},\n\t\t},\n\t} {\n\t\tq, err := NewGeoShapeQuery(\n\t\t\t[][][][]float64{{{test.coordinates}}},\n\t\t\tgeo.PointType,\n\t\t\t\"contains\",\n\t\t)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"test: %d, query err: %v\", testi+1, err)\n\t\t}\n\t\tq.SetField(\"geometry\")\n\n\t\tres, err := idx.Search(NewSearchRequest(q))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"test: %d, search err: %v\", testi+1, err)\n\t\t}\n\n\t\tif len(res.Hits) != len(test.expectHits) {\n\t\t\tt.Errorf(\"test: %d, unexpected hits: %v\", testi+1, len(res.Hits))\n\t\t}\n\n\tOUTER:\n\t\tfor _, expect := range test.expectHits {\n\t\t\tfor _, got := range res.Hits {\n\t\t\t\tif got.ID == expect {\n\t\t\t\t\tcontinue OUTER\n\t\t\t\t}\n\t\t\t}\n\t\t\tt.Errorf(\"test: %d, couldn't get: %v\", testi+1, expect)\n\t\t}\n\t}\n}\n\nfunc TestAnalyzerInheritanceForDefaultDynamicMapping(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping.DefaultAnalyzer = keyword.Name\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdoc := map[string]interface{}{\n\t\t\"fieldX\": \"AbCdEf\",\n\t}\n\n\tif err = idx.Index(\"doc\", doc); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Match query to apply keyword analyzer to fieldX.\n\tmq := NewMatchQuery(\"AbCdEf\")\n\tmq.SetField(\"fieldX\")\n\n\tsr := NewSearchRequest(mq)\n\tresults, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(results.Hits) != 1 {\n\t\tt.Fatalf(\"expected 1 hit, got %d\", len(results.Hits))\n\t}\n}\n\nfunc TestCustomDateTimeParserLayoutValidation(t *testing.T) {\n\tflexiblegoName := flexible.Name\n\tsanitizedgoName := sanitized.Name\n\timap := mapping.NewIndexMapping()\n\tcorrectConfig := map[string]interface{}{\n\t\t\"type\": sanitizedgoName,\n\t\t\"layouts\": []interface{}{\n\t\t\t// some custom layouts\n\t\t\t\"2006-01-02 15:04:05.0000\",\n\t\t\t\"2006\\\\01\\\\02T03:04:05PM\",\n\t\t\t\"2006/01/02\",\n\t\t\t\"2006-01-02T15:04:05.999Z0700PMMST\",\n\t\t\t\"15:04:05.0000Z07:00 Monday\",\n\n\t\t\t// standard layouts\n\t\t\ttime.Layout,\n\t\t\ttime.ANSIC,\n\t\t\ttime.UnixDate,\n\t\t\ttime.RubyDate,\n\t\t\ttime.RFC822,\n\t\t\ttime.RFC822Z,\n\t\t\ttime.RFC850,\n\t\t\ttime.RFC1123,\n\t\t\ttime.RFC1123Z,\n\t\t\ttime.RFC3339,\n\t\t\ttime.RFC3339Nano,\n\t\t\ttime.Kitchen,\n\t\t\ttime.Stamp,\n\t\t\ttime.StampMilli,\n\t\t\ttime.StampMicro,\n\t\t\ttime.StampNano,\n\t\t\t\"2006-01-02 15:04:05\", // time.DateTime\n\t\t\t\"2006-01-02\",          // time.DateOnly\n\t\t\t\"15:04:05\",            // time.TimeOnly\n\n\t\t\t// Corrected layouts to the incorrect ones below.\n\t\t\t\"2006-01-02 03:04:05 -0700\",\n\t\t\t\"2006-01-02 15:04:05 -0700\",\n\t\t\t\"3:04PM\",\n\t\t\t\"2006-01-02 15:04:05.000 -0700 MST\",\n\t\t\t\"January 2 2006 3:04 PM\",\n\t\t\t\"02/Jan/06 3:04PM\",\n\t\t\t\"Mon 02 Jan 3:04:05 PM\",\n\t\t},\n\t}\n\n\t// Correct layouts - sanitizedgo should work without errors.\n\terr := imap.AddCustomDateTimeParser(\"custDT\", correctConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"expected no error, got: %v\", err)\n\t}\n\t// Flexiblego should work without errors as well.\n\tcorrectConfig[\"type\"] = flexiblegoName\n\terr = imap.AddCustomDateTimeParser(\"custDT_Flexi\", correctConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"expected no error, got: %v\", err)\n\t}\n\n\tincorrectLayouts := [][]interface{}{\n\t\t{\n\t\t\t\"2000-03-31 01:33:51 +0300\",\n\t\t},\n\t\t{\n\t\t\t\"2006-01-02 15:04:51 +0300\",\n\t\t},\n\t\t{\n\t\t\t\"2000-03-31 01:33:05 +0300\",\n\t\t},\n\t\t{\n\t\t\t\"4:45PM\",\n\t\t},\n\t\t{\n\t\t\t\"2006-01-02 15:04:05.445 -0700 MST\",\n\t\t},\n\t\t{\n\t\t\t\"August 20 2001 8:55 AM\",\n\t\t},\n\t\t{\n\t\t\t\"28/Jul/23 12:48PM\",\n\t\t},\n\t\t{\n\t\t\t\"Tue 22 Aug 6:37:30 AM\",\n\t\t},\n\t}\n\n\t// first check sanitizedgo, should throw error for each of the incorrect layouts.\n\tnumExpectedErrors := len(incorrectLayouts)\n\tnumActualErrors := 0\n\tfor idx, badLayout := range incorrectLayouts {\n\t\tincorrectConfig := map[string]interface{}{\n\t\t\t\"type\":    sanitizedgoName,\n\t\t\t\"layouts\": badLayout,\n\t\t}\n\t\terr := imap.AddCustomDateTimeParser(fmt.Sprintf(\"%d_DT\", idx), incorrectConfig)\n\t\tif err != nil {\n\t\t\tnumActualErrors++\n\t\t}\n\t}\n\t// Expecting all layouts to be incorrect, since sanitizedgo is being used.\n\tif numActualErrors != numExpectedErrors {\n\t\tt.Fatalf(\"expected %d errors, got: %d\", numExpectedErrors, numActualErrors)\n\t}\n\n\t// sanity test - flexiblego should still allow the incorrect layouts, for legacy purposes\n\tfor idx, badLayout := range incorrectLayouts {\n\t\tincorrectConfig := map[string]interface{}{\n\t\t\t\"type\":    flexiblegoName,\n\t\t\t\"layouts\": badLayout,\n\t\t}\n\t\terr := imap.AddCustomDateTimeParser(fmt.Sprintf(\"%d_DT_Flexi\", idx), incorrectConfig)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"expected no error, got: %v\", err)\n\t\t}\n\t}\n}\n\nfunc TestDateRangeStringQuery(t *testing.T) {\n\tidxMapping := NewIndexMapping()\n\n\terr := idxMapping.AddCustomDateTimeParser(\"customDT\", map[string]interface{}{\n\t\t\"type\": sanitized.Name,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"02/01/2006 15:04:05\",\n\t\t\t\"2006/01/02 3:04PM\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idxMapping.AddCustomDateTimeParser(\"queryDT\", map[string]interface{}{\n\t\t\"type\": sanitized.Name,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"02/01/2006 3:04PM\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdtmap := NewDateTimeFieldMapping()\n\tdtmap.DateFormat = \"customDT\"\n\tidxMapping.DefaultMapping.AddFieldMappingsAt(\"date\", dtmap)\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\tdocuments := map[string]map[string]interface{}{\n\t\t\"doc1\": {\n\t\t\t\"date\": \"2001/08/20 6:00PM\",\n\t\t},\n\t\t\"doc2\": {\n\t\t\t\"date\": \"20/08/2001 18:00:20\",\n\t\t},\n\t\t\"doc3\": {\n\t\t\t\"date\": \"20/08/2001 18:10:00\",\n\t\t},\n\t\t\"doc4\": {\n\t\t\t\"date\": \"2001/08/20 6:15PM\",\n\t\t},\n\t\t\"doc5\": {\n\t\t\t\"date\": \"20/08/2001 18:20:00\",\n\t\t},\n\t}\n\n\tbatch := idx.NewBatch()\n\tfor docID, doc := range documents {\n\t\terr := batch.Index(docID, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttype testResult struct {\n\t\tdocID    string // doc ID of the hit\n\t\thitField string // fields returned as part of the hit\n\t}\n\n\ttype testStruct struct {\n\t\tstart          string\n\t\tend            string\n\t\tfield          string\n\t\tdateTimeParser string // name of the custom date time parser to use if nil, use QueryDateTimeParser\n\t\tincludeStart   bool\n\t\tincludeEnd     bool\n\t\texpectedHits   []testResult\n\t\terr            error\n\t}\n\n\ttestQueries := []testStruct{\n\t\t// test cases with RFC3339 parser and toggling includeStart and includeEnd\n\t\t{\n\t\t\tstart:        \"2001-08-20T18:00:00\",\n\t\t\tend:          \"2001-08-20T18:10:00\",\n\t\t\tfield:        \"date\",\n\t\t\tincludeStart: true,\n\t\t\tincludeEnd:   true,\n\t\t\texpectedHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc1\",\n\t\t\t\t\thitField: \"2001/08/20 6:00PM\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc2\",\n\t\t\t\t\thitField: \"20/08/2001 18:00:20\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc3\",\n\t\t\t\t\thitField: \"20/08/2001 18:10:00\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tstart:        \"2001-08-20T18:00:00\",\n\t\t\tend:          \"2001-08-20T18:10:00\",\n\t\t\tfield:        \"date\",\n\t\t\tincludeStart: false,\n\t\t\tincludeEnd:   true,\n\t\t\texpectedHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc2\",\n\t\t\t\t\thitField: \"20/08/2001 18:00:20\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc3\",\n\t\t\t\t\thitField: \"20/08/2001 18:10:00\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tstart:        \"2001-08-20T18:00:00\",\n\t\t\tend:          \"2001-08-20T18:10:00\",\n\t\t\tfield:        \"date\",\n\t\t\tincludeStart: false,\n\t\t\tincludeEnd:   false,\n\t\t\texpectedHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc2\",\n\t\t\t\t\thitField: \"20/08/2001 18:00:20\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// test cases with custom parser and omitting start and end\n\t\t{\n\t\t\tstart:          \"20/08/2001 18:00:00\",\n\t\t\tend:            \"2001/08/20 6:10PM\",\n\t\t\tfield:          \"date\",\n\t\t\tdateTimeParser: \"customDT\",\n\t\t\tincludeStart:   true,\n\t\t\tincludeEnd:     true,\n\t\t\texpectedHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc1\",\n\t\t\t\t\thitField: \"2001/08/20 6:00PM\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc2\",\n\t\t\t\t\thitField: \"20/08/2001 18:00:20\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc3\",\n\t\t\t\t\thitField: \"20/08/2001 18:10:00\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tend:            \"20/08/2001 18:15:00\",\n\t\t\tfield:          \"date\",\n\t\t\tdateTimeParser: \"customDT\",\n\t\t\tincludeStart:   true,\n\t\t\tincludeEnd:     true,\n\t\t\texpectedHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc1\",\n\t\t\t\t\thitField: \"2001/08/20 6:00PM\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc2\",\n\t\t\t\t\thitField: \"20/08/2001 18:00:20\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc3\",\n\t\t\t\t\thitField: \"20/08/2001 18:10:00\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc4\",\n\t\t\t\t\thitField: \"2001/08/20 6:15PM\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tstart:          \"2001/08/20 6:15PM\",\n\t\t\tfield:          \"date\",\n\t\t\tdateTimeParser: \"customDT\",\n\t\t\tincludeStart:   true,\n\t\t\tincludeEnd:     true,\n\t\t\texpectedHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc4\",\n\t\t\t\t\thitField: \"2001/08/20 6:15PM\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc5\",\n\t\t\t\t\thitField: \"20/08/2001 18:20:00\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tstart:          \"20/08/2001 6:15PM\",\n\t\t\tfield:          \"date\",\n\t\t\tdateTimeParser: \"queryDT\",\n\t\t\tincludeStart:   true,\n\t\t\tincludeEnd:     true,\n\t\t\texpectedHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc4\",\n\t\t\t\t\thitField: \"2001/08/20 6:15PM\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:    \"doc5\",\n\t\t\t\t\thitField: \"20/08/2001 18:20:00\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// error path test cases\n\t\t{\n\t\t\tfield:          \"date\",\n\t\t\tdateTimeParser: \"customDT\",\n\t\t\tincludeStart:   true,\n\t\t\tincludeEnd:     true,\n\t\t\terr:            fmt.Errorf(\"date range query must specify at least one of start/end\"),\n\t\t},\n\t\t{\n\t\t\tfield:        \"date\",\n\t\t\tincludeStart: true,\n\t\t\tincludeEnd:   true,\n\t\t\terr:          fmt.Errorf(\"date range query must specify at least one of start/end\"),\n\t\t},\n\t\t{\n\t\t\tstart:          \"2001-08-20T18:00:00\",\n\t\t\tend:            \"2001-08-20T18:10:00\",\n\t\t\tfield:          \"date\",\n\t\t\tdateTimeParser: \"customDT\",\n\t\t\terr:            fmt.Errorf(\"unable to parse datetime with any of the layouts, date time parser name: customDT\"),\n\t\t},\n\t\t{\n\t\t\tstart: \"3001-08-20T18:00:00\",\n\t\t\tend:   \"2001-08-20T18:10:00\",\n\t\t\tfield: \"date\",\n\t\t\terr:   fmt.Errorf(\"invalid/unsupported date range, start: 3001-08-20T18:00:00\"),\n\t\t},\n\t\t{\n\t\t\tstart:          \"2001/08/20 6:00PM\",\n\t\t\tend:            \"3001/08/20 6:30PM\",\n\t\t\tfield:          \"date\",\n\t\t\tdateTimeParser: \"customDT\",\n\t\t\terr:            fmt.Errorf(\"invalid/unsupported date range, end: 3001/08/20 6:30PM\"),\n\t\t},\n\t}\n\n\tfor _, dtq := range testQueries {\n\t\tvar err error\n\t\tdateQuery := NewDateRangeInclusiveStringQuery(dtq.start, dtq.end, &dtq.includeStart, &dtq.includeEnd)\n\t\tdateQuery.SetDateTimeParser(dtq.dateTimeParser)\n\t\tdateQuery.SetField(dtq.field)\n\n\t\tsr := NewSearchRequest(dateQuery)\n\t\tsr.SortBy([]string{dtq.field})\n\t\tsr.Fields = []string{dtq.field}\n\n\t\tres, err := idx.Search(sr)\n\t\tif err != nil {\n\t\t\tif dtq.err == nil {\n\t\t\t\tt.Fatalf(\"expected no error, got: %v\", err)\n\t\t\t}\n\t\t\tif dtq.err.Error() != err.Error() {\n\t\t\t\tt.Fatalf(\"expected error: %v, got: %v\", dtq.err, err)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif len(res.Hits) != len(dtq.expectedHits) {\n\t\t\tt.Fatalf(\"expected %d hits, got %d\", len(dtq.expectedHits), len(res.Hits))\n\t\t}\n\t\tfor i, hit := range res.Hits {\n\t\t\tif hit.ID != dtq.expectedHits[i].docID {\n\t\t\t\tt.Fatalf(\"expected docID %s, got %s\", dtq.expectedHits[i].docID, hit.ID)\n\t\t\t}\n\t\t\tif hit.Fields[dtq.field].(string) != dtq.expectedHits[i].hitField {\n\t\t\t\tt.Fatalf(\"expected hit field %s, got %s\", dtq.expectedHits[i].hitField, hit.Fields[dtq.field])\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDateRangeFacetQueriesWithCustomDateTimeParser(t *testing.T) {\n\tidxMapping := NewIndexMapping()\n\n\terr := idxMapping.AddCustomDateTimeParser(\"customDT\", map[string]interface{}{\n\t\t\"type\": sanitized.Name,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"02/01/2006 15:04:05\",\n\t\t\t\"2006/01/02 3:04PM\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = idxMapping.AddCustomDateTimeParser(\"queryDT\", map[string]interface{}{\n\t\t\"type\": sanitized.Name,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"02/01/2006 3:04PM\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdtmap := NewDateTimeFieldMapping()\n\tdtmap.DateFormat = \"customDT\"\n\tidxMapping.DefaultMapping.AddFieldMappingsAt(\"date\", dtmap)\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, idxMapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\tdocuments := map[string]map[string]interface{}{\n\t\t\"doc1\": {\n\t\t\t\"date\": \"2001/08/20 6:00PM\",\n\t\t},\n\t\t\"doc2\": {\n\t\t\t\"date\": \"20/08/2001 18:00:20\",\n\t\t},\n\t\t\"doc3\": {\n\t\t\t\"date\": \"20/08/2001 18:10:00\",\n\t\t},\n\t\t\"doc4\": {\n\t\t\t\"date\": \"2001/08/20 6:15PM\",\n\t\t},\n\t\t\"doc5\": {\n\t\t\t\"date\": \"20/08/2001 18:20:00\",\n\t\t},\n\t}\n\n\tbatch := idx.NewBatch()\n\tfor docID, doc := range documents {\n\t\terr := batch.Index(docID, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tquery := NewMatchAllQuery()\n\n\ttype testFacetResult struct {\n\t\tname  string\n\t\tstart string\n\t\tend   string\n\t\tcount int\n\t\terr   error\n\t}\n\n\ttype testFacetRequest struct {\n\t\tname   string\n\t\tstart  string\n\t\tend    string\n\t\tparser string\n\t\tresult testFacetResult\n\t}\n\n\ttests := []testFacetRequest{\n\t\t{\n\t\t\t// Test without a query time override of the parser (use default parser)\n\t\t\tname:  \"test\",\n\t\t\tstart: \"2001-08-20 18:00:00\",\n\t\t\tend:   \"2001-08-20 18:10:00\",\n\t\t\tresult: testFacetResult{\n\t\t\t\tname:  \"test\",\n\t\t\t\tstart: \"2001-08-20T18:00:00Z\",\n\t\t\t\tend:   \"2001-08-20T18:10:00Z\",\n\t\t\t\tcount: 2,\n\t\t\t\terr:   nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:   \"test\",\n\t\t\tstart:  \"20/08/2001 6:00PM\",\n\t\t\tend:    \"20/08/2001 6:10PM\",\n\t\t\tparser: \"queryDT\",\n\t\t\tresult: testFacetResult{\n\t\t\t\tname:  \"test\",\n\t\t\t\tstart: \"2001-08-20T18:00:00Z\",\n\t\t\t\tend:   \"2001-08-20T18:10:00Z\",\n\t\t\t\tcount: 2,\n\t\t\t\terr:   nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:   \"test\",\n\t\t\tstart:  \"20/08/2001 15:00:00\",\n\t\t\tend:    \"2001/08/20 6:10PM\",\n\t\t\tparser: \"customDT\",\n\t\t\tresult: testFacetResult{\n\t\t\t\tname:  \"test\",\n\t\t\t\tstart: \"2001-08-20T15:00:00Z\",\n\t\t\t\tend:   \"2001-08-20T18:10:00Z\",\n\t\t\t\tcount: 2,\n\t\t\t\terr:   nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:   \"test\",\n\t\t\tend:    \"2001/08/20 6:15PM\",\n\t\t\tparser: \"customDT\",\n\t\t\tresult: testFacetResult{\n\t\t\t\tname:  \"test\",\n\t\t\t\tend:   \"2001-08-20T18:15:00Z\",\n\t\t\t\tcount: 3,\n\t\t\t\terr:   nil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:   \"test\",\n\t\t\tstart:  \"20/08/2001 6:15PM\",\n\t\t\tparser: \"queryDT\",\n\t\t\tresult: testFacetResult{\n\t\t\t\tname:  \"test\",\n\t\t\t\tstart: \"2001-08-20T18:15:00Z\",\n\t\t\t\tcount: 2,\n\t\t\t\terr:   nil,\n\t\t\t},\n\t\t},\n\t\t// some error cases\n\t\t{\n\t\t\tname:   \"test\",\n\t\t\tparser: \"queryDT\",\n\t\t\tresult: testFacetResult{\n\t\t\t\tname: \"test\",\n\t\t\t\terr:  fmt.Errorf(\"date range query must specify either start, end or both for date range name 'test'\"),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t// default parser is used for the query, but the start time is not in the correct format (RFC3339),\n\t\t\t// so it should throw an error\n\t\t\tname:  \"test\",\n\t\t\tstart: \"20/08/2001 6:15PM\",\n\t\t\tresult: testFacetResult{\n\t\t\t\tname: \"test\",\n\t\t\t\terr:  fmt.Errorf(\"ParseDates err: error parsing start date '20/08/2001 6:15PM' for date range name 'test': unable to parse datetime with any of the layouts, using date time parser named dateTimeOptional\"),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tsearchRequest := NewSearchRequest(query)\n\n\t\tfr := NewFacetRequest(\"date\", 100)\n\t\tstart := &test.start\n\t\tif test.start == \"\" {\n\t\t\tstart = nil\n\t\t}\n\t\tend := &test.end\n\t\tif test.end == \"\" {\n\t\t\tend = nil\n\t\t}\n\n\t\tfr.AddDateTimeRangeStringWithParser(test.name, start, end, test.parser)\n\t\tsearchRequest.AddFacet(\"dateFacet\", fr)\n\n\t\tsearchResults, err := idx.Search(searchRequest)\n\t\tif err != nil {\n\t\t\tif test.result.err == nil {\n\t\t\t\tt.Fatalf(\"Unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif err.Error() != test.result.err.Error() {\n\t\t\t\tt.Fatalf(\"Expected error %v, got %v\", test.result.err, err)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tfor _, facetResult := range searchResults.Facets {\n\t\t\tif len(facetResult.DateRanges) != 1 {\n\t\t\t\tt.Fatal(\"Expected 1 date range facet\")\n\t\t\t}\n\t\t\tresult := facetResult.DateRanges[0]\n\t\t\tif result.Name != test.result.name {\n\t\t\t\tt.Fatalf(\"Expected name %s, got %s\", test.result.name, result.Name)\n\t\t\t}\n\t\t\tif result.Start != nil && *result.Start != test.result.start {\n\t\t\t\tt.Fatalf(\"Expected start %s, got %s\", test.result.start, *result.Start)\n\t\t\t}\n\t\t\tif result.End != nil && *result.End != test.result.end {\n\t\t\t\tt.Fatalf(\"Expected end %s, got %s\", test.result.end, *result.End)\n\t\t\t}\n\t\t\tif result.Start == nil && test.result.start != \"\" {\n\t\t\t\tt.Fatalf(\"Expected start %s, got nil\", test.result.start)\n\t\t\t}\n\t\t\tif result.End == nil && test.result.end != \"\" {\n\t\t\t\tt.Fatalf(\"Expected end %s, got nil\", test.result.end)\n\t\t\t}\n\t\t\tif result.Count != test.result.count {\n\t\t\t\tt.Fatalf(\"Expected count %d, got %d\", test.result.count, result.Count)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestDateRangeTimestampQueries(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\timap := mapping.NewIndexMapping()\n\n\t// add a date field with a valid format to the default mapping\n\t// for good measure\n\n\tdtParserConfig := map[string]interface{}{\n\t\t\"type\":    flexible.Name,\n\t\t\"layouts\": []interface{}{\"2006/01/02 15:04:05\"},\n\t}\n\terr := imap.AddCustomDateTimeParser(\"custDT\", dtParserConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdateField := mapping.NewDateTimeFieldMapping()\n\tdateField.DateFormat = \"custDT\"\n\n\tunixSecField := mapping.NewDateTimeFieldMapping()\n\tunixSecField.DateFormat = seconds.Name\n\n\tunixMilliSecField := mapping.NewDateTimeFieldMapping()\n\tunixMilliSecField.DateFormat = milliseconds.Name\n\n\tunixMicroSecField := mapping.NewDateTimeFieldMapping()\n\tunixMicroSecField.DateFormat = microseconds.Name\n\n\tunixNanoSecField := mapping.NewDateTimeFieldMapping()\n\tunixNanoSecField.DateFormat = nanoseconds.Name\n\n\timap.DefaultMapping.AddFieldMappingsAt(\"date\", dateField)\n\timap.DefaultMapping.AddFieldMappingsAt(\"seconds\", unixSecField)\n\timap.DefaultMapping.AddFieldMappingsAt(\"milliseconds\", unixMilliSecField)\n\timap.DefaultMapping.AddFieldMappingsAt(\"microseconds\", unixMicroSecField)\n\timap.DefaultMapping.AddFieldMappingsAt(\"nanoseconds\", unixNanoSecField)\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocuments := map[string]map[string]string{\n\t\t\"doc1\": {\n\t\t\t\"date\":         \"2001/08/20 03:00:10\",\n\t\t\t\"seconds\":      \"998276410\",\n\t\t\t\"milliseconds\": \"998276410100\",\n\t\t\t\"microseconds\": \"998276410100300\",\n\t\t\t\"nanoseconds\":  \"998276410100300400\",\n\t\t},\n\t\t\"doc2\": {\n\t\t\t\"date\":         \"2001/08/20 03:00:20\",\n\t\t\t\"seconds\":      \"998276420\",\n\t\t\t\"milliseconds\": \"998276410200\",\n\t\t\t\"microseconds\": \"998276410100400\",\n\t\t\t\"nanoseconds\":  \"998276410100300500\",\n\t\t},\n\t\t\"doc3\": {\n\t\t\t\"date\":         \"2001/08/20 03:00:30\",\n\t\t\t\"seconds\":      \"998276430\",\n\t\t\t\"milliseconds\": \"998276410300\",\n\t\t\t\"microseconds\": \"998276410100500\",\n\t\t\t\"nanoseconds\":  \"998276410100300600\",\n\t\t},\n\t\t\"doc4\": {\n\t\t\t\"date\":         \"2001/08/20 03:00:40\",\n\t\t\t\"seconds\":      \"998276440\",\n\t\t\t\"milliseconds\": \"998276410400\",\n\t\t\t\"microseconds\": \"998276410100600\",\n\t\t\t\"nanoseconds\":  \"998276410100300700\",\n\t\t},\n\t\t\"doc5\": {\n\t\t\t\"date\":         \"2001/08/20 03:00:50\",\n\t\t\t\"seconds\":      \"998276450\",\n\t\t\t\"milliseconds\": \"998276410500\",\n\t\t\t\"microseconds\": \"998276410100700\",\n\t\t\t\"nanoseconds\":  \"998276410100300800\",\n\t\t},\n\t}\n\n\tbatch := idx.NewBatch()\n\tfor docID, doc := range documents {\n\t\terr := batch.Index(docID, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttype testStruct struct {\n\t\tstart        string\n\t\tend          string\n\t\tfield        string\n\t\texpectedHits []string\n\t}\n\n\ttestQueries := []testStruct{\n\t\t{\n\t\t\tstart: \"2001-08-20T03:00:05\",\n\t\t\tend:   \"2001-08-20T03:00:25\",\n\t\t\tfield: \"date\",\n\t\t\texpectedHits: []string{\n\t\t\t\t\"doc1\",\n\t\t\t\t\"doc2\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tstart: \"2001-08-20T03:00:15\",\n\t\t\tend:   \"2001-08-20T03:00:35\",\n\t\t\tfield: \"seconds\",\n\t\t\texpectedHits: []string{\n\t\t\t\t\"doc2\",\n\t\t\t\t\"doc3\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tstart: \"2001-08-20T03:00:10.150\",\n\t\t\tend:   \"2001-08-20T03:00:10.450\",\n\t\t\tfield: \"milliseconds\",\n\t\t\texpectedHits: []string{\n\t\t\t\t\"doc2\",\n\t\t\t\t\"doc3\",\n\t\t\t\t\"doc4\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tstart: \"2001-08-20T03:00:10.100450\",\n\t\t\tend:   \"2001-08-20T03:00:10.100650\",\n\t\t\tfield: \"microseconds\",\n\t\t\texpectedHits: []string{\n\t\t\t\t\"doc3\",\n\t\t\t\t\"doc4\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tstart: \"2001-08-20T03:00:10.100300550\",\n\t\t\tend:   \"2001-08-20T03:00:10.100300850\",\n\t\t\tfield: \"nanoseconds\",\n\t\t\texpectedHits: []string{\n\t\t\t\t\"doc3\",\n\t\t\t\t\"doc4\",\n\t\t\t\t\"doc5\",\n\t\t\t},\n\t\t},\n\t}\n\ttestLayout := \"2006-01-02T15:04:05\"\n\tfor _, dtq := range testQueries {\n\t\tstartTime, err := time.Parse(testLayout, dtq.start)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tendTime, err := time.Parse(testLayout, dtq.end)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tdrq := NewDateRangeQuery(startTime, endTime)\n\t\tdrq.SetField(dtq.field)\n\n\t\tsr := NewSearchRequest(drq)\n\t\tsr.SortBy([]string{dtq.field})\n\t\tsr.Fields = []string{\"*\"}\n\n\t\tres, err := idx.Search(sr)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif len(res.Hits) != len(dtq.expectedHits) {\n\t\t\tt.Fatalf(\"expected %d hits, got %d\", len(dtq.expectedHits), len(res.Hits))\n\t\t}\n\t\tfor i, hit := range res.Hits {\n\t\t\tif hit.ID != dtq.expectedHits[i] {\n\t\t\t\tt.Fatalf(\"expected docID %s, got %s\", dtq.expectedHits[i], hit.ID)\n\t\t\t}\n\t\t\tif len(hit.Fields) != len(documents[hit.ID]) {\n\t\t\t\tt.Fatalf(\"expected hit %s to have %d fields, got %d\", hit.ID, len(documents[hit.ID]), len(hit.Fields))\n\t\t\t}\n\t\t\tfor k, v := range documents[hit.ID] {\n\t\t\t\tif hit.Fields[k] != v {\n\t\t\t\t\tt.Fatalf(\"expected field %s to be %s, got %s\", k, v, hit.Fields[k])\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestPercentAndIsoStyleDates(t *testing.T) {\n\tpercentName := percent.Name\n\tisoName := iso.Name\n\n\timap := mapping.NewIndexMapping()\n\tpercentConfig := map[string]interface{}{\n\t\t\"type\": percentName,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"%Y/%m/%d %l:%M%p\",                // doc 1\n\t\t\t\"%d/%m/%Y %H:%M:%S\",               // doc 2\n\t\t\t\"%Y-%m-%dT%H:%M:%S%z\",             // doc 3\n\t\t\t\"%d %B %y %l%p %Z\",                // doc 4\n\t\t\t\"%Y; %b %d (%a) %I:%M:%S.%N%P %z\", // doc 5\n\t\t},\n\t}\n\tisoConfig := map[string]interface{}{\n\t\t\"type\": isoName,\n\t\t\"layouts\": []interface{}{\n\t\t\t\"yyyy/MM/dd h:mma\",                       // doc 1\n\t\t\t\"dd/MM/yyyy HH:mm:ss\",                    // doc 2\n\t\t\t\"yyyy-MM-dd'T'HH:mm:ssXX\",                // doc 3\n\t\t\t\"dd MMMM yy ha z\",                        // doc 4\n\t\t\t\"yyyy; MMM dd (EEE) hh:mm:ss.SSSSSaa xx\", // doc 5\n\t\t},\n\t}\n\n\terr := imap.AddCustomDateTimeParser(\"percentDate\", percentConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = imap.AddCustomDateTimeParser(\"isoDate\", isoConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpercentField := mapping.NewDateTimeFieldMapping()\n\tpercentField.DateFormat = \"percentDate\"\n\n\tisoField := mapping.NewDateTimeFieldMapping()\n\tisoField.DateFormat = \"isoDate\"\n\n\timap.DefaultMapping.AddFieldMappingsAt(\"percentDate\", percentField)\n\timap.DefaultMapping.AddFieldMappingsAt(\"isoDate\", isoField)\n\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocuments := map[string]map[string]interface{}{\n\t\t\"doc1\": {\n\t\t\t\"percentDate\": \"2001/08/20 6:00PM\",\n\t\t\t\"isoDate\":     \"2001/08/20 6:00PM\",\n\t\t},\n\t\t\"doc2\": {\n\t\t\t\"percentDate\": \"20/08/2001 18:05:00\",\n\t\t\t\"isoDate\":     \"20/08/2001 18:05:00\",\n\t\t},\n\t\t\"doc3\": {\n\t\t\t\"percentDate\": \"2001-08-20T18:10:00Z\",\n\t\t\t\"isoDate\":     \"2001-08-20T18:10:00Z\",\n\t\t},\n\t\t\"doc4\": {\n\t\t\t\"percentDate\": \"20 August 01 6PM UTC\",\n\t\t\t\"isoDate\":     \"20 August 01 6PM UTC\",\n\t\t},\n\t\t\"doc5\": {\n\t\t\t\"percentDate\": \"2001; Aug 20 (Mon) 06:15:15.23456pm +0000\",\n\t\t\t\"isoDate\":     \"2001; Aug 20 (Mon) 06:15:15.23456pm +0000\",\n\t\t},\n\t}\n\n\tbatch := idx.NewBatch()\n\tfor docID, doc := range documents {\n\t\terr := batch.Index(docID, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttype testStruct struct {\n\t\tstart string\n\t\tend   string\n\t\tfield string\n\t}\n\n\tfor _, field := range []string{\"percentDate\", \"isoDate\"} {\n\t\ttestQueries := []testStruct{\n\t\t\t{\n\t\t\t\tstart: \"2001/08/20 6:00PM\",\n\t\t\t\tend:   \"2001/08/20 6:20PM\",\n\t\t\t\tfield: field,\n\t\t\t},\n\t\t\t{\n\t\t\t\tstart: \"20/08/2001 18:00:00\",\n\t\t\t\tend:   \"20/08/2001 18:20:00\",\n\t\t\t\tfield: field,\n\t\t\t},\n\t\t\t{\n\t\t\t\tstart: \"2001-08-20T18:00:00Z\",\n\t\t\t\tend:   \"2001-08-20T18:20:00Z\",\n\t\t\t\tfield: field,\n\t\t\t},\n\t\t\t{\n\t\t\t\tstart: \"20 August 01 6PM UTC\",\n\t\t\t\tend:   \"20 August 01 7PM UTC\",\n\t\t\t\tfield: field,\n\t\t\t},\n\t\t\t{\n\t\t\t\tstart: \"2001; Aug 20 (Mon) 06:00:00.00000pm +0000\",\n\t\t\t\tend:   \"2001; Aug 20 (Mon) 06:20:20.00000pm +0000\",\n\t\t\t\tfield: field,\n\t\t\t},\n\t\t}\n\t\tincludeStart := true\n\t\tincludeEnd := true\n\t\tfor _, dtq := range testQueries {\n\t\t\tdrq := NewDateRangeInclusiveStringQuery(dtq.start, dtq.end, &includeStart, &includeEnd)\n\t\t\tdrq.SetField(dtq.field)\n\t\t\tdrq.SetDateTimeParser(field)\n\t\t\tsr := NewSearchRequest(drq)\n\t\t\tres, err := idx.Search(sr)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif len(res.Hits) != 5 {\n\t\t\t\tt.Fatalf(\"expected %d hits, got %d\", 5, len(res.Hits))\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc roundToDecimalPlace(num float64, decimalPlaces int) float64 {\n\tprecision := math.Pow(10, float64(decimalPlaces))\n\treturn math.Round(num*precision) / precision\n}\n\nfunc TestScoreBreakdown(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\timap := mapping.NewIndexMapping()\n\ttextField := mapping.NewTextFieldMapping()\n\ttextField.Analyzer = simple.Name\n\timap.DefaultMapping.AddFieldMappingsAt(\"text\", textField)\n\n\tdocuments := map[string]map[string]interface{}{\n\t\t\"doc1\": {\n\t\t\t\"text\": \"lorem ipsum dolor sit amet consectetur adipiscing elit do eiusmod tempor\",\n\t\t},\n\t\t\"doc2\": {\n\t\t\t\"text\": \"lorem dolor amet adipiscing sed eiusmod\",\n\t\t},\n\t\t\"doc3\": {\n\t\t\t\"text\": \"ipsum sit consectetur elit do tempor\",\n\t\t},\n\t\t\"doc4\": {\n\t\t\t\"text\": \"lorem ipsum sit amet adipiscing elit do eiusmod\",\n\t\t},\n\t}\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch := idx.NewBatch()\n\tfor docID, doc := range documents {\n\t\terr := batch.Index(docID, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttype testResult struct {\n\t\tdocID          string // doc ID of the hit\n\t\tscore          float64\n\t\tscoreBreakdown map[int]float64\n\t}\n\ttype testStruct struct {\n\t\tquery      string\n\t\ttyp        string\n\t\texpectHits []testResult\n\t}\n\ttestQueries := []testStruct{\n\t\t{\n\t\t\t// trigger disjunction heap searcher (>10 searchers)\n\t\t\t// expect score breakdown to have a 0 at BLANK\n\t\t\tquery: `{\"disjuncts\":[{\"term\":\"lorem\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"ipsum\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"dolor\",\"field\":\"text\"},{\"term\":\"sit\",\"field\":\"text\"},{\"term\":\"amet\",\"field\":\"text\"},{\"term\":\"consectetur\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"adipiscing\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"elit\",\"field\":\"text\"},{\"term\":\"sed\",\"field\":\"text\"},{\"term\":\"do\",\"field\":\"text\"},{\"term\":\"eiusmod\",\"field\":\"text\"},{\"term\":\"tempor\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"}]}`,\n\t\t\ttyp:   \"disjunction\",\n\t\t\texpectHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:          \"doc1\",\n\t\t\t\t\tscore:          0.3034548543819603,\n\t\t\t\t\tscoreBreakdown: map[int]float64{0: 0.040398807605268316, 2: 0.040398807605268316, 5: 0.0669862776967768, 6: 0.040398807605268316, 7: 0.040398807605268316, 8: 0.0669862776967768, 10: 0.040398807605268316, 12: 0.040398807605268316, 14: 0.040398807605268316, 15: 0.040398807605268316, 16: 0.0669862776967768},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:          \"doc2\",\n\t\t\t\t\tscore:          0.14725661652397853,\n\t\t\t\t\tscoreBreakdown: map[int]float64{0: 0.05470024557900147, 5: 0.09069985124905133, 7: 0.05470024557900147, 10: 0.05470024557900147, 13: 0.15681178542754148, 15: 0.05470024557900147},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:          \"doc3\",\n\t\t\t\t\tscore:          0.12637916362550797,\n\t\t\t\t\tscoreBreakdown: map[int]float64{2: 0.05470024557900147, 6: 0.05470024557900147, 8: 0.09069985124905133, 12: 0.05470024557900147, 14: 0.05470024557900147, 16: 0.09069985124905133},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:          \"doc4\",\n\t\t\t\t\tscore:          0.15956816751152955,\n\t\t\t\t\tscoreBreakdown: map[int]float64{0: 0.04737179972998534, 2: 0.04737179972998534, 6: 0.04737179972998534, 7: 0.04737179972998534, 10: 0.04737179972998534, 12: 0.04737179972998534, 14: 0.04737179972998534, 15: 0.04737179972998534},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t// trigger disjunction slice searcher (< 10 searchers)\n\t\t\t// expect BLANK to give a 0 in score breakdown\n\t\t\tquery: `{\"disjuncts\":[{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"lorem\",\"field\":\"text\"},{\"term\":\"ipsum\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"},{\"term\":\"dolor\",\"field\":\"text\"},{\"term\":\"sit\",\"field\":\"text\"},{\"term\":\"blank\",\"field\":\"text\"}]}`,\n\t\t\ttyp:   \"disjunction\",\n\t\t\texpectHits: []testResult{\n\t\t\t\t{\n\t\t\t\t\tdocID:          \"doc1\",\n\t\t\t\t\tscore:          0.1340684440934241,\n\t\t\t\t\tscoreBreakdown: map[int]float64{1: 0.05756326446708409, 2: 0.05756326446708409, 5: 0.09544709478559595, 6: 0.05756326446708409},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:          \"doc2\",\n\t\t\t\t\tscore:          0.05179425287147191,\n\t\t\t\t\tscoreBreakdown: map[int]float64{1: 0.0779410306721006, 5: 0.129235980813787},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:          \"doc3\",\n\t\t\t\t\tscore:          0.0389705153360503,\n\t\t\t\t\tscoreBreakdown: map[int]float64{2: 0.0779410306721006, 6: 0.0779410306721006},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tdocID:          \"doc4\",\n\t\t\t\t\tscore:          0.07593627256602972,\n\t\t\t\t\tscoreBreakdown: map[int]float64{1: 0.06749890894758198, 2: 0.06749890894758198, 6: 0.06749890894758198},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, dtq := range testQueries {\n\t\tvar q query.Query\n\t\tvar rv query.DisjunctionQuery\n\t\terr := json.Unmarshal([]byte(dtq.query), &rv)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\trv.RetrieveScoreBreakdown(true)\n\t\tq = &rv\n\t\tsr := NewSearchRequest(q)\n\t\tsr.SortBy([]string{\"_id\"})\n\t\tsr.Explain = true\n\t\tres, err := idx.Search(sr)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif len(res.Hits) != len(dtq.expectHits) {\n\t\t\tt.Fatalf(\"expected %d hits, got %d\", len(dtq.expectHits), len(res.Hits))\n\t\t}\n\t\tfor i, hit := range res.Hits {\n\t\t\tif hit.ID != dtq.expectHits[i].docID {\n\t\t\t\tt.Fatalf(\"expected docID %s, got %s\", dtq.expectHits[i].docID, hit.ID)\n\t\t\t}\n\t\t\tif len(hit.ScoreBreakdown) != len(dtq.expectHits[i].scoreBreakdown) {\n\t\t\t\tt.Fatalf(\"expected %d score breakdown, got %d\", len(dtq.expectHits[i].scoreBreakdown), len(hit.ScoreBreakdown))\n\t\t\t}\n\t\t\tfor j, score := range hit.ScoreBreakdown {\n\t\t\t\tactualScore := roundToDecimalPlace(score, 3)\n\t\t\t\texpectScore := roundToDecimalPlace(dtq.expectHits[i].scoreBreakdown[j], 3)\n\t\t\t\tif actualScore != expectScore {\n\t\t\t\t\tt.Fatalf(\"expected score breakdown %f, got %f\", dtq.expectHits[i].scoreBreakdown[j], score)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestAutoFuzzy(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\timap := mapping.NewIndexMapping()\n\n\tif err := imap.AddCustomAnalyzer(\"splitter\", map[string]interface{}{\n\t\t\"type\":          custom.Name,\n\t\t\"tokenizer\":     whitespace.Name,\n\t\t\"token_filters\": []interface{}{lowercase.Name},\n\t}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttextField := mapping.NewTextFieldMapping()\n\ttextField.Analyzer = \"splitter\"\n\ttextField.Store = true\n\ttextField.IncludeTermVectors = true\n\ttextField.IncludeInAll = true\n\n\timap.DefaultMapping.Dynamic = false\n\timap.DefaultMapping.AddFieldMappingsAt(\"model\", textField)\n\n\tdocuments := map[string]map[string]interface{}{\n\t\t\"product1\": {\n\t\t\t\"model\": \"apple iphone 12\",\n\t\t},\n\t\t\"product2\": {\n\t\t\t\"model\": \"apple iphone 13\",\n\t\t},\n\t\t\"product3\": {\n\t\t\t\"model\": \"samsung galaxy s22\",\n\t\t},\n\t\t\"product4\": {\n\t\t\t\"model\": \"samsung galaxy note\",\n\t\t},\n\t\t\"product5\": {\n\t\t\t\"model\": \"google pixel 5\",\n\t\t},\n\t\t\"product6\": {\n\t\t\t\"model\": \"oneplus 9 pro\",\n\t\t},\n\t\t\"product7\": {\n\t\t\t\"model\": \"xiaomi mi 11\",\n\t\t},\n\t\t\"product8\": {\n\t\t\t\"model\": \"oppo find x3\",\n\t\t},\n\t\t\"product9\": {\n\t\t\t\"model\": \"vivo x60 pro\",\n\t\t},\n\t\t\"product10\": {\n\t\t\t\"model\": \"oneplus 8t pro\",\n\t\t},\n\t\t\"product11\": {\n\t\t\t\"model\": \"nokia xr20\",\n\t\t},\n\t\t\"product12\": {\n\t\t\t\"model\": \"poco f1\",\n\t\t},\n\t\t\"product13\": {\n\t\t\t\"model\": \"asus rog 5\",\n\t\t},\n\t\t\"product14\": {\n\t\t\t\"model\": \"samsung galaxy a15 5g\",\n\t\t},\n\t\t\"product15\": {\n\t\t\t\"model\": \"tecno camon 17\",\n\t\t},\n\t}\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tbatch := idx.NewBatch()\n\tfor docID, doc := range documents {\n\t\terr := batch.Index(docID, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttype testStruct struct {\n\t\tquery      string\n\t\texpectHits []string\n\t}\n\ttestQueries := []testStruct{\n\t\t{\n\t\t\t// match query with fuzziness set to 2\n\t\t\tquery: `{\n\t\t\t\t\t\"match\" : \"applle iphone 12\",\n\t\t\t\t\t\"fuzziness\": 2,\n\t\t\t\t\t\"field\" : \"model\"\n\t\t\t\t}`,\n\t\t\texpectHits: []string{\"product1\", \"product2\", \"product7\", \"product14\", \"product15\", \"product12\", \"product10\", \"product3\", \"product6\", \"product8\"},\n\t\t},\n\t\t{\n\t\t\t// match query with fuzziness set to \"auto\"\n\t\t\tquery: `{\n\t\t\t\t\t\"match\" : \"applle iphone 12\",\n\t\t\t\t\t\"fuzziness\": \"auto\",\n\t\t\t\t\t\"field\" : \"model\"\n\t\t\t\t}`,\n\t\t\texpectHits: []string{\"product1\", \"product2\"},\n\t\t},\n\t\t{\n\t\t\t// match query with fuzziness set to 2 with `and` operator\n\t\t\tquery: `{\n\t\t\t\t\t\"match\" : \"applle iphone 12\",\n\t\t\t\t\t\"fuzziness\": 2,\n\t\t\t\t\t\"field\" : \"model\",\n\t\t\t\t\t\"operator\": \"and\"\n\t\t\t\t}`,\n\t\t\texpectHits: []string{\"product1\", \"product2\"},\n\t\t},\n\t\t{\n\t\t\t// match query with fuzziness set to \"auto\" with `and`` operator\n\t\t\tquery: `{\n\t\t\t\t\t\"match\" : \"applle iphone 12\",\n\t\t\t\t\t\"fuzziness\": \"auto\",\n\t\t\t\t\t\"field\" : \"model\",\n\t\t\t\t\t\"operator\": \"and\"\n\t\t\t\t}`,\n\t\t\texpectHits: []string{\"product1\"},\n\t\t},\n\t\t// match phrase query with fuzziness set to 2\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\t\"match_phrase\" : \"onplus 9 pro\",\n\t\t\t\t\t\"fuzziness\": 2,\n\t\t\t\t\t\"field\" : \"model\"\n\t\t\t\t}`,\n\t\t\texpectHits: []string{\"product6\", \"product10\"},\n\t\t},\n\t\t// match phrase query with fuzziness set to \"auto\"\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\"match_phrase\" : \"onplus 9 pro\",\n\t\t\t\t\"fuzziness\": \"auto\",\n\t\t\t\t\"field\" : \"model\"\n\t\t\t}`,\n\t\t\texpectHits: []string{\"product6\"},\n\t\t},\n\t}\n\n\tfor _, dtq := range testQueries {\n\t\tq, err := query.ParseQuery([]byte(dtq.query))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tsr := NewSearchRequest(q)\n\t\tsr.Highlight = NewHighlightWithStyle(ansi.Name)\n\t\tsr.SortBy([]string{\"-_score\", \"_id\"})\n\t\tsr.Fields = []string{\"*\"}\n\t\tsr.Explain = true\n\n\t\tres, err := idx.Search(sr)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif len(res.Hits) != len(dtq.expectHits) {\n\t\t\tt.Fatalf(\"expected %d hits, got %d\", len(dtq.expectHits), len(res.Hits))\n\t\t}\n\t\tfor i, hit := range res.Hits {\n\t\t\tif hit.ID != dtq.expectHits[i] {\n\t\t\t\tt.Fatalf(\"expected docID %s, got %s\", dtq.expectHits[i], hit.ID)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestThesaurusTermReader(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tsynonymCollection := \"collection1\"\n\n\tsynonymSourceName := \"english\"\n\n\tanalyzer := simple.Name\n\n\tsynonymSourceConfig := map[string]interface{}{\n\t\t\"collection\": synonymCollection,\n\t\t\"analyzer\":   analyzer,\n\t}\n\n\ttextField := mapping.NewTextFieldMapping()\n\ttextField.Analyzer = analyzer\n\ttextField.SynonymSource = synonymSourceName\n\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping.AddFieldMappingsAt(\"text\", textField)\n\terr := imap.AddSynonymSource(synonymSourceName, synonymSourceConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = imap.Validate()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocuments := map[string]map[string]interface{}{\n\t\t\"doc1\": {\n\t\t\t\"text\": \"quick brown fox eats\",\n\t\t},\n\t\t\"doc2\": {\n\t\t\t\"text\": \"fast red wolf jumps\",\n\t\t},\n\t\t\"doc3\": {\n\t\t\t\"text\": \"quick red cat runs\",\n\t\t},\n\t\t\"doc4\": {\n\t\t\t\"text\": \"speedy brown dog barks\",\n\t\t},\n\t\t\"doc5\": {\n\t\t\t\"text\": \"fast green rabbit hops\",\n\t\t},\n\t}\n\n\tbatch := idx.NewBatch()\n\tfor docID, doc := range documents {\n\t\terr := batch.Index(docID, doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tsynonymDocuments := map[string]*SynonymDefinition{\n\t\t\"synDoc1\": {\n\t\t\tSynonyms: []string{\"quick\", \"fast\", \"speedy\"},\n\t\t},\n\t\t\"synDoc2\": {\n\t\t\tInput:    []string{\"color\", \"colour\"},\n\t\t\tSynonyms: []string{\"red\", \"green\", \"blue\", \"yellow\", \"brown\"},\n\t\t},\n\t\t\"synDoc3\": {\n\t\t\tInput:    []string{\"animal\", \"creature\"},\n\t\t\tSynonyms: []string{\"fox\", \"wolf\", \"cat\", \"dog\", \"rabbit\"},\n\t\t},\n\t\t\"synDoc4\": {\n\t\t\tSynonyms: []string{\"eats\", \"jumps\", \"runs\", \"barks\", \"hops\"},\n\t\t},\n\t}\n\n\tfor synName, synDef := range synonymDocuments {\n\t\terr := batch.IndexSynonym(synName, synonymCollection, synDef)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\terr = idx.Batch(batch)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tsco, err := idx.Advanced()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treader, err := sco.Reader()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = reader.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tthesReader, ok := reader.(index.ThesaurusReader)\n\tif !ok {\n\t\tt.Fatal(\"expected thesaurus reader\")\n\t}\n\n\ttype testStruct struct {\n\t\tqueryTerm        string\n\t\texpectedSynonyms []string\n\t}\n\n\ttestQueries := []testStruct{\n\t\t{\n\t\t\tqueryTerm:        \"quick\",\n\t\t\texpectedSynonyms: []string{\"fast\", \"speedy\"},\n\t\t},\n\t\t{\n\t\t\tqueryTerm:        \"red\",\n\t\t\texpectedSynonyms: []string{},\n\t\t},\n\t\t{\n\t\t\tqueryTerm:        \"color\",\n\t\t\texpectedSynonyms: []string{\"red\", \"green\", \"blue\", \"yellow\", \"brown\"},\n\t\t},\n\t\t{\n\t\t\tqueryTerm:        \"colour\",\n\t\t\texpectedSynonyms: []string{\"red\", \"green\", \"blue\", \"yellow\", \"brown\"},\n\t\t},\n\t\t{\n\t\t\tqueryTerm:        \"animal\",\n\t\t\texpectedSynonyms: []string{\"fox\", \"wolf\", \"cat\", \"dog\", \"rabbit\"},\n\t\t},\n\t\t{\n\t\t\tqueryTerm:        \"creature\",\n\t\t\texpectedSynonyms: []string{\"fox\", \"wolf\", \"cat\", \"dog\", \"rabbit\"},\n\t\t},\n\t\t{\n\t\t\tqueryTerm:        \"fox\",\n\t\t\texpectedSynonyms: []string{},\n\t\t},\n\t\t{\n\t\t\tqueryTerm:        \"eats\",\n\t\t\texpectedSynonyms: []string{\"jumps\", \"runs\", \"barks\", \"hops\"},\n\t\t},\n\t\t{\n\t\t\tqueryTerm:        \"jumps\",\n\t\t\texpectedSynonyms: []string{\"eats\", \"runs\", \"barks\", \"hops\"},\n\t\t},\n\t}\n\n\tfor _, test := range testQueries {\n\t\tstr, err := thesReader.ThesaurusTermReader(context.Background(), synonymSourceName, []byte(test.queryTerm))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tvar gotSynonyms []string\n\t\tfor {\n\t\t\tsynonym, err := str.Next()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif synonym == \"\" {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tgotSynonyms = append(gotSynonyms, string(synonym))\n\t\t}\n\t\tif len(gotSynonyms) != len(test.expectedSynonyms) {\n\t\t\tt.Fatalf(\"expected %d synonyms, got %d\", len(test.expectedSynonyms), len(gotSynonyms))\n\t\t}\n\t\tsort.Strings(gotSynonyms)\n\t\tsort.Strings(test.expectedSynonyms)\n\t\tfor i, syn := range gotSynonyms {\n\t\t\tif syn != test.expectedSynonyms[i] {\n\t\t\t\tt.Fatalf(\"expected synonym %s, got %s\", test.expectedSynonyms[i], syn)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestSynonymSearchQueries(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tsynonymCollection := \"collection1\"\n\n\tsynonymSourceName := \"english\"\n\n\tanalyzer := en.AnalyzerName\n\n\tsynonymSourceConfig := map[string]interface{}{\n\t\t\"collection\": synonymCollection,\n\t\t\"analyzer\":   analyzer,\n\t}\n\n\ttextField := mapping.NewTextFieldMapping()\n\ttextField.Analyzer = analyzer\n\ttextField.SynonymSource = synonymSourceName\n\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping.AddFieldMappingsAt(\"text\", textField)\n\terr := imap.AddSynonymSource(synonymSourceName, synonymSourceConfig)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = imap.Validate()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocuments := map[string]map[string]interface{}{\n\t\t\"doc1\": {\n\t\t\t\"text\": `The hardworking employee consistently strives to exceed expectations.\n\t\t\t\t\tHis industrious nature makes him a valuable asset to any team.\n\t\t\t\t\tHis conscientious attention to detail ensures that projects are completed efficiently and accurately.\n\t\t\t\t\tHe remains persistent even in the face of challenges.`,\n\t\t},\n\t\t\"doc2\": {\n\t\t\t\"text\": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life.\n\t\t\t\t\tGuests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation.\n\t\t\t\t\tThe calm environment offers the ideal place to meditate and connect with nature.\n\t\t\t\t\tEven the most stressed individuals find themselves feeling relaxed and at ease.`,\n\t\t},\n\t\t\"doc3\": {\n\t\t\t\"text\": `The house was burned down, leaving only a charred shell behind.\n\t\t\t\t\tThe intense heat of the flames caused the walls to warp and the roof to cave in.\n\t\t\t\t\tThe seared remains of the furniture told the story of the blaze.\n\t\t\t\t\tThe incinerated remains left little more than ashes to remember what once was.`,\n\t\t},\n\t\t\"doc4\": {\n\t\t\t\"text\": `The faithful dog followed its owner everywhere, always loyal and steadfast.\n\t\t\t\t\tIt was devoted to protecting its family, and its reliable nature meant it could always be trusted.\n\t\t\t\t\tIn the face of danger, the dog remained calm, knowing its role was to stay vigilant.\n\t\t\t\t\tIts trustworthy companionship provided comfort and security.`,\n\t\t},\n\t\t\"doc5\": {\n\t\t\t\"text\": `The lively market is bustling with activity from morning to night.\n\t\t\t\t\tThe dynamic energy of the crowd fills the air as vendors sell their wares.\n\t\t\t\t\tShoppers wander from stall to stall, captivated by the vibrant colors and energetic atmosphere.\n\t\t\t\t\tThis place is alive with movement and life.`,\n\t\t},\n\t\t\"doc6\": {\n\t\t\t\"text\": `In moments of crisis, bravery shines through.\n\t\t\t\t\tIt takes valor to step forward when others are afraid to act.\n\t\t\t\t\tHeroes are defined by their guts and nerve, taking risks to protect others.\n\t\t\t\t\tBoldness in the face of danger is what sets them apart.`,\n\t\t},\n\t\t\"doc7\": {\n\t\t\t\"text\": `Innovation is the driving force behind progress in every industry.\n\t\t\t\t\tThe company fosters an environment of invention, encouraging creativity at every level.\n\t\t\t\t\tThe focus on novelty and improvement means that ideas are always evolving.\n\t\t\t\t\tThe development of new solutions is at the core of the company's mission.`,\n\t\t},\n\t\t\"doc8\": {\n\t\t\t\"text\": `The blazing sunset cast a radiant glow over the horizon, painting the sky with hues of red and orange.\n\t\t\t\t\tThe intense heat of the day gave way to a fiery display of color.\n\t\t\t\t\tAs the sun set, the glowing light illuminated the landscape, creating a breathtaking scene.\n\t\t\t\t\tThe fiery sky was a sight to behold.`,\n\t\t},\n\t\t\"doc9\": {\n\t\t\t\"text\": `The fertile soil of the valley makes it perfect for farming.\n\t\t\t\t\tThe productive land yields abundant crops year after year.\n\t\t\t\t\tFarmers rely on the rich, fruitful ground to sustain their livelihoods.\n\t\t\t\t\tThe area is known for its plentiful harvests, supporting both local communities and export markets.`,\n\t\t},\n\t\t\"doc10\": {\n\t\t\t\"text\": `The arid desert is a vast, dry expanse with little water or vegetation.\n\t\t\t\t\tThe barren landscape stretches as far as the eye can see, offering little respite from the scorching sun.\n\t\t\t\t\tThe desolate environment is unforgiving to those who venture too far without preparation.\n\t\t\t\t\tThe parched earth cracks under the heat, creating a harsh, unyielding terrain.`,\n\t\t},\n\t\t\"doc11\": {\n\t\t\t\"text\": `The fox is known for its cunning and intelligence.\n\t\t\t\t\tAs a predator, it relies on its sharp instincts to outwit its prey.\n\t\t\t\t\tIts vulpine nature makes it both mysterious and fascinating.\n\t\t\t\t\tThe fox's ability to hunt with precision and stealth is what makes it such a formidable hunter.`,\n\t\t},\n\t\t\"doc12\": {\n\t\t\t\"text\": `The dog is often considered man's best friend due to its loyal nature.\n\t\t\t\t\tAs a companion, the hound provides both protection and affection.\n\t\t\t\t\tThe puppy quickly becomes a member of the family, always by your side.\n\t\t\t\t\tIts playful energy and unshakable loyalty make it a beloved pet.`,\n\t\t},\n\t\t\"doc13\": {\n\t\t\t\"text\": `He worked tirelessly through the night, always persistent in his efforts.\n\t\t\t\t\tHis industrious approach to problem-solving kept the project moving forward.\n\t\t\t\t\tNo matter how difficult the task, he remained focused, always giving his best.\n\t\t\t\t\tHis dedication paid off when the project was completed ahead of schedule.`,\n\t\t},\n\t\t\"doc14\": {\n\t\t\t\"text\": `The river flowed calmly through the valley, its peaceful current offering a sense of tranquility.\n\t\t\t\t\tFishermen relaxed by the banks, enjoying the calm waters that reflected the sky above.\n\t\t\t\t\tThe tranquil nature of the river made it a perfect spot for meditation.\n\t\t\t\t\tAs the day ended, the river's quiet flow brought a sense of peace.`,\n\t\t},\n\t\t\"doc15\": {\n\t\t\t\"text\": `After the fire, all that was left was the charred remains of what once was.\n\t\t\t\t\tThe seared walls of the house told a tragic story.\n\t\t\t\t\tThe intensity of the blaze had burned everything in its path, leaving only the smoldering wreckage behind.\n\t\t\t\t\tThe incinerated objects could not be salvaged, and the damage was beyond repair.`,\n\t\t},\n\t\t\"doc16\": {\n\t\t\t\"text\": `The devoted employee always went above and beyond to complete his tasks.\n\t\t\t\t\tHis steadfast commitment to the company made him a valuable team member.\n\t\t\t\t\tHe was reliable, never failing to meet deadlines.\n\t\t\t\t\tHis trustworthiness earned him the respect of his colleagues, and was considered an\n\t\t\t\t\tingenious expert in his field.`,\n\t\t},\n\t\t\"doc17\": {\n\t\t\t\"text\": `The city is vibrant, full of life and energy.\n\t\t\t\t\tThe dynamic pace of the streets reflects the diverse culture of its inhabitants.\n\t\t\t\t\tPeople from all walks of life contribute to the energetic atmosphere.\n\t\t\t\t\tThe city's lively spirit can be felt in every corner, from the bustling markets to the lively festivals.`,\n\t\t},\n\t\t\"doc18\": {\n\t\t\t\"text\": `In a moment of uncertainty, he made a bold decision that would change his life forever.\n\t\t\t\t\tIt took courage and nerve to take the leap, but his bravery paid off.\n\t\t\t\t\tThe guts to face the unknown allowed him to achieve something remarkable.\n\t\t\t\t\tBeing an bright scholar, the skill he demonstrated inspired those around him.`,\n\t\t},\n\t\t\"doc19\": {\n\t\t\t\"text\": `Innovation is often born from necessity, and the lightbulb is a prime example.\n\t\t\t\t\tThomas Edison's invention changed the world, offering a new way to see the night.\n\t\t\t\t\tThe creativity involved in developing such a groundbreaking product sparked a wave of\n\t\t\t\t\tnovelty in the scientific community. This improvement in technology continues to shape the modern world.\n\t\t\t\t\tHe was a clever academic and a smart researcher.`,\n\t\t},\n\t\t\"doc20\": {\n\t\t\t\"text\": `The fiery volcano erupted with a force that shook the earth. Its radiant lava flowed down the sides,\n\t\t\t\t\tilluminating the night sky. The intense heat from the eruption could be felt miles away, as the\n\t\t\t\t\tglowing lava burned everything in its path. The fiery display was both terrifying and mesmerizing.`,\n\t\t},\n\t}\n\n\tsynonymDocuments := map[string]*SynonymDefinition{\n\t\t\"synDoc1\": {\n\t\t\tSynonyms: []string{\"hardworking\", \"industrious\", \"conscientious\", \"persistent\", \"focused\", \"devoted\"},\n\t\t},\n\t\t\"synDoc2\": {\n\t\t\tSynonyms: []string{\"tranquil\", \"peaceful\", \"calm\", \"relaxed\", \"unruffled\"},\n\t\t},\n\t\t\"synDoc3\": {\n\t\t\tSynonyms: []string{\"burned\", \"charred\", \"seared\", \"incinerated\", \"singed\"},\n\t\t},\n\t\t\"synDoc4\": {\n\t\t\tSynonyms: []string{\"faithful\", \"steadfast\", \"devoted\", \"reliable\", \"trustworthy\"},\n\t\t},\n\t\t\"synDoc5\": {\n\t\t\tSynonyms: []string{\"lively\", \"dynamic\", \"energetic\", \"vivid\", \"vibrating\"},\n\t\t},\n\t\t\"synDoc6\": {\n\t\t\tSynonyms: []string{\"bravery\", \"valor\", \"guts\", \"nerve\", \"boldness\"},\n\t\t},\n\t\t\"synDoc7\": {\n\t\t\tInput:    []string{\"innovation\"},\n\t\t\tSynonyms: []string{\"invention\", \"creativity\", \"novelty\", \"improvement\", \"development\"},\n\t\t},\n\t\t\"synDoc8\": {\n\t\t\tInput:    []string{\"blazing\"},\n\t\t\tSynonyms: []string{\"intense\", \"radiant\", \"burning\", \"fiery\", \"glowing\"},\n\t\t},\n\t\t\"synDoc9\": {\n\t\t\tInput:    []string{\"fertile\"},\n\t\t\tSynonyms: []string{\"productive\", \"fruitful\", \"rich\", \"abundant\", \"plentiful\"},\n\t\t},\n\t\t\"synDoc10\": {\n\t\t\tInput:    []string{\"arid\"},\n\t\t\tSynonyms: []string{\"dry\", \"barren\", \"desolate\", \"parched\", \"unfertile\"},\n\t\t},\n\t\t\"synDoc11\": {\n\t\t\tInput:    []string{\"fox\"},\n\t\t\tSynonyms: []string{\"vulpine\", \"canine\", \"predator\", \"hunter\", \"pursuer\"},\n\t\t},\n\t\t\"synDoc12\": {\n\t\t\tInput:    []string{\"dog\"},\n\t\t\tSynonyms: []string{\"canine\", \"hound\", \"puppy\", \"pup\", \"companion\"},\n\t\t},\n\t\t\"synDoc13\": {\n\t\t\tSynonyms: []string{\"researcher\", \"scientist\", \"scholar\", \"academic\", \"expert\"},\n\t\t},\n\t\t\"synDoc14\": {\n\t\t\tSynonyms: []string{\"bright\", \"clever\", \"ingenious\", \"sharp\", \"astute\", \"smart\"},\n\t\t},\n\t}\n\n\t// Combine both maps into a slice of map entries (as they both have similar structure)\n\tvar combinedDocIDs []string\n\tfor id := range synonymDocuments {\n\t\tcombinedDocIDs = append(combinedDocIDs, id)\n\t}\n\tfor id := range documents {\n\t\tcombinedDocIDs = append(combinedDocIDs, id)\n\t}\n\trand.Shuffle(len(combinedDocIDs), func(i, j int) {\n\t\tcombinedDocIDs[i], combinedDocIDs[j] = combinedDocIDs[j], combinedDocIDs[i]\n\t})\n\n\t// Function to create batches of 5\n\tcreateDocBatches := func(docs []string, batchSize int) [][]string {\n\t\tvar batches [][]string\n\t\tfor i := 0; i < len(docs); i += batchSize {\n\t\t\tend := i + batchSize\n\t\t\tif end > len(docs) {\n\t\t\t\tend = len(docs)\n\t\t\t}\n\t\t\tbatches = append(batches, docs[i:end])\n\t\t}\n\t\treturn batches\n\t}\n\t// Create batches of 5 documents\n\tbatchSize := 5\n\tdocBatches := createDocBatches(combinedDocIDs, batchSize)\n\tif len(docBatches) == 0 {\n\t\tt.Fatal(\"expected batches\")\n\t}\n\ttotalDocs := 0\n\tfor _, batch := range docBatches {\n\t\ttotalDocs += len(batch)\n\t}\n\tif totalDocs != len(combinedDocIDs) {\n\t\tt.Fatalf(\"expected %d documents, got %d\", len(combinedDocIDs), totalDocs)\n\t}\n\n\tvar batches []*Batch\n\tfor _, docBatch := range docBatches {\n\t\tbatch := idx.NewBatch()\n\t\tfor _, docID := range docBatch {\n\t\t\tif synDef, ok := synonymDocuments[docID]; ok {\n\t\t\t\terr := batch.IndexSynonym(docID, synonymCollection, synDef)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\terr := batch.Index(docID, documents[docID])\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tbatches = append(batches, batch)\n\t}\n\tfor _, batch := range batches {\n\t\terr = idx.Batch(batch)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\ttype testStruct struct {\n\t\tquery      string\n\t\texpectHits []string\n\t}\n\n\ttestQueries := []testStruct{\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\"match\": \"hardworking employee\",\n\t\t\t\t\"field\": \"text\"\n\t\t\t}`,\n\t\t\texpectHits: []string{\"doc1\", \"doc13\", \"doc16\", \"doc4\", \"doc7\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\"match\": \"Hardwork and industrius efforts bring lovely and tranqual moments, with a glazing blow of valour.\",\n\t\t\t\t\"field\": \"text\",\n\t\t\t\t\"fuzziness\": \"auto\"\n\t\t\t}`,\n\t\t\texpectHits: []string{\n\t\t\t\t\"doc1\", \"doc13\", \"doc14\", \"doc15\", \"doc16\",\n\t\t\t\t\"doc17\", \"doc18\", \"doc2\", \"doc20\", \"doc3\",\n\t\t\t\t\"doc4\", \"doc5\", \"doc6\", \"doc7\", \"doc8\", \"doc9\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\"prefix\": \"in\",\n\t\t\t\t\"field\": \"text\"\n\t\t\t}`,\n\t\t\texpectHits: []string{\n\t\t\t\t\"doc1\", \"doc11\", \"doc13\", \"doc15\", \"doc16\",\n\t\t\t\t\"doc17\", \"doc18\", \"doc19\", \"doc2\", \"doc20\",\n\t\t\t\t\"doc3\", \"doc4\", \"doc7\", \"doc8\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\"prefix\": \"vivid\",\n\t\t\t\t\"field\": \"text\"\n\t\t\t}`,\n\t\t\texpectHits: []string{\n\t\t\t\t\"doc17\", \"doc5\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\"match_phrase\": \"smart academic\",\n\t\t\t\t\"field\": \"text\"\n\t\t\t}`,\n\t\t\texpectHits: []string{\"doc16\", \"doc18\", \"doc19\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\"match_phrase\": \"smrat acedemic\",\n\t\t\t\t\"field\": \"text\",\n\t\t\t\t\"fuzziness\": \"auto\"\n\t\t\t}`,\n\t\t\texpectHits: []string{\"doc16\", \"doc18\", \"doc19\"},\n\t\t},\n\t\t{\n\t\t\tquery: `{\n\t\t\t\t\"wildcard\": \"br*\",\n\t\t\t\t\"field\": \"text\"\n\t\t\t}`,\n\t\t\texpectHits: []string{\"doc11\", \"doc14\", \"doc16\", \"doc18\", \"doc19\", \"doc6\", \"doc8\"},\n\t\t},\n\t}\n\n\tgetTotalSynonymSearchStat := func(idx Index) int {\n\t\tir, err := idx.Advanced()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tstat := ir.StatsMap()[\"synonym_searches\"].(uint64)\n\t\treturn int(stat)\n\t}\n\n\trunTestQueries := func(idx Index) error {\n\t\tfor _, dtq := range testQueries {\n\t\t\tq, err := query.ParseQuery([]byte(dtq.query))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tsr := NewSearchRequest(q)\n\t\t\tsr.Highlight = NewHighlightWithStyle(ansi.Name)\n\t\t\tsr.SortBy([]string{\"_id\"})\n\t\t\tsr.Fields = []string{\"*\"}\n\t\t\tsr.Size = 30\n\t\t\tsr.Explain = true\n\t\t\tres, err := idx.Search(sr)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif len(res.Hits) != len(dtq.expectHits) {\n\t\t\t\treturn fmt.Errorf(\"expected %d hits, got %d\", len(dtq.expectHits), len(res.Hits))\n\t\t\t}\n\t\t\t// sort the expected hits to match the order of the search results\n\t\t\tsort.Strings(dtq.expectHits)\n\t\t\tfor i, hit := range res.Hits {\n\t\t\t\tif hit.ID != dtq.expectHits[i] {\n\t\t\t\t\treturn fmt.Errorf(\"expected docID %s, got %s\", dtq.expectHits[i], hit.ID)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\terr = runTestQueries(idx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// now verify that the stat for number of synonym enabled queries is correct\n\ttotalSynonymSearchStat := getTotalSynonymSearchStat(idx)\n\tif totalSynonymSearchStat != len(testQueries) {\n\t\tt.Fatalf(\"expected %d synonym searches, got %d\", len(testQueries), totalSynonymSearchStat)\n\t}\n\n\t// test with index alias - with 1 batch per index\n\tnumIndexes := len(batches)\n\tindexes := make([]Index, numIndexes)\n\tindexesPath := make([]string, numIndexes)\n\tfor i := 0; i < numIndexes; i++ {\n\t\ttmpIndexPath := createTmpIndexPath(t)\n\t\tidx, err := New(tmpIndexPath, imap)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr = idx.Batch(batches[i])\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tindexes[i] = idx\n\t\tindexesPath[i] = tmpIndexPath\n\t}\n\tdefer func() {\n\t\tfor i := 0; i < numIndexes; i++ {\n\t\t\terr = indexes[i].Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tcleanupTmpIndexPath(t, indexesPath[i])\n\t\t}\n\t}()\n\talias := NewIndexAlias(indexes...)\n\n\tif err := alias.SetIndexMapping(imap); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = runTestQueries(alias)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// verify the synonym search stat for the alias\n\ttotalSynonymSearchStat = getTotalSynonymSearchStat(indexes[0])\n\tif totalSynonymSearchStat != len(testQueries) {\n\t\tt.Fatalf(\"expected %d synonym searches, got %d\", len(testQueries), totalSynonymSearchStat)\n\t}\n\tfor i := 1; i < numIndexes; i++ {\n\t\tidxStat := getTotalSynonymSearchStat(indexes[i])\n\t\tif idxStat != totalSynonymSearchStat {\n\t\t\tt.Fatalf(\"expected %d synonym searches, got %d\", totalSynonymSearchStat, idxStat)\n\t\t}\n\t}\n\tif totalSynonymSearchStat != len(testQueries) {\n\t\tt.Fatalf(\"expected %d synonym searches, got %d\", len(testQueries), totalSynonymSearchStat)\n\t}\n\t// test with multi-level alias now with two index per alias\n\t// and having any extra index being in the final alias\n\tnumAliases := numIndexes / 2\n\textraIndex := numIndexes % 2\n\taliases := make([]IndexAlias, numAliases)\n\tfor i := 0; i < numAliases; i++ {\n\t\talias := NewIndexAlias(indexes[i*2], indexes[i*2+1])\n\t\taliases[i] = alias\n\t}\n\tif extraIndex > 0 {\n\t\taliases[numAliases-1].Add(indexes[numIndexes-1])\n\t}\n\talias = NewIndexAlias()\n\n\tif err := alias.SetIndexMapping(imap); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := 0; i < numAliases; i++ {\n\t\talias.Add(aliases[i])\n\t}\n\terr = runTestQueries(alias)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// verify the synonym searches stat for the alias\n\ttotalSynonymSearchStat = getTotalSynonymSearchStat(indexes[0])\n\tif totalSynonymSearchStat != 2*len(testQueries) {\n\t\tt.Fatalf(\"expected %d synonym searches, got %d\", len(testQueries), totalSynonymSearchStat)\n\t}\n\tfor i := 1; i < numIndexes; i++ {\n\t\tidxStat := getTotalSynonymSearchStat(indexes[i])\n\t\tif idxStat != totalSynonymSearchStat {\n\t\t\tt.Fatalf(\"expected %d synonym searches, got %d\", totalSynonymSearchStat, idxStat)\n\t\t}\n\t}\n}\n\nfunc TestGeoDistanceInSort(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath)\n\n\tfm := mapping.NewGeoPointFieldMapping()\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping.AddFieldMappingsAt(\"geo\", fm)\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tqp := []float64{0, 0}\n\n\tdocs := []struct {\n\t\tid       string\n\t\tpoint    []float64\n\t\tdistance float64\n\t}{\n\t\t{\n\t\t\tid:       \"1\",\n\t\t\tpoint:    []float64{1, 1},\n\t\t\tdistance: geo.Haversin(1, 1, qp[0], qp[1]) * 1000,\n\t\t},\n\t\t{\n\t\t\tid:       \"2\",\n\t\t\tpoint:    []float64{2, 2},\n\t\t\tdistance: geo.Haversin(2, 2, qp[0], qp[1]) * 1000,\n\t\t},\n\t\t{\n\t\t\tid:       \"3\",\n\t\t\tpoint:    []float64{3, 3},\n\t\t\tdistance: geo.Haversin(3, 3, qp[0], qp[1]) * 1000,\n\t\t},\n\t}\n\n\tfor _, doc := range docs {\n\t\tif err := idx.Index(doc.id, map[string]interface{}{\"geo\": doc.point}); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tq := NewGeoDistanceQuery(qp[0], qp[1], \"1000000m\")\n\tq.SetField(\"geo\")\n\treq := NewSearchRequest(q)\n\treq.Sort = make(search.SortOrder, 0)\n\treq.Sort = append(req.Sort, &search.SortGeoDistance{\n\t\tField: \"geo\",\n\t\tDesc:  false,\n\t\tUnit:  \"m\",\n\t\tLon:   qp[0],\n\t\tLat:   qp[1],\n\t})\n\tres, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i, doc := range res.Hits {\n\t\thitDist, err := strconv.ParseFloat(doc.DecodedSort[0], 64)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif math.Abs(hitDist-docs[i].distance) > 1 {\n\t\t\tt.Fatalf(\"distance error greater than 1 meter, expected distance - %v, got - %v\", docs[i].distance, hitDist)\n\t\t}\n\t}\n}\n\nfunc TestFilteredBooleanQuery(t *testing.T) {\n\ttmpIndexPath := createTmpIndexPath(t)\n\n\timap := mapping.NewIndexMapping()\n\n\tgenreMapping := mapping.NewTextFieldMapping()\n\tgenreMapping.Analyzer = keyword.Name\n\n\tauthorMapping := mapping.NewTextFieldMapping()\n\tauthorMapping.Analyzer = keyword.Name\n\n\ttitleMapping := mapping.NewTextFieldMapping()\n\ttitleMapping.Analyzer = en.AnalyzerName\n\n\ttagsMapping := mapping.NewNumericFieldMapping()\n\ttagsMapping.Store = false\n\ttagsMapping.IncludeInAll = false\n\n\tpriceMapping := mapping.NewNumericFieldMapping()\n\timap.DefaultMapping.AddFieldMappingsAt(\"genre\", genreMapping)\n\timap.DefaultMapping.AddFieldMappingsAt(\"author\", authorMapping)\n\timap.DefaultMapping.AddFieldMappingsAt(\"title\", titleMapping)\n\timap.DefaultMapping.AddFieldMappingsAt(\"price\", priceMapping)\n\timap.DefaultMapping.AddFieldMappingsAt(\"tags\", tagsMapping)\n\n\tidx, err := New(tmpIndexPath, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\terr := os.RemoveAll(tmpIndexPath)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\t// Book dataset\n\tvar docs = []map[string]interface{}{\n\t\t{\n\t\t\t\"title\":  \"The Catcher in the Rye\",\n\t\t\t\"author\": \"J.D. Salinger\",\n\t\t\t\"genre\":  \"fiction\",\n\t\t\t\"price\":  9.99,\n\t\t\t\"tags\":   []int{1, 2, 3},\n\t\t},\n\t\t{\n\t\t\t\"title\":  \"Sapiens\",\n\t\t\t\"author\": \"Yuval Noah Harari\",\n\t\t\t\"genre\":  \"non-fiction\",\n\t\t\t\"price\":  14.29,\n\t\t\t\"tags\":   []int{2},\n\t\t},\n\t\t{\n\t\t\t\"title\":  \"To Kill a Mockingbird\",\n\t\t\t\"author\": \"Harper Lee\",\n\t\t\t\"genre\":  \"fiction\",\n\t\t\t\"price\":  12,\n\t\t\t\"tags\":   []int{},\n\t\t},\n\t\t{\n\t\t\t\"title\":  \"The Power of Habit\",\n\t\t\t\"author\": \"Charles Duhigg\",\n\t\t\t\"genre\":  \"self-help\",\n\t\t\t\"price\":  26,\n\t\t\t\"tags\":   []int{1, 2},\n\t\t},\n\t\t{\n\t\t\t\"title\":  \"The Great Gatsby\",\n\t\t\t\"author\": \"F. Scott Fitzgerald\",\n\t\t\t\"genre\":  \"fiction\",\n\t\t\t\"price\":  22,\n\t\t\t\"tags\":   []int{1, 2},\n\t\t},\n\t\t{\n\t\t\t\"title\":  \"Atomic Habits\",\n\t\t\t\"author\": \"James Clear\",\n\t\t\t\"genre\":  \"self-help\",\n\t\t\t\"price\":  15,\n\t\t\t\"tags\":   []int{3},\n\t\t},\n\t\t{\n\t\t\t\"title\":  \"Educated\",\n\t\t\t\"author\": \"Tara Westover\",\n\t\t\t\"genre\":  \"non-fiction\",\n\t\t\t\"price\":  18,\n\t\t},\n\t\t{\n\t\t\t\"title\":  \"1984\",\n\t\t\t\"author\": \"George Orwell\",\n\t\t\t\"genre\":  \"fiction\",\n\t\t\t\"price\":  20,\n\t\t},\n\t}\n\n\tb := idx.NewBatch()\n\tfor i, doc := range docs {\n\t\terr := b.Index(strconv.Itoa(i), doc)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\t// execute the batch\n\terr = idx.Batch(b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Suppose the user is interested in books in the fiction genre\n\t// and is only interested in books within their budget of 20\n\tfictionQuery := NewTermQuery(\"fiction\")\n\tfictionQuery.SetField(\"genre\")\n\n\t// A numeric range query for books with a price less than or equal to 20\n\tmax := float64(20)\n\tmaxInclusive := true\n\tpriceFilterQuery := NewNumericRangeQuery(nil, &max)\n\tpriceFilterQuery.InclusiveMax = &maxInclusive\n\tpriceFilterQuery.SetField(\"price\")\n\n\t// An unfiltered boolean query requesting all books in the fiction genre\n\t// All 4 books in the fiction genre should be returned with the\n\t// same score as they are all in the same genre\n\tq := NewBooleanQuery()\n\tq.AddMust(fictionQuery)\n\n\treq := NewSearchRequest(q)\n\treq.Explain = true\n\treq.Fields = []string{\"title\"}\n\t// sort by book titles in ascending order\n\treq.Sort = make(search.SortOrder, 0)\n\ttitleSort := &search.SortField{\n\t\tField: \"price\",\n\t\tDesc:  false,\n\t}\n\treq.Sort = append(req.Sort, titleSort)\n\n\tres, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(res.Hits) != 4 {\n\t\tt.Fatalf(\"expected 4 hits, got %d\", len(res.Hits))\n\t}\n\t// Verify the results are in the correct order\n\texpectedTitleOrder := []string{\n\t\t\"The Catcher in the Rye\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\",\n\t\t\"The Great Gatsby\",\n\t}\n\tfor i, doc := range res.Hits {\n\t\tif doc.Fields[\"title\"] != expectedTitleOrder[i] {\n\t\t\tt.Fatalf(\"expected title %s, got %s\", expectedTitleOrder[i], doc.Fields[\"title\"])\n\t\t}\n\t}\n\t// Ensure that the scores are the same for all documents\n\tunfilteredScore := res.Hits[0].Score\n\tfor i := 1; i < len(res.Hits); i++ {\n\t\tif res.Hits[i].Score != unfilteredScore {\n\t\t\tt.Fatalf(\"expected score %f, got %f\", unfilteredScore, res.Hits[i].Score)\n\t\t}\n\t}\n\t// A filtered boolean query requesting all books satisfying the\n\t// filterQuery and the priceFilterQuery\n\t// But the filter query is in the Must clause\n\tq = NewBooleanQuery()\n\tq.AddMust(fictionQuery)\n\tq.AddMust(priceFilterQuery)\n\treq = NewSearchRequest(q)\n\treq.Explain = true\n\treq.Fields = []string{\"title\"}\n\t// sort by book titles in ascending order\n\treq.Sort = make(search.SortOrder, 0)\n\treq.Sort = append(req.Sort, titleSort)\n\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// here the score must not be the same for all documents\n\t// as the price filter is applied in the Must clause\n\t// and the score is different compared to the previous unfiltered boolean query\n\tif len(res.Hits) != 3 {\n\t\tt.Fatalf(\"expected 3 hits, got %d\", len(res.Hits))\n\t}\n\t// Verify the results are in the correct order\n\texpectedTitleOrder = []string{\n\t\t\"The Catcher in the Rye\",\n\t\t\"To Kill a Mockingbird\",\n\t\t\"1984\",\n\t}\n\tfor i, doc := range res.Hits {\n\t\tif doc.Fields[\"title\"] != expectedTitleOrder[i] {\n\t\t\tt.Fatalf(\"expected title %s, got %s\", expectedTitleOrder[i], doc.Fields[\"title\"])\n\t\t}\n\t}\n\t// Ensure that the scores are different for all documents\n\tfor i := 0; i < len(res.Hits); i++ {\n\t\tif res.Hits[i].Score == unfilteredScore {\n\t\t\tt.Fatalf(\"expected different score, got %f\", res.Hits[i].Score)\n\t\t}\n\t}\n\t// A filtered boolean query requesting all books satisfying the\n\t// filterQuery and the priceFilterQuery\n\t// But the filter query is in the Filter clause\n\tq = NewBooleanQuery()\n\tq.AddMust(fictionQuery)\n\tq.AddFilter(priceFilterQuery)\n\n\treq = NewSearchRequest(q)\n\treq.Explain = true\n\treq.Fields = []string{\"title\"}\n\treq.Sort = make(search.SortOrder, 0)\n\treq.Sort = append(req.Sort, titleSort)\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(res.Hits) != 3 {\n\t\tt.Fatalf(\"expected 3 hits, got %d\", len(res.Hits))\n\t}\n\t// Verify the results are in the correct order\n\tfor i, doc := range res.Hits {\n\t\tif doc.Fields[\"title\"] != expectedTitleOrder[i] {\n\t\t\tt.Fatalf(\"expected title %s, got %s\", expectedTitleOrder[i], doc.Fields[\"title\"])\n\t\t}\n\t}\n\t// Ensure that the scores are the same for all documents\n\tfor i := 0; i < len(res.Hits); i++ {\n\t\tif res.Hits[i].Score != unfilteredScore {\n\t\t\tt.Fatalf(\"expected score %f, got %f\", unfilteredScore, res.Hits[i].Score)\n\t\t}\n\t}\n\t// A filtered boolean query requesting all books with tag value 3\n\t// The filter is in the Filter clause\n\t// Two books have tag value 3\n\tp := float64(3)\n\tincl := true\n\teqlFilter := NewNumericRangeInclusiveQuery(&p, &p, &incl, &incl)\n\teqlFilter.SetField(\"tags\")\n\tq = NewBooleanQuery()\n\tq.AddFilter(eqlFilter)\n\n\treq = NewSearchRequest(q)\n\treq.Fields = []string{\"title\"}\n\tres, err = idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif res.Hits.Len() != 2 {\n\t\tt.Fatalf(\"expected two hits, found '%d'\", res.Hits.Len())\n\t}\n\tif res.Total != 2 {\n\t\tt.Fatalf(\"expected two total, found '%d'\", res.Total)\n\t}\n}\n\nfunc TestGeoDistanceInSortAlias(t *testing.T) {\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath1)\n\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath2)\n\n\tfm := mapping.NewGeoPointFieldMapping()\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping.AddFieldMappingsAt(\"geo\", fm)\n\n\tidx1, err := New(tmpIndexPath1, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tidx2, err := New(tmpIndexPath2, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tqp := []float64{0, 0}\n\n\tdocs := []struct {\n\t\tid       string\n\t\tpoint    []float64\n\t\tdistance float64\n\t}{\n\t\t{\n\t\t\tid:       \"1\",\n\t\t\tpoint:    []float64{1, 1},\n\t\t\tdistance: geo.Haversin(1, 1, qp[0], qp[1]) * 1000,\n\t\t},\n\t\t{\n\t\t\tid:       \"2\",\n\t\t\tpoint:    []float64{2, 2},\n\t\t\tdistance: geo.Haversin(2, 2, qp[0], qp[1]) * 1000,\n\t\t},\n\t\t{\n\t\t\tid:       \"3\",\n\t\t\tpoint:    []float64{3, 3},\n\t\t\tdistance: geo.Haversin(3, 3, qp[0], qp[1]) * 1000,\n\t\t},\n\t}\n\tif err := idx1.Index(docs[0].id, map[string]interface{}{\"geo\": docs[0].point}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idx2.Index(docs[1].id, map[string]interface{}{\"geo\": docs[1].point}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idx1.Index(docs[2].id, map[string]interface{}{\"geo\": docs[2].point}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx := NewIndexAlias(idx1, idx2)\n\n\tq := NewGeoDistanceQuery(qp[0], qp[1], \"1000000m\")\n\tq.SetField(\"geo\")\n\treq := NewSearchRequest(q)\n\treq.Sort = make(search.SortOrder, 0)\n\treq.Sort = append(req.Sort, &search.SortGeoDistance{\n\t\tField: \"geo\",\n\t\tLon:   qp[0],\n\t\tLat:   qp[1],\n\t})\n\tres, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i, doc := range res.Hits {\n\t\thitDist, err := strconv.ParseFloat(doc.DecodedSort[0], 64)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif math.Abs(hitDist-docs[i].distance) > 1 {\n\t\t\tt.Fatalf(\"distance error greater than 1 meter, expected distance - %v, got - %v\", docs[i].distance, hitDist)\n\t\t}\n\t}\n}\n\nfunc TestDateSortAlias(t *testing.T) {\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath1)\n\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath2)\n\n\tfm := mapping.NewDateTimeFieldMapping()\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping.AddFieldMappingsAt(\"date\", fm)\n\n\tidx1, err := New(tmpIndexPath1, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tidx2, err := New(tmpIndexPath2, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocs := []struct {\n\t\tid   string\n\t\tdate string\n\t}{\n\t\t{\n\t\t\tid:   \"1\",\n\t\t\tdate: \"2023-01-01\",\n\t\t},\n\t\t{\n\t\t\tid:   \"2\",\n\t\t\tdate: \"2023-02-01\",\n\t\t},\n\t\t{\n\t\t\tid:   \"3\",\n\t\t\tdate: \"2023-03-01\",\n\t\t},\n\t}\n\n\tif err := idx1.Index(docs[0].id, map[string]interface{}{\"date\": docs[0].date}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idx2.Index(docs[1].id, map[string]interface{}{\"date\": docs[1].date}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idx1.Index(docs[2].id, map[string]interface{}{\"date\": docs[2].date}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx := NewIndexAlias(idx1, idx2)\n\n\tq := query.NewMatchAllQuery()\n\treq := NewSearchRequest(q)\n\treq.Sort = make(search.SortOrder, 0)\n\treq.Sort = append(req.Sort, &search.SortField{\n\t\tField: \"date\",\n\t\tType:  search.SortFieldAsDate,\n\t})\n\tres, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i, doc := range res.Hits {\n\t\texpectedDate, err := time.Parse(\"2006-01-02\", docs[i].date)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\texpectedDateStr := expectedDate.UTC().Format(time.RFC3339Nano)\n\t\tif doc.DecodedSort[0] != expectedDateStr {\n\t\t\tt.Fatalf(\"expected date %s, got %s\", doc.DecodedSort[0], expectedDateStr)\n\t\t}\n\t}\n}\n\nfunc TestNumericSortAlias(t *testing.T) {\n\ttmpIndexPath1 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath1)\n\n\ttmpIndexPath2 := createTmpIndexPath(t)\n\tdefer cleanupTmpIndexPath(t, tmpIndexPath2)\n\n\tfm := mapping.NewNumericFieldMapping()\n\timap := mapping.NewIndexMapping()\n\timap.DefaultMapping.AddFieldMappingsAt(\"num\", fm)\n\n\tidx1, err := New(tmpIndexPath1, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx1.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tidx2, err := New(tmpIndexPath2, imap)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() {\n\t\terr = idx2.Close()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tdocs := []struct {\n\t\tid  string\n\t\tnum int\n\t}{\n\t\t{\n\t\t\tid:  \"1\",\n\t\t\tnum: 10,\n\t\t},\n\t\t{\n\t\t\tid:  \"2\",\n\t\t\tnum: 20,\n\t\t},\n\t\t{\n\t\t\tid:  \"3\",\n\t\t\tnum: 30,\n\t\t},\n\t}\n\n\tif err := idx1.Index(docs[0].id, map[string]interface{}{\"num\": docs[0].num}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idx2.Index(docs[1].id, map[string]interface{}{\"num\": docs[1].num}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := idx1.Index(docs[2].id, map[string]interface{}{\"num\": docs[2].num}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tidx := NewIndexAlias(idx1, idx2)\n\n\tq := query.NewMatchAllQuery()\n\treq := NewSearchRequest(q)\n\treq.Sort = make(search.SortOrder, 0)\n\treq.Sort = append(req.Sort, &search.SortField{\n\t\tField: \"num\",\n\t\tType:  search.SortFieldAsNumber,\n\t})\n\tres, err := idx.Search(req)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i, doc := range res.Hits {\n\t\thitNum, err := strconv.Atoi(doc.DecodedSort[0])\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif hitNum != docs[i].num {\n\t\t\tt.Fatalf(\"expected num %d, got %d\", docs[i].num, hitNum)\n\t\t}\n\t}\n}\n\nfunc TestSearchRequestValidatePagination(t *testing.T) {\n\ttests := []struct {\n\t\tname      string\n\t\treq       *SearchRequest\n\t\texpectErr error\n\t}{\n\t\t{\n\t\t\tname: \"invalid search after with numeric sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortField{Field: \"num\", Type: search.SortFieldAsNumber},\n\t\t\t\t},\n\t\t\t\tSearchAfter: []string{\"not-a-number\"},\n\t\t\t},\n\t\t\texpectErr: fmt.Errorf(\"invalid search after value for sort field 'num': 'not-a-number'. strconv.ParseFloat: parsing \\\"not-a-number\\\": invalid syntax\"),\n\t\t},\n\t\t{\n\t\t\tname: \"invalid search before with numeric sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortField{Field: \"num\", Type: search.SortFieldAsNumber},\n\t\t\t\t},\n\t\t\t\tSearchBefore: []string{\"not-a-number\"},\n\t\t\t},\n\t\t\texpectErr: fmt.Errorf(\"invalid search before value for sort field 'num': 'not-a-number'. strconv.ParseFloat: parsing \\\"not-a-number\\\": invalid syntax\"),\n\t\t},\n\t\t{\n\t\t\tname: \"invalid search after with date sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortField{Field: \"date\", Type: search.SortFieldAsDate},\n\t\t\t\t},\n\t\t\t\tSearchAfter: []string{\"1 March 2023\"},\n\t\t\t},\n\t\t\texpectErr: fmt.Errorf(\"invalid search after value for sort field 'date': '1 March 2023'. parsing time \\\"1 March 2023\\\" as \\\"2006-01-02T15:04:05.999999999Z07:00\\\": cannot parse \\\"1 March 2023\\\" as \\\"2006\\\"\"),\n\t\t},\n\t\t{\n\t\t\tname: \"invalid search before with date sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortField{Field: \"date\", Type: search.SortFieldAsDate},\n\t\t\t\t},\n\t\t\t\tSearchBefore: []string{\"1 March 2023\"},\n\t\t\t},\n\t\t\texpectErr: fmt.Errorf(\"invalid search before value for sort field 'date': '1 March 2023'. parsing time \\\"1 March 2023\\\" as \\\"2006-01-02T15:04:05.999999999Z07:00\\\": cannot parse \\\"1 March 2023\\\" as \\\"2006\\\"\"),\n\t\t},\n\t\t{\n\t\t\tname: \"invalid search after with geo distance sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortGeoDistance{Field: \"geo\"},\n\t\t\t\t},\n\t\t\t\tSearchAfter: []string{\"not-a-number\"},\n\t\t\t},\n\t\t\texpectErr: fmt.Errorf(\"invalid search after value for sort field 'geo': 'not-a-number'. strconv.ParseFloat: parsing \\\"not-a-number\\\": invalid syntax\"),\n\t\t},\n\t\t{\n\t\t\tname: \"invalid search before with geo distance sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortGeoDistance{Field: \"geo\"},\n\t\t\t\t},\n\t\t\t\tSearchBefore: []string{\"not-a-number\"},\n\t\t\t},\n\t\t\texpectErr: fmt.Errorf(\"invalid search before value for sort field 'geo': 'not-a-number'. strconv.ParseFloat: parsing \\\"not-a-number\\\": invalid syntax\"),\n\t\t},\n\t\t{\n\t\t\tname: \"valid search after with text sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortField{Field: \"text\", Type: search.SortFieldAsString},\n\t\t\t\t},\n\t\t\t\tSearchAfter: []string{\"anything\"},\n\t\t\t},\n\t\t\texpectErr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"valid search after with numeric sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortField{Field: \"num\", Type: search.SortFieldAsNumber},\n\t\t\t\t},\n\t\t\t\tSearchAfter: []string{\"50.5\"},\n\t\t\t},\n\t\t\texpectErr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"valid search after with date sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortField{Field: \"date\", Type: search.SortFieldAsDate},\n\t\t\t\t},\n\t\t\t\tSearchAfter: []string{time.Now().UTC().Format(time.RFC3339Nano)},\n\t\t\t},\n\t\t\texpectErr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"valid search after with geo distance sort\",\n\t\t\treq: &SearchRequest{\n\t\t\t\tQuery: NewMatchAllQuery(),\n\t\t\t\tSort: search.SortOrder{\n\t\t\t\t\t&search.SortGeoDistance{Field: \"geo\"},\n\t\t\t\t},\n\t\t\t\tSearchAfter: []string{\"1.234\"},\n\t\t\t},\n\t\t\texpectErr: nil,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\terr := test.req.Validate()\n\t\t\tif test.expectErr != nil {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Fatalf(\"expected error: %v, got nil\", test.expectErr)\n\t\t\t\t}\n\t\t\t\tif err.Error() != test.expectErr.Error() {\n\t\t\t\t\tt.Fatalf(\"expected error: %v, got: %v\", test.expectErr, err)\n\t\t\t\t}\n\t\t\t} else if err != nil {\n\t\t\t\tt.Fatalf(\"expected no error, got: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "size/sizes.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage size\n\nimport (\n\t\"reflect\"\n)\n\nfunc init() {\n\tvar b bool\n\tSizeOfBool = int(reflect.TypeOf(b).Size())\n\tvar f32 float32\n\tSizeOfFloat32 = int(reflect.TypeOf(f32).Size())\n\tvar f64 float64\n\tSizeOfFloat64 = int(reflect.TypeOf(f64).Size())\n\tvar i int\n\tSizeOfInt = int(reflect.TypeOf(i).Size())\n\tvar m map[int]int\n\tSizeOfMap = int(reflect.TypeOf(m).Size())\n\tvar ptr *int\n\tSizeOfPtr = int(reflect.TypeOf(ptr).Size())\n\tvar slice []int\n\tSizeOfSlice = int(reflect.TypeOf(slice).Size())\n\tvar str string\n\tSizeOfString = int(reflect.TypeOf(str).Size())\n\tvar u8 uint8\n\tSizeOfUint8 = int(reflect.TypeOf(u8).Size())\n\tvar u16 uint16\n\tSizeOfUint16 = int(reflect.TypeOf(u16).Size())\n\tvar u32 uint32\n\tSizeOfUint32 = int(reflect.TypeOf(u32).Size())\n\tvar u64 uint64\n\tSizeOfUint64 = int(reflect.TypeOf(u64).Size())\n}\n\nvar SizeOfBool int\nvar SizeOfFloat32 int\nvar SizeOfFloat64 int\nvar SizeOfInt int\nvar SizeOfMap int\nvar SizeOfPtr int\nvar SizeOfSlice int\nvar SizeOfString int\nvar SizeOfUint8 int\nvar SizeOfUint16 int\nvar SizeOfUint32 int\nvar SizeOfUint64 int\n"
  },
  {
    "path": "test/integration.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"github.com/blevesearch/bleve/v2\"\n)\n\ntype SearchTest struct {\n\tSearch  *bleve.SearchRequest `json:\"search\"`\n\tResult  *bleve.SearchResult  `json:\"result\"`\n\tComment string               `json:\"comment\"`\n}\n\ntype SearchTests []*SearchTest\n"
  },
  {
    "path": "test/integration_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"encoding/json\"\n\t\"flag\"\n\t\"fmt\"\n\t\"math\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\n\t// allow choosing alternate kvstores\n\t_ \"github.com/blevesearch/bleve/v2/config\"\n)\n\nvar dataset = flag.String(\"dataset\", \"\", \"only test datasets matching this regex\")\nvar onlynum = flag.Int(\"testnum\", -1, \"only run the test with this number\")\nvar keepIndex = flag.Bool(\"keepIndex\", false, \"keep the index after testing\")\n\nvar indexType = flag.String(\"indexType\", bleve.Config.DefaultIndexType, \"index type to build\")\nvar kvType = flag.String(\"kvType\", bleve.Config.DefaultKVStore, \"kv store type to build\")\nvar segType = flag.String(\"segType\", \"\", \"force scorch segment type\")\nvar segVer = flag.Int(\"segVer\", 0, \"force scorch segment version\")\n\nfunc TestIntegration(t *testing.T) {\n\n\tflag.Parse()\n\n\tt.Logf(\"using index type %s and kv type %s\", *indexType, *kvType)\n\tif *segType != \"\" {\n\t\tt.Logf(\"forcing segment type: %s\", *segType)\n\t}\n\tif *segVer != 0 {\n\t\tt.Logf(\"forcing segment version: %d\", *segVer)\n\t}\n\n\tvar err error\n\tvar datasetRegexp *regexp.Regexp\n\tif *dataset != \"\" {\n\t\tdatasetRegexp, err = regexp.Compile(*dataset)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tentries, err := os.ReadDir(\"tests\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor _, f := range entries {\n\t\tif datasetRegexp != nil {\n\t\t\tif !datasetRegexp.MatchString(f.Name()) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif f.IsDir() {\n\t\t\tt.Logf(\"Running test: %s\", f.Name())\n\t\t\trunTestDir(t, \"tests\"+string(filepath.Separator)+f.Name(), f.Name())\n\t\t}\n\t}\n}\n\nfunc runTestDir(t *testing.T, dir, datasetName string) {\n\t// read the mapping\n\tmappingBytes, err := os.ReadFile(dir + string(filepath.Separator) + \"mapping.json\")\n\tif err != nil {\n\t\tt.Errorf(\"error reading mapping: %v\", err)\n\t\treturn\n\t}\n\tvar mapping mapping.IndexMappingImpl\n\terr = json.Unmarshal(mappingBytes, &mapping)\n\tif err != nil {\n\t\tt.Errorf(\"error unmarshalling mapping: %v\", err)\n\t\treturn\n\t}\n\n\tvar index bleve.Index\n\tvar cleanup func()\n\n\t// if there is a dir named 'data' open single index\n\t_, err = os.Stat(dir + string(filepath.Separator) + \"data\")\n\tif !os.IsNotExist(err) {\n\n\t\tindex, cleanup, err = loadDataSet(t, datasetName, mapping, dir+string(filepath.Separator)+\"data\")\n\t\tif err != nil {\n\t\t\tt.Errorf(\"error loading dataset: %v\", err)\n\t\t\treturn\n\t\t}\n\t\tdefer cleanup()\n\t} else {\n\t\t// if there is a dir named 'datasets' build alias over each index\n\t\t_, err = os.Stat(dir + string(filepath.Separator) + \"datasets\")\n\t\tif !os.IsNotExist(err) {\n\t\t\tindex, cleanup, err = loadDataSets(t, datasetName, mapping, dir+string(filepath.Separator)+\"datasets\")\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"error loading dataset: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tdefer cleanup()\n\t\t}\n\t}\n\n\t// read the searches\n\tsearchBytes, err := os.ReadFile(dir + string(filepath.Separator) + \"searches.json\")\n\tif err != nil {\n\t\tt.Errorf(\"error reading searches: %v\", err)\n\t\treturn\n\t}\n\tvar searches SearchTests\n\terr = json.Unmarshal(searchBytes, &searches)\n\tif err != nil {\n\t\tt.Errorf(\"error unmarshalling searches: %v\", err)\n\t\treturn\n\t}\n\n\t// run the searches\n\tfor testNum, search := range searches {\n\t\tif *onlynum < 0 || (*onlynum > 0 && testNum == *onlynum) {\n\t\t\tres, err := index.Search(search.Search)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"error running search: %v\", err)\n\t\t\t}\n\t\t\tif res.Total != search.Result.Total {\n\t\t\t\tt.Errorf(\"test error - %s\", search.Comment)\n\t\t\t\tt.Errorf(\"test %d - expected total: %d got %d\", testNum, search.Result.Total, res.Total)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(res.Hits) != len(search.Result.Hits) {\n\t\t\t\tt.Errorf(\"test error - %s\", search.Comment)\n\t\t\t\tt.Errorf(\"test %d - expected hits len: %d got %d\", testNum, len(search.Result.Hits), len(res.Hits))\n\t\t\t\tt.Errorf(\"got hits: %v\", res.Hits)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor hi, hit := range search.Result.Hits {\n\t\t\t\tif hit.ID != res.Hits[hi].ID {\n\t\t\t\t\tt.Errorf(\"test error - %s\", search.Comment)\n\t\t\t\t\tt.Errorf(\"test %d - expected hit %d to have ID %s got %s\", testNum, hi, hit.ID, res.Hits[hi].ID)\n\t\t\t\t}\n\t\t\t\tif hit.Fields != nil {\n\t\t\t\t\tif !reflect.DeepEqual(hit.Fields, res.Hits[hi].Fields) {\n\t\t\t\t\t\tt.Errorf(\"test error - %s\", search.Comment)\n\t\t\t\t\t\tt.Errorf(\"test  %d - expected hit %d to have fields %#v got %#v\", testNum, hi, hit.Fields, res.Hits[hi].Fields)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif hit.Fragments != nil {\n\t\t\t\t\tif !reflect.DeepEqual(hit.Fragments, res.Hits[hi].Fragments) {\n\t\t\t\t\t\tt.Errorf(\"test error - %s\", search.Comment)\n\t\t\t\t\t\tt.Errorf(\"test %d - expected hit %d to have fragments %#v got %#v\", testNum, hi, hit.Fragments, res.Hits[hi].Fragments)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif hit.Locations != nil {\n\t\t\t\t\tif !reflect.DeepEqual(hit.Locations, res.Hits[hi].Locations) {\n\t\t\t\t\t\tt.Errorf(\"test error - %s\", search.Comment)\n\t\t\t\t\t\tt.Errorf(\"test %d - expected hit %d to have locations %#v got %#v\", testNum, hi, hit.Locations, res.Hits[hi].Locations)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// assert that none of the scores were NaN,+Inf,-Inf\n\t\t\t\tif math.IsInf(res.Hits[hi].Score, 0) || math.IsNaN(res.Hits[hi].Score) {\n\t\t\t\t\tt.Errorf(\"test error - %s\", search.Comment)\n\t\t\t\t\tt.Errorf(\"test %d - invalid score %f\", testNum, res.Hits[hi].Score)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif search.Result.Facets != nil {\n\t\t\t\tif !reflect.DeepEqual(search.Result.Facets, res.Facets) {\n\t\t\t\t\tt.Errorf(\"test error - %s\", search.Comment)\n\t\t\t\t\tt.Errorf(\"test %d - expected facets: %#v got %#v\", testNum, search.Result.Facets, res.Facets)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, ok := index.(bleve.IndexAlias); !ok {\n\t\t\t\t// check that custom index name is in results\n\t\t\t\tfor _, hit := range res.Hits {\n\t\t\t\t\tif hit.Index != datasetName {\n\t\t\t\t\t\tt.Fatalf(\"expected name: %s, got: %s\", datasetName, hit.Index)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc loadDataSet(t *testing.T, datasetName string, mapping mapping.IndexMappingImpl, path string) (bleve.Index, func(), error) {\n\tidxPath := fmt.Sprintf(\"test-%s.bleve\", datasetName)\n\tcfg := map[string]interface{}{}\n\tif *segType != \"\" {\n\t\tcfg[\"forceSegmentType\"] = *segType\n\t}\n\tif *segVer != 0 {\n\t\tcfg[\"forceSegmentVersion\"] = *segVer\n\t}\n\n\tindex, err := bleve.NewUsing(idxPath, &mapping, *indexType, *kvType, cfg)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"error creating new index: %v\", err)\n\t}\n\t// set a custom index name\n\tindex.SetName(datasetName)\n\n\t// index data\n\tentries, err := os.ReadDir(path)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"error reading data dir: %v\", err)\n\t}\n\tfor _, f := range entries {\n\t\tfileBytes, err := os.ReadFile(path + string(filepath.Separator) + f.Name())\n\t\tif err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"error reading data file: %v\", err)\n\t\t}\n\t\tvar fileDoc interface{}\n\t\terr = json.Unmarshal(fileBytes, &fileDoc)\n\t\tif err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"error parsing data file as json: %v\", err)\n\t\t}\n\t\tfilename := f.Name()\n\t\text := filepath.Ext(filename)\n\t\tid := filename[0 : len(filename)-len(ext)]\n\t\terr = index.Index(id, fileDoc)\n\t\tif err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"error indexing data: %v\", err)\n\t\t}\n\t}\n\tcleanup := func() {\n\t\terr := index.Close()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error closing index: %v\", err)\n\t\t}\n\t\tif !*keepIndex {\n\t\t\terr := os.RemoveAll(idxPath)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"error removing index: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\treturn index, cleanup, nil\n}\n\nfunc loadDataSets(t *testing.T, datasetName string, mapping mapping.IndexMappingImpl, path string) (bleve.Index, func(), error) {\n\tentries, err := os.ReadDir(path)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"error reading datasets dir: %v\", err)\n\t}\n\tvar cleanups []func()\n\talias := bleve.NewIndexAlias()\n\tfor _, f := range entries {\n\t\tidx, idxCleanup, err := loadDataSet(t, f.Name(), mapping, path+string(filepath.Separator)+f.Name())\n\t\tif err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"error loading dataset: %v\", err)\n\t\t}\n\t\tcleanups = append(cleanups, idxCleanup)\n\t\talias.Add(idx)\n\t}\n\talias.SetName(datasetName)\n\n\tcleanupAll := func() {\n\t\tfor _, cleanup := range cleanups {\n\t\t\tcleanup()\n\t\t}\n\t}\n\n\treturn alias, cleanupAll, nil\n}\n"
  },
  {
    "path": "test/ip_field_test.go",
    "content": "//  Copyright (c) 2021 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"net\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n)\n\ntype doc struct {\n\tIP string `json:\"ip\"`\n}\n\nfunc createIdx(t *testing.T) bleve.Index {\n\tipIndexed := mapping.NewIPFieldMapping()\n\tipIndexed.Name = \"ip\"\n\n\tlineMapping := bleve.NewDocumentStaticMapping()\n\tlineMapping.AddFieldMappingsAt(\"ip\", ipIndexed)\n\n\tmapping := bleve.NewIndexMapping()\n\tmapping.DefaultMapping = lineMapping\n\tmapping.DefaultAnalyzer = \"standard\"\n\n\tidx, err := bleve.NewMemOnly(mapping)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\treturn idx\n}\n\nfunc Test_ipv4CidrQuery(t *testing.T) {\n\tidx := createIdx(t)\n\tdefer idx.Close()\n\n\terr := idx.Index(\"id1\", doc{\"192.168.1.21\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treqStr := `192.168.1.0/24`\n\tquery := bleve.NewIPRangeQuery(reqStr)\n\tquery.FieldVal = \"ip\"\n\n\tsearch := bleve.NewSearchRequest(query)\n\tres, err := idx.Search(search)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 1 {\n\t\tt.Fatalf(\"failed to find %q, res -> %s\", reqStr, res)\n\t}\n\tif res.Hits[0].ID != \"id1\" {\n\t\tt.Fatalf(\"expected %q got %q\", \"id1\", res.Hits[0].Index)\n\t}\n}\n\nfunc Test_ipv6CidrQuery(t *testing.T) {\n\tidx := createIdx(t)\n\tdefer idx.Close()\n\n\terr := idx.Index(\"id1\", doc{\"2a00:23c8:7283:ff00:1fa8:2af6:9dec:6b19\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treqStr := `2a00:23c8:7283:ff00:1fa8:0:0:0/80`\n\tquery := bleve.NewIPRangeQuery(reqStr)\n\tquery.FieldVal = \"ip\"\n\n\tsearch := bleve.NewSearchRequest(query)\n\tres, err := idx.Search(search)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 1 {\n\t\tt.Fatalf(\"failed to find %q, res -> %s\", reqStr, res)\n\t}\n\tif res.Hits[0].ID != \"id1\" {\n\t\tt.Fatalf(\"expected %q got %q\", \"id1\", res.Hits[0].Index)\n\t}\n}\n\nfunc Test_MultiIPvr4CidrQuery(t *testing.T) {\n\tidx := createIdx(t)\n\tdefer idx.Close()\n\n\terr := idx.Index(\"id1\", doc{\"192.168.1.0\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Index(\"id2\", doc{\"192.168.1.255\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Index(\"id3\", doc{\"192.168.2.22\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treqStr := `192.168.1.0/24`\n\tquery := bleve.NewIPRangeQuery(reqStr)\n\tquery.FieldVal = \"ip\"\n\n\tsearch := bleve.NewSearchRequest(query)\n\tres, err := idx.Search(search)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 2 {\n\t\tt.Fatalf(\"failed to find %q, res -> %s\", reqStr, res)\n\t}\n\tif res.Hits[0].ID != \"id1\" {\n\t\tt.Fatalf(\"expected %q got %q\", \"id1\", res.Hits[0].ID)\n\t}\n\tif res.Hits[1].ID != \"id2\" {\n\t\tt.Fatalf(\"expected %q got %q\", \"id2\", res.Hits[0].Index)\n\t}\n}\n\nfunc Test_CidrQueryNonDivisibleBy8(t *testing.T) {\n\tidx := createIdx(t)\n\tdefer idx.Close()\n\n\terr := idx.Index(\"id1\", doc{\"192.168.1.1\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Index(\"id2\", doc{\"192.168.1.2\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Index(\"id3\", doc{\"192.168.2.5\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = idx.Index(\"id4\", doc{\"192.168.2.6\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treqStr := `192.168.1.0/30`\n\tquery := bleve.NewIPRangeQuery(reqStr)\n\tquery.FieldVal = \"ip\"\n\n\tsearch := bleve.NewSearchRequest(query)\n\tres, err := idx.Search(search)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 2 {\n\t\tt.Fatalf(\"failed to find %q, res -> %s\", reqStr, res)\n\t}\n\tif res.Hits[0].ID != \"id1\" {\n\t\tt.Fatalf(\"expected %q got %q\", \"id1\", res.Hits[0].ID)\n\t}\n\tif res.Hits[1].ID != \"id2\" {\n\t\tt.Fatalf(\"expected %q got %q\", \"id2\", res.Hits[0].Index)\n\t}\n}\n\nfunc Test_simpleIPv4MatchQuery(t *testing.T) {\n\tidx := createIdx(t)\n\tdefer idx.Close()\n\n\terr := idx.Index(\"id1\", doc{\"192.168.1.21\"})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treqStr := `192.168.1.21`\n\tquery := bleve.NewIPRangeQuery(reqStr)\n\tquery.FieldVal = \"ip\"\n\n\tsearch := bleve.NewSearchRequest(query)\n\tres, err := idx.Search(search)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 1 {\n\t\tt.Fatalf(\"failed to find %q, res -> %s\", reqStr, res)\n\t}\n\tif res.Hits[0].ID != \"id1\" {\n\t\tt.Fatalf(\"expected %q got %q\", \"id1\", res.Hits[0].Index)\n\t}\n}\n\nfunc Test_ipv4LiteralData(t *testing.T) {\n\tidx := createIdx(t)\n\tdefer idx.Close()\n\n\ttype stronglyTyped struct {\n\t\tIP net.IP `json:\"ip\"`\n\t}\n\n\terr := idx.Index(\"id1\", stronglyTyped{net.ParseIP(\"192.168.1.21\")})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\treqStr := `192.168.1.0/24`\n\tquery := bleve.NewIPRangeQuery(reqStr)\n\tquery.FieldVal = \"ip\"\n\n\tsearch := bleve.NewSearchRequest(query)\n\tres, err := idx.Search(search)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Total != 1 {\n\t\tt.Fatalf(\"failed to find %q, res -> %s\", reqStr, res)\n\t}\n\tif res.Hits[0].ID != \"id1\" {\n\t\tt.Fatalf(\"expected %q got %q\", \"id1\", res.Hits[0].Index)\n\t}\n}\n\nfunc Test_badIPFmt(t *testing.T) {\n\tidx := createIdx(t)\n\tdefer idx.Close()\n\n\treqStr := `192.168.1.`\n\tquery := bleve.NewIPRangeQuery(reqStr)\n\tquery.FieldVal = \"ip\"\n\n\tsearch := bleve.NewSearchRequest(query)\n\t_, err := idx.Search(search)\n\tif err == nil {\n\t\tt.Errorf(\"%q is not a valid IP\", reqStr)\n\t}\n}\n\nfunc Test_badCIDRFmt(t *testing.T) {\n\tidx := createIdx(t)\n\tdefer idx.Close()\n\n\treqStr := `/`\n\tquery := bleve.NewIPRangeQuery(reqStr)\n\tquery.FieldVal = \"ip\"\n\n\terr := query.Validate()\n\tif err == nil {\n\t\tt.Errorf(\"%q is not a valid CIDR\", reqStr)\n\t}\n\n\tsearch := bleve.NewSearchRequest(query)\n\t_, err = idx.Search(search)\n\tif err == nil {\n\t\tt.Errorf(\"%q is not a valid CIDR\", reqStr)\n\t}\n}\n"
  },
  {
    "path": "test/tests/alias/datasets/shard0/a.json",
    "content": "{\n  \"name\": \"a\"\n}"
  },
  {
    "path": "test/tests/alias/datasets/shard0/c.json",
    "content": "{\n  \"name\": \"c\"\n}"
  },
  {
    "path": "test/tests/alias/datasets/shard1/b.json",
    "content": "{\n  \"name\": \"b\"\n}"
  },
  {
    "path": "test/tests/alias/datasets/shard1/d.json",
    "content": "{\n  \"name\": \"d\"\n}"
  },
  {
    "path": "test/tests/alias/mapping.json",
    "content": "{\n  \"default_analyzer\": \"keyword\"\n}\n"
  },
  {
    "path": "test/tests/alias/searches.json",
    "content": "[\n  {\n    \"comment\": \"match all across shards\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"sort\": [\"-_score\", \"_id\"],\n      \"query\": {\n        \"match_all\": {}\n      }\n    },\n    \"result\": {\n      \"total_hits\": 4,\n      \"hits\": [\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"d\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search after b (page 2 when size=2)\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 2,\n      \"sort\": [\"name\"],\n      \"search_after\": [\"b\"],\n      \"query\": {\n        \"match_all\": {}\n      }\n    },\n    \"result\": {\n      \"total_hits\": 4,\n      \"hits\": [\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"d\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search before c (page 1 when size=2)\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 2,\n      \"sort\": [\"name\"],\n      \"search_before\": [\"c\"],\n      \"query\": {\n        \"match_all\": {}\n      }\n    },\n    \"result\": {\n      \"total_hits\": 4,\n      \"hits\": [\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"b\"\n        }\n      ]\n    }\n  }\n]"
  },
  {
    "path": "test/tests/basic/data/a.json",
    "content": "{\n\t\"id\": \"a\",\n\t\"name\": \"marty\",\n\t\"age\": 19,\n\t\"title\": \"mista\",\n\t\"tags\": [\"gopher\", \"belieber\"]\n}"
  },
  {
    "path": "test/tests/basic/data/b.json",
    "content": "{\n\t\"id\": \"b\",\n\t\"name\": \"steve has <a> long & complicated name\",\n\t\"age\": 27,\n\t\"birthday\": \"2001-09-09T01:46:40Z\",\n\t\"title\": \"missess\"\n}"
  },
  {
    "path": "test/tests/basic/data/c.json",
    "content": "{\n\t\"id\": \"c\",\n\t\"name\": \"bob walks home\",\n\t\"age\": 64,\n\t\"birthday\": \"2014-05-13T16:53:20Z\",\n\t\"title\": \"masta\"\n}"
  },
  {
    "path": "test/tests/basic/data/d.json",
    "content": "{\n\t\"id\": \"d\",\n\t\"name\": \"bobbleheaded wings top the phone\",\n\t\"age\": 72,\n\t\"birthday\": \"2014-05-13T16:53:20Z\",\n\t\"title\": \"mizz\"\n}"
  },
  {
    "path": "test/tests/basic/mapping.json",
    "content": "{\n  \"types\": {\n    \"person\": {\n      \"properties\": {\n        \"name\": {\n          \"fields\": [\n            {\n              \"include_term_vectors\": true,\n              \"include_in_all\": true,\n              \"index\": true,\n              \"store\": true,\n              \"analyzer\": \"en\",\n              \"type\": \"text\"\n            }\n          ],\n          \"dynamic\": true,\n          \"enabled\": true\n        },\n        \"id\": {\n          \"dynamic\": false,\n          \"enabled\": false\n        }\n      }\n    }\n  },\n  \"default_type\": \"person\"\n}"
  },
  {
    "path": "test/tests/basic/searches.json",
    "content": "[\n\t{\n\t\t\"comment\": \"test term search, exact match\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"term\": \"marti\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test term search, no match\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"term\": \"noone\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 0,\n\t\t\t\"hits\": []\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test match phrase search\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"match_phrase\": \"steve has\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test term search, no match\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"term\": \"walking\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 0,\n\t\t\t\"hits\": []\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test match search, matching due to analysis\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"fuzziness\": 0,\n\t\t\t\t\"prefix_length\": 0,\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"match\": \"walking\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"c\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test term prefix search\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"prefix\": \"bobble\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test simple query string\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"query\": \"+name:phone\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test numeric range, no lower bound\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"age\",\n\t\t\t\t\"max\": 30\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test numeric range, upper and lower bounds\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"age\",\n\t\t\t\t\"max\": 30,\n\t\t\t\t\"min\": 20\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test conjunction of numeric range, upper and lower bounds\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"conjuncts\": [\n\t\t\t\t\t{\n\t\t\t\t\t\t\"boost\": 1,\n\t\t\t\t\t\t\"field\": \"age\",\n\t\t\t\t\t\t\"min\": 20\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t\"boost\": 1,\n\t\t\t\t\t\t\"field\": \"age\",\n\t\t\t\t\t\t\"max\": 30\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test date range, no upper bound\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"birthday\",\n\t\t\t\t\"start\": \"2010-01-01\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"c\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test numeric range, no lower bound\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"birthday\",\n\t\t\t\t\"end\": \"2010-01-01\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test term search, matching inside an array\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"tags\",\n\t\t\t\t\"term\": \"gopher\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test term search, matching another element inside array\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"tags\",\n\t\t\t\t\"term\": \"belieber\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test term search, not present in array\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"tags\",\n\t\t\t\t\"term\": \"notintagsarray\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 0,\n\t\t\t\"hits\": []\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"with size 0, total should be 1, but hits empty\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 0,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"term\": \"marti\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": []\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"a search for doc a that includes tags field, verifies both values come back\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"fields\": [\"tags\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"term\": \"marti\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\",\n\t\t\t\t\t\"fields\": {\n\t\t\t\t\t\t\"tags\": [\"gopher\", \"belieber\"]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test fuzzy search, fuzziness 1 with match\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"term\": \"msrti\",\n\t\t\t\t\"fuzziness\": 1\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"highlight results\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"match\": \"long\"\n\t\t\t},\n\t\t\t\"highlight\": {\n\t\t\t\t\"fields\": [\"name\"]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\",\n\t\t\t\t\t\"fragments\": {\n\t\t\t\t\t\t\"name\": [\"steve has &lt;a&gt; <mark>long</mark> &amp; complicated name\"]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"highlight results without specifying fields\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"match\": \"long\"\n\t\t\t},\n\t\t\t\"highlight\": {}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\",\n\t\t\t\t\t\"fragments\": {\n\t\t\t\t\t\t\"name\": [\"steve has &lt;a&gt; <mark>long</mark> &amp; complicated name\"]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"request fields\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"fields\": [\"age\",\"birthday\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"match\": \"long\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\",\n\t\t\t\t\t\"fields\": {\n\t\t\t\t\t\t\"age\": 27,\n\t\t\t\t\t\t\"birthday\": \"2001-09-09T01:46:40Z\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"tests query string only containing MUST NOT clause, bug #193\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"query\": \"-title:mista\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 3,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"c\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"highlight results including non-matching field (which should be produced in its entirety, though unhighlighted)\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"match\": \"long\"\n\t\t\t},\n\t\t\t\"highlight\": {\n\t\t\t\t\"fields\": [\"name\", \"title\"]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\",\n\t\t\t\t\t\"fragments\": {\n\t\t\t\t\t\t\"name\": [\"steve has &lt;a&gt; <mark>long</mark> &amp; complicated name\"],\n\t\t\t\t\t\t\"title\": [\"missess\"]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"search and highlight an array field\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"tags\",\n\t\t\t\t\"match\": \"gopher\"\n\t\t\t},\n\t\t\t\"highlight\": {\n\t\t\t\t\"fields\": [\"tags\"]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\",\n\t\t\t\t\t\"fragments\": {\n\t\t\t\t\t\t\"tags\": [\"<mark>gopher</mark>\"]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"reproduce bug in prefix search\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"title\",\n\t\t\t\t\"prefix\": \"miss\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test match none\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"match_none\": {}\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 0,\n\t\t\t\"hits\": []\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test match all\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\": {}\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 4,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"c\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test doc id query\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"ids\": [\"b\", \"c\"]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"c\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test query string MUST and SHOULD\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"query\": \"+age:>20 missess\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 3,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"c\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"d\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test regexp matching term\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"regexp\": \"mar.*\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test regexp that should not match when properly anchored\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"regexp\": \"mar.\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 0,\n\t\t\t\"hits\": []\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test wildcard matching term\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"name\",\n\t\t\t\t\"wildcard\": \"mar*\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test boost - term query\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t{\n\t\t\t\t\t\t\"field\": \"name\",\n\t\t\t\t\t\t\"term\": \"marti\",\n\t\t\t\t\t\t\"boost\": 1.0\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t\"field\": \"name\",\n\t\t\t\t\t\t\"term\": \"steve\",\n\t\t\t\t\t\t\"boost\": 5.0\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test boost - term query\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t{\n\t\t\t\t\t\t\"field\": \"name\",\n\t\t\t\t\t\t\"term\": \"marti\",\n\t\t\t\t\t\t\"boost\": 1.0\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t\"fuzziness\": 1,\n\t\t\t\t\t\t\"field\": \"name\",\n\t\t\t\t\t\t\"term\": \"steve\",\n\t\t\t\t\t\t\"boost\": 5.0\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test boost - numeric range query\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t{\n\t\t\t\t\t\t\"field\": \"name\",\n\t\t\t\t\t\t\"term\": \"marti\",\n\t\t\t\t\t\t\"boost\": 1.0\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t\"field\": \"age\",\n\t\t\t\t\t\t\"min\": 25,\n\t\t\t\t\t\t\"max\": 29,\n\t\t\t\t\t\t\"boost\": 50.0\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test boost - regexp query\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"disjuncts\": [\n\t\t\t\t\t{\n\t\t\t\t\t\t\"field\": \"name\",\n\t\t\t\t\t\t\"term\": \"marti\",\n\t\t\t\t\t\t\"boost\": 1.0\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t\"field\": \"name\",\n\t\t\t\t\t\t\"regexp\": \"stev.*\",\n\t\t\t\t\t\t\"boost\": 5.0\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test wildcard inside query string\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"query\": \"name:mar*\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test regexp inside query string\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"query\": \"name:/mar.*/\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"comment\": \"test term range\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"title\",\n\t\t\t\t\"max\": \"miz\",\n\t\t\t\t\"min\": \"mis\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}\n]\n"
  },
  {
    "path": "test/tests/employee/data/emp10508560.json",
    "content": "{\n  \"salary\": 104561.8,\n  \"_type\": \"emp\",\n  \"name\": \"Deirdre Reed\",\n  \"mutated\": 0,\n  \"is_manager\": true,\n  \"dept\": \"Accounts\",\n  \"join_date\": \"2003-05-28T21:29:00\",\n  \"manages\": {\n    \"team_size\": 9,\n    \"reports\": [\n      \"Gallia Julián\",\n      \"Duvessa Nicolás\",\n      \"Beryl Thomas\",\n      \"Deirdre Julián\",\n      \"Antonia Gerónimo\",\n      \"Ciara Young\",\n      \"Riona Richardson IX\",\n      \"Severin Jr.\",\n      \"Perdita Morgan\"\n    ]\n  },\n  \"languages_known\": [\n    \"English\",\n    \"Spanish\",\n    \"German\",\n    \"Italian\",\n    \"French\",\n    \"Arabic\",\n    \"Africans\",\n    \"Hindi\",\n    \"Vietnamese\",\n    \"Urdu\",\n    \"Dutch\",\n    \"Quechua\",\n    \"Japanese\",\n    \"Chinese\",\n    \"Nepalese\",\n    \"Thai\",\n    \"Malay\"\n  ],\n  \"emp_id\": \"10508560\",\n  \"email\": \"deirdre@mcdiabetes.com\"\n}"
  },
  {
    "path": "test/tests/employee/mapping.json",
    "content": "{}"
  },
  {
    "path": "test/tests/employee/searches.json",
    "content": "[\n\t{\n\t\t\"comment\": \"test array position output\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"manages.reports\",\n\t\t\t\t\"term\": \"julián\"\n\t\t\t},\n            \"includeLocations\": true\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"emp10508560\",\n\t\t\t\t\t\"locations\": {\n\t\t\t\t\t\t\"manages.reports\": {\n\t\t\t\t\t\t\t\"julián\": [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\"pos\": 2,\n\t\t\t\t\t\t\t\t\t\"start\": 7,\n\t\t\t\t\t\t\t\t\t\"end\": 14,\n\t\t\t\t\t\t\t\t\t\"array_positions\":[0]\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\"pos\": 2,\n\t\t\t\t\t\t\t\t\t\"start\": 8,\n\t\t\t\t\t\t\t\t\t\"end\": 15,\n\t\t\t\t\t\t\t\t\t\"array_positions\":[3]\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}\n]"
  },
  {
    "path": "test/tests/facet/data/a.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"book\",\n\t\"rating\": 2,\n\t\"updated\": \"2014-11-25\"\n}"
  },
  {
    "path": "test/tests/facet/data/b.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"book\",\n\t\"rating\": 7,\n\t\"updated\": \"2013-07-25\"\n}"
  },
  {
    "path": "test/tests/facet/data/c.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"book\",\n\t\"rating\": 1,\n\t\"updated\": \"2014-03-03\"\n}"
  },
  {
    "path": "test/tests/facet/data/d.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"book\",\n\t\"rating\": 9,\n\t\"updated\": \"2014-09-16\"\n}"
  },
  {
    "path": "test/tests/facet/data/e.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"book\",\n\t\"rating\": 5,\n\t\"updated\": \"2014-11-15\"\n}"
  },
  {
    "path": "test/tests/facet/data/f.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"movie\",\n\t\"rating\": 3,\n\t\"updated\": \"2017-06-05\"\n}"
  },
  {
    "path": "test/tests/facet/data/g.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"movie\",\n\t\"rating\": 9,\n\t\"updated\": \"2011-10-03\"\n}"
  },
  {
    "path": "test/tests/facet/data/h.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"movie\",\n\t\"rating\": 9,\n\t\"updated\": \"2019-08-26\"\n}"
  },
  {
    "path": "test/tests/facet/data/i.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"movie\",\n\t\"rating\": 1,\n\t\"updated\": \"2014-12-14\"\n}"
  },
  {
    "path": "test/tests/facet/data/j.json",
    "content": "{\n\t\"category\": \"inventory\",\n\t\"type\": \"game\",\n\t\"rating\": 9,\n\t\"updated\": \"2013-10-20\"\n}"
  },
  {
    "path": "test/tests/facet/mapping.json",
    "content": "{}"
  },
  {
    "path": "test/tests/facet/searches.json",
    "content": "[\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 0,\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"category\",\n\t\t\t\t\"term\": \"inventory\"\n\t\t\t},\n\t\t\t\"facets\": {\n\t\t\t\t\"types\": {\n\t\t\t\t\t\"size\": 3,\n\t\t\t\t\t\"field\": \"type\"\n\t\t\t\t}\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 10,\n\t\t\t\"hits\": [],\n\t\t\t\"facets\": {\n\t\t\t\t\"types\": {\n\t\t\t\t\t\"field\": \"type\",\n\t\t\t\t\t\"total\": 10,\n\t\t\t\t\t\"missing\": 0,\n\t\t\t\t\t\"other\": 0,\n\t\t\t\t\t\"terms\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"term\": \"book\",\n\t\t\t\t\t\t\t\"count\": 5\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"term\": \"movie\",\n\t\t\t\t\t\t\t\"count\": 4\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"term\": \"game\",\n\t\t\t\t\t\t\t\"count\": 1\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 0,\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"category\",\n\t\t\t\t\"term\": \"inventory\"\n\t\t\t},\n\t\t\t\"facets\": {\n\t\t\t\t\"types\": {\n\t\t\t\t\t\"size\": 3,\n\t\t\t\t\t\"field\": \"rating\",\n\t\t\t\t\t\"numeric_ranges\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"low\",\n\t\t\t\t\t\t\t\"max\": 5\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"high\",\n\t\t\t\t\t\t\t\"min\": 5\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 10,\n\t\t\t\"hits\": [],\n\t\t\t\"facets\": {\n\t\t\t\t\"types\": {\n\t\t\t\t\t\"field\": \"rating\",\n\t\t\t\t\t\"total\": 10,\n\t\t\t\t\t\"missing\": 0,\n\t\t\t\t\t\"other\": 0,\n\t\t\t\t\t\"numeric_ranges\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"high\",\n\t\t\t\t\t\t\t\"count\": 6,\n\t\t\t\t\t\t\t\"min\": 5\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"low\",\n\t\t\t\t\t\t\t\"count\": 4,\n\t\t\t\t\t\t\t\"max\": 5\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 0,\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"category\",\n\t\t\t\t\"term\": \"inventory\"\n\t\t\t},\n\t\t\t\"facets\": {\n\t\t\t\t\"types\": {\n\t\t\t\t\t\"size\": 3,\n\t\t\t\t\t\"field\": \"updated\",\n\t\t\t\t\t\"date_ranges\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"old\",\n\t\t\t\t\t\t\t\"end\": \"2012-01-01\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"new\",\n\t\t\t\t\t\t\t\"start\": \"2012-01-01\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 10,\n\t\t\t\"hits\": [],\n\t\t\t\"facets\": {\n\t\t\t\t\"types\": {\n\t\t\t\t\t\"field\": \"updated\",\n\t\t\t\t\t\"total\": 10,\n\t\t\t\t\t\"missing\": 0,\n\t\t\t\t\t\"other\": 0,\n\t\t\t\t\t\"date_ranges\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"new\",\n\t\t\t\t\t\t\t\"count\": 9,\n\t\t\t\t\t\t\t\"start\": \"2012-01-01T00:00:00Z\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"name\": \"old\",\n\t\t\t\t\t\t\t\"count\": 1,\n\t\t\t\t\t\t\t\"end\": \"2012-01-01T00:00:00Z\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n]"
  },
  {
    "path": "test/tests/fosdem/data/3311@FOSDEM15@fosdem.org.json",
    "content": "{\n\t\"description\": \"From Prolog to Erlang to Haskell to Lisp to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful\",\n\t\"category\": \"Word\"\n}"
  },
  {
    "path": "test/tests/fosdem/data/3492@FOSDEM15@fosdem.org.json",
    "content": "{\n\t\"description\": \"different cats\",\n\t\"category\": \"Perl\"\n}"
  },
  {
    "path": "test/tests/fosdem/data/3496@FOSDEM15@fosdem.org.json",
    "content": "{\n\t\"description\": \"many cats\",\n\t\"category\": \"Perl\"\n}"
  },
  {
    "path": "test/tests/fosdem/data/3505@FOSDEM15@fosdem.org.json",
    "content": "{\n\t\"description\": \"From Prolog to Erlang to Haskell to Lisp to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful\",\n\t\"category\": \"Perl\"\n}"
  },
  {
    "path": "test/tests/fosdem/data/3507@FOSDEM15@fosdem.org.json",
    "content": "{\n\t\"description\": \"From Prolog to Erlang to Haskell to Gel to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful\",\n\t\"category\": \"Perl\"\n}"
  },
  {
    "path": "test/tests/fosdem/mapping.json",
    "content": "{\n  \"default_mapping\": {\n    \"enabled\": true,\n    \"dynamic\": true,\n    \"properties\": {\n      \"category\": {\n        \"enabled\": true,\n        \"dynamic\": true,\n        \"fields\": [\n          {\n            \"type\": \"text\",\n            \"analyzer\": \"keyword\",\n            \"store\": true,\n            \"index\": true,\n            \"include_term_vectors\": true,\n            \"include_in_all\": true\n          }\n        ],\n        \"default_analyzer\": \"\"\n      },\n      \"description\": {\n        \"enabled\": true,\n        \"dynamic\": true,\n        \"fields\": [\n          {\n            \"type\": \"text\",\n            \"analyzer\": \"en\",\n            \"store\": true,\n            \"index\": true,\n            \"include_term_vectors\": true,\n            \"include_in_all\": true\n          }\n        ],\n        \"default_analyzer\": \"\"\n      },\n      \"summary\": {\n        \"enabled\": true,\n        \"dynamic\": true,\n        \"fields\": [\n          {\n            \"type\": \"text\",\n            \"analyzer\": \"en\",\n            \"store\": true,\n            \"index\": true,\n            \"include_term_vectors\": true,\n            \"include_in_all\": true\n          }\n        ],\n        \"default_analyzer\": \"\"\n      },\n      \"url\": {\n        \"enabled\": true,\n        \"dynamic\": true,\n        \"fields\": [\n          {\n            \"type\": \"text\",\n            \"analyzer\": \"keyword\",\n            \"store\": true,\n            \"index\": true,\n            \"include_term_vectors\": true,\n            \"include_in_all\": true\n          }\n        ],\n        \"default_analyzer\": \"\"\n      }\n    },\n    \"default_analyzer\": \"\"\n  },\n  \"type_field\": \"_type\",\n  \"default_type\": \"_default\",\n  \"default_analyzer\": \"en\",\n  \"default_datetime_parser\": \"dateTimeOptional\",\n  \"default_field\": \"_all\",\n  \"byte_array_converter\": \"json\",\n  \"analysis\": {}\n}"
  },
  {
    "path": "test/tests/fosdem/searches.json",
    "content": "[\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"category\",\n\t\t\t\t\"match_phrase\": \"Perl\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 4,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3492@FOSDEM15@fosdem.org\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3496@FOSDEM15@fosdem.org\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3505@FOSDEM15@fosdem.org\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3507@FOSDEM15@fosdem.org\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"match\": \"lisp\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3311@FOSDEM15@fosdem.org\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3505@FOSDEM15@fosdem.org\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\"boost\":1,\"query\":\"+lisp +category:Perl\"}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3505@FOSDEM15@fosdem.org\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\"boost\":1,\"query\":\"+lisp +category:\\\"Perl\\\"\"}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3505@FOSDEM15@fosdem.org\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n                \"must\": {\n                    \"conjuncts\":[\n                    {\"boost\":1,\"query\":\"+cats\"},\n                    {\"field\":\"category\",\"match_phrase\":\"Perl\"}\n                    ]\n                }\n            }\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 2,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3492@FOSDEM15@fosdem.org\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"3496@FOSDEM15@fosdem.org\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}\n]\n"
  },
  {
    "path": "test/tests/geo/data/amoeba_brewery.json",
    "content": "{\"name\":\"amoeba brewery\",\"city\":\"bangalore\",\"state\":\"KAR\",\"code\":\"\",\"country\":\"India\",\"phone\":\"\",\"website\":\"\",\"type\":\"brewery\",\"updated\":\"2019-09-17 20:00:20\",\"description\":\"brewery near cb office\",\"address\":[],\"geo\":{\"accuracy\":\"APPROXIMATE\",\"lat\":12.97467,\"lon\":77.60490}}"
  },
  {
    "path": "test/tests/geo/data/brewpub_on_the_green.json",
    "content": "{\"name\":\"Brewpub-on-the-Green\",\"city\":\"Fremont\",\"state\":\"California\",\"code\":\"\",\"country\":\"United States\",\"phone\":\"\",\"website\":\"\",\"type\":\"brewery\",\"updated\":\"2010-07-22 20:00:20\",\"description\":\"\",\"address\":[],\"geo\":{\"accuracy\":\"APPROXIMATE\",\"lat\":37.5483,\"lon\":-121.989}}"
  },
  {
    "path": "test/tests/geo/data/capital_city_brewing_company.json",
    "content": "{\"name\":\"Capital City Brewing Company\",\"city\":\"Washington\",\"state\":\"District of Columbia\",\"code\":\"20005\",\"country\":\"United States\",\"phone\":\"202.628.2222\",\"website\":\"http://www.capcitybrew.com\",\"type\":\"brewery\",\"updated\":\"2010-07-22 20:00:20\",\"description\":\"Washington DC's first brewpub since prohibition, Capitol City Brewing Co. opened its doors in 1992. Our first location still stands in Downtown DC, at 11th and H St., NW. Our company policy is to bring the fine craft of brewing to every person who lives and visits our region, as well as treating them to a wonderful meal and a great experience.\",\"address\":[\"1100 New York Ave, NW\"],\"geo\":{\"accuracy\":\"ROOFTOP\",\"lat\":38.8999,\"lon\":-77.0272}}"
  },
  {
    "path": "test/tests/geo/data/communiti_brewery.json",
    "content": "{\"name\":\"communiti brewery\",\"city\":\"bangalore\",\"state\":\"KAR\",\"code\":\"\",\"country\":\"India\",\"phone\":\"\",\"website\":\"\",\"type\":\"brewery\",\"updated\":\"2019-09-17 20:00:20\",\"description\":\"brewery near cb office\",\"address\":[],\"geo\":{\"accuracy\":\"APPROXIMATE\",\"lat\":12.97237,\"lon\":77.608237}}"
  },
  {
    "path": "test/tests/geo/data/firehouse_grill_brewery.json",
    "content": "{\"name\":\"Firehouse Grill & Brewery\",\"city\":\"Sunnyvale\",\"state\":\"California\",\"code\":\"94086\",\"country\":\"United States\",\"phone\":\"1-408-773-9500\",\"website\":\"\",\"type\":\"brewery\",\"updated\":\"2010-07-22 20:00:20\",\"description\":\"\",\"address\":[\"111 South Murphy Avenue\"],\"geo\":{\"accuracy\":\"RANGE_INTERPOLATED\",\"lat\":37.3775,\"lon\":-122.03}}"
  },
  {
    "path": "test/tests/geo/data/hook_ladder_brewing_company.json",
    "content": "{\"name\":\"Hook & Ladder Brewing Company\",\"city\":\"Silver Spring\",\"state\":\"Maryland\",\"code\":\"20910\",\"country\":\"United States\",\"phone\":\"301.565.4522\",\"website\":\"http://www.hookandladderbeer.com\",\"type\":\"brewery\",\"updated\":\"2010-07-22 20:00:20\",\"description\":\"At Hook & Ladder Brewing we believe in great beer in the company of good friends, so we bring you three great beers for your drinking pleasure (please drink responsibly). Each of our beers is carefully crafted with the finest quality ingredients for a distinctive taste we know you will enjoy. Try one tonight, you just might get hooked. Through our own experiences in the fire and rescue service we have chosen the Hook & Ladder as a symbol of pride and honor to pay tribute to the brave men and women who serve and protect our communities.\",\"address\":[\"8113 Fenton St.\"],\"geo\":{\"accuracy\":\"ROOFTOP\",\"lat\":38.9911,\"lon\":-77.0237}}"
  },
  {
    "path": "test/tests/geo/data/jack_s_brewing.json",
    "content": "{\"name\":\"Jack's Brewing\",\"city\":\"Fremont\",\"state\":\"California\",\"code\":\"94538\",\"country\":\"United States\",\"phone\":\"1-510-796-2036\",\"website\":\"\",\"type\":\"brewery\",\"updated\":\"2010-07-22 20:00:20\",\"description\":\"\",\"address\":[\"39176 Argonaut Way\"],\"geo\":{\"accuracy\":\"ROOFTOP\",\"lat\":37.5441,\"lon\":-121.988}}"
  },
  {
    "path": "test/tests/geo/data/social_brewery.json",
    "content": "{\"name\":\"social brewery\",\"city\":\"bangalore\",\"state\":\"KAR\",\"code\":\"\",\"country\":\"India\",\"phone\":\"\",\"website\":\"\",\"type\":\"brewery\",\"updated\":\"2019-09-17 20:00:20\",\"description\":\"brewery near cb office, but outside the polygon\",\"address\":[],\"geo\":{\"accuracy\":\"APPROXIMATE\",\"lat\":12.9736946,\"lon\":77.6042133}}"
  },
  {
    "path": "test/tests/geo/data/sweet_water_tavern_and_brewery.json",
    "content": "{\"name\":\"Sweet Water Tavern and Brewery\",\"city\":\"Sterling\",\"state\":\"Virginia\",\"code\":\"20121\",\"country\":\"United States\",\"phone\":\"(703) 449-1108\",\"website\":\"http://www.greatamericanrestaurants.com/sweetMainSter/index.htm\",\"type\":\"brewery\",\"updated\":\"2010-07-22 20:00:20\",\"description\":\"\",\"address\":[\"45980 Waterview Plaza\"],\"geo\":{\"accuracy\":\"RANGE_INTERPOLATED\",\"lat\":39.0324,\"lon\":-77.4097}}"
  },
  {
    "path": "test/tests/geo/mapping.json",
    "content": "{\n  \"types\": {\n    \"brewery\": {\n      \"properties\": {\n        \"name\": {\n          \"fields\": [\n            {\n              \"include_term_vectors\": true,\n              \"include_in_all\": true,\n              \"index\": true,\n              \"store\": true,\n              \"analyzer\": \"keyword\",\n              \"type\": \"text\"\n            }\n          ],\n          \"dynamic\": true,\n          \"enabled\": true\n        },\n        \"geo\": {\n          \"fields\": [\n            {\n              \"include_term_vectors\": true,\n              \"include_in_all\": true,\n              \"index\": true,\n              \"store\": true,\n              \"type\": \"geopoint\"\n            }\n          ],\n          \"dynamic\": true,\n          \"enabled\": true\n        }\n      }\n    }\n  },\n  \"default_type\": \"brewery\"\n}\n"
  },
  {
    "path": "test/tests/geo/searches.json",
    "content": "[\n  {\n    \"comment\": \"breweries near the couchbase office\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"location\": {\n          \"lon\": -122.107799,\n          \"lat\": 37.399285\n        },\n        \"distance\": \"100mi\",\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        {\n          \"by\": \"geo_distance\",\n          \"field\": \"geo\",\n          \"unit\": \"mi\",\n          \"location\": {\n            \"lon\": -122.107799,\n            \"lat\": 37.399285\n          }\n        }\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 3,\n      \"hits\": [\n        {\n          \"id\": \"firehouse_grill_brewery\"\n        },\n        {\n          \"id\": \"jack_s_brewing\"\n        },\n        {\n          \"id\": \"brewpub_on_the_green\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"breweries near the whitehouse\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"location\": {\n          \"lon\": -77.0365,\n          \"lat\": 38.8977\n        },\n        \"distance\": \"100mi\",\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        {\n          \"by\": \"geo_distance\",\n          \"field\": \"geo\",\n          \"unit\": \"mi\",\n          \"location\": {\n            \"lon\": -77.0365,\n            \"lat\": 38.8977\n          }\n        }\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 3,\n      \"hits\": [\n        {\n          \"id\": \"capital_city_brewing_company\"\n        },\n        {\n          \"id\": \"hook_ladder_brewing_company\"\n        },\n        {\n          \"id\": \"sweet_water_tavern_and_brewery\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"bounding box of USA\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"top_left\": {\n          \"lon\": -125.0011,\n          \"lat\": 49.5904\n        },\n        \"bottom_right\": {\n          \"lon\": -66.9326,\n          \"lat\": 24.9493\n        },\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        \"name\"\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 6,\n      \"hits\": [\n        {\n          \"id\": \"brewpub_on_the_green\"\n        },\n        {\n          \"id\": \"capital_city_brewing_company\"\n        },\n        {\n          \"id\": \"firehouse_grill_brewery\"\n        },\n        {\n          \"id\": \"hook_ladder_brewing_company\"\n        },\n        {\n          \"id\": \"jack_s_brewing\"\n        },\n        {\n          \"id\": \"sweet_water_tavern_and_brewery\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"bounding box around DC area\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"top_left\": {\n          \"lon\": -78,\n          \"lat\": 39.5\n        },\n        \"bottom_right\": {\n          \"lon\": -76,\n          \"lat\": 38.5\n        },\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        \"name\"\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 3,\n      \"hits\": [\n        {\n          \"id\": \"capital_city_brewing_company\"\n        },\n        {\n          \"id\": \"hook_ladder_brewing_company\"\n        },\n        {\n          \"id\": \"sweet_water_tavern_and_brewery\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"breweries near the couchbase office, using GeoJSON style points\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"location\": [-122.107799,37.399285],\n        \"distance\": \"100mi\",\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        {\n          \"by\": \"geo_distance\",\n          \"field\": \"geo\",\n          \"unit\": \"mi\",\n          \"location\": [-122.107799,37.399285]\n        }\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 3,\n      \"hits\": [\n        {\n          \"id\": \"firehouse_grill_brewery\"\n        },\n        {\n          \"id\": \"jack_s_brewing\"\n        },\n        {\n          \"id\": \"brewpub_on_the_green\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"bounding box around DC area, using GeoJSON style\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"top_left\": [-78,39.5],\n        \"bottom_right\": [-76,38.5],\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        \"name\"\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 3,\n      \"hits\": [\n        {\n          \"id\": \"capital_city_brewing_company\"\n        },\n        {\n          \"id\": \"hook_ladder_brewing_company\"\n        },\n        {\n          \"id\": \"sweet_water_tavern_and_brewery\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"polygon around cb office area, using GeoJSON lat/lon as array\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"polygon_points\": [[77.607749,12.974872],[77.6101101,12.971725],[77.606912,12.972530],[77.603780,12.975112]],\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        \"name\"\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 2,\n      \"hits\": [\n        {\n          \"id\": \"amoeba_brewery\"\n        },\n        {\n          \"id\": \"communiti_brewery\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"polygon around cb office area, using GeoJSON lat/lon as string\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"polygon_points\": [\"12.974872, 77.607749\",\"12.971725, 77.6101101\",\"12.972530, 77.606912\",\"12.975112, 77.603780\"],\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        \"name\"\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 2,\n      \"hits\": [\n        {\n          \"id\": \"amoeba_brewery\"\n        },\n        {\n          \"id\": \"communiti_brewery\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"polygon around cb office area\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"polygon_points\": [{\"lat\":12.974872, \"lon\":77.607749}, {\"lat\":12.971725, \"lon\":77.6101101},\n          {\"lat\":12.972530, \"lon\":77.606912}, {\"lat\":12.975112, \"lon\":77.603780}],\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        \"name\"\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 2,\n      \"hits\": [\n        {\n          \"id\": \"amoeba_brewery\"\n        },\n        {\n          \"id\": \"communiti_brewery\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"polygon around cb office area as geohash\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"polygon_points\": [\"tdr1y40\", \"tdr1y13\", \"tdr1vcx\", \"tdr1vfj\"],\n        \"field\": \"geo\"\n      },\n      \"sort\": [\n        \"name\"\n      ]\n    },\n    \"result\": {\n      \"total_hits\": 2,\n      \"hits\": [\n        {\n          \"id\": \"amoeba_brewery\"\n        },\n        {\n          \"id\": \"communiti_brewery\"\n        }\n      ]\n    }\n  }\n]\n\n"
  },
  {
    "path": "test/tests/geoshapes/data/circle_halairport.json",
    "content": "{\n  \"name\": \"hal airpork circular region\",\n  \"city\": \"bangalore\",\n  \"type\": \"geoshapes\",\n  \"description\": \"circle covering the hal airport\",\n  \"region\": {\n    \"type\": \"Circle\",\n    \"coordinates\": [\n      77.6698637008667,\n      12.951865687866821\n    ],\n    \"radius\": \"2.4km\"\n  }\n}"
  },
  {
    "path": "test/tests/geoshapes/data/envelope_brockwell_park.json",
    "content": "{\n  \"name\": \"brockwell park envelope\",\n  \"city\": \"london\",\n  \"type\": \"geoshapes\",\n  \"description\": \"brockwell park envelope\",\n  \"region\": {\n    \"type\": \"envelope\",\n    \"coordinates\": [\n      [\n        -0.11278152465820314,\n        51.44579626059569\n      ],\n      [\n        -0.10037899017333984,\n        51.45566490761856\n      ]\n    ]\n  }\n}"
  },
  {
    "path": "test/tests/geoshapes/data/geometrycollection_tvm.json",
    "content": "{\n    \"name\": \"geometrycollection comprised of various shapes\",\n    \"city\": \"bangalore\",\n    \"type\": \"geoshapes\",\n    \"description\": \"geometrycollection comprised of various shapes\",\n    \"region\": {\n        \"type\": \"geometrycollection\",\n        \"geometries\": [\n            {\n                \"type\": \"point\",\n                \"coordinates\": [\n                    76.92815780639648,\n                    8.525851789596233\n                ]\n            },\n            {\n                \"type\": \"LineString\",\n                \"coordinates\": [\n                    [\n                        76.92867279052734,\n                        8.490369393806219\n                    ],\n                    [\n                        76.94377899169922,\n                        8.494104537551882\n                    ]\n                ]\n            },\n            {\n                \"type\": \"polygon\",\n                \"coordinates\": [\n                    [\n                        [\n                            76.92815780639648,\n                            8.525851789596233\n                        ],\n                        [\n                            76.92060470581055,\n                            8.520504174874656\n                        ],\n                        [\n                            76.92206382751465,\n                            8.519061154914393\n                        ],\n                        [\n                            76.92824363708496,\n                            8.519061154914393\n                        ],\n                        [\n                            76.92970275878906,\n                            8.523475081176768\n                        ],\n                        [\n                            76.92815780639648,\n                            8.525851789596233\n                        ]\n                    ]\n                ]\n            },\n            {\n                \"type\": \"multipoint\",\n                \"coordinates\": [\n                    [\n                        76.90670013427733,\n                        8.497839644932787\n                    ],\n                    [\n                        76.94137573242188,\n                        8.485275957394883\n                    ]\n                ]\n            },\n            {\n                \"type\": \"multiLineString\",\n                \"coordinates\": [\n                    [\n                        [\n                            76.89322471618651,\n                            8.521522773921424\n                        ],\n                        [\n                            76.89648628234863,\n                            8.518042549311815\n                        ]\n                    ],\n                    [\n                        [\n                            76.9068717956543,\n                            8.494783650690053\n                        ],\n                        [\n                            76.93296432495117,\n                            8.468552033040881\n                        ]\n                    ]\n                ]\n            },\n            {\n                \"type\": \"multipolygon\",\n                \"coordinates\": [\n                    [\n                        [\n                            [\n                                76.90249443054199,\n                                8.546138091708775\n                            ],\n                            [\n                                76.89983367919922,\n                                8.541300033890494\n                            ],\n                            [\n                                76.90498352050781,\n                                8.53985709248573\n                            ],\n                            [\n                                76.90858840942383,\n                                8.54520443620746\n                            ],\n                            [\n                                76.90712928771973,\n                                8.548090273095957\n                            ],\n                            [\n                                76.90249443054199,\n                                8.546138091708775\n                            ]\n                        ]\n                    ],\n                    [\n                        [\n                            [\n                                76.88326835632324,\n                                8.564131732621458\n                            ],\n                            [\n                                76.88429832458496,\n                                8.555729147617923\n                            ],\n                            [\n                                76.88893318176268,\n                                8.552079482230221\n                            ],\n                            [\n                                76.89339637756348,\n                                8.55369212938781\n                            ],\n                            [\n                                76.89494132995605,\n                                8.56133089156368\n                            ],\n                            [\n                                76.89116477966309,\n                                8.566423314514562\n                            ],\n                            [\n                                76.88326835632324,\n                                8.564131732621458\n                            ]\n                        ]\n                    ]\n                ]\n            }\n        ]\n    }\n}\n"
  },
  {
    "path": "test/tests/geoshapes/data/linestring_putney_bridge.json",
    "content": "{\n    \"name\": \"linestring for putney bridge\",\n    \"city\": \"london\",\n    \"type\": \"geoshapes\",\n    \"description\": \"linestring for putney bridge\",\n    \"region\": {\n        \"type\": \"linestring\",\n        \"coordinates\": [\n            [\n                -0.21183013916015625,\n                51.46791083061189\n            ],\n            [\n                -0.21431922912597656,\n                51.465504685939706\n            ]\n        ]\n    }\n}"
  },
  {
    "path": "test/tests/geoshapes/data/multilinestring_old_airport_road.json",
    "content": "{\n    \"name\": \"road routes\",\n    \"city\": \"bangalore\",\n    \"type\": \"geoshapes\",\n    \"description\": \"multilinestrings approximating the roads indiranagar 100ft and old airport port road\",\n    \"region\": {\n        \"type\": \"multilinestring\",\n        \"coordinates\": [\n            [\n                [\n                    77.64081001281738,\n                    12.983398626256326\n                ],\n                [\n                    77.64166831970213,\n                    12.960648472679763\n                ]\n            ],\n             [   [\n                    77.64192581176758,\n                    12.960564828571133\n                ],\n                [\n                    77.66990661621094,\n                    12.958390071883693\n                ]\n            ],\n            [    [\n                    77.67016410827637,\n                    12.958055492245812\n                ],\n                [\n                    77.68106460571289,\n                    12.954626025039444\n                ]\n            ],\n              [  [\n                    77.68149375915527,\n                    12.954542378907867\n                ],\n                [\n                    77.7011489868164,\n                    12.957219041184294\n                ]\n            ]\n        ]\n    }\n}\n"
  },
  {
    "path": "test/tests/geoshapes/data/multipoint_blr_stadiums.json",
    "content": "{\n  \"name\": \"multipoints for stadiums\",\n  \"city\": \"bangalore\",\n  \"type\": \"geoshapes\",\n  \"description\": \"contains 3 points\",\n  \"region\": {\n    \"type\": \"multipoint\",\n    \"coordinates\": [\n      [\n        77.5929594039917,\n        12.969347306502671\n      ],\n      [\n        77.6004695892334,\n        12.979007674139009\n      ],\n      [\n        77.60068416595459,\n        12.961735843534306\n      ]\n    ]\n  }\n}"
  },
  {
    "path": "test/tests/geoshapes/data/multipolygon_london_parks.json",
    "content": "{\n  \"name\": \"london parks as multipolygon\",\n  \"city\": \"london\",\n  \"type\": \"geoshapes\",\n  \"description\": \"multipolygon with london\",\n  \"region\": {\n    \"type\": \"MultiPolygon\",\n    \"coordinates\": [\n      [\n        [\n          [\n            -0.163421630859375,\n            51.531600743186644\n          ],\n          [\n            -0.15277862548828125,\n            51.52455221546295\n          ],\n          [\n            -0.14556884765625,\n            51.524979430024345\n          ],\n          [\n            -0.14591217041015625,\n            51.536085601784755\n          ],\n          [\n            -0.15895843505859375,\n            51.53693981046689\n          ],\n          [\n            -0.163421630859375,\n            51.531600743186644\n          ]\n        ]\n      ],\n      [\n        [\n          [\n            -0.1902008056640625,\n            51.5091698216777\n          ],\n          [\n            -0.1888275146484375,\n            51.50147667659363\n          ],\n          [\n            -0.15071868896484375,\n            51.503186376638006\n          ],\n          [\n            -0.1599884033203125,\n            51.51322956905176\n          ],\n          [\n            -0.1902008056640625,\n            51.5091698216777\n          ]\n        ]\n      ],\n      [\n        [\n          [\n            -0.16582489013671875,\n            51.4811690848672\n          ],\n          [\n            -0.1635932922363281,\n            51.474861202507434\n          ],\n          [\n            -0.14883041381835938,\n            51.47764105478667\n          ],\n          [\n            -0.14951705932617188,\n            51.48352095330697\n          ],\n          [\n            -0.16582489013671875,\n            51.4811690848672\n          ]\n        ]\n      ]\n    ]\n  }\n}"
  },
  {
    "path": "test/tests/geoshapes/data/point_museum_of_london.json",
    "content": "{\n  \"name\": \"geopoint for the museum of london\",\n  \"city\": \"london\",\n  \"type\": \"geoshapes\",\n  \"description\": \"geopoint for the museum of london\",\n  \"region\": {\n    \"type\": \"point\",\n    \"coordinates\": [\n      -0.09613037109375,\n      51.51803669675129\n    ]\n  }\n}"
  },
  {
    "path": "test/tests/geoshapes/data/polygon_cubbonpark.json",
    "content": "{\n  \"name\": \"cubbon park polygon\",\n  \"city\": \"bangalore\",\n  \"type\": \"geoshapes\",\n  \"description\": \"polygon inside cubbon park\",\n  \"region\": {\n    \"type\": \"Polygon\",\n    \"coordinates\": [\n      [\n        [\n          77.58894681930542,\n          12.976498523818783\n        ],\n        [\n          77.58677959442139,\n          12.974533005048169\n        ],\n        [\n          77.5879168510437,\n          12.971333776381767\n        ],\n        [\n          77.58849620819092,\n          12.96800904416803\n        ],\n        [\n          77.59371042251587,\n          12.972128359891645\n        ],\n        [\n          77.59512662887573,\n          12.973842978816679\n        ],\n        [\n          77.59253025054932,\n          12.976853988320428\n        ],\n        [\n          77.58894681930542,\n          12.976498523818783\n        ]\n      ]\n    ]\n  }\n}"
  },
  {
    "path": "test/tests/geoshapes/mapping.json",
    "content": "{\n  \"types\": {\n    \"geoshapes\": {\n      \"properties\": {\n        \"name\": {\n          \"fields\": [\n            {\n              \"include_term_vectors\": true,\n              \"include_in_all\": true,\n              \"index\": true,\n              \"store\": true,\n              \"analyzer\": \"keyword\",\n              \"type\": \"text\"\n            }\n          ],\n          \"dynamic\": true,\n          \"enabled\": true\n        },\n        \"region\": {\n          \"fields\": [\n            {\n              \"include_term_vectors\": true,\n              \"include_in_all\": true,\n              \"index\": true,\n              \"store\": true,\n              \"type\": \"geoshape\"\n            }\n          ],\n          \"dynamic\": true,\n          \"enabled\": true\n        }\n      }\n    }\n  },\n  \"default_type\": \"geoshapes\"\n}"
  },
  {
    "path": "test/tests/geoshapes/searches.json",
    "content": "[\n  {\n    \"comment\": \"search with a circular shape within cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"Circle\",\n            \"coordinates\": [\n              77.59092092514038,\n              12.975494856600474\n            ],\n            \"radius\": \"0.1km\"\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a circular shape within cubbon park polygon, (circle doesn't fully contained within)\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"circle\",\n            \"coordinates\": [\n              77.59092092514038,\n              12.975494856600474\n            ],\n            \"radius\": \"150m\"\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 0,\n      \"hits\": []\n    }\n  },\n  {\n    \"comment\": \"search with a polygon that contains the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"Polygon\",\n            \"coordinates\": [\n              [\n                [\n                  77.58617877960205,\n                  12.9772303619447\n                ],\n                [\n                  77.58630752563477,\n                  12.966419848296587\n                ],\n                [\n                  77.59802341461182,\n                  12.968887279637073\n                ],\n                [\n                  77.5989246368408,\n                  12.980304058548604\n                ],\n                [\n                  77.58617877960205,\n                  12.9772303619447\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"within\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a multipolygon that intersects the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"multipolygon\",\n            \"coordinates\": [\n              [\n                [\n                  [\n                    77.58268117904663,\n                    12.980513152175025\n                  ],\n                  [\n                    77.58147954940794,\n                    12.977983107483992\n                  ],\n                  [\n                    77.58708000183104,\n                    12.97886130773254\n                  ],\n                  [\n                    77.58268117904663,\n                    12.980513152175025\n                  ]\n                ]\n              ],\n              [\n                [\n                  [\n                    77.5864577293396,\n                    12.97762764459667\n                  ],\n                  [\n                    77.58879661560059,\n                    12.975076660730531\n                  ],\n                  [\n                    77.59115695953369,\n                    12.979216768855913\n                  ],\n                  [\n                    77.5864577293396,\n                    12.97762764459667\n                  ]\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with multilinestrings that intersects the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"multilinestring\",\n            \"coordinates\": [\n              [\n                [\n                  77.58761644363403,\n                  12.974302996517075\n                ],\n                [\n                  77.59319543838501,\n                  12.978401298465434\n                ]\n              ],\n              [\n                [\n                  77.5947618484497,\n                  12.98500862259466\n                ],\n                [\n                  77.59808778762817,\n                  12.983565899088745\n                ]\n              ],\n              [\n                [\n                  77.60109186172485,\n                  12.973529329896703\n                ],\n                [\n                  77.59943962097168,\n                  12.970225537247586\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with multilinestrings that aren't contained within the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"multilinestring\",\n            \"coordinates\": [\n              [\n                [\n                  77.58761644363403,\n                  12.974302996517075\n                ],\n                [\n                  77.59319543838501,\n                  12.978401298465434\n                ]\n              ],\n              [\n                [\n                  77.5947618484497,\n                  12.98500862259466\n                ],\n                [\n                  77.59808778762817,\n                  12.983565899088745\n                ]\n              ],\n              [\n                [\n                  77.60109186172485,\n                  12.973529329896703\n                ],\n                [\n                  77.59943962097168,\n                  12.970225537247586\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 0,\n      \"hits\": []\n    }\n  },\n  {\n    \"comment\": \"search with multilinestrings that are all contained within the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"multilinestring\",\n            \"coordinates\": [\n              [\n                [\n                  77.59107112884521,\n                  12.975243939162915\n                ],\n                [\n                  77.59190797805786,\n                  12.973842978816679\n                ]\n              ],\n              [\n                [\n                  77.58954763412476,\n                  12.970685561638497\n                ],\n                [\n                  77.59117841720581,\n                  12.971835618893842\n                ]\n              ],\n              [\n                [\n                  77.58851766586304,\n                  12.973152950670608\n                ],\n                [\n                  77.58937597274779,\n                  12.972212000113458\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with point that is contained within the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"Point\",\n            \"coordinates\": [\n              77.59107112884521,\n              12.975243939162915\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with an envelope that is within the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"Envelope\",\n            \"coordinates\": [\n              [\n                77.59158611297607,\n                12.9720028995062035\n              ],\n              [\n                77.59263753890991,\n                12.973173860642571\n              ]\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with an envelope that contains the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"Envelope\",\n            \"coordinates\": [\n              [\n                77.57969856262207,\n                12.9641614998626\n              ],\n              [\n                77.60295867919922,\n                12.989336742847172\n              ]\n            ]\n          },\n          \"relation\": \"within\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a geometrycollection that is within the cubbon park polygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"geometrycollection\",\n            \"geometries\": [\n              {\n                \"type\": \"point\",\n                \"coordinates\": [\n                  77.59158611297607,\n                  12.972002899506203\n                ]\n              },\n              {\n                \"type\": \"LineString\",\n                \"coordinates\": [\n                  [\n                    77.58851766586304,\n                    12.973152950670608\n                  ],\n                  [\n                    77.58937597274779,\n                    12.972212000113458\n                  ]\n                ]\n              },\n              {\n                \"type\": \"polygon\",\n                \"coordinates\": [\n                  [\n                    [\n                      77.59055614471436,\n                      12.974721193688106\n                    ],\n                    [\n                      77.58954763412476,\n                      12.97350841995465\n                    ],\n                    [\n                      77.59141445159912,\n                      12.973382960265356\n                    ],\n                    [\n                      77.59055614471436,\n                      12.974721193688106\n                    ]\n                  ]\n                ]\n              }\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a polygon that intersects the hal airport region\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"polygon\",\n            \"coordinates\": [\n              [\n                [\n                  77.67934799194336,\n                  12.938147195017896\n                ],\n                [\n                  77.66793251037598,\n                  12.930492951786736\n                ],\n                [\n                  77.67711639404297,\n                  12.922127390141315\n                ],\n                [\n                  77.67934799194336,\n                  12.938147195017896\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"circle_halairport\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a linestring that intersects the hal airport and cubbon park region\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"linestring\",\n            \"coordinates\": [\n              [\n                77.59042739868164,\n                12.973529329896703\n              ],\n              [\n                77.65892028808594,\n                12.950109093741462\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 2,\n      \"hits\": [\n        {\n          \"id\": \"circle_halairport\"\n        },\n        {\n          \"id\": \"polygon_cubbonpark\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with an envelope within the circle_halairport\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"envelope\",\n            \"coordinates\": [\n              [\n                77.65625953674316,\n                12.943249893344905\n              ],\n              [\n                77.68355369567871,\n                12.945843027882455\n              ]\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"circle_halairport\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a circle which intersects the road multilinestring and the hal circle\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"circle\",\n            \"coordinates\": [\n              77.68132209777832,\n              12.954918786278716\n            ],\n            \"radius\": \"50m\"\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 2,\n      \"hits\": [\n        {\n          \"id\": \"circle_halairport\"\n        },\n        {\n          \"id\": \"multilinestring_old_airport_road\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a polygon which intersects the road multilinestring\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"polygon\",\n            \"coordinates\": [\n              [\n                [\n                  77.64102458953856,\n                  12.97751264178902\n                ],\n                [\n                  77.64109969139099,\n                  12.975317123441693\n                ],\n                [\n                  77.64338493347168,\n                  12.976728530319054\n                ],\n                [\n                  77.64102458953856,\n                  12.97751264178902\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"multilinestring_old_airport_road\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a linestring which intersects the road multilinestring\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"linestring\",\n            \"coordinates\": [\n              [\n                77.63969421386717,\n                12.978265386473618\n              ],\n              [\n                77.64354586601257,\n                12.978453572288663\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"multilinestring_old_airport_road\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with an envelope which intersects the road multilinestring\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"envelope\",\n            \"coordinates\": [\n              [\n                77.64100313186644,\n                12.95902786307307\n              ],\n              [\n                77.6419472694397,\n                12.96069029472353\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"multilinestring_old_airport_road\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with multipoint which are contained within the multipolygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"multipoint\",\n            \"coordinates\": [\n              [\n                -0.14797210693359375,\n                51.52615424940099\n              ],\n              [\n                -0.16857147216796875,\n                51.50863561745838\n              ],\n              [\n                -0.15535354614257812,\n                51.48010001366223\n              ]\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"multipolygon_london_parks\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with multilinestring that are contained within the multipolygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"multilinestring\",\n            \"coordinates\": [\n              [\n                [\n                  -0.17063140869140625,\n                  51.50884929989774\n                ],\n                [\n                  -0.15655517578125,\n                  51.5072466571743\n                ]\n              ],\n              [\n                [\n                  -0.16222000122070312,\n                  51.47988619641402\n                ],\n                [\n                  -0.15466690063476562,\n                  51.48074145939243\n                ]\n              ],\n              [\n                [\n                  -0.15844345092773438,\n                  51.53245503603458\n                ],\n                [\n                  -0.15123367309570312,\n                  51.53170753066937\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"multipolygon_london_parks\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with multilinestring out of which one isn't contained within the multipolygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"multilinestring\",\n            \"coordinates\": [\n              [\n                [\n                  -0.17063140869140625,\n                  51.50884929989774\n                ],\n                [\n                  -0.15655517578125,\n                  51.5072466571743\n                ]\n              ],\n              [\n                [\n                  -0.16222000122070312,\n                  51.47988619641402\n                ],\n                [\n                  -0.15466690063476562,\n                  51.48074145939243\n                ]\n              ],\n              [\n                [\n                  -0.15844345092773438,\n                  51.53245503603458\n                ],\n                [\n                  -0.15123367309570312,\n                  51.53170753066937\n                ]\n              ],\n              [\n                [\n                  -0.08651733398437499,\n                  51.51013137348817\n                ],\n                [\n                  -0.08909225463867188,\n                  51.50543026060529\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 0,\n      \"hits\": []\n    }\n  },\n  {\n    \"comment\": \"search with a geometrycollection that contains the london_parks_multipolygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"geometrycollection\",\n            \"geometries\": [\n              {\n                \"type\": \"multipolygon\",\n                \"coordinates\": [\n                  [\n                    [\n                      [\n                        -0.19517898559570312,\n                        51.51344322994464\n                      ],\n                      [\n                        -0.19277572631835938,\n                        51.49292721420451\n                      ],\n                      [\n                        -0.14110565185546875,\n                        51.49773648412071\n                      ],\n                      [\n                        -0.14471054077148438,\n                        51.51889124411907\n                      ],\n                      [\n                        -0.19517898559570312,\n                        51.51344322994464\n                      ]\n                    ]\n                  ],\n                  [\n                    [\n                      [\n                        -0.16925811767578122,\n                        51.48373475351443\n                      ],\n                      [\n                        -0.16925811767578122,\n                        51.47004951935931\n                      ],\n                      [\n                        -0.14608383178710938,\n                        51.472722739318336\n                      ],\n                      [\n                        -0.14453887939453125,\n                        51.48758298584306\n                      ],\n                      [\n                        -0.16925811767578122,\n                        51.48373475351443\n                      ]\n                    ]\n                  ]\n                ]\n              },\n              {\n                \"type\": \"LineString\",\n                \"coordinates\": [\n                  [\n                    77.58851766586304,\n                    12.973152950670608\n                  ],\n                  [\n                    77.58937597274779,\n                    12.972212000113458\n                  ]\n                ]\n              },\n              {\n                \"type\": \"polygon\",\n                \"coordinates\": [\n                  [\n                    [\n                      -0.17337799072265625,\n                      51.54323910441573\n                    ],\n                    [\n                      -0.1668548583984375,\n                      51.51889124411907\n                    ],\n                    [\n                      -0.09286880493164062,\n                      51.53341609632549\n                    ],\n                    [\n                      -0.17337799072265625,\n                      51.54323910441573\n                    ]\n                  ]\n                ]\n              }\n            ]\n          },\n          \"relation\": \"within\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"multipolygon_london_parks\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a circle that intersects with one of the polygons in the multipolygon_london_parks\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"circle\",\n            \"coordinates\": [\n              -0.14265060424804688,\n              51.53298896092339\n            ],\n            \"radius\": \"550m\"\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"multipolygon_london_parks\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with a circle that contains london museum geopoint\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"circle\",\n            \"coordinates\": [\n              -0.09115219116210938,\n              51.516487788780005\n            ],\n            \"radius\": \"1050m\"\n          },\n          \"relation\": \"within\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"point_museum_of_london\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with brockwell park polygon that is contained within brockwell park envelope\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"polygon\",\n            \"coordinates\": [\n              [\n                [\n                  -0.11149406433105469,\n                  51.454942883825744\n                ],\n                [\n                  -0.11230945587158205,\n                  51.45218839188088\n                ],\n                [\n                  -0.11136531829833984,\n                  51.450530268053605\n                ],\n                [\n                  -0.1117086410522461,\n                  51.44873835686053\n                ],\n                [\n                  -0.11016368865966797,\n                  51.446010237625224\n                ],\n                [\n                  -0.10497093200683594,\n                  51.446705656046376\n                ],\n                [\n                  -0.10192394256591797,\n                  51.4490058107573\n                ],\n                [\n                  -0.1007223129272461,\n                  51.45085119994589\n                ],\n                [\n                  -0.10188102722167967,\n                  51.45218839188088\n                ],\n                [\n                  -0.10681629180908203,\n                  51.45368600035086\n                ],\n                [\n                  -0.10715961456298828,\n                  51.453338345620416\n                ],\n                [\n                  -0.11149406433105469,\n                  51.454942883825744\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"envelope_brockwell_park\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with point that is contained within brockwell park envelope\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"point\",\n            \"coordinates\": [\n              -0.10074377059936523,\n              51.450824455707696\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"envelope_brockwell_park\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with linestring that intersects the putney bridge\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"LineString\",\n            \"coordinates\": [\n              [\n                -0.2171945571899414,\n                51.46876631814087\n              ],\n              [\n                -0.2064228057861328,\n                51.464943233925986\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"linestring_putney_bridge\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search with polygon that contains the blr stadiums/multipoint\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"polygon\",\n            \"coordinates\": [\n              [\n                [\n                  77.60107040405273,\n                  12.981349524921757\n                ],\n                [\n                  77.59270191192627,\n                  12.969180024104505\n                ],\n                [\n                  77.60089874267577,\n                  12.961024870820744\n                ],\n                [\n                  77.60107040405273,\n                  12.981349524921757\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"within\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"multipoint_blr_stadiums\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search a point that is within the multipolygon of the geometrycollection\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"point\",\n            \"coordinates\": [\n              76.88919067382812,\n              8.556238400473156\n            ]\n          },\n          \"relation\": \"contains\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"geometrycollection_tvm\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search an envelope that intersects with the polygon of the geometrycollection\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"envelope\",\n            \"coordinates\": [\n              [\n                76.91880226135254,\n                8.515665792358828\n              ],\n              [\n                76.92523956298828,\n                8.525427378462332\n              ]\n            ]\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"geometrycollection_tvm\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search a circle that intersects with the linestring of the geometrycollection\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"circle\",\n            \"coordinates\": [\n              76.91305160522461,\n              8.477890354619287\n            ],\n            \"radius\": \"1mi\"\n          },\n          \"relation\": \"intersects\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"geometrycollection_tvm\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search a circle that contains the entire geometrycollection\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"circle\",\n            \"coordinates\": [\n              76.93622589111328,\n              8.501574715933401\n            ],\n            \"radius\": \"10mi\"\n          },\n          \"relation\": \"within\"\n        }\n      }\n    },\n    \"result\": {\n      \"total_hits\": 1,\n      \"hits\": [\n        {\n          \"id\": \"geometrycollection_tvm\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search a polygon that contains the entire geometrycollection, circle, multilinestring, polygon, multipoint\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"circle\",\n            \"coordinates\": [\n              77.71728515624999,\n              12.060809058367294\n            ],\n            \"radius\": \"1000mi\"\n          },\n          \"relation\": \"within\"\n        }\n      },\n      \"sort\": [\"-_id\"]\n    },\n    \"result\": {\n      \"total_hits\": 5,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        },\n        {\n          \"id\": \"multipoint_blr_stadiums\"\n        },\n        {\n          \"id\": \"multilinestring_old_airport_road\"\n        },\n        {\n          \"id\": \"geometrycollection_tvm\"\n        },\n        {\n          \"id\": \"circle_halairport\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search circle that contains the envelope, linestring, point, multipolygon\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"circle\",\n            \"coordinates\": [\n              -0.23277282714843747,\n              51.45828549061808\n            ],\n            \"radius\": \"1000mi\"\n          },\n          \"relation\": \"within\"\n        }\n      },\n      \"sort\": [\"-_id\"]\n    },\n    \"result\": {\n      \"total_hits\": 4,\n      \"hits\": [\n        {\n          \"id\": \"point_museum_of_london\"\n        },\n        {\n          \"id\": \"multipolygon_london_parks\"\n        },\n        {\n          \"id\": \"linestring_putney_bridge\"\n        },\n        {\n          \"id\": \"envelope_brockwell_park\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"search a polygon(almost the whole earth surface) that contains every indexed shape\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"geometry\": {\n          \"shape\": {\n            \"type\": \"polygon\",\n            \"coordinates\": [\n              [\n                [\n                  -135.0, -38.0\n                ],\n                [\n                  149.0, -38.0\n                ],\n                [\n                  149.0, 77.0\n                ],\n                [\n                  -135.0, 77.0\n                ]\n              ]\n            ]\n          },\n          \"relation\": \"within\"\n        }\n      },\n      \"sort\": [\"-_id\"]\n    },\n    \"result\": {\n      \"total_hits\": 9,\n      \"hits\": [\n        {\n          \"id\": \"polygon_cubbonpark\"\n        },\n        {\n          \"id\": \"point_museum_of_london\"\n        },\n        {\n          \"id\": \"multipolygon_london_parks\"\n        },\n        {\n          \"id\": \"multipoint_blr_stadiums\"\n        },\n        {\n          \"id\": \"multilinestring_old_airport_road\"\n        },\n        {\n          \"id\": \"linestring_putney_bridge\"\n        },\n        {\n          \"id\": \"geometrycollection_tvm\"\n        },\n        {\n          \"id\": \"envelope_brockwell_park\"\n        },\n        {\n          \"id\": \"circle_halairport\"\n        }\n      ]\n    }\n  }\n]\n"
  },
  {
    "path": "test/tests/phrase/data/a.json",
    "content": "{\n\t\"body\": \"Twenty Thousand Leagues Under The Sea\"\n}"
  },
  {
    "path": "test/tests/phrase/data/b.json",
    "content": "{\n\t\"body\": [\"bad call\", \"defenseless receiver\"]\n}"
  },
  {
    "path": "test/tests/phrase/mapping.json",
    "content": "{\n  \"types\": {\n    \"book\": {\n      \"properties\": {\n        \"body\": {\n          \"fields\": [\n            {\n              \"include_term_vectors\": true,\n              \"include_in_all\": true,\n              \"index\": true,\n              \"store\": true,\n              \"analyzer\": \"en\",\n              \"type\": \"text\"\n            }\n          ],\n          \"dynamic\": true,\n          \"enabled\": true\n        }\n      }\n    }\n  },\n  \"default_type\": \"book\"\n}"
  },
  {
    "path": "test/tests/phrase/searches.json",
    "content": "[\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Twenty\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Twenty Thousand\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Twenty Thousand Leagues\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Twenty Thousand Leagues Under\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Twenty Thousand Leagues Under the\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Twenty Thousand Leagues Under the Sea\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Thousand\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Thousand Leagues\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Thousand Leagues Under\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Thousand Leagues Under the\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Thousand Leagues Under the Sea\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Leagues\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Leagues Under\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Leagues Under the\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Leagues Under the Sea\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n \t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Under the Sea\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"the Sea\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"Sea\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"bad call\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"defenseless receiver\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"b\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t{\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"match_phrase\": \"bad receiver\"\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 0,\n\t\t\t\"hits\": []\n\t\t}\n\t},\n  {\n    \"comment\": \"multi-phrase terms\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"sort\": [\"-_score\", \"_id\"],\n\t\t\t\"query\": {\n\t\t\t\t\"field\": \"body\",\n\t\t\t\t\"terms\": [[\"twenti\",\"thirti\"],[\"thousand\"]]\n\t\t\t}\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 1,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}\n]\n"
  },
  {
    "path": "test/tests/sort/data/a.json",
    "content": "{\n\t\"id\": \"a\",\n\t\"name\": \"marty\",\n\t\"age\": 19,\n  \"born\": \"2014-11-25\",\n\t\"title\": \"mista\",\n\t\"tags\": [\"gopher\", \"belieber\"]\n}\n"
  },
  {
    "path": "test/tests/sort/data/b.json",
    "content": "{\n\t\"id\": \"b\",\n\t\"name\": \"steve\",\n\t\"age\": 21,\n  \"born\": \"2000-09-11\",\n\t\"title\": \"zebra\",\n\t\"tags\": [\"thought-leader\", \"futurist\"]\n}\n"
  },
  {
    "path": "test/tests/sort/data/c.json",
    "content": "{\n\t\"id\": \"c\",\n\t\"name\": \"aster\",\n\t\"age\": 21,\n  \"born\": \"1954-02-02\",\n\t\"title\": \"blogger\",\n\t\"tags\": [\"red\", \"blue\", \"green\"]\n}\n"
  },
  {
    "path": "test/tests/sort/data/d.json",
    "content": "{\n\t\"id\": \"d\",\n\t\"age\": 65,\n  \"born\": \"1978-12-02\",\n\t\"title\": \"agent d is desperately trying out to be successful rapster!\",\n\t\"tags\": [\"cats\"]\n}\n"
  },
  {
    "path": "test/tests/sort/data/e.json",
    "content": "{\n\t\"id\": \"e\",\n\t\"name\": \"nancy\",\n  \"born\": \"1954-10-22\",\n\t\"title\": \"rapstar nancy rapster\",\n\t\"tags\": [\"pain\"]\n}\n"
  },
  {
    "path": "test/tests/sort/data/f.json",
    "content": "{\n\t\"id\": \"f\",\n\t\"name\": \"frank\",\n\t\"age\": 1,\n\t\"title\": \"frank the taxman of cb, Rapster!\",\n\t\"tags\": [\"vitamin\",\"purple\"]\n}\n"
  },
  {
    "path": "test/tests/sort/mapping.json",
    "content": "{\n\n}\n"
  },
  {
    "path": "test/tests/sort/searches.json",
    "content": "[\n  {\n    \"comment\": \"sort by name, ascending\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [\"name\"]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"c\"\n\t\t\t\t},\n        {\n          \"id\": \"f\"\n        },\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"d\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort by name, descending\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [\"-name\"]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"f\"\n        },\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"d\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort by name, descending, missing first\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [{\"by\":\"field\",\"field\":\"name\",\"missing\":\"first\",\"desc\":true}]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n        {\n          \"id\": \"d\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"f\"\n        },\n        {\n          \"id\": \"c\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort by age, ascending, _id,  ascending\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [\"age\", \"_id\"]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"f\"\n\t\t\t\t},\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"d\"\n        },\n        {\n          \"id\": \"e\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort by age, descending, _id,  ascending\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [\"-age\", \"_id\"]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"d\"\n\t\t\t\t},\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"f\"\n        },\n        {\n          \"id\": \"e\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort by age, descending, missing first, id, ascending\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [{\"by\":\"field\",\"field\":\"age\",\"missing\":\"first\",\"desc\":true},{\"by\":\"id\",\"desc\":false}]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n        {\n          \"id\": \"e\"\n        },\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"d\"\n\t\t\t\t},\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"f\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort by born, ascending\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [\"born\"]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"c\"\n\t\t\t\t},\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"d\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"f\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort by born, descending\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [\"-born\"]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t},\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"d\"\n        },\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"f\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort by born, descending, missing first\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [{\"by\":\"field\",\"field\":\"born\",\"missing\":\"first\",\"desc\":true}]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n        {\n          \"id\": \"f\"\n        },\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"a\"\n\t\t\t\t},\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"d\"\n        },\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"c\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n    \"comment\": \"sort on multi-valued field\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"match_all\":{}\n      },\n      \"sort\": [{\"by\":\"field\",\"field\":\"tags\",\"mode\":\"min\"}]\n    },\n    \"result\": {\n      \"total_hits\": 6,\n      \"hits\": [\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"d\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"f\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"multi-column sort by age, ascending, name, ascending (flips b and c which have same age)\",\n\t\t\"search\": {\n\t\t\t\"from\": 0,\n\t\t\t\"size\": 10,\n\t\t\t\"query\": {\n\t\t\t\t\"match_all\":{}\n\t\t\t},\n      \"sort\": [\"age\", \"name\"]\n\t\t},\n\t\t\"result\": {\n\t\t\t\"total_hits\": 6,\n\t\t\t\"hits\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"f\"\n\t\t\t\t},\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"d\"\n        },\n        {\n          \"id\": \"e\"\n        }\n\t\t\t]\n\t\t}\n\t},\n  {\n      \"comment\": \"sort by docid descending\",\n  \t\t\"search\": {\n  \t\t\t\"from\": 0,\n  \t\t\t\"size\": 10,\n  \t\t\t\"query\": {\n  \t\t\t\t\"match_all\":{}\n  \t\t\t},\n        \"sort\": [\"-_id\"]\n  \t\t},\n  \t\t\"result\": {\n  \t\t\t\"total_hits\": 6,\n  \t\t\t\"hits\": [\n  \t\t\t\t{\n  \t\t\t\t\t\"id\": \"f\"\n  \t\t\t\t},\n          {\n            \"id\": \"e\"\n          },\n          {\n            \"id\": \"d\"\n          },\n          {\n            \"id\": \"c\"\n          },\n          {\n            \"id\": \"b\"\n          },\n          {\n            \"id\": \"a\"\n          }\n  \t\t\t]\n  \t\t}\n  \t},\n  {\n    \"comment\": \"sort by name, ascending, after marty\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"match_all\":{}\n      },\n      \"sort\": [\"name\"],\n      \"search_after\": [\"marty\"]\n    },\n    \"result\": {\n      \"total_hits\": 6,\n      \"hits\": [\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"d\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"sort by name, ascending, before nancy\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"match_all\":{}\n      },\n      \"sort\": [\"name\"],\n      \"search_before\": [\"nancy\"]\n    },\n    \"result\": {\n      \"total_hits\": 6,\n      \"hits\": [\n        {\n          \"id\": \"c\"\n        },\n        {\n          \"id\": \"f\"\n        },\n        {\n          \"id\": \"a\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"sort by ID, after doc d\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"match_all\":{}\n      },\n      \"sort\": [\"_id\"],\n      \"search_after\": [\"d\"]\n    },\n    \"result\": {\n      \"total_hits\": 6,\n      \"hits\": [\n        {\n          \"id\": \"e\"\n        },\n        {\n          \"id\": \"f\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"sort by ID, before doc d\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"match_all\":{}\n      },\n      \"sort\": [\"_id\"],\n      \"search_before\": [\"d\"]\n    },\n    \"result\": {\n      \"total_hits\": 6,\n      \"hits\": [\n        {\n          \"id\": \"a\"\n        },\n        {\n          \"id\": \"b\"\n        },\n        {\n          \"id\": \"c\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"sort by score, after score 0.286889[ e(299646) > f(286889) > d(222224)]\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"query\":\"rapster\"\n      },\n      \"sort\": [\"_score\"],\n      \"search_after\": [\"0.286889\"]\n    },\n    \"result\": {\n      \"total_hits\": 3,\n      \"hits\": [\n        {\n          \"id\": \"f\"\n        },\n        {\n          \"id\": \"e\"\n        }\n      ]\n    }\n  },\n  {\n    \"comment\": \"sort by score, before score f/0.286889[ e(299646) > f(286889) > d(222224)]\",\n    \"search\": {\n      \"from\": 0,\n      \"size\": 10,\n      \"query\": {\n        \"query\":\"rapster\"\n      },\n      \"sort\": [\"_score\"],\n      \"search_before\": [\"0.286889\"]\n    },\n    \"result\": {\n      \"total_hits\": 3,\n      \"hits\": [\n        {\n          \"id\": \"d\"\n        }\n      ]\n    }\n  }\n]\n"
  },
  {
    "path": "test/versus_score_test.go",
    "content": "//  Copyright (c) 2018 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\n\t\"github.com/blevesearch/bleve/v2\"\n\t\"github.com/blevesearch/bleve/v2/document\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n\tindex \"github.com/blevesearch/bleve_index_api\"\n)\n\nfunc TestDisjunctionSearchScoreIndexWithCompositeFields(t *testing.T) {\n\tupHits := disjunctionQueryiOnIndexWithCompositeFields(upsidedown.Name, t)\n\tscHits := disjunctionQueryiOnIndexWithCompositeFields(scorch.Name, t)\n\n\tif upHits[0].ID != scHits[0].ID || upHits[1].ID != scHits[1].ID {\n\t\tt.Errorf(\"upsidedown, scorch returned different docs;\\n\"+\n\t\t\t\"upsidedown: (%s, %s), scorch: (%s, %s)\\n\",\n\t\t\tupHits[0].ID, upHits[1].ID, scHits[0].ID, scHits[1].ID)\n\t}\n\n\tif scHits[0].Score != upHits[0].Score || scHits[1].Score != upHits[1].Score {\n\t\tt.Errorf(\"upsidedown, scorch showing different scores;\\n\"+\n\t\t\t\"upsidedown: (%+v, %+v), scorch: (%+v, %+v)\\n\",\n\t\t\t*upHits[0].Expl, *upHits[1].Expl, *scHits[0].Expl, *scHits[1].Expl)\n\t}\n}\n\nfunc disjunctionQueryiOnIndexWithCompositeFields(indexName string,\n\tt *testing.T,\n) []*search.DocumentMatch {\n\ttmpIndexPath, err := os.MkdirTemp(\"\", \"bleve-testidx\")\n\tif err != nil {\n\t\tt.Fatalf(\"error creating temp dir: %v\", err)\n\t}\n\tdefer func() {\n\t\terr := os.RemoveAll(tmpIndexPath)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error removing temp dir: %v\", err)\n\t\t}\n\t}()\n\t// create an index\n\tidxMapping := mapping.NewIndexMapping()\n\tidx, err := bleve.NewUsing(tmpIndexPath, idxMapping, indexName,\n\t\tbleve.Config.DefaultKVStore, nil)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tdefer func() {\n\t\terr = idx.Close()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}()\n\n\t// create and insert documents as a batch\n\tbatch := idx.NewBatch()\n\tdocs := []struct {\n\t\tfield1 string\n\t\tfield2 int\n\t}{\n\t\t{\n\t\t\tfield1: \"one\",\n\t\t\tfield2: 1,\n\t\t},\n\t\t{\n\t\t\tfield1: \"two\",\n\t\t\tfield2: 2,\n\t\t},\n\t}\n\n\tfor i := 0; i < len(docs); i++ {\n\t\tdoc := document.NewDocument(strconv.Itoa(docs[i].field2))\n\t\tdoc.Fields = []document.Field{\n\t\t\tdocument.NewTextField(\"field1\", []uint64{}, []byte(docs[i].field1)),\n\t\t\tdocument.NewNumericField(\"field2\", []uint64{}, float64(docs[i].field2)),\n\t\t}\n\t\tdoc.CompositeFields = []*document.CompositeField{\n\t\t\tdocument.NewCompositeFieldWithIndexingOptions(\n\t\t\t\t\"_all\", true, []string{\"field1\"}, []string{},\n\t\t\t\tindex.IndexField|index.IncludeTermVectors),\n\t\t}\n\t\tif err = batch.IndexAdvanced(doc); err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\tif err = idx.Batch(batch); err != nil {\n\t\tt.Error(err)\n\t}\n\n\t/*\n\t\tQuery:\n\t\t\t\t DISJ\n\t\t\t        /    \\\n\t\t\t     CONJ    TERM(two)\n\t\t\t     /\n\t\t          TERM(one)\n\t*/\n\n\ttq1 := bleve.NewTermQuery(\"one\")\n\ttq1.SetBoost(2)\n\ttq2 := bleve.NewTermQuery(\"two\")\n\ttq2.SetBoost(3)\n\n\tcq := bleve.NewConjunctionQuery(tq1)\n\tcq.SetBoost(4)\n\n\tq := bleve.NewDisjunctionQuery(tq1, tq2)\n\tsr := bleve.NewSearchRequestOptions(q, 2, 0, true)\n\tres, err := idx.Search(sr)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(res.Hits) != 2 {\n\t\tt.Errorf(\"indexType: %s Expected 2 hits, but got: %v\", indexName, len(res.Hits))\n\t}\n\n\treturn res.Hits\n}\n"
  },
  {
    "path": "test/versus_test.go",
    "content": "//  Copyright (c) 2014 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage test\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"text/template\"\n\t\"time\"\n\n\t\"github.com/blevesearch/bleve/v2\"\n\t\"github.com/blevesearch/bleve/v2/index/scorch\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown\"\n\t\"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb\"\n\t\"github.com/blevesearch/bleve/v2/mapping\"\n\t\"github.com/blevesearch/bleve/v2/search\"\n)\n\n// Tests scorch indexer versus upsidedown/bolt indexer against various\n// templated queries.  Example usage from the bleve top-level directory...\n//\n//     go test -v -run TestScorchVersusUpsideDownBolt ./test\n//     VERBOSE=1 FOCUS=Trista go test -v -run TestScorchVersusUpsideDownBolt ./test\n//\n\nfunc init() {\n\t// override for tests\n\tscorch.DefaultPersisterNapTimeMSec = 1\n}\n\nfunc TestScorchVersusUpsideDownBoltAll(t *testing.T) {\n\t(&VersusTest{\n\t\tt:                    t,\n\t\tNumDocs:              1000,\n\t\tMaxWordsPerDoc:       20,\n\t\tNumWords:             10,\n\t\tBatchSize:            1000,\n\t\tNumAttemptsPerSearch: 100,\n\t}).run(scorch.Name, boltdb.Name, upsidedown.Name, boltdb.Name, nil, nil)\n}\n\nfunc TestScorchVersusUpsideDownBoltSmallMNSAM(t *testing.T) {\n\t(&VersusTest{\n\t\tt:                    t,\n\t\tFocus:                \"must-not-same-as-must\",\n\t\tNumDocs:              5,\n\t\tMaxWordsPerDoc:       2,\n\t\tNumWords:             1,\n\t\tBatchSize:            1,\n\t\tNumAttemptsPerSearch: 1,\n\t}).run(scorch.Name, boltdb.Name, upsidedown.Name, boltdb.Name, nil, nil)\n}\n\nfunc TestScorchVersusUpsideDownBoltSmallCMP11(t *testing.T) {\n\t(&VersusTest{\n\t\tt:                    t,\n\t\tFocus:                \"conjuncts-match-phrase-1-1\",\n\t\tNumDocs:              30,\n\t\tMaxWordsPerDoc:       8,\n\t\tNumWords:             2,\n\t\tBatchSize:            1,\n\t\tNumAttemptsPerSearch: 1,\n\t}).run(scorch.Name, boltdb.Name, upsidedown.Name, boltdb.Name, nil, nil)\n}\n\n// -------------------------------------------------------\n\n// Templates used to compare search results in the \"versus\" tests.\nvar testVersusSearchTemplates = []string{\n\t`{\n      \"about\": \"expected to return zero hits\",\n      \"query\": {\n       \"query\": \"title:notARealTitle\"\n      }\n     }`,\n\t`{\n      \"about\": \"try straight word()'s\",\n      \"query\": {\n       \"query\": \"body:{{word}}\"\n      }\n     }`,\n\t`{\n      \"about\": \"conjuncts on same term\",\n      \"query\": {\n        \"conjuncts\": [\n          { \"field\": \"body\", \"term\": \"{{word}}\", \"boost\": 1.0 },\n          { \"field\": \"body\", \"term\": \"{{word}}\", \"boost\": 1.0 }\n        ]\n      }\n     }`,\n\t`{\n      \"about\": \"disjuncts on same term\",\n      \"query\": {\n        \"disjuncts\": [\n          { \"field\": \"body\", \"term\": \"{{word}}\", \"boost\": 1.0 },\n          { \"field\": \"body\", \"term\": \"{{word}}\", \"boost\": 1.0 }\n        ]\n      }\n     }`,\n\t`{\n      \"about\": \"never-matching-title-conjuncts\",\n      \"query\": {\n        \"conjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{word}}\"},\n          {\"field\": \"body\", \"match\": \"{{word}}\"},\n          {\"field\": \"title\", \"match\": \"notAnActualTitle\"}\n        ]\n      }\n     }`,\n\t`{\n      \"about\": \"never-matching-title-disjuncts\",\n      \"query\": {\n        \"disjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{word}}\"},\n          {\"field\": \"body\", \"match\": \"{{word}}\"},\n          {\"field\": \"title\", \"match\": \"notAnActualTitle\"}\n        ]\n      }\n     }`,\n\t`{\n      \"about\": \"must-not-never-matches\",\n      \"query\": {\n        \"must_not\": {\"disjuncts\": [\n          {\"field\": \"title\", \"match\": \"notAnActualTitle\"}\n        ]},\n        \"should\": {\"disjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{word}}\"}\n        ]}\n      }\n     }`,\n\t`{\n      \"about\": \"must-not-only\",\n      \"query\": {\n        \"must_not\": {\"disjuncts\": [\n          {\"field\": \"body\", \"term\": \"{{word}}\"}\n        ]}\n      }\n     }`,\n\t`{\n      \"about\": \"must-not-same-as-must -- see: MB-27291\",\n      \"query\": {\n        \"must_not\": {\"disjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{word}}\"}\n        ]},\n        \"must\": {\"conjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{word}}\"}\n        ]}\n      }\n     }`,\n\t`{\n      \"about\": \"must-not-same-as-should\",\n      \"query\": {\n        \"must_not\": {\"disjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{word}}\"}\n        ]},\n        \"should\": {\"disjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{word}}\"}\n        ]}\n      }\n     }`,\n\t`{\n      \"about\": \"inspired by testrunner RQG issue -- see: MB-27291\",\n      \"query\": {\n        \"must_not\": {\"disjuncts\": [\n          {\"field\": \"title\", \"match\": \"Trista Allen\"},\n          {\"field\": \"body\", \"match\": \"{{word}}\"}\n        ]},\n        \"should\": {\"disjuncts\": [\n          {\"field\": \"title\", \"match\": \"Kallie Safiya Amara\"},\n          {\"field\": \"body\", \"match\": \"{{word}}\"}\n        ]}\n      }\n     }`,\n\t`{\n      \"about\": \"conjuncts-match-phrase-1-1 inspired by testrunner RQG issue -- see: MB-27291\",\n      \"query\": {\n        \"conjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{bodyWord 0}}\"},\n          {\"field\": \"body\", \"match_phrase\": \"{{bodyWord 1}} {{bodyWord 1}}\"}\n        ]\n      }\n     }`,\n\t`{\n      \"about\": \"conjuncts-match-phrase-1-2 inspired by testrunner RQG issue -- see: MB-27291 -- FAILS!!\",\n      \"query\": {\n        \"conjuncts\": [\n          {\"field\": \"body\", \"match\": \"{{bodyWord 0}}\"},\n          {\"field\": \"body\", \"match_phrase\": \"{{bodyWord 1}} {{bodyWord 2}}\"}\n        ]\n      }\n     }`,\n}\n\n// -------------------------------------------------------\n\ntype VersusTest struct {\n\tt *testing.T\n\n\t// Use environment variable VERBOSE=<integer> that's > 0 for more\n\t// verbose output.\n\tVerbose int\n\n\t// Allow user to focus on particular search templates, where\n\t// where the search template must contain the Focus string.\n\tFocus string\n\n\tNumDocs              int // Number of docs to insert.\n\tMaxWordsPerDoc       int // Max number words in each doc's Body field.\n\tNumWords             int // Total number of words in the dictionary.\n\tBatchSize            int // Batch size when inserting docs.\n\tNumAttemptsPerSearch int // For each search template, number of searches to try.\n\n\t// The Bodies is an array with length NumDocs, where each entry\n\t// is the words in a doc's Body field.\n\tBodies [][]string\n\n\tCurAttempt  int\n\tTotAttempts int\n}\n\n// -------------------------------------------------------\n\nfunc testVersusSearches(vt *VersusTest, searchTemplates []string, idxA, idxB bleve.Index) {\n\tt := vt.t\n\n\tfuncMap := template.FuncMap{\n\t\t// Returns a word.  The word may or may not be in any\n\t\t// document's body.\n\t\t\"word\": func() string {\n\t\t\treturn vt.genWord(vt.CurAttempt % vt.NumWords)\n\t\t},\n\t\t// Picks a document and returns the i'th word in that\n\t\t// document's body.  You can use this in searches to\n\t\t// definitely find at least one document.\n\t\t\"bodyWord\": func(i int) string {\n\t\t\tbody := vt.Bodies[vt.CurAttempt%len(vt.Bodies)]\n\t\t\tif len(body) == 0 {\n\t\t\t\treturn \"\"\n\t\t\t}\n\t\t\treturn body[i%len(body)]\n\t\t},\n\t}\n\n\t// Optionally allow call to focus on a particular search templates,\n\t// where the search template must contain the vt.Focus string.\n\tif vt.Focus == \"\" {\n\t\tvt.Focus = os.Getenv(\"FOCUS\")\n\t}\n\n\tfor i, searchTemplate := range searchTemplates {\n\t\tif vt.Focus != \"\" && !strings.Contains(searchTemplate, vt.Focus) {\n\t\t\tcontinue\n\t\t}\n\n\t\ttmpl, err := template.New(\"search\").Funcs(funcMap).Parse(searchTemplate)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"could not parse search template: %s, err: %v\", searchTemplate, err)\n\t\t}\n\t\tfor j := 0; j < vt.NumAttemptsPerSearch; j++ {\n\t\t\tvt.CurAttempt = j\n\n\t\t\tvar buf bytes.Buffer\n\t\t\terr = tmpl.Execute(&buf, vt)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"could not execute search template: %s, err: %v\", searchTemplate, err)\n\t\t\t}\n\n\t\t\tbufBytes := buf.Bytes()\n\n\t\t\tif vt.Verbose > 0 {\n\t\t\t\tfmt.Printf(\"  %s\\n\", bufBytes)\n\t\t\t}\n\n\t\t\tvar search bleve.SearchRequest\n\t\t\terr = json.Unmarshal(bufBytes, &search)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"could not unmarshal search: %s, err: %v\", bufBytes, err)\n\t\t\t}\n\n\t\t\tsearch.Size = vt.NumDocs * 10 // Crank up limit to get all results.\n\n\t\t\tsearchA := search\n\t\t\tsearchB := search\n\n\t\t\tresA, errA := idxA.Search(&searchA)\n\t\t\tresB, errB := idxB.Search(&searchB)\n\t\t\tif errA != errB {\n\t\t\t\tt.Errorf(\"search: (%d) %s,\\n err mismatch, errA: %v, errB: %v\",\n\t\t\t\t\ti, bufBytes, errA, errB)\n\t\t\t}\n\n\t\t\t// Scores might have float64 vs float32 wobbles, so truncate precision.\n\t\t\tresA.MaxScore = math.Trunc(resA.MaxScore*1000.0) / 1000.0\n\t\t\tresB.MaxScore = math.Trunc(resB.MaxScore*1000.0) / 1000.0\n\n\t\t\t// Timings may be different between A & B, so force equality.\n\t\t\tresA.Took = resB.Took\n\n\t\t\t// Hits might have different ordering since some indexers\n\t\t\t// (like upsidedown) have a natural secondary sort on id\n\t\t\t// while others (like scorch) don't.  So, we compare by\n\t\t\t// putting the hits from A & B into maps.\n\t\t\thitsA := hitsById(resA)\n\t\t\thitsB := hitsById(resB)\n\t\t\tfor id, hitA := range hitsA {\n\t\t\t\thitB := hitsB[id]\n\t\t\t\tif len(hitA.FieldTermLocations) == 0 {\n\t\t\t\t\thitA.FieldTermLocations = nil\n\t\t\t\t}\n\t\t\t\tif len(hitB.FieldTermLocations) == 0 {\n\t\t\t\t\thitB.FieldTermLocations = nil\n\t\t\t\t}\n\t\t\t\tif !reflect.DeepEqual(hitA, hitB) {\n\t\t\t\t\tt.Errorf(\"\\n  driving from hitsA\\n    hitA: %#v,\\n    hitB: %#v\", hitA, hitB)\n\t\t\t\t\tidx, _ := strconv.Atoi(id)\n\t\t\t\t\tt.Errorf(\"\\n    doc: %d, body: %s\", idx, strings.Join(vt.Bodies[idx], \" \"))\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor id, hitB := range hitsB {\n\t\t\t\thitA := hitsA[id]\n\t\t\t\tif len(hitA.FieldTermLocations) == 0 {\n\t\t\t\t\thitA.FieldTermLocations = nil\n\t\t\t\t}\n\t\t\t\tif len(hitB.FieldTermLocations) == 0 {\n\t\t\t\t\thitB.FieldTermLocations = nil\n\t\t\t\t}\n\t\t\t\tif !reflect.DeepEqual(hitA, hitB) {\n\t\t\t\t\tt.Errorf(\"\\n  driving from hitsB\\n    hitA: %#v,\\n    hitB: %#v\", hitA, hitB)\n\t\t\t\t\tidx, _ := strconv.Atoi(id)\n\t\t\t\t\tt.Errorf(\"\\n    doc: %d, body: %s\", idx, strings.Join(vt.Bodies[idx], \" \"))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(hitsA, hitsB) {\n\t\t\t\tt.Errorf(\"=========\\nsearch: (%d) %s,\\n res hits mismatch,\\n len(hitsA): %d,\\n len(hitsB): %d\",\n\t\t\t\t\ti, bufBytes, len(hitsA), len(hitsB))\n\t\t\t\tt.Errorf(\"\\n  hitsA: %#v,\\n  hitsB: %#v\",\n\t\t\t\t\thitsA, hitsB)\n\t\t\t}\n\n\t\t\tresA.Hits = nil\n\t\t\tresB.Hits = nil\n\t\t\tresA.Cost = 0\n\t\t\tresB.Cost = 0\n\n\t\t\tif !reflect.DeepEqual(resA, resB) {\n\t\t\t\tresAj, _ := json.Marshal(resA)\n\t\t\t\tresBj, _ := json.Marshal(resB)\n\t\t\t\tt.Errorf(\"search: (%d) %s,\\n  res mismatch,\\n  resA: %s,\\n  resB: %s\",\n\t\t\t\t\ti, bufBytes, resAj, resBj)\n\t\t\t}\n\n\t\t\tif vt.Verbose > 0 {\n\t\t\t\tfmt.Printf(\"  Total: (%t) %d\\n\", resA.Total == resB.Total, resA.Total)\n\t\t\t}\n\n\t\t\tvt.TotAttempts++\n\t\t}\n\t}\n}\n\n// Organizes the hits into a map keyed by id.\nfunc hitsById(res *bleve.SearchResult) map[string]*search.DocumentMatch {\n\trv := make(map[string]*search.DocumentMatch, len(res.Hits))\n\n\tfor _, hit := range res.Hits {\n\t\t// Clear out or truncate precision of hit fields that might be\n\t\t// different across different indexer implementations.\n\t\thit.Index = \"\"\n\t\thit.Score = math.Trunc(hit.Score*1000.0) / 1000.0\n\t\thit.IndexInternalID = nil\n\t\thit.HitNumber = 0\n\n\t\trv[hit.ID] = hit\n\t}\n\n\treturn rv\n}\n\n// -------------------------------------------------------\n\nfunc (vt *VersusTest) run(indexTypeA, kvStoreA, indexTypeB, kvStoreB string,\n\tcb func(versusTest *VersusTest, searchTemplates []string, idxA, idxB bleve.Index),\n\tsearchTemplates []string,\n) {\n\tif cb == nil {\n\t\tcb = testVersusSearches\n\t}\n\n\tif searchTemplates == nil {\n\t\tsearchTemplates = testVersusSearchTemplates\n\t}\n\n\tif vt.Verbose <= 0 {\n\t\tvt.Verbose, _ = strconv.Atoi(os.Getenv(\"VERBOSE\"))\n\t}\n\n\tdirA := \"/tmp/bleve-versus-test-a\"\n\tdirB := \"/tmp/bleve-versus-test-b\"\n\n\tdefer func() {\n\t\t_ = os.RemoveAll(dirA)\n\t\t_ = os.RemoveAll(dirB)\n\t}()\n\n\t_ = os.RemoveAll(dirA)\n\t_ = os.RemoveAll(dirB)\n\n\timA := vt.makeIndexMapping()\n\timB := vt.makeIndexMapping()\n\n\tkvConfigA := map[string]interface{}{}\n\tkvConfigB := map[string]interface{}{}\n\n\tidxA, err := bleve.NewUsing(dirA, imA, indexTypeA, kvStoreA, kvConfigA)\n\tif err != nil || idxA == nil {\n\t\tvt.t.Fatalf(\"new using err: %v\", err)\n\t}\n\tdefer func() { _ = idxA.Close() }()\n\n\tidxB, err := bleve.NewUsing(dirB, imB, indexTypeB, kvStoreB, kvConfigB)\n\tif err != nil || idxB == nil {\n\t\tvt.t.Fatalf(\"new using err: %v\", err)\n\t}\n\tdefer func() { _ = idxB.Close() }()\n\n\tif vt.Bodies == nil {\n\t\tvt.Bodies = vt.genBodies()\n\t}\n\n\tvt.insertBodies(idxA)\n\tvt.insertBodies(idxB)\n\n\tcb(vt, searchTemplates, idxA, idxB)\n}\n\n// -------------------------------------------------------\n\nfunc (vt *VersusTest) makeIndexMapping() mapping.IndexMapping {\n\tstandardFM := bleve.NewTextFieldMapping()\n\tstandardFM.Store = false\n\tstandardFM.IncludeInAll = false\n\tstandardFM.IncludeTermVectors = true\n\tstandardFM.Analyzer = \"standard\"\n\n\tdm := bleve.NewDocumentMapping()\n\tdm.AddFieldMappingsAt(\"title\", standardFM)\n\tdm.AddFieldMappingsAt(\"body\", standardFM)\n\n\tim := bleve.NewIndexMapping()\n\tim.DefaultMapping = dm\n\tim.DefaultAnalyzer = \"standard\"\n\n\treturn im\n}\n\nfunc (vt *VersusTest) insertBodies(idx bleve.Index) {\n\tbatch := idx.NewBatch()\n\tfor i, bodyWords := range vt.Bodies {\n\t\ttitle := fmt.Sprintf(\"%d\", i)\n\t\tbody := strings.Join(bodyWords, \" \")\n\t\terr := batch.Index(title, map[string]interface{}{\"title\": title, \"body\": body})\n\t\tif err != nil {\n\t\t\tvt.t.Fatalf(\"batch.Index err: %v\", err)\n\t\t}\n\t\tif i%vt.BatchSize == 0 {\n\t\t\terr = idx.Batch(batch)\n\t\t\tif err != nil {\n\t\t\t\tvt.t.Fatalf(\"batch err: %v\", err)\n\t\t\t}\n\t\t\tbatch.Reset()\n\t\t}\n\t}\n\terr := idx.Batch(batch)\n\tif err != nil {\n\t\tvt.t.Fatalf(\"last batch err: %v\", err)\n\t}\n}\n\nfunc (vt *VersusTest) genBodies() (rv [][]string) {\n\tfor i := 0; i < vt.NumDocs; i++ {\n\t\trv = append(rv, vt.genBody())\n\t}\n\treturn rv\n}\n\nfunc (vt *VersusTest) genBody() (rv []string) {\n\trng := rand.New(rand.NewSource(time.Now().UnixNano()))\n\n\tm := rng.Intn(vt.MaxWordsPerDoc)\n\tfor j := 0; j < m; j++ {\n\t\trv = append(rv, vt.genWord(rng.Intn(vt.NumWords)))\n\t}\n\treturn rv\n}\n\nfunc (vt *VersusTest) genWord(i int) string {\n\treturn fmt.Sprintf(\"%x\", i)\n}\n"
  },
  {
    "path": "util/extract.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage util\n\nimport (\n\t\"math\"\n\t\"reflect\"\n)\n\n// extract numeric value (if possible) and returns a float64\nfunc ExtractNumericValFloat64(v interface{}) (float64, bool) {\n\tval := reflect.ValueOf(v)\n\tif !val.IsValid() {\n\t\treturn 0, false\n\t}\n\n\tswitch {\n\tcase val.CanFloat():\n\t\treturn val.Float(), true\n\tcase val.CanInt():\n\t\treturn float64(val.Int()), true\n\tcase val.CanUint():\n\t\treturn float64(val.Uint()), true\n\t}\n\n\treturn 0, false\n}\n\n// extract numeric value (if possible) and returns a float32\nfunc ExtractNumericValFloat32(v interface{}) (float32, bool) {\n\tval := reflect.ValueOf(v)\n\tif !val.IsValid() {\n\t\treturn 0, false\n\t}\n\n\tswitch {\n\tcase val.CanFloat():\n\t\tfloatVal := val.Float()\n\t\tif !IsValidFloat32(floatVal) {\n\t\t\treturn 0, false\n\t\t}\n\t\treturn float32(floatVal), true\n\tcase val.CanInt():\n\t\treturn float32(val.Int()), true\n\tcase val.CanUint():\n\t\treturn float32(val.Uint()), true\n\t}\n\n\treturn 0, false\n}\n\nfunc IsValidFloat32(val float64) bool {\n\treturn !math.IsNaN(val) && !math.IsInf(val, 0) && val <= math.MaxFloat32\n}\n"
  },
  {
    "path": "util/json.go",
    "content": "//  Copyright (c) 2023 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage util\n\nimport (\n\t\"encoding/json\"\n)\n\n// Should only be overwritten during process init()'ialization.\nvar (\n\tMarshalJSON   = json.Marshal\n\tUnmarshalJSON = json.Unmarshal\n)\n"
  },
  {
    "path": "util/keys.go",
    "content": "//  Copyright (c) 2025 Couchbase, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// \t\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage util\n\nvar (\n\t// Bolt keys\n\tBoltSnapshotsBucket           = []byte{'s'}\n\tBoltPathKey                   = []byte{'p'}\n\tBoltDeletedKey                = []byte{'d'}\n\tBoltInternalKey               = []byte{'i'}\n\tBoltMetaDataKey               = []byte{'m'}\n\tBoltMetaDataSegmentTypeKey    = []byte(\"type\")\n\tBoltMetaDataSegmentVersionKey = []byte(\"version\")\n\tBoltMetaDataTimeStamp         = []byte(\"timeStamp\")\n\tBoltStatsKey                  = []byte(\"stats\")\n\tBoltUpdatedFieldsKey          = []byte(\"fields\")\n\tTotBytesWrittenKey            = []byte(\"TotBytesWritten\")\n\n\tMappingInternalKey = []byte(\"_mapping\")\n)\n"
  }
]