[ { "path": ".github/CODEOWNERS", "content": "@Workiva/skreams\n" }, { "path": ".github/workflows/tests.yaml", "content": "name: \"Tests\"\n\non:\n pull_request:\n push:\n branches:\n - 'master'\n tags:\n - '*'\n\npermissions:\n pull-requests: write\n contents: write\n id-token: write\n\njobs:\n Tests:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n go: [ '1.15', 'stable' ]\n name: Tests on Go ${{ matrix.go }}\n steps:\n - name: Checkout Repo\n uses: actions/checkout@v4\n with:\n path: go/src/github.com/Workiva/go-datastructures\n\n - name: Setup Go\n uses: actions/setup-go@v5.0.0\n with:\n go-version: ${{ matrix.go }}\n\n # go install does not work because it needs credentials\n - name: install go2xunit\n run: |\n git clone https://github.com/tebeka/go2xunit.git\n cd go2xunit\n git checkout v1.4.10\n go install \n cd ..\n\n - name: Run Tests\n timeout-minutes: 10\n run: |\n cd go/src/github.com/Workiva/go-datastructures\n go test ./... | tee ${{github.workspace}}/go-test.txt \n\n - name: XML output\n run: |\n mkdir artifacts\n go2xunit -input ./go-test.txt -output ./artifacts/tests_go_version-${{ matrix.go }}.xml\n\n - name: Upload Test Results\n uses: actions/upload-artifact@v4\n with:\n name: go-datastructures test go ${{ matrix.go }}\n path: ./artifacts/tests_go_version-${{ matrix.go }}.xml\n retention-days: 7\n\n - uses: anchore/sbom-action@v0\n with:\n path: ./ \n format: cyclonedx-json\n artifact-name: ${{ matrix.go }}-sbom.spdx\n" }, { "path": ".gitignore", "content": "*.out\n*.test\n.idea\n" }, { "path": "Dockerfile", "content": "FROM golang:1.16-alpine3.13 AS build-go\n\nARG GIT_SSH_KEY\nARG KNOWN_HOSTS_CONTENT\nWORKDIR /go/src/github.com/Workiva/go-datastructures/\nADD . /go/src/github.com/Workiva/go-datastructures/\n\nARG GOPATH=/go/\nENV PATH $GOPATH/bin:$PATH\nRUN echo \"Starting the script section\" && \\\n go mod vendor && \\\n echo \"script section completed\"\n\nARG BUILD_ARTIFACTS_DEPENDENCIES=/go/src/github.com/Workiva/go-datastructures/go.mod\n\nFROM scratch\n" }, { "path": "LICENSE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License." }, { "path": "README.md", "content": "go-datastructures\n=================\n\nGo-datastructures is a collection of useful, performant, and threadsafe Go\ndatastructures.\n\n### NOTE: only tested with Go 1.3+.\n\n#### Augmented Tree\n\nInterval tree for collision in n-dimensional ranges. Implemented via a\nred-black augmented tree. Extra dimensions are handled in simultaneous\ninserts/queries to save space although this may result in suboptimal time\ncomplexity. Intersection determined using bit arrays. In a single dimension,\ninserts, deletes, and queries should be in O(log n) time.\n\n#### Bitarray\n\nBitarray used to detect existence without having to resort to hashing with\nhashmaps. Requires entities have a uint64 unique identifier. Two\nimplementations exist, regular and sparse. Sparse saves a great deal of space\nbut insertions are O(log n). There are some useful functions on the BitArray\ninterface to detect intersection between two bitarrays. This package also\nincludes bitmaps of length 32 and 64 that provide increased speed and O(1) for\nall operations by storing the bitmaps in unsigned integers rather than arrays.\n\n#### Futures\n\nA helpful tool to send a \"broadcast\" message to listeners. Channels have the\nissue that once one listener takes a message from a channel the other listeners\naren't notified. There were many cases when I wanted to notify many listeners\nof a single event and this package helps.\n\n#### Queue\n\nPackage contains both a normal and priority queue. Both implementations never\nblock on send and grow as much as necessary. Both also only return errors if\nyou attempt to push to a disposed queue and will not panic like sending a\nmessage on a closed channel. The priority queue also allows you to place items\nin priority order inside the queue. If you give a useful hint to the regular\nqueue, it is actually faster than a channel. The priority queue is somewhat\nslow currently and targeted for an update to a Fibonacci heap.\n\nAlso included in the queue package is a MPMC threadsafe ring buffer. This is a\nblock full/empty queue, but will return a blocked thread if the queue is\ndisposed while a thread is blocked. This can be used to synchronize goroutines\nand ensure goroutines quit so objects can be GC'd. Threadsafety is achieved\nusing only CAS operations making this queue quite fast. Benchmarks can be found\nin that package.\n\n#### Fibonacci Heap\n\nA standard Fibonacci heap providing the usual operations. Can be useful in executing Dijkstra or Prim's algorithms in the theoretically minimal time. Also useful as a general-purpose priority queue. The special thing about Fibonacci heaps versus other heap variants is the cheap decrease-key operation. This heap has a constant complexity for find minimum, insert and merge of two heaps, an amortized constant complexity for decrease key and O(log(n)) complexity for a deletion or dequeue minimum. In practice the constant factors are large, so Fibonacci heaps could be slower than Pairing heaps, depending on usage. Benchmarks - in the project subfolder. The heap has not been designed for thread-safety.\n\n#### Range Tree\n\nUseful to determine if n-dimensional points fall within an n-dimensional range.\nNot a typical range tree however, as we are actually using an n-dimensional\nsorted list of points as this proved to be simpler and faster than attempting a\ntraditional range tree while saving space on any dimension greater than one.\nInserts are typical BBST times at O(log n^d) where d is the number of\ndimensions.\n\n#### Set\nOur Set implementation is very simple, accepts items of type `interface{}` and\nincludes only a few methods. If your application requires a richer Set\nimplementation over lists of type `sort.Interface`, see\n[xtgo/set](https://github.com/xtgo/set) and\n[goware/set](https://github.com/goware/set).\n\n#### Threadsafe\nA package that is meant to contain some commonly used items but in a threadsafe\nway. Example: there's a threadsafe error in there as I commonly found myself\nwanting to set an error in many threads at the same time (yes, I know, but\nchannels are slow).\n\n#### AVL Tree\n\nThis is an example of a branch copy immutable AVL BBST. Any operation on a node\nmakes a copy of that node's branch. Because of this, this tree is inherently\nthreadsafe although the writes will likely still need to be serialized. This\nstructure is good if your use case is a large number of reads and infrequent\nwrites as reads will be highly available but writes somewhat slow due to the\ncopying. This structure serves as a basis for a large number of functional data\nstructures.\n\n#### X-Fast Trie\n\nAn interesting design that treats integers as words and uses a trie structure to\nreduce time complexities by matching prefixes. This structure is really fast\nfor finding values or making predecessor/successor types of queries, but also\nresults in greater than linear space consumption. The exact time complexities\ncan be found in that package.\n\n#### Y-Fast Trie\n\nAn extension of the X-Fast trie in which an X-Fast trie is combined with some\nother ordered data structure to reduce space consumption and improve CRUD types\nof operations. These secondary structures are often BSTs, but our implementation\nuses a simple ordered list as I believe this improves cache locality. We also\nuse fixed size buckets to aid in parallelization of operations. Exact time\ncomplexities are in that package.\n\n#### Fast integer hashmap\n\nA datastructure used for checking existence but without knowing the bounds of\nyour data. If you have a limited small bounds, the bitarray package might be a\nbetter choice. This implementation uses a fairly simple hashing algorithm\ncombined with linear probing and a flat datastructure to provide optimal\nperformance up to a few million integers (faster than the native Golang\nimplementation). Beyond that, the native implementation is faster (I believe\nthey are using a large -ary B-tree). In the future, this will be implemented\nwith a B-tree for scale.\n\n#### Skiplist\n\nAn ordered structure that provides amortized logarithmic operations but without\nthe complication of rotations that are required by BSTs. In testing, however,\nthe performance of the skip list is often far worse than the guaranteed log n\ntime of a BBST. Tall nodes tend to \"cast shadows\", especially when large\nbitsizes are required as the optimum maximum height for a node is often based on\nthis. More detailed performance characteristics are provided in that package.\n\n#### Sort\n\nThe sort package implements a multithreaded bucket sort that can be up to 3x\nfaster than the native Golang sort package. These buckets are then merged using\na symmetrical merge, similar to the stable sort in the Golang package. However,\nour algorithm is modified so that two sorted lists can be merged by using\nsymmetrical decomposition.\n\n#### Numerics\n\nEarly work on some nonlinear optimization problems. The initial implementation\nallows a simple use case with either linear or nonlinear constraints. You can\nfind min/max or target an optimal value. The package currently employs a\nprobabilistic global restart system in an attempt to avoid local critical points.\nMore details can be found in that package.\n\n#### B+ Tree\n\nInitial implementation of a B+ tree. Delete method still needs added as well as\nsome performance optimization. Specific performance characteristics can be\nfound in that package. Despite the theoretical superiority of BSTs, the B-tree\noften has better all around performance due to cache locality. The current\nimplementation is mutable, but the immutable AVL tree can be used to build an\nimmutable version. Unfortunately, to make the B-tree generic we require an\ninterface and the most expensive operation in CPU profiling is the interface\nmethod which in turn calls into runtime.assertI2T. We need generics.\n\n#### Immutable B Tree\nA btree based on two principles, immutability and concurrency. \nSomewhat slow for single value lookups and puts, it is very fast for bulk operations.\nA persister can be injected to make this index persistent.\n\n#### Ctrie\n\nA concurrent, lock-free hash array mapped trie with efficient non-blocking\nsnapshots. For lookups, Ctries have comparable performance to concurrent skip\nlists and concurrent hashmaps. One key advantage of Ctries is they are\ndynamically allocated. Memory consumption is always proportional to the number\nof keys in the Ctrie, while hashmaps typically have to grow and shrink. Lookups,\ninserts, and removes are O(logn).\n\nOne interesting advantage Ctries have over traditional concurrent data\nstructures is support for lock-free, linearizable, constant-time snapshots.\nMost concurrent data structures do not support snapshots, instead opting for\nlocks or requiring a quiescent state. This allows Ctries to have O(1) iterator\ncreation and clear operations and O(logn) size retrieval.\n\n#### Dtrie\n\nA persistent hash trie that dynamically expands or shrinks to provide efficient\nmemory allocation. Being persistent, the Dtrie is immutable and any modification\nyields a new version of the Dtrie rather than changing the original. Bitmapped\nnodes allow for O(log32(n)) get, remove, and update operations. Insertions are\nO(n) and iteration is O(1).\n\n#### Persistent List\n\nA persistent, immutable linked list. All write operations yield a new, updated\nstructure which preserve and reuse previous versions. This uses a very\nfunctional, cons-style of list manipulation. Insert, get, remove, and size\noperations are O(n) as you would expect.\n\n#### Simple Graph\n\nA mutable, non-persistent undirected graph where parallel edges and self-loops are \nnot permitted. Operations to add an edge as well as retrieve the total number of \nvertices/edges are O(1) while the operation to retrieve the vertices adjacent to a\ntarget is O(n). For more details see [wikipedia](https://en.wikipedia.org/wiki/Graph_(discrete_mathematics)#Simple_graph)\n\n### Installation\n\n 1. Install Go 1.3 or higher.\n 2. Run `go get github.com/Workiva/go-datastructures/...`\n\n### Updating\n\nWhen new code is merged to master, you can use\n\n\tgo get -u github.com/Workiva/go-datastructures/...\n\nTo retrieve the latest version of go-datastructures.\n\n### Testing\n\nTo run all the unit tests use these commands:\n\n\tcd $GOPATH/src/github.com/Workiva/go-datastructures\n\tgo get -t -u ./...\n\tgo test ./...\n\nOnce you've done this once, you can simply use this command to run all unit tests:\n\n\tgo test ./...\n\n\n### Contributing\n\nRequirements to commit here:\n\n - Branch off master, PR back to master.\n - `gofmt`'d code.\n - Compliance with [these guidelines](https://code.google.com/p/go-wiki/wiki/CodeReviewComments)\n - Unit test coverage\n - [Good commit messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)\n\n\n### Maintainers\n - Alexander Campbell <[alexander.campbell@workiva.com](mailto:alexander.campbell@workiva.com)>\n - Dustin Hiatt <[dustin.hiatt@workiva.com](mailto:dustin.hiatt@workiva.com)>\n - Ryan Jackson <[ryan.jackson@workiva.com](mailto:ryan.jackson@workiva.com)>\n" }, { "path": "augmentedtree/atree.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage augmentedtree\n\nfunc intervalOverlaps(n *node, low, high int64, interval Interval, maxDimension uint64) bool {\n\tif !overlaps(n.interval.HighAtDimension(1), high, n.interval.LowAtDimension(1), low) {\n\t\treturn false\n\t}\n\n\tif interval == nil {\n\t\treturn true\n\t}\n\n\tfor i := uint64(2); i <= maxDimension; i++ {\n\t\tif !n.interval.OverlapsAtDimension(interval, i) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc overlaps(high, otherHigh, low, otherLow int64) bool {\n\treturn high >= otherLow && low <= otherHigh\n}\n\n// compare returns an int indicating which direction the node\n// should go.\nfunc compare(nodeLow, ivLow int64, nodeID, ivID uint64) int {\n\tif ivLow > nodeLow {\n\t\treturn 1\n\t}\n\n\tif ivLow < nodeLow {\n\t\treturn 0\n\t}\n\n\treturn intFromBool(ivID > nodeID)\n}\n\ntype node struct {\n\tinterval Interval\n\tmax, min int64 // max value held by children\n\tchildren [2]*node // array to hold left/right\n\tred bool // indicates if this node is red\n\tid uint64 // we store the id locally to reduce the number of calls to the method on the interface\n}\n\nfunc (n *node) query(low, high int64, interval Interval, maxDimension uint64, fn func(node *node)) {\n\tif n.children[0] != nil && overlaps(n.children[0].max, high, n.children[0].min, low) {\n\t\tn.children[0].query(low, high, interval, maxDimension, fn)\n\t}\n\n\tif intervalOverlaps(n, low, high, interval, maxDimension) {\n\t\tfn(n)\n\t}\n\n\tif n.children[1] != nil && overlaps(n.children[1].max, high, n.children[1].min, low) {\n\t\tn.children[1].query(low, high, interval, maxDimension, fn)\n\t}\n}\n\nfunc (n *node) adjustRanges() {\n\tfor i := 0; i <= 1; i++ {\n\t\tif n.children[i] != nil {\n\t\t\tn.children[i].adjustRanges()\n\t\t}\n\t}\n\n\tn.adjustRange()\n}\n\nfunc (n *node) adjustRange() {\n\tsetMin(n)\n\tsetMax(n)\n}\n\nfunc newDummy() node {\n\treturn node{\n\t\tchildren: [2]*node{},\n\t}\n}\n\nfunc newNode(interval Interval, min, max int64, dimension uint64) *node {\n\titn := &node{\n\t\tinterval: interval,\n\t\tmin: min,\n\t\tmax: max,\n\t\tred: true,\n\t\tchildren: [2]*node{},\n\t}\n\tif interval != nil {\n\t\titn.id = interval.ID()\n\t}\n\n\treturn itn\n}\n\ntype tree struct {\n\troot *node\n\tmaxDimension, number uint64\n\tdummy node\n}\n\nfunc (t *tree) Traverse(fn func(id Interval)) {\n\tnodes := []*node{t.root}\n\n\tfor len(nodes) != 0 {\n\t\tc := nodes[len(nodes)-1]\n\t\tnodes = nodes[:len(nodes)-1]\n\t\tif c != nil {\n\t\t\tfn(c.interval)\n\t\t\tif c.children[0] != nil {\n\t\t\t\tnodes = append(nodes, c.children[0])\n\t\t\t}\n\t\t\tif c.children[1] != nil {\n\t\t\t\tnodes = append(nodes, c.children[1])\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (tree *tree) resetDummy() {\n\ttree.dummy.children[0], tree.dummy.children[1] = nil, nil\n\ttree.dummy.red = false\n}\n\n// Len returns the number of items in this tree.\nfunc (tree *tree) Len() uint64 {\n\treturn tree.number\n}\n\n// add will add the provided interval to the tree.\nfunc (tree *tree) add(iv Interval) {\n\tif tree.root == nil {\n\t\ttree.root = newNode(\n\t\t\tiv, iv.LowAtDimension(1),\n\t\t\tiv.HighAtDimension(1),\n\t\t\t1,\n\t\t)\n\t\ttree.root.red = false\n\t\ttree.number++\n\t\treturn\n\t}\n\n\ttree.resetDummy()\n\tvar (\n\t\tdummy = tree.dummy\n\t\tparent, grandParent *node\n\t\tnode = tree.root\n\t\tdir, last int\n\t\totherLast = 1\n\t\tid = iv.ID()\n\t\tmax = iv.HighAtDimension(1)\n\t\tivLow = iv.LowAtDimension(1)\n\t\thelper = &dummy\n\t)\n\n\t// set this AFTER clearing dummy\n\thelper.children[1] = tree.root\n\tfor {\n\t\tif node == nil {\n\t\t\tnode = newNode(iv, ivLow, max, 1)\n\t\t\tparent.children[dir] = node\n\t\t\ttree.number++\n\t\t} else if isRed(node.children[0]) && isRed(node.children[1]) {\n\t\t\tnode.red = true\n\t\t\tnode.children[0].red = false\n\t\t\tnode.children[1].red = false\n\t\t}\n\t\tif max > node.max {\n\t\t\tnode.max = max\n\t\t}\n\n\t\tif ivLow < node.min {\n\t\t\tnode.min = ivLow\n\t\t}\n\n\t\tif isRed(parent) && isRed(node) {\n\t\t\tlocalDir := intFromBool(helper.children[1] == grandParent)\n\n\t\t\tif node == parent.children[last] {\n\t\t\t\thelper.children[localDir] = rotate(grandParent, otherLast)\n\t\t\t} else {\n\t\t\t\thelper.children[localDir] = doubleRotate(grandParent, otherLast)\n\t\t\t}\n\t\t}\n\n\t\tif node.id == id {\n\t\t\tbreak\n\t\t}\n\n\t\tlast = dir\n\t\totherLast = takeOpposite(last)\n\t\tdir = compare(node.interval.LowAtDimension(1), ivLow, node.id, id)\n\n\t\tif grandParent != nil {\n\t\t\thelper = grandParent\n\t\t}\n\t\tgrandParent, parent, node = parent, node, node.children[dir]\n\t}\n\n\ttree.root = dummy.children[1]\n\ttree.root.red = false\n}\n\n// Add will add the provided intervals to this tree.\nfunc (tree *tree) Add(intervals ...Interval) {\n\tfor _, iv := range intervals {\n\t\ttree.add(iv)\n\t}\n}\n\n// delete will remove the provided interval from the tree.\nfunc (tree *tree) delete(iv Interval) {\n\tif tree.root == nil {\n\t\treturn\n\t}\n\n\ttree.resetDummy()\n\tvar (\n\t\tdummy = tree.dummy\n\t\tfound, parent, grandParent *node\n\t\tlast, otherDir, otherLast int // keeping track of last direction\n\t\tid = iv.ID()\n\t\tdir = 1\n\t\tnode = &dummy\n\t\tivLow = iv.LowAtDimension(1)\n\t)\n\n\tnode.children[1] = tree.root\n\tfor node.children[dir] != nil {\n\t\tlast = dir\n\t\totherLast = takeOpposite(last)\n\n\t\tgrandParent, parent, node = parent, node, node.children[dir]\n\n\t\tdir = compare(node.interval.LowAtDimension(1), ivLow, node.id, id)\n\t\totherDir = takeOpposite(dir)\n\n\t\tif node.id == id {\n\t\t\tfound = node\n\t\t}\n\n\t\tif !isRed(node) && !isRed(node.children[dir]) {\n\t\t\tif isRed(node.children[otherDir]) {\n\t\t\t\tparent.children[last] = rotate(node, dir)\n\t\t\t\tparent = parent.children[last]\n\t\t\t} else if !isRed(node.children[otherDir]) {\n\t\t\t\tt := parent.children[otherLast]\n\n\t\t\t\tif t != nil {\n\t\t\t\t\tif !isRed(t.children[otherLast]) && !isRed(t.children[last]) {\n\t\t\t\t\t\tparent.red = false\n\t\t\t\t\t\tnode.red = true\n\t\t\t\t\t\tt.red = true\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlocalDir := intFromBool(grandParent.children[1] == parent)\n\n\t\t\t\t\t\tif isRed(t.children[last]) {\n\t\t\t\t\t\t\tgrandParent.children[localDir] = doubleRotate(\n\t\t\t\t\t\t\t\tparent, last,\n\t\t\t\t\t\t\t)\n\t\t\t\t\t\t} else if isRed(t.children[otherLast]) {\n\t\t\t\t\t\t\tgrandParent.children[localDir] = rotate(\n\t\t\t\t\t\t\t\tparent, last,\n\t\t\t\t\t\t\t)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tnode.red = true\n\t\t\t\t\t\tgrandParent.children[localDir].red = true\n\t\t\t\t\t\tgrandParent.children[localDir].children[0].red = false\n\t\t\t\t\t\tgrandParent.children[localDir].children[1].red = false\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif found != nil {\n\t\ttree.number--\n\t\tfound.interval, found.max, found.min, found.id = node.interval, node.max, node.min, node.id\n\t\tparentDir := intFromBool(parent.children[1] == node)\n\t\tchildDir := intFromBool(node.children[0] == nil)\n\n\t\tparent.children[parentDir] = node.children[childDir]\n\t}\n\n\ttree.root = dummy.children[1]\n\tif tree.root != nil {\n\t\ttree.root.red = false\n\t}\n}\n\n// Delete will remove the provided intervals from this tree.\nfunc (tree *tree) Delete(intervals ...Interval) {\n\tfor _, iv := range intervals {\n\t\ttree.delete(iv)\n\t}\n\tif tree.root != nil {\n\t\ttree.root.adjustRanges()\n\t}\n}\n\n// Query will return a list of intervals that intersect the provided\n// interval. The provided interval's ID method is ignored so the\n// provided ID is irrelevant.\nfunc (tree *tree) Query(interval Interval) Intervals {\n\tif tree.root == nil {\n\t\treturn nil\n\t}\n\n\tvar (\n\t\tIntervals = intervalsPool.Get().(Intervals)\n\t\tivLow = interval.LowAtDimension(1)\n\t\tivHigh = interval.HighAtDimension(1)\n\t)\n\n\ttree.root.query(ivLow, ivHigh, interval, tree.maxDimension, func(node *node) {\n\t\tIntervals = append(Intervals, node.interval)\n\t})\n\n\treturn Intervals\n}\n\nfunc isRed(node *node) bool {\n\treturn node != nil && node.red\n}\n\nfunc setMax(parent *node) {\n\tparent.max = parent.interval.HighAtDimension(1)\n\n\tif parent.children[0] != nil && parent.children[0].max > parent.max {\n\t\tparent.max = parent.children[0].max\n\t}\n\n\tif parent.children[1] != nil && parent.children[1].max > parent.max {\n\t\tparent.max = parent.children[1].max\n\t}\n}\n\nfunc setMin(parent *node) {\n\tparent.min = parent.interval.LowAtDimension(1)\n\tif parent.children[0] != nil && parent.children[0].min < parent.min {\n\t\tparent.min = parent.children[0].min\n\t}\n\n\tif parent.children[1] != nil && parent.children[1].min < parent.min {\n\t\tparent.min = parent.children[1].min\n\t}\n\n\tif parent.interval.LowAtDimension(1) < parent.min {\n\t\tparent.min = parent.interval.LowAtDimension(1)\n\t}\n}\n\nfunc rotate(parent *node, dir int) *node {\n\totherDir := takeOpposite(dir)\n\n\tchild := parent.children[otherDir]\n\tparent.children[otherDir] = child.children[dir]\n\tchild.children[dir] = parent\n\tparent.red = true\n\tchild.red = false\n\tchild.max = parent.max\n\tsetMax(child)\n\tsetMax(parent)\n\tsetMin(child)\n\tsetMin(parent)\n\n\treturn child\n}\n\nfunc doubleRotate(parent *node, dir int) *node {\n\totherDir := takeOpposite(dir)\n\n\tparent.children[otherDir] = rotate(parent.children[otherDir], otherDir)\n\treturn rotate(parent, dir)\n}\n\nfunc intFromBool(value bool) int {\n\tif value {\n\t\treturn 1\n\t}\n\n\treturn 0\n}\n\nfunc takeOpposite(value int) int {\n\treturn 1 - value\n}\n\nfunc newTree(maxDimension uint64) *tree {\n\treturn &tree{\n\t\tmaxDimension: maxDimension,\n\t\tdummy: newDummy(),\n\t}\n}\n\n// New constructs and returns a new interval tree with the max\n// dimensions provided.\nfunc New(dimensions uint64) Tree {\n\treturn newTree(dimensions)\n}\n" }, { "path": "augmentedtree/atree_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage augmentedtree\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc min(one, two int64) int64 {\n\tif one == -1 {\n\t\treturn two\n\t}\n\n\tif two == -1 {\n\t\treturn one\n\t}\n\n\tif one > two {\n\t\treturn two\n\t}\n\n\treturn one\n}\n\nfunc max(one, two int64) int64 {\n\tif one == -1 {\n\t\treturn two\n\t}\n\n\tif two == -1 {\n\t\treturn one\n\t}\n\n\tif one > two {\n\t\treturn one\n\t}\n\n\treturn two\n}\n\nfunc checkRedBlack(tb testing.TB, node *node, dimension int) (int64, int64, int64) {\n\tlh, rh := 0, 0\n\tif node == nil {\n\t\treturn 1, -1, -1\n\t}\n\n\tif isRed(node) {\n\t\tif isRed(node.children[0]) || isRed(node.children[1]) {\n\t\t\ttb.Errorf(`Node is red and has red children: %+v`, node)\n\t\t}\n\t}\n\n\tfn := func(min, max int64) {\n\t\tif min != -1 && min < node.min {\n\t\t\ttb.Errorf(`Min not set correctly: %+v, node: %+v`, min, node)\n\t\t}\n\n\t\tif max != -1 && max > node.max {\n\t\t\ttb.Errorf(`Max not set correctly: %+v, node: %+v`, max, node)\n\t\t}\n\t}\n\n\tleft, minL, maxL := checkRedBlack(tb, node.children[0], dimension)\n\tfn(minL, maxL)\n\tright, minR, maxR := checkRedBlack(tb, node.children[1], dimension)\n\tfn(minR, maxR)\n\n\tmin := min(minL, minR)\n\tif min == -1 && node.min != node.interval.LowAtDimension(1) {\n\t\ttb.Errorf(`Min not set correctly, node: %+v`, node)\n\t} else if min != -1 && node.children[0] != nil && node.children[0].min != node.min {\n\t\ttb.Errorf(`Min not set correctly: node: %+v, child: %+v`, node, node.children[0])\n\t} else if min != -1 && node.children[0] == nil && node.min != node.interval.LowAtDimension(1) {\n\t\ttb.Errorf(`Min not set correctly: %+v`, node)\n\t}\n\n\tmax := max(maxL, maxR)\n\tif max == -1 && node.max != node.interval.HighAtDimension(1) {\n\t\ttb.Errorf(`Max not set correctly, node: %+v`, node)\n\t} else if max > node.interval.HighAtDimension(1) && max != node.max {\n\t\ttb.Errorf(`Max not set correctly, max: %+v, node: %+v`, max, node)\n\t}\n\n\tif left != 0 && right != 0 && lh != rh {\n\t\ttb.Errorf(`Black violation: left: %d, right: %d`, left, right)\n\t}\n\n\tif left != 0 && right != 0 {\n\t\tif isRed(node) {\n\t\t\treturn left, node.min, node.max\n\t\t}\n\n\t\treturn left + 1, node.min, node.max\n\t}\n\n\treturn 0, node.min, node.max\n}\n\nfunc constructSingleDimensionTestTree(number int) (*tree, Intervals) {\n\ttree := newTree(1)\n\n\tivs := make(Intervals, 0, number)\n\tfor i := 0; i < number; i++ {\n\t\tiv := constructSingleDimensionInterval(int64(i), int64(i)+10, uint64(i))\n\t\tivs = append(ivs, iv)\n\t}\n\n\ttree.Add(ivs...)\n\treturn tree, ivs\n}\n\nfunc TestSimpleAddNilRoot(t *testing.T) {\n\tit := newTree(1)\n\n\tiv := constructSingleDimensionInterval(5, 10, 0)\n\n\tit.Add(iv)\n\n\texpected := newNode(iv, 5, 10, 1)\n\texpected.red = false\n\n\tassert.Equal(t, expected, it.root)\n\tassert.Equal(t, uint64(1), it.Len())\n\tcheckRedBlack(t, it.root, 1)\n}\n\nfunc TestSimpleAddRootLeft(t *testing.T) {\n\tit := newTree(1)\n\n\tiv := constructSingleDimensionInterval(5, 10, 0)\n\tit.Add(iv)\n\n\texpectedRoot := newNode(iv, 4, 11, 1)\n\texpectedRoot.red = false\n\n\tiv = constructSingleDimensionInterval(4, 11, 1)\n\tit.Add(iv)\n\n\texpectedChild := newNode(iv, 4, 11, 1)\n\texpectedRoot.children[0] = expectedChild\n\n\tassert.Equal(t, expectedRoot, it.root)\n\tassert.Equal(t, uint64(2), it.Len())\n\tcheckRedBlack(t, it.root, 1)\n}\n\nfunc TestSimpleAddRootRight(t *testing.T) {\n\tit := newTree(1)\n\n\tiv := constructSingleDimensionInterval(5, 10, 0)\n\tit.Add(iv)\n\n\texpectedRoot := newNode(iv, 5, 11, 1)\n\texpectedRoot.red = false\n\n\tiv = constructSingleDimensionInterval(7, 11, 1)\n\tit.Add(iv)\n\n\texpectedChild := newNode(iv, 7, 11, 1)\n\texpectedRoot.children[1] = expectedChild\n\n\tassert.Equal(t, expectedRoot, it.root)\n\tassert.Equal(t, uint64(2), it.Len())\n\tcheckRedBlack(t, it.root, 1)\n}\n\nfunc TestAddRootLeftAndRight(t *testing.T) {\n\tit := newTree(1)\n\n\tiv := constructSingleDimensionInterval(5, 10, 0)\n\tit.Add(iv)\n\n\texpectedRoot := newNode(iv, 4, 12, 1)\n\texpectedRoot.red = false\n\n\tiv = constructSingleDimensionInterval(4, 11, 1)\n\tit.Add(iv)\n\n\texpectedLeft := newNode(iv, 4, 11, 1)\n\texpectedRoot.children[0] = expectedLeft\n\n\tiv = constructSingleDimensionInterval(7, 12, 1)\n\tit.Add(iv)\n\n\texpectedRight := newNode(iv, 7, 12, 1)\n\texpectedRoot.children[1] = expectedRight\n\n\tassert.Equal(t, expectedRoot, it.root)\n\tassert.Equal(t, uint64(3), it.Len())\n\tcheckRedBlack(t, it.root, 1)\n}\n\nfunc TestAddRebalanceInOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tfor i := int64(0); i < 10; i++ {\n\t\tiv := constructSingleDimensionInterval(i, i+1, uint64(i))\n\t\tit.add(iv)\n\t}\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tassert.Len(t, result, 10)\n\tassert.Equal(t, uint64(10), it.Len())\n}\n\nfunc TestAddRebalanceOutOfOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tfor i := int64(9); i >= 0; i-- {\n\t\tiv := constructSingleDimensionInterval(i, i+1, uint64(i))\n\t\tit.add(iv)\n\t}\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tassert.Len(t, result, 10)\n\tassert.Equal(t, uint64(10), it.Len())\n}\n\nfunc TestAddRebalanceRandomOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tstarts := []int64{0, 4, 2, 1, 3}\n\n\tfor _, start := range starts {\n\t\tiv := constructSingleDimensionInterval(start, start+1, uint64(start))\n\t\tit.add(iv)\n\t}\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tassert.Len(t, result, 5)\n\tassert.Equal(t, uint64(5), it.Len())\n}\n\nfunc TestAddLargeNumberOfItems(t *testing.T) {\n\tnumItems := int64(1000)\n\tit := newTree(1)\n\n\tfor i := int64(0); i < numItems; i++ {\n\t\tiv := constructSingleDimensionInterval(i, i+1, uint64(i))\n\t\tit.add(iv)\n\t}\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(constructSingleDimensionInterval(0, numItems, 0))\n\tassert.Len(t, result, int(numItems))\n\tassert.Equal(t, uint64(numItems), it.Len())\n}\n\nfunc BenchmarkAddItems(b *testing.B) {\n\tnumItems := int64(1000)\n\tintervals := make(Intervals, 0, numItems)\n\n\tfor i := int64(0); i < numItems; i++ {\n\t\tiv := constructSingleDimensionInterval(i, i+1, uint64(i))\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tit := newTree(1)\n\t\tit.Add(intervals...)\n\t}\n}\n\nfunc BenchmarkQueryItems(b *testing.B) {\n\tnumItems := int64(1000)\n\tintervals := make(Intervals, 0, numItems)\n\n\tfor i := int64(0); i < numItems; i++ {\n\t\tiv := constructSingleDimensionInterval(i, i+1, uint64(i))\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit := newTree(1)\n\tit.Add(intervals...)\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tit.Query(constructSingleDimensionInterval(0, numItems, 0))\n\t}\n}\n\nfunc constructSingleDimensionQueryTestTree() (\n\t*tree, Interval, Interval, Interval) {\n\n\tit := newTree(1)\n\n\tiv1 := constructSingleDimensionInterval(6, 10, 0)\n\tit.Add(iv1)\n\n\tiv2 := constructSingleDimensionInterval(4, 5, 1)\n\tit.Add(iv2)\n\n\tiv3 := constructSingleDimensionInterval(7, 12, 2)\n\tit.Add(iv3)\n\n\treturn it, iv1, iv2, iv3\n}\n\nfunc TestSimpleQuery(t *testing.T) {\n\tit, iv1, iv2, _ := constructSingleDimensionQueryTestTree()\n\n\tresult := it.Query(constructSingleDimensionInterval(3, 6, 0))\n\n\texpected := Intervals{iv2, iv1}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestRightQuery(t *testing.T) {\n\tit, iv1, _, iv3 := constructSingleDimensionQueryTestTree()\n\n\tresult := it.Query(constructSingleDimensionInterval(6, 8, 0))\n\n\texpected := Intervals{iv1, iv3}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestLeftQuery(t *testing.T) {\n\tit, _, iv2, _ := constructSingleDimensionQueryTestTree()\n\n\tresult := it.Query(constructSingleDimensionInterval(3, 5, 0))\n\n\texpected := Intervals{iv2}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestMatchingQuery(t *testing.T) {\n\tit, _, iv2, _ := constructSingleDimensionQueryTestTree()\n\n\tresult := it.Query(constructSingleDimensionInterval(4, 5, 0))\n\n\texpected := Intervals{iv2}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestNoMatchLeft(t *testing.T) {\n\tit, _, _, _ := constructSingleDimensionQueryTestTree()\n\n\tresult := it.Query(constructSingleDimensionInterval(1, 3, 0))\n\n\texpected := Intervals{}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestNoMatchRight(t *testing.T) {\n\tit, _, _, _ := constructSingleDimensionQueryTestTree()\n\n\tresult := it.Query(constructSingleDimensionInterval(13, 13, 0))\n\n\texpected := Intervals{}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestAllQuery(t *testing.T) {\n\tit, iv1, iv2, iv3 := constructSingleDimensionQueryTestTree()\n\n\tresult := it.Query(constructSingleDimensionInterval(1, 14, 0))\n\n\texpected := Intervals{iv2, iv1, iv3}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestQueryDuplicate(t *testing.T) {\n\tit, _, iv2, _ := constructSingleDimensionQueryTestTree()\n\tiv4 := constructSingleDimensionInterval(4, 5, 3)\n\tit.Add(iv4)\n\n\tresult := it.Query(constructSingleDimensionInterval(4, 5, 0))\n\n\texpected := Intervals{iv2, iv4}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestRootDelete(t *testing.T) {\n\tit := newTree(1)\n\tiv := constructSingleDimensionInterval(1, 5, 1)\n\tit.add(iv)\n\n\tit.Delete(iv)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(constructSingleDimensionInterval(1, 10, 0))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc TestDeleteLeft(t *testing.T) {\n\tit, iv1, iv2, iv3 := constructSingleDimensionQueryTestTree()\n\n\tit.Delete(iv2)\n\n\texpected := Intervals{iv1, iv3}\n\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tcheckRedBlack(t, it.root, 1)\n\tassert.Equal(t, expected, result)\n\tassert.Equal(t, uint64(2), it.Len())\n}\n\nfunc TestDeleteRight(t *testing.T) {\n\tit, iv1, iv2, iv3 := constructSingleDimensionQueryTestTree()\n\n\tit.Delete(iv3)\n\n\texpected := Intervals{iv2, iv1}\n\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tcheckRedBlack(t, it.root, 1)\n\tassert.Equal(t, expected, result)\n\tassert.Equal(t, uint64(2), it.Len())\n}\n\nfunc TestDeleteCenter(t *testing.T) {\n\tit, iv1, iv2, iv3 := constructSingleDimensionQueryTestTree()\n\n\tit.Delete(iv1)\n\n\texpected := Intervals{iv2, iv3}\n\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tcheckRedBlack(t, it.root, 1)\n\tassert.Equal(t, expected, result)\n\tassert.Equal(t, uint64(2), it.Len())\n}\n\nfunc TestDeleteRebalanceInOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tvar toDelete *mockInterval\n\n\tfor i := int64(0); i < 10; i++ {\n\t\tiv := constructSingleDimensionInterval(i, i+1, uint64(i))\n\t\tit.add(iv)\n\t\tif i == 5 {\n\t\t\ttoDelete = iv\n\t\t}\n\t}\n\n\tit.Delete(toDelete)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tassert.Len(t, result, 9)\n\tassert.Equal(t, uint64(9), it.Len())\n}\n\nfunc TestDeleteRebalanceOutOfOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tvar toDelete *mockInterval\n\tfor i := int64(9); i >= 0; i-- {\n\t\tiv := constructSingleDimensionInterval(i, i+1, uint64(i))\n\t\tit.add(iv)\n\t\tif i == 5 {\n\t\t\ttoDelete = iv\n\t\t}\n\t}\n\n\tit.Delete(toDelete)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tassert.Len(t, result, 9)\n\tassert.Equal(t, uint64(9), it.Len())\n}\n\nfunc TestDeleteRebalanceRandomOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tstarts := []int64{0, 4, 2, 1, 3}\n\n\tvar toDelete *mockInterval\n\tfor _, start := range starts {\n\t\tiv := constructSingleDimensionInterval(start, start+1, uint64(start))\n\t\tit.add(iv)\n\t\tif start == 1 {\n\t\t\ttoDelete = iv\n\t\t}\n\t}\n\n\tit.Delete(toDelete)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(constructSingleDimensionInterval(0, 10, 0))\n\tassert.Len(t, result, 4)\n\tassert.Equal(t, uint64(4), it.Len())\n}\n\nfunc TestDeleteEmptyTree(t *testing.T) {\n\tit := newTree(1)\n\n\tit.Delete(constructSingleDimensionInterval(0, 1, 1))\n\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc BenchmarkDeleteItems(b *testing.B) {\n\tnumItems := int64(1000)\n\n\tintervals := make(Intervals, 0, numItems)\n\tfor i := int64(0); i < numItems; i++ {\n\t\tiv := constructSingleDimensionInterval(i, i+1, uint64(i))\n\t\tintervals = append(intervals, iv)\n\t}\n\n\ttrees := make([]*tree, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tit := newTree(1)\n\t\tit.Add(intervals...)\n\t\ttrees = append(trees, it)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttrees[i].Delete(intervals...)\n\t}\n}\n\nfunc TestAddDuplicateRanges(t *testing.T) {\n\tit := newTree(1)\n\tiv1 := constructSingleDimensionInterval(0, 10, 1)\n\tiv2 := constructSingleDimensionInterval(0, 10, 2)\n\tiv3 := constructSingleDimensionInterval(0, 10, 3)\n\n\tit.Add(iv1, iv2, iv3)\n\tit.Delete(iv1, iv2, iv3)\n\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc TestAddDeleteDuplicatesRebalanceInOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tintervals := make(Intervals, 0, 10)\n\n\tfor i := 0; i < 10; i++ {\n\t\tiv := constructSingleDimensionInterval(0, 10, uint64(i))\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit.Add(intervals...)\n\tit.Delete(intervals...)\n\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc TestAddDeleteDuplicatesRebalanceReverseOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tintervals := make(Intervals, 0, 10)\n\n\tfor i := 9; i >= 0; i-- {\n\t\tiv := constructSingleDimensionInterval(0, 10, uint64(i))\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit.Add(intervals...)\n\tit.Delete(intervals...)\n\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc TestAddDeleteDuplicatesRebalanceRandomOrder(t *testing.T) {\n\tit := newTree(1)\n\n\tstarts := []int{0, 4, 2, 1, 3}\n\tintervals := make(Intervals, 0, 5)\n\n\tfor _, start := range starts {\n\t\tiv := constructSingleDimensionInterval(0, 10, uint64(start))\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit.Add(intervals...)\n\tit.Delete(intervals...)\n\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc TestInsertDuplicateIntervalsToRoot(t *testing.T) {\n\ttree := newTree(1)\n\tiv1 := constructSingleDimensionInterval(0, 10, 1)\n\tiv2 := constructSingleDimensionInterval(0, 10, 1)\n\tiv3 := constructSingleDimensionInterval(0, 10, 1)\n\n\ttree.Add(iv1, iv2, iv3)\n\n\tcheckRedBlack(t, tree.root, 1)\n}\n\nfunc TestInsertDuplicateIntervalChildren(t *testing.T) {\n\ttree, _ := constructSingleDimensionTestTree(20)\n\n\tiv1 := constructSingleDimensionInterval(0, 10, 21)\n\tiv2 := constructSingleDimensionInterval(0, 10, 21)\n\n\ttree.Add(iv1, iv2)\n\n\tcheckRedBlack(t, tree.root, 1)\n\n\tresult := tree.Query(constructSingleDimensionInterval(0, 10, 0))\n\tassert.Contains(t, result, iv1)\n}\n\nfunc TestTraverse(t *testing.T) {\n\ttree := newTree(1)\n\n\ttree.Traverse(func(i Interval) {\n\t\tassert.Fail(t, `traverse should not be called for empty tree`)\n\t})\n\n\ttop := 30\n\tfor i := 0; i <= top; i++ {\n\t\ttree.Add(constructSingleDimensionInterval(int64(i*10), int64((i+1)*10), uint64(i)))\n\t}\n\tfound := map[uint64]bool{}\n\ttree.Traverse(func(id Interval) {\n\t\tfound[id.ID()] = true\n\t})\n\tfor i := 0; i <= top; i++ {\n\t\tif found, _ := found[uint64(i)]; !found {\n\t\t\tt.Errorf(\"could not find expected interval %d\", i)\n\t\t}\n\t}\n}\n" }, { "path": "augmentedtree/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage augmentedtree is designed to be useful when checking\nfor intersection of ranges in n-dimensions. For instance, if you imagine\nan xy plane then the augmented tree is for telling you if\nplane defined by the points (0, 0) and (10, 10). The augmented\ntree can tell you if that plane overlaps with a plane defined by\n(-5, -5) and (5, 5) (true in this case). You can also check\nintersections against a point by constructing a range of encompassed\nsolely if a single point.\n\nThe current tree is a simple top-down red-black binary search tree.\n\nTODO: Add a bottom-up implementation to assist with duplicate\nrange handling.\n*/\npackage augmentedtree\n\n// Interval is the interface that must be implemented by any\n// item added to the interval tree. This interface is similar to the\n// interval found in the rangetree package and it should be possible\n// for the same struct to implement both interfaces. Note that ranges\n// here are inclusive. It is also expected that the provided interval\n// is immutable and that the returned values never change. Doing so\n// results in undefined behavior.\ntype Interval interface {\n\t// LowAtDimension returns an integer representing the lower bound\n\t// at the requested dimension.\n\tLowAtDimension(uint64) int64\n\t// HighAtDimension returns an integer representing the higher bound\n\t// at the requested dimension.\n\tHighAtDimension(uint64) int64\n\t// OverlapsAtDimension should return a bool indicating if the provided\n\t// interval overlaps this interval at the dimension requested.\n\tOverlapsAtDimension(Interval, uint64) bool\n\t// ID should be a unique ID representing this interval. This\n\t// is used to identify which interval to delete from the tree if\n\t// there are duplicates.\n\tID() uint64\n}\n\n// Tree defines the object that is returned from the\n// tree constructor. We use a Tree interface here because\n// the returned tree could be a single dimension or many\n// dimensions.\ntype Tree interface {\n\t// Add will add the provided intervals to the tree.\n\tAdd(intervals ...Interval)\n\t// Len returns the number of intervals in the tree.\n\tLen() uint64\n\t// Delete will remove the provided intervals from the tree.\n\tDelete(intervals ...Interval)\n\t// Query will return a list of intervals that intersect the provided\n\t// interval. The provided interval's ID method is ignored so the\n\t// provided ID is irrelevant.\n\tQuery(interval Interval) Intervals\n\t// Traverse will traverse tree and give alls intervals\n\t// found in an undefined order\n\tTraverse(func(Interval))\n}\n" }, { "path": "augmentedtree/intervals.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage augmentedtree\n\nimport \"sync\"\n\nvar intervalsPool = sync.Pool{\n\tNew: func() interface{} {\n\t\treturn make(Intervals, 0, 10)\n\t},\n}\n\n// Intervals represents a list of Intervals.\ntype Intervals []Interval\n\n// Dispose will free any consumed resources and allow this list to be\n// re-allocated.\nfunc (ivs *Intervals) Dispose() {\n\tfor i := 0; i < len(*ivs); i++ {\n\t\t(*ivs)[i] = nil\n\t}\n\n\t*ivs = (*ivs)[:0]\n\tintervalsPool.Put(*ivs)\n}\n" }, { "path": "augmentedtree/intervals_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage augmentedtree\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestIntervalsDispose(t *testing.T) {\n\tintervals := intervalsPool.Get().(Intervals)\n\tintervals = append(intervals, constructSingleDimensionInterval(0, 1, 0))\n\n\tintervals.Dispose()\n\n\tassert.Len(t, intervals, 0)\n}\n" }, { "path": "augmentedtree/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage augmentedtree\n\nimport \"fmt\"\n\ntype dimension struct {\n\tlow, high int64\n}\n\ntype mockInterval struct {\n\tdimensions []*dimension\n\tid uint64\n}\n\nfunc (mi *mockInterval) checkDimension(dimension uint64) {\n\tif dimension > uint64(len(mi.dimensions)) {\n\t\tpanic(fmt.Sprintf(`Dimension: %d out of range.`, dimension))\n\t}\n}\n\nfunc (mi *mockInterval) LowAtDimension(dimension uint64) int64 {\n\treturn mi.dimensions[dimension-1].low\n}\n\nfunc (mi *mockInterval) HighAtDimension(dimension uint64) int64 {\n\treturn mi.dimensions[dimension-1].high\n}\n\nfunc (mi *mockInterval) OverlapsAtDimension(iv Interval, dimension uint64) bool {\n\treturn mi.HighAtDimension(dimension) > iv.LowAtDimension(dimension) &&\n\t\tmi.LowAtDimension(dimension) < iv.HighAtDimension(dimension)\n}\n\nfunc (mi mockInterval) ID() uint64 {\n\treturn mi.id\n}\n\nfunc constructSingleDimensionInterval(low, high int64, id uint64) *mockInterval {\n\treturn &mockInterval{[]*dimension{&dimension{low: low, high: high}}, id}\n}\n\nfunc constructMultiDimensionInterval(id uint64, dimensions ...*dimension) *mockInterval {\n\treturn &mockInterval{dimensions: dimensions, id: id}\n}\n" }, { "path": "augmentedtree/multidimensional_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage augmentedtree\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc constructMultiDimensionQueryTestTree() (\n\t*tree, Interval, Interval, Interval) {\n\n\tit := newTree(2)\n\n\tiv1 := constructMultiDimensionInterval(\n\t\t0, &dimension{low: 5, high: 10}, &dimension{low: 5, high: 10},\n\t)\n\tit.Add(iv1)\n\n\tiv2 := constructMultiDimensionInterval(\n\t\t1, &dimension{low: 4, high: 5}, &dimension{low: 4, high: 5},\n\t)\n\tit.Add(iv2)\n\n\tiv3 := constructMultiDimensionInterval(\n\t\t2, &dimension{low: 7, high: 12}, &dimension{low: 7, high: 12},\n\t)\n\tit.Add(iv3)\n\n\treturn it, iv1, iv2, iv3\n}\n\nfunc TestRootAddMultipleDimensions(t *testing.T) {\n\tit := newTree(2)\n\tiv := constructMultiDimensionInterval(\n\t\t1, &dimension{low: 0, high: 5}, &dimension{low: 1, high: 6},\n\t)\n\n\tit.Add(iv)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 10}, &dimension{0, 10},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{100, 200}, &dimension{100, 200},\n\t\t),\n\t)\n\tassert.Len(t, result, 0)\n}\n\nfunc TestMultipleAddMultipleDimensions(t *testing.T) {\n\tit, iv1, iv2, iv3 := constructMultiDimensionQueryTestTree()\n\n\tcheckRedBlack(t, it.root, 1)\n\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 100}, &dimension{0, 100},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv2, iv1, iv3}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{3, 5}, &dimension{3, 5},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv2}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{5, 8}, &dimension{5, 8},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv1, iv3}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{11, 15}, &dimension{11, 15},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv3}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{15, 20}, &dimension{15, 20},\n\t\t),\n\t)\n\tassert.Len(t, result, 0)\n}\n\nfunc TestAddRebalanceInOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tfor i := int64(0); i < 10; i++ {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{i, i + 1}, &dimension{i, i + 1},\n\t\t)\n\t\tit.Add(iv)\n\t}\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 10}, &dimension{0, 10},\n\t\t),\n\t)\n\tassert.Len(t, result, 10)\n\tassert.Equal(t, uint64(10), it.Len())\n}\n\nfunc TestAddRebalanceReverseOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tfor i := int64(9); i >= 0; i-- {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{i, i + 1}, &dimension{i, i + 1},\n\t\t)\n\t\tit.Add(iv)\n\t}\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 10}, &dimension{0, 10},\n\t\t),\n\t)\n\tassert.Len(t, result, 10)\n\tassert.Equal(t, uint64(10), it.Len())\n}\n\nfunc TestAddRebalanceRandomOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tstarts := []int64{0, 4, 2, 1, 3}\n\n\tfor i, start := range starts {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{start, start + 1}, &dimension{start, start + 1},\n\t\t)\n\t\tit.Add(iv)\n\t}\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 10}, &dimension{0, 10},\n\t\t),\n\t)\n\tassert.Len(t, result, 5)\n\tassert.Equal(t, uint64(5), it.Len())\n}\n\nfunc TestAddLargeNumbersMultiDimensions(t *testing.T) {\n\tnumItems := int64(1000)\n\tit := newTree(2)\n\n\tfor i := int64(0); i < numItems; i++ {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{i, i + 1}, &dimension{i, i + 1},\n\t\t)\n\t\tit.Add(iv)\n\t}\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, numItems}, &dimension{0, numItems},\n\t\t),\n\t)\n\tassert.Len(t, result, int(numItems))\n\tassert.Equal(t, uint64(numItems), it.Len())\n}\n\nfunc BenchmarkAddItemsMultiDimensions(b *testing.B) {\n\tnumItems := int64(b.N)\n\tintervals := make(Intervals, 0, numItems)\n\n\tfor i := int64(0); i < numItems; i++ {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{i, i + 1}, &dimension{i, i + 1},\n\t\t)\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit := newTree(2)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tit.Add(intervals[int64(i)%numItems])\n\t}\n}\n\nfunc BenchmarkQueryItemsMultiDimensions(b *testing.B) {\n\tnumItems := int64(1000)\n\tintervals := make(Intervals, 0, numItems)\n\n\tfor i := int64(0); i < numItems; i++ {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{i, i + 1}, &dimension{i, i + 1},\n\t\t)\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit := newTree(2)\n\tit.Add(intervals...)\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tit.Query(\n\t\t\tconstructMultiDimensionInterval(\n\t\t\t\t0, &dimension{0, numItems}, &dimension{0, numItems},\n\t\t\t),\n\t\t)\n\t}\n}\n\nfunc TestRootDeleteMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\tiv := constructMultiDimensionInterval(\n\t\t0, &dimension{low: 5, high: 10}, &dimension{low: 5, high: 10},\n\t)\n\tit.Add(iv)\n\n\tit.Delete(iv)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 100}, &dimension{0, 100},\n\t\t),\n\t)\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc TestDeleteMultiDimensions(t *testing.T) {\n\tit, iv1, iv2, iv3 := constructMultiDimensionQueryTestTree()\n\n\tcheckRedBlack(t, it.root, 1)\n\n\tit.Delete(iv1)\n\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 100}, &dimension{0, 100},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv2, iv3}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{3, 5}, &dimension{3, 5},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv2}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{5, 8}, &dimension{5, 8},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv3}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{11, 15}, &dimension{11, 15},\n\t\t),\n\t)\n\tassert.Equal(t, Intervals{iv3}, result)\n\n\tresult = it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{15, 20}, &dimension{15, 20},\n\t\t),\n\t)\n\tassert.Len(t, result, 0)\n}\n\nfunc TestDeleteRebalanceInOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tvar toDelete *mockInterval\n\n\tfor i := int64(0); i < 10; i++ {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{i, i + 1}, &dimension{i, i + 1},\n\t\t)\n\t\tit.Add(iv)\n\t\tif i == 5 {\n\t\t\ttoDelete = iv\n\t\t}\n\t}\n\n\tit.Delete(toDelete)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 10}, &dimension{0, 10},\n\t\t),\n\t)\n\tassert.Len(t, result, 9)\n\tassert.Equal(t, uint64(9), it.Len())\n}\n\nfunc TestDeleteRebalanceReverseOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tvar toDelete *mockInterval\n\n\tfor i := int64(9); i >= 0; i-- {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{i, i + 1}, &dimension{i, i + 1},\n\t\t)\n\t\tit.Add(iv)\n\t\tif i == 5 {\n\t\t\ttoDelete = iv\n\t\t}\n\t}\n\n\tit.Delete(toDelete)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 10}, &dimension{0, 10},\n\t\t),\n\t)\n\tassert.Len(t, result, 9)\n\tassert.Equal(t, uint64(9), it.Len())\n}\n\nfunc TestDeleteRebalanceRandomOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tstarts := []int64{0, 4, 2, 1, 3}\n\n\tvar toDelete *mockInterval\n\n\tfor i, start := range starts {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{start, start + 1}, &dimension{start, start + 1},\n\t\t)\n\t\tit.Add(iv)\n\t\tif start == 1 {\n\t\t\ttoDelete = iv\n\t\t}\n\t}\n\n\tit.Delete(toDelete)\n\n\tcheckRedBlack(t, it.root, 1)\n\tresult := it.Query(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 10}, &dimension{0, 10},\n\t\t),\n\t)\n\tassert.Len(t, result, 4)\n\tassert.Equal(t, uint64(4), it.Len())\n}\n\nfunc TestDeleteEmptyTreeMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tit.Delete(\n\t\tconstructMultiDimensionInterval(\n\t\t\t0, &dimension{0, 10}, &dimension{0, 10},\n\t\t),\n\t)\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc BenchmarkDeleteItemsMultiDimensions(b *testing.B) {\n\tnumItems := int64(1000)\n\tintervals := make(Intervals, 0, numItems)\n\n\tfor i := int64(0); i < numItems; i++ {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{i, i + 1}, &dimension{i, i + 1},\n\t\t)\n\t\tintervals = append(intervals, iv)\n\t}\n\n\ttrees := make([]*tree, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tit := newTree(2)\n\t\tit.Add(intervals...)\n\t\ttrees = append(trees, it)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttrees[i].Delete(intervals...)\n\t}\n}\n\nfunc TestAddDeleteDuplicatesRebalanceInOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tintervals := make(Intervals, 0, 10)\n\n\tfor i := 0; i < 10; i++ {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{0, 10}, &dimension{0, 10},\n\t\t)\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit.Add(intervals...)\n\tit.Delete(intervals...)\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc TestAddDeleteDuplicatesRebalanceReverseOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tintervals := make(Intervals, 0, 10)\n\n\tfor i := 9; i >= 0; i-- {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(i), &dimension{0, 10}, &dimension{0, 10},\n\t\t)\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit.Add(intervals...)\n\tit.Delete(intervals...)\n\tassert.Equal(t, uint64(0), it.Len())\n}\n\nfunc TestAddDeleteDuplicatesRebalanceRandomOrderMultiDimensions(t *testing.T) {\n\tit := newTree(2)\n\n\tintervals := make(Intervals, 0, 5)\n\tstarts := []int{0, 4, 2, 1, 3}\n\n\tfor _, start := range starts {\n\t\tiv := constructMultiDimensionInterval(\n\t\t\tuint64(start), &dimension{0, 10}, &dimension{0, 10},\n\t\t)\n\t\tintervals = append(intervals, iv)\n\t}\n\n\tit.Add(intervals...)\n\tit.Delete(intervals...)\n\tassert.Equal(t, uint64(0), it.Len())\n}\n" }, { "path": "aviary.yaml", "content": "version: 1\n\nexclude:\n - tests?/\n\nraven_monitored_classes: null\n\nraven_monitored_files: null\n\nraven_monitored_functions: null\n\nraven_monitored_keywords: null\n\n" }, { "path": "batcher/batcher.go", "content": "/*\nCopyright 2015 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batcher\n\nimport (\n\t\"errors\"\n\t\"time\"\n)\n\n// I honestly can't believe I'm doing this, but go's sync package doesn't\n// have a TryLock function.\n// Could probably do this with atomics\ntype mutex struct {\n\t// This is really more of a semaphore design, but eh\n\t// Full -> locked, empty -> unlocked\n\tlock chan struct{}\n}\n\nfunc newMutex() *mutex {\n\treturn &mutex{lock: make(chan struct{}, 1)}\n}\n\nfunc (m *mutex) Lock() {\n\tm.lock <- struct{}{}\n}\n\nfunc (m *mutex) Unlock() {\n\t<-m.lock\n}\n\nfunc (m *mutex) TryLock() bool {\n\tselect {\n\tcase m.lock <- struct{}{}:\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n\n// Batcher provides an API for accumulating items into a batch for processing.\ntype Batcher interface {\n\t// Put adds items to the batcher.\n\tPut(interface{}) error\n\n\t// Get retrieves a batch from the batcher. This call will block until\n\t// one of the conditions for a \"complete\" batch is reached.\n\tGet() ([]interface{}, error)\n\n\t// Flush forcibly completes the batch currently being built\n\tFlush() error\n\n\t// Dispose will dispose of the batcher. Any calls to Put or Flush\n\t// will return ErrDisposed, calls to Get will return an error iff\n\t// there are no more ready batches.\n\tDispose()\n\n\t// IsDisposed will determine if the batcher is disposed\n\tIsDisposed() bool\n}\n\n// ErrDisposed is the error returned for a disposed Batcher\nvar ErrDisposed = errors.New(\"batcher: disposed\")\n\n// CalculateBytes evaluates the number of bytes in an item added to a Batcher.\ntype CalculateBytes func(interface{}) uint\n\ntype basicBatcher struct {\n\tmaxTime time.Duration\n\tmaxItems uint\n\tmaxBytes uint\n\tcalculateBytes CalculateBytes\n\tdisposed bool\n\titems []interface{}\n\tbatchChan chan []interface{}\n\tavailableBytes uint\n\tlock *mutex\n}\n\n// New creates a new Batcher using the provided arguments.\n// Batch readiness can be determined in three ways:\n// - Maximum number of bytes per batch\n// - Maximum number of items per batch\n// - Maximum amount of time waiting for a batch\n// Values of zero for one of these fields indicate they should not be\n// taken into account when evaluating the readiness of a batch.\n// This provides an ordering guarantee for any given thread such that if a\n// thread places two items in the batcher, Get will guarantee the first\n// item is returned before the second, whether before the second in the same\n// batch, or in an earlier batch.\nfunc New(maxTime time.Duration, maxItems, maxBytes, queueLen uint, calculate CalculateBytes) (Batcher, error) {\n\tif maxBytes > 0 && calculate == nil {\n\t\treturn nil, errors.New(\"batcher: must provide CalculateBytes function\")\n\t}\n\n\treturn &basicBatcher{\n\t\tmaxTime: maxTime,\n\t\tmaxItems: maxItems,\n\t\tmaxBytes: maxBytes,\n\t\tcalculateBytes: calculate,\n\t\titems: make([]interface{}, 0, maxItems),\n\t\tbatchChan: make(chan []interface{}, queueLen),\n\t\tlock: newMutex(),\n\t}, nil\n}\n\n// Put adds items to the batcher.\nfunc (b *basicBatcher) Put(item interface{}) error {\n\tb.lock.Lock()\n\tif b.disposed {\n\t\tb.lock.Unlock()\n\t\treturn ErrDisposed\n\t}\n\n\tb.items = append(b.items, item)\n\tif b.calculateBytes != nil {\n\t\tb.availableBytes += b.calculateBytes(item)\n\t}\n\tif b.ready() {\n\t\t// To guarantee ordering this MUST be in the lock, otherwise multiple\n\t\t// flush calls could be blocked at the same time, in which case\n\t\t// there's no guarantee each batch is placed into the channel in\n\t\t// the proper order\n\t\tb.flush()\n\t}\n\n\tb.lock.Unlock()\n\treturn nil\n}\n\n// Get retrieves a batch from the batcher. This call will block until\n// one of the conditions for a \"complete\" batch is reached.\nfunc (b *basicBatcher) Get() ([]interface{}, error) {\n\t// Don't check disposed yet so any items remaining in the queue\n\t// will be returned properly.\n\n\tvar timeout <-chan time.Time\n\tif b.maxTime > 0 {\n\t\ttimeout = time.After(b.maxTime)\n\t}\n\n\tselect {\n\tcase items, ok := <-b.batchChan:\n\t\t// If there's something on the batch channel, we definitely want that.\n\t\tif !ok {\n\t\t\treturn nil, ErrDisposed\n\t\t}\n\t\treturn items, nil\n\tcase <-timeout:\n\t\t// It's possible something was added to the channel after something\n\t\t// was received on the timeout channel, in which case that must\n\t\t// be returned first to satisfy our ordering guarantees.\n\t\t// We can't just grab the lock here in case the batch channel is full,\n\t\t// in which case a Put or Flush will be blocked and holding\n\t\t// onto the lock. In that case, there should be something on the\n\t\t// batch channel\n\t\tfor {\n\t\t\tif b.lock.TryLock() {\n\t\t\t\t// We have a lock, try to read from channel first in case\n\t\t\t\t// something snuck in\n\t\t\t\tselect {\n\t\t\t\tcase items, ok := <-b.batchChan:\n\t\t\t\t\tb.lock.Unlock()\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn nil, ErrDisposed\n\t\t\t\t\t}\n\t\t\t\t\treturn items, nil\n\t\t\t\tdefault:\n\t\t\t\t}\n\n\t\t\t\t// If that is unsuccessful, nothing was added to the channel,\n\t\t\t\t// and the temp buffer can't have changed because of the lock,\n\t\t\t\t// so grab that\n\t\t\t\titems := b.items\n\t\t\t\tb.items = make([]interface{}, 0, b.maxItems)\n\t\t\t\tb.availableBytes = 0\n\t\t\t\tb.lock.Unlock()\n\t\t\t\treturn items, nil\n\t\t\t} else {\n\t\t\t\t// If we didn't get a lock, there are two cases:\n\t\t\t\t// 1) The batch chan is full.\n\t\t\t\t// 2) A Put or Flush temporarily has the lock.\n\t\t\t\t// In either case, trying to read something off the batch chan,\n\t\t\t\t// and going back to trying to get a lock if unsuccessful\n\t\t\t\t// works.\n\t\t\t\tselect {\n\t\t\t\tcase items, ok := <-b.batchChan:\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\treturn nil, ErrDisposed\n\t\t\t\t\t}\n\t\t\t\t\treturn items, nil\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Flush forcibly completes the batch currently being built\nfunc (b *basicBatcher) Flush() error {\n\t// This is the same pattern as a Put\n\tb.lock.Lock()\n\tif b.disposed {\n\t\tb.lock.Unlock()\n\t\treturn ErrDisposed\n\t}\n\tb.flush()\n\tb.lock.Unlock()\n\treturn nil\n}\n\n// Dispose will dispose of the batcher. Any calls to Put or Flush\n// will return ErrDisposed, calls to Get will return an error iff\n// there are no more ready batches. Any items not flushed and retrieved\n// by a Get may or may not be retrievable after calling this.\nfunc (b *basicBatcher) Dispose() {\n\tfor {\n\t\tif b.lock.TryLock() {\n\t\t\t// We've got a lock\n\t\t\tif b.disposed {\n\t\t\t\tb.lock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tb.disposed = true\n\t\t\tb.items = nil\n\t\t\tb.drainBatchChan()\n\t\t\tclose(b.batchChan)\n\t\t\tb.lock.Unlock()\n\t\t} else {\n\t\t\t// Two cases here:\n\t\t\t// 1) Something is blocked and holding onto the lock\n\t\t\t// 2) Something temporarily has a lock\n\t\t\t// For case 1, we have to clear at least some space so the blocked\n\t\t\t// Put/Flush can release the lock. For case 2, nothing bad\n\t\t\t// will happen here\n\t\t\tb.drainBatchChan()\n\t\t}\n\n\t}\n}\n\n// IsDisposed will determine if the batcher is disposed\nfunc (b *basicBatcher) IsDisposed() bool {\n\tb.lock.Lock()\n\tdisposed := b.disposed\n\tb.lock.Unlock()\n\treturn disposed\n}\n\n// flush adds the batch currently being built to the queue of completed batches.\n// flush is not threadsafe, so should be synchronized externally.\nfunc (b *basicBatcher) flush() {\n\tb.batchChan <- b.items\n\tb.items = make([]interface{}, 0, b.maxItems)\n\tb.availableBytes = 0\n}\n\nfunc (b *basicBatcher) ready() bool {\n\tif b.maxItems != 0 && uint(len(b.items)) >= b.maxItems {\n\t\treturn true\n\t}\n\tif b.maxBytes != 0 && b.availableBytes >= b.maxBytes {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (b *basicBatcher) drainBatchChan() {\n\tfor {\n\t\tselect {\n\t\tcase <-b.batchChan:\n\t\tdefault:\n\t\t\treturn\n\t\t}\n\t}\n}\n" }, { "path": "batcher/batcher_test.go", "content": "/*\nCopyright 2015 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batcher\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestNoCalculateBytes(t *testing.T) {\n\t_, err := New(0, 0, 100, 5, nil)\n\tassert.Error(t, err)\n}\n\nfunc TestMaxItems(t *testing.T) {\n\tassert := assert.New(t)\n\tb, err := New(0, 100, 100000, 10, func(str interface{}) uint {\n\t\treturn uint(len(str.(string)))\n\t})\n\tassert.Nil(err)\n\n\tfor i := 0; i < 1000; i++ {\n\t\tassert.Nil(b.Put(\"foo bar baz\"))\n\t}\n\n\tbatch, err := b.Get()\n\tassert.Len(batch, 100)\n\tassert.Nil(err)\n}\n\nfunc TestMaxBytes(t *testing.T) {\n\tassert := assert.New(t)\n\tb, err := New(0, 10000, 100, 10, func(str interface{}) uint {\n\t\treturn uint(len(str.(string)))\n\t})\n\tassert.Nil(err)\n\n\tgo func() {\n\t\tfor i := 0; i < 1000; i++ {\n\t\t\tb.Put(\"a\")\n\t\t}\n\t}()\n\n\tbatch, err := b.Get()\n\tassert.Len(batch, 100)\n\tassert.Nil(err)\n}\n\nfunc TestMaxTime(t *testing.T) {\n\tassert := assert.New(t)\n\tb, err := New(time.Millisecond*200, 100000, 100000, 10,\n\t\tfunc(str interface{}) uint {\n\t\t\treturn uint(len(str.(string)))\n\t\t},\n\t)\n\tassert.Nil(err)\n\n\tgo func() {\n\t\tfor i := 0; i < 10000; i++ {\n\t\t\tb.Put(\"a\")\n\t\t\ttime.Sleep(time.Millisecond)\n\t\t}\n\t}()\n\n\tbefore := time.Now()\n\tbatch, err := b.Get()\n\n\t// This delta is normally 1-3 ms but running tests in CI with -race causes\n\t// this to run much slower. For now, just bump up the threshold.\n\tassert.InDelta(200, time.Since(before).Seconds()*1000, 100)\n\tassert.True(len(batch) > 0)\n\tassert.Nil(err)\n}\n\nfunc TestFlush(t *testing.T) {\n\tassert := assert.New(t)\n\tb, err := New(0, 10, 10, 10, func(str interface{}) uint {\n\t\treturn uint(len(str.(string)))\n\t})\n\tassert.Nil(err)\n\tb.Put(\"a\")\n\twait := make(chan bool)\n\tgo func() {\n\t\tbatch, err := b.Get()\n\t\tassert.Equal([]interface{}{\"a\"}, batch)\n\t\tassert.Nil(err)\n\t\twait <- true\n\t}()\n\n\tb.Flush()\n\t<-wait\n}\n\nfunc TestMultiConsumer(t *testing.T) {\n\tassert := assert.New(t)\n\tb, err := New(0, 100, 100000, 10, func(str interface{}) uint {\n\t\treturn uint(len(str.(string)))\n\t})\n\tassert.Nil(err)\n\n\tvar wg sync.WaitGroup\n\twg.Add(5)\n\tfor i := 0; i < 5; i++ {\n\t\tgo func() {\n\t\t\tbatch, err := b.Get()\n\t\t\tassert.Len(batch, 100)\n\t\t\tassert.Nil(err)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\tgo func() {\n\t\tfor i := 0; i < 500; i++ {\n\t\t\tb.Put(\"a\")\n\t\t}\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestDispose(t *testing.T) {\n\tassert := assert.New(t)\n\tb, err := New(1, 2, 100000, 2, func(str interface{}) uint {\n\t\treturn uint(len(str.(string)))\n\t})\n\tassert.Nil(err)\n\tb.Put(\"a\")\n\tb.Put(\"b\")\n\tb.Put(\"c\")\n\n\tbatch1, err := b.Get()\n\tassert.Equal([]interface{}{\"a\", \"b\"}, batch1)\n\tassert.Nil(err)\n\n\tbatch2, err := b.Get()\n\tassert.Equal([]interface{}{\"c\"}, batch2)\n\tassert.Nil(err)\n\n\tb.Put(\"d\")\n\tb.Put(\"e\")\n\tb.Put(\"f\")\n\n\tb.Dispose()\n\n\t_, err = b.Get()\n\tassert.Equal(ErrDisposed, err)\n\n\tassert.Equal(ErrDisposed, b.Put(\"j\"))\n\tassert.Equal(ErrDisposed, b.Flush())\n\n}\n\nfunc TestIsDisposed(t *testing.T) {\n\tassert := assert.New(t)\n\tb, err := New(0, 10, 10, 10, func(str interface{}) uint {\n\t\treturn uint(len(str.(string)))\n\t})\n\tassert.Nil(err)\n\tassert.False(b.IsDisposed())\n\tb.Dispose()\n\tassert.True(b.IsDisposed())\n}\n" }, { "path": "bitarray/and.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nfunc andSparseWithSparseBitArray(sba, other *sparseBitArray) BitArray {\n\tmax := maxInt64(int64(len(sba.indices)), int64(len(other.indices)))\n\tindices := make(uintSlice, 0, max)\n\tblocks := make(blocks, 0, max)\n\n\tselfIndex := 0\n\totherIndex := 0\n\tvar resultBlock block\n\n\t// move through the array and compare the blocks if they happen to\n\t// intersect\n\tfor {\n\t\tif selfIndex == len(sba.indices) || otherIndex == len(other.indices) {\n\t\t\t// One of the arrays has been exhausted. We don't need\n\t\t\t// to compare anything else for a bitwise and; the\n\t\t\t// operation is complete.\n\t\t\tbreak\n\t\t}\n\n\t\tselfValue := sba.indices[selfIndex]\n\t\totherValue := other.indices[otherIndex]\n\n\t\tswitch {\n\t\tcase otherValue < selfValue:\n\t\t\t// The `sba` bitarray has a block with a index position\n\t\t\t// greater than us. We want to compare with that block\n\t\t\t// if possible, so move our `other` index closer to that\n\t\t\t// block's index.\n\t\t\totherIndex++\n\n\t\tcase otherValue > selfValue:\n\t\t\t// This is the exact logical inverse of the above case.\n\t\t\tselfIndex++\n\n\t\tdefault:\n\t\t\t// Here, our indices match for both `sba` and `other`.\n\t\t\t// Time to do the bitwise AND operation and add a block\n\t\t\t// to our result list if the block has values in it.\n\t\t\tresultBlock = sba.blocks[selfIndex].and(other.blocks[otherIndex])\n\t\t\tif resultBlock > 0 {\n\t\t\t\tindices = append(indices, selfValue)\n\t\t\t\tblocks = append(blocks, resultBlock)\n\t\t\t}\n\t\t\tselfIndex++\n\t\t\totherIndex++\n\t\t}\n\t}\n\n\treturn &sparseBitArray{\n\t\tindices: indices,\n\t\tblocks: blocks,\n\t}\n}\n\nfunc andSparseWithDenseBitArray(sba *sparseBitArray, other *bitArray) BitArray {\n\tif other.IsEmpty() {\n\t\treturn newSparseBitArray()\n\t}\n\n\t// Use a duplicate of the sparse array to store the results of the\n\t// bitwise and. More memory-efficient than allocating a new dense bit\n\t// array.\n\t//\n\t// NOTE: this could be faster if we didn't copy the values as well\n\t// (since they are overwritten), but I don't want this method to know\n\t// too much about the internals of sparseBitArray. The performance hit\n\t// should be minor anyway.\n\tba := sba.copy()\n\n\t// Run through the sparse array and attempt comparisons wherever\n\t// possible against the dense bit array.\n\tfor selfIndex, selfValue := range ba.indices {\n\n\t\tif selfValue >= uint64(len(other.blocks)) {\n\t\t\t// The dense bit array has been exhausted. This is the\n\t\t\t// annoying case because we have to trim the sparse\n\t\t\t// array to the size of the dense array.\n\t\t\tba.blocks = ba.blocks[:selfIndex-1]\n\t\t\tba.indices = ba.indices[:selfIndex-1]\n\n\t\t\t// once this is done, there are no more comparisons.\n\t\t\t// We're ready to return\n\t\t\tbreak\n\t\t}\n\t\tba.blocks[selfIndex] = ba.blocks[selfIndex].and(\n\t\t\tother.blocks[selfValue])\n\n\t}\n\n\t// Ensure any zero'd blocks in the resulting sparse\n\t// array are deleted\n\tfor i := 0; i < len(ba.blocks); i++ {\n\t\tif ba.blocks[i] == 0 {\n\t\t\tba.blocks.deleteAtIndex(int64(i))\n\t\t\tba.indices.deleteAtIndex(int64(i))\n\t\t\ti--\n\t\t}\n\t}\n\n\treturn ba\n}\n\nfunc andDenseWithDenseBitArray(dba, other *bitArray) BitArray {\n\tmin := minUint64(uint64(len(dba.blocks)), uint64(len(other.blocks)))\n\n\tba := newBitArray(min * s)\n\n\tfor i := uint64(0); i < min; i++ {\n\t\tba.blocks[i] = dba.blocks[i].and(other.blocks[i])\n\t}\n\n\tba.setLowest()\n\tba.setHighest()\n\n\treturn ba\n}\n" }, { "path": "bitarray/and_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\n// checkBit is a helper method for these unit tests\nfunc checkBit(t *testing.T, ba BitArray, position uint64, expected bool) {\n\tok, err := ba.GetBit(position)\n\tif assert.NoError(t, err) {\n\t\tif expected {\n\t\t\tassert.True(t, ok, \"Bitarray at position %d should be set\", position)\n\t\t} else {\n\t\t\tassert.False(t, ok, \"Bitarray at position %d should be unset\", position)\n\t\t}\n\t}\n}\n\nfunc TestAndSparseWithSparseBitArray(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\t// bits for which only one of the arrays is set\n\tsba.SetBit(3)\n\tsba.SetBit(280)\n\tother.SetBit(9)\n\tother.SetBit(100)\n\tsba.SetBit(1000)\n\tother.SetBit(1001)\n\n\t// bits for which both arrays are set\n\tsba.SetBit(1)\n\tother.SetBit(1)\n\tsba.SetBit(2680)\n\tother.SetBit(2680)\n\tsba.SetBit(30)\n\tother.SetBit(30)\n\n\tba := andSparseWithSparseBitArray(sba, other)\n\n\t// Bits in both\n\tcheckBit(t, ba, 1, true)\n\tcheckBit(t, ba, 30, true)\n\tcheckBit(t, ba, 2680, true)\n\n\t// Bits in sba but not other\n\tcheckBit(t, ba, 3, false)\n\tcheckBit(t, ba, 280, false)\n\tcheckBit(t, ba, 1000, false)\n\n\t// Bits in other but not sba\n\tcheckBit(t, ba, 9, false)\n\tcheckBit(t, ba, 100, false)\n\tcheckBit(t, ba, 2, false)\n\n\tnums := ba.ToNums()\n\tassert.Equal(t, []uint64{1, 30, 2680}, nums)\n}\n\nfunc TestAndSparseWithDenseBitArray(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newBitArray(300)\n\n\tother.SetBit(1)\n\tsba.SetBit(1)\n\tother.SetBit(150)\n\tsba.SetBit(150)\n\tsba.SetBit(155)\n\tother.SetBit(156)\n\tsba.SetBit(300)\n\tother.SetBit(300)\n\n\tba := andSparseWithDenseBitArray(sba, other)\n\n\t// Bits in both\n\tcheckBit(t, ba, 1, true)\n\tcheckBit(t, ba, 150, true)\n\tcheckBit(t, ba, 300, true)\n\n\t// Bits in sba but not other\n\tcheckBit(t, ba, 155, false)\n\n\t// Bits in other but not sba\n\tcheckBit(t, ba, 156, false)\n\n}\n\n// Make sure that the sparse array is trimmed correctly if compared against a\n// smaller dense bit array.\nfunc TestAndSparseWithSmallerDenseBitArray(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newBitArray(512)\n\n\tother.SetBit(1)\n\tsba.SetBit(1)\n\tother.SetBit(150)\n\tsba.SetBit(150)\n\tsba.SetBit(155)\n\tsba.SetBit(500)\n\n\tother.SetBit(128)\n\tsba.SetBit(1500)\n\tsba.SetBit(1200)\n\n\tba := andSparseWithDenseBitArray(sba, other)\n\n\t// Bits in both\n\tcheckBit(t, ba, 1, true)\n\tcheckBit(t, ba, 150, true)\n\n\t// Bits in sba but not other\n\tcheckBit(t, ba, 155, false)\n\tcheckBit(t, ba, 500, false)\n\tcheckBit(t, ba, 1200, false)\n\tcheckBit(t, ba, 1500, false)\n\n\t// Bits in other but not sba\n\tcheckBit(t, ba, 128, false)\n}\n\nfunc TestAndDenseWithDenseBitArray(t *testing.T) {\n\tdba := newBitArray(1000)\n\tother := newBitArray(2000)\n\n\tdba.SetBit(1)\n\tother.SetBit(18)\n\tdba.SetBit(222)\n\tother.SetBit(222)\n\tother.SetBit(1501)\n\n\tba := andDenseWithDenseBitArray(dba, other)\n\n\tcheckBit(t, ba, 0, false)\n\tcheckBit(t, ba, 1, false)\n\tcheckBit(t, ba, 3, false)\n\tcheckBit(t, ba, 18, false)\n\tcheckBit(t, ba, 222, true)\n\n\t// check that the ba is the minimum of the size of `dba` and `other`\n\t// (dense bitarrays return an error on an out-of-bounds access)\n\t_, err := ba.GetBit(1500)\n\tassert.Equal(t, OutOfRangeError(1500), err)\n\t_, err = ba.GetBit(1501)\n\tassert.Equal(t, OutOfRangeError(1501), err)\n}\n\nfunc TestAndSparseWithEmptySparse(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tsba.SetBit(5)\n\n\tba := andSparseWithSparseBitArray(sba, other)\n\n\tcheckBit(t, ba, 0, false)\n\tcheckBit(t, ba, 5, false)\n\tcheckBit(t, ba, 100, false)\n}\n\nfunc TestAndSparseWithEmptyDense(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newBitArray(1000)\n\n\tsba.SetBit(5)\n\tba := andSparseWithDenseBitArray(sba, other)\n\tcheckBit(t, ba, 5, false)\n\n\tsba.Reset()\n\tother.SetBit(5)\n\n\tba = andSparseWithDenseBitArray(sba, other)\n\tcheckBit(t, ba, 5, false)\n}\n\nfunc TestAndDenseWithEmptyDense(t *testing.T) {\n\tdba := newBitArray(1000)\n\tother := newBitArray(1000)\n\n\tdba.SetBit(5)\n\tba := andDenseWithDenseBitArray(dba, other)\n\tcheckBit(t, ba, 5, false)\n\n\tdba.Reset()\n\tother.SetBit(5)\n\tba = andDenseWithDenseBitArray(dba, other)\n\tcheckBit(t, ba, 5, false)\n}\n" }, { "path": "bitarray/bitarray.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage bitarray implements a bit array. Useful for tracking bool type values in a space\nefficient way. This is *NOT* a threadsafe package.\n*/\npackage bitarray\n\nimport \"math/bits\"\n\n// bitArray is a struct that maintains state of a bit array.\ntype bitArray struct {\n\tblocks []block\n\tlowest uint64\n\thighest uint64\n\tanyset bool\n}\n\nfunc getIndexAndRemainder(k uint64) (uint64, uint64) {\n\treturn k / s, k % s\n}\n\nfunc (ba *bitArray) setLowest() {\n\tfor i := uint64(0); i < uint64(len(ba.blocks)); i++ {\n\t\tif ba.blocks[i] == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tpos := ba.blocks[i].findRightPosition()\n\t\tba.lowest = (i * s) + pos\n\t\tba.anyset = true\n\t\treturn\n\t}\n\n\tba.anyset = false\n\tba.lowest = 0\n\tba.highest = 0\n}\n\nfunc (ba *bitArray) setHighest() {\n\tfor i := len(ba.blocks) - 1; i >= 0; i-- {\n\t\tif ba.blocks[i] == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tpos := ba.blocks[i].findLeftPosition()\n\t\tba.highest = (uint64(i) * s) + pos\n\t\tba.anyset = true\n\t\treturn\n\t}\n\n\tba.anyset = false\n\tba.highest = 0\n\tba.lowest = 0\n}\n\n// capacity returns the total capacity of the bit array.\nfunc (ba *bitArray) Capacity() uint64 {\n\treturn uint64(len(ba.blocks)) * s\n}\n\n// ToNums converts this bitarray to a list of numbers contained within it.\nfunc (ba *bitArray) ToNums() []uint64 {\n\tnums := make([]uint64, 0, ba.highest-ba.lowest/4)\n\tfor i, block := range ba.blocks {\n\t\tblock.toNums(uint64(i)*s, &nums)\n\t}\n\n\treturn nums\n}\n\n// SetBit sets a bit at the given index to true.\nfunc (ba *bitArray) SetBit(k uint64) error {\n\tif k >= ba.Capacity() {\n\t\treturn OutOfRangeError(k)\n\t}\n\n\tif !ba.anyset {\n\t\tba.lowest = k\n\t\tba.highest = k\n\t\tba.anyset = true\n\t} else {\n\t\tif k < ba.lowest {\n\t\t\tba.lowest = k\n\t\t} else if k > ba.highest {\n\t\t\tba.highest = k\n\t\t}\n\t}\n\n\ti, pos := getIndexAndRemainder(k)\n\tba.blocks[i] = ba.blocks[i].insert(pos)\n\treturn nil\n}\n\n// GetBit returns a bool indicating if the value at the given\n// index has been set.\nfunc (ba *bitArray) GetBit(k uint64) (bool, error) {\n\tif k >= ba.Capacity() {\n\t\treturn false, OutOfRangeError(k)\n\t}\n\n\ti, pos := getIndexAndRemainder(k)\n\tresult := ba.blocks[i]&block(1<>= fromOffset\n\t\t}\n\n\t\tfor block != 0 {\n\t\t\ttrailing := bits.TrailingZeros64(uint64(block))\n\n\t\t\tif isFirstBlock {\n\t\t\t\tresults[resultSize] = uint64(trailing) + (blockIndex << 6) + fromOffset\n\t\t\t} else {\n\t\t\t\tresults[resultSize] = uint64(trailing) + (blockIndex << 6)\n\t\t\t}\n\t\t\tresultSize++\n\n\t\t\tif resultSize == cap(results) {\n\t\t\t\treturn results[:resultSize]\n\t\t\t}\n\n\t\t\t// Clear the bit we just added to the result, which is the last bit set in the block. Ex.:\n\t\t\t// block 01001100\n\t\t\t// ^block 10110011\n\t\t\t// (^block) + 1 10110100\n\t\t\t// block & (^block) + 1 00000100\n\t\t\t// block ^ mask 01001000\n\t\t\tmask := block & ((^block) + 1)\n\t\t\tblock = block ^ mask\n\t\t}\n\t}\n\n\treturn results[:resultSize]\n}\n\n// ClearBit will unset a bit at the given index if it is set.\nfunc (ba *bitArray) ClearBit(k uint64) error {\n\tif k >= ba.Capacity() {\n\t\treturn OutOfRangeError(k)\n\t}\n\n\tif !ba.anyset { // nothing is set, might as well bail\n\t\treturn nil\n\t}\n\n\ti, pos := getIndexAndRemainder(k)\n\tba.blocks[i] &^= block(1 << pos)\n\n\tif k == ba.highest {\n\t\tba.setHighest()\n\t} else if k == ba.lowest {\n\t\tba.setLowest()\n\t}\n\treturn nil\n}\n\n// Count returns the number of set bits in this array.\nfunc (ba *bitArray) Count() int {\n\tcount := 0\n\tfor _, block := range ba.blocks {\n\t\tcount += bits.OnesCount64(uint64(block))\n\t}\n\treturn count\n}\n\n// Or will bitwise or two bit arrays and return a new bit array\n// representing the result.\nfunc (ba *bitArray) Or(other BitArray) BitArray {\n\tif dba, ok := other.(*bitArray); ok {\n\t\treturn orDenseWithDenseBitArray(ba, dba)\n\t}\n\n\treturn orSparseWithDenseBitArray(other.(*sparseBitArray), ba)\n}\n\n// And will bitwise and two bit arrays and return a new bit array\n// representing the result.\nfunc (ba *bitArray) And(other BitArray) BitArray {\n\tif dba, ok := other.(*bitArray); ok {\n\t\treturn andDenseWithDenseBitArray(ba, dba)\n\t}\n\n\treturn andSparseWithDenseBitArray(other.(*sparseBitArray), ba)\n}\n\n// Nand will return the result of doing a bitwise and not of the bit array\n// with the other bit array on each block.\nfunc (ba *bitArray) Nand(other BitArray) BitArray {\n\tif dba, ok := other.(*bitArray); ok {\n\t\treturn nandDenseWithDenseBitArray(ba, dba)\n\t}\n\n\treturn nandDenseWithSparseBitArray(ba, other.(*sparseBitArray))\n}\n\n// Reset clears out the bit array.\nfunc (ba *bitArray) Reset() {\n\tfor i := uint64(0); i < uint64(len(ba.blocks)); i++ {\n\t\tba.blocks[i] &= block(0)\n\t}\n\tba.anyset = false\n}\n\n// Equals returns a bool indicating if these two bit arrays are equal.\nfunc (ba *bitArray) Equals(other BitArray) bool {\n\tif other.Capacity() == 0 && ba.highest > 0 {\n\t\treturn false\n\t}\n\n\tif other.Capacity() == 0 && !ba.anyset {\n\t\treturn true\n\t}\n\n\tvar selfIndex uint64\n\tfor iter := other.Blocks(); iter.Next(); {\n\t\ttoIndex, otherBlock := iter.Value()\n\t\tif toIndex > selfIndex {\n\t\t\tfor i := selfIndex; i < toIndex; i++ {\n\t\t\t\tif ba.blocks[i] > 0 {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tselfIndex = toIndex\n\t\tif !ba.blocks[selfIndex].equals(otherBlock) {\n\t\t\treturn false\n\t\t}\n\t\tselfIndex++\n\t}\n\n\tlastIndex, _ := getIndexAndRemainder(ba.highest)\n\tif lastIndex >= selfIndex {\n\t\treturn false\n\t}\n\n\treturn true\n}\n\n// Intersects returns a bool indicating if the supplied bitarray intersects\n// this bitarray. This will check for intersection up to the length of the supplied\n// bitarray. If the supplied bitarray is longer than this bitarray, this\n// function returns false.\nfunc (ba *bitArray) Intersects(other BitArray) bool {\n\tif other.Capacity() > ba.Capacity() {\n\t\treturn false\n\t}\n\n\tif sba, ok := other.(*sparseBitArray); ok {\n\t\treturn ba.intersectsSparseBitArray(sba)\n\t}\n\n\treturn ba.intersectsDenseBitArray(other.(*bitArray))\n}\n\n// Blocks will return an iterator over this bit array.\nfunc (ba *bitArray) Blocks() Iterator {\n\treturn newBitArrayIterator(ba)\n}\n\nfunc (ba *bitArray) IsEmpty() bool {\n\treturn !ba.anyset\n}\n\n// complement flips all bits in this array.\nfunc (ba *bitArray) complement() {\n\tfor i := uint64(0); i < uint64(len(ba.blocks)); i++ {\n\t\tba.blocks[i] = ^ba.blocks[i]\n\t}\n\n\tba.setLowest()\n\tif ba.anyset {\n\t\tba.setHighest()\n\t}\n}\n\nfunc (ba *bitArray) intersectsSparseBitArray(other *sparseBitArray) bool {\n\tfor i, index := range other.indices {\n\t\tif !ba.blocks[index].intersects(other.blocks[i]) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc (ba *bitArray) intersectsDenseBitArray(other *bitArray) bool {\n\tfor i, block := range other.blocks {\n\t\tif !ba.blocks[i].intersects(block) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc (ba *bitArray) copy() BitArray {\n\tblocks := make(blocks, len(ba.blocks))\n\tcopy(blocks, ba.blocks)\n\treturn &bitArray{\n\t\tblocks: blocks,\n\t\tlowest: ba.lowest,\n\t\thighest: ba.highest,\n\t\tanyset: ba.anyset,\n\t}\n}\n\n// newBitArray returns a new dense BitArray at the specified size. This is a\n// separate private constructor so unit tests don't have to constantly cast the\n// BitArray interface to the concrete type.\nfunc newBitArray(size uint64, args ...bool) *bitArray {\n\ti, r := getIndexAndRemainder(size)\n\tif r > 0 {\n\t\ti++\n\t}\n\n\tba := &bitArray{\n\t\tblocks: make([]block, i),\n\t\tanyset: false,\n\t}\n\n\tif len(args) > 0 && args[0] == true {\n\t\tfor i := uint64(0); i < uint64(len(ba.blocks)); i++ {\n\t\t\tba.blocks[i] = maximumBlock\n\t\t}\n\n\t\tba.lowest = 0\n\t\tba.highest = i*s - 1\n\t\tba.anyset = true\n\t}\n\n\treturn ba\n}\n\n// NewBitArray returns a new BitArray at the specified size. The\n// optional arg denotes whether this bitarray should be set to the\n// bitwise complement of the empty array, ie. sets all bits.\nfunc NewBitArray(size uint64, args ...bool) BitArray {\n\treturn newBitArray(size, args...)\n}\n" }, { "path": "bitarray/bitarray_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestBitOperations(t *testing.T) {\n\tba := newBitArray(10)\n\n\terr := ba.SetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresult, err := ba.GetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif !result {\n\t\tt.Errorf(`Expected true at position: %d`, 5)\n\t}\n\n\tresult, err = ba.GetBit(3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif result {\n\t\tt.Errorf(`Expected false at position %d`, 3)\n\t}\n\n\terr = ba.ClearBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresult, err = ba.GetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif result {\n\t\tt.Errorf(`Expected false at position: %d`, 5)\n\t}\n\n\tba = newBitArray(24)\n\terr = ba.SetBit(16)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresult, err = ba.GetBit(16)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif !result {\n\t\tt.Errorf(`Expected true at position: %d`, 16)\n\t}\n}\n\nfunc TestDuplicateOperation(t *testing.T) {\n\tba := newBitArray(10)\n\n\terr := ba.SetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = ba.SetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresult, err := ba.GetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif !result {\n\t\tt.Errorf(`Expected true at position: %d`, 5)\n\t}\n\n\terr = ba.ClearBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = ba.ClearBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresult, err = ba.GetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif result {\n\t\tt.Errorf(`Expected false at position: %d`, 5)\n\t}\n}\n\nfunc TestOutOfBounds(t *testing.T) {\n\tba := newBitArray(4)\n\n\terr := ba.SetBit(s + 1)\n\n\tif _, ok := err.(OutOfRangeError); !ok {\n\t\tt.Errorf(`Expected out of range error.`)\n\t}\n\n\t_, err = ba.GetBit(s + 1)\n\tif _, ok := err.(OutOfRangeError); !ok {\n\t\tt.Errorf(`Expected out of range error.`)\n\t}\n}\n\nfunc TestIsEmpty(t *testing.T) {\n\tba := newBitArray(10)\n\tassert.True(t, ba.IsEmpty())\n\n\tba.SetBit(5)\n\tassert.False(t, ba.IsEmpty())\n}\n\nfunc TestCount(t *testing.T) {\n\tba := newBitArray(500)\n\tassert.Equal(t, 0, ba.Count())\n\n\trequire.NoError(t, ba.SetBit(0))\n\tassert.Equal(t, 1, ba.Count())\n\n\trequire.NoError(t, ba.SetBit(40))\n\trequire.NoError(t, ba.SetBit(64))\n\trequire.NoError(t, ba.SetBit(100))\n\trequire.NoError(t, ba.SetBit(200))\n\trequire.NoError(t, ba.SetBit(469))\n\trequire.NoError(t, ba.SetBit(500))\n\tassert.Equal(t, 7, ba.Count())\n\n\trequire.NoError(t, ba.ClearBit(200))\n\tassert.Equal(t, 6, ba.Count())\n\n\tba.Reset()\n\tassert.Equal(t, 0, ba.Count())\n}\n\nfunc TestClear(t *testing.T) {\n\tba := newBitArray(10)\n\n\terr := ba.SetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = ba.SetBit(9)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tba.Reset()\n\n\tassert.False(t, ba.anyset)\n\tresult, err := ba.GetBit(5)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif result {\n\t\tt.Errorf(`BA not reset.`)\n\t}\n\n\tresult, err = ba.GetBit(9)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif result {\n\t\tt.Errorf(`BA not reset.`)\n\t}\n}\n\nfunc BenchmarkGetBit(b *testing.B) {\n\tnumItems := uint64(168000)\n\n\tba := newBitArray(numItems)\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tba.SetBit(i)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tfor j := uint64(0); j < numItems; j++ {\n\t\t\tba.GetBit(j)\n\t\t}\n\t}\n}\n\nfunc TestGetSetBits(t *testing.T) {\n\tba := newBitArray(1000)\n\tbuf := make([]uint64, 0, 5)\n\n\trequire.NoError(t, ba.SetBit(1))\n\trequire.NoError(t, ba.SetBit(4))\n\trequire.NoError(t, ba.SetBit(8))\n\trequire.NoError(t, ba.SetBit(63))\n\trequire.NoError(t, ba.SetBit(64))\n\trequire.NoError(t, ba.SetBit(200))\n\trequire.NoError(t, ba.SetBit(1000))\n\n\tassert.Equal(t, []uint64(nil), ba.GetSetBits(0, nil))\n\tassert.Equal(t, []uint64{}, ba.GetSetBits(0, []uint64{}))\n\n\tassert.Equal(t, []uint64{1, 4, 8, 63, 64}, ba.GetSetBits(0, buf))\n\tassert.Equal(t, []uint64{63, 64, 200, 1000}, ba.GetSetBits(10, buf))\n\tassert.Equal(t, []uint64{63, 64, 200, 1000}, ba.GetSetBits(63, buf))\n\tassert.Equal(t, []uint64{200, 1000}, ba.GetSetBits(128, buf))\n\n\trequire.NoError(t, ba.ClearBit(4))\n\trequire.NoError(t, ba.ClearBit(64))\n\tassert.Equal(t, []uint64{1, 8, 63, 200, 1000}, ba.GetSetBits(0, buf))\n\tassert.Empty(t, ba.GetSetBits(1001, buf))\n\n\tba.Reset()\n\tassert.Empty(t, ba.GetSetBits(0, buf))\n}\n\nfunc BenchmarkGetSetBits(b *testing.B) {\n\tnumItems := uint64(168000)\n\n\tba := newBitArray(numItems)\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tif i%13 == 0 || i%5 == 0 {\n\t\t\trequire.NoError(b, ba.SetBit(i))\n\t\t}\n\t}\n\n\tbuf := make([]uint64, 0, ba.Capacity())\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tba.GetSetBits(0, buf)\n\t}\n}\n\nfunc TestEquality(t *testing.T) {\n\tba := newBitArray(s + 1)\n\tother := newBitArray(s + 1)\n\n\tif !ba.Equals(other) {\n\t\tt.Errorf(`Expected equality.`)\n\t}\n\n\tba.SetBit(s + 1)\n\tother.SetBit(s + 1)\n\n\tif !ba.Equals(other) {\n\t\tt.Errorf(`Expected equality.`)\n\t}\n\n\tother.SetBit(0)\n\n\tif ba.Equals(other) {\n\t\tt.Errorf(`Expected inequality.`)\n\t}\n}\n\nfunc BenchmarkEquality(b *testing.B) {\n\tba := newBitArray(160000)\n\tother := newBitArray(ba.Capacity())\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tba.Equals(other)\n\t}\n}\n\nfunc TestIntersects(t *testing.T) {\n\tba := newBitArray(10)\n\tother := newBitArray(ba.Capacity())\n\n\tba.SetBit(1)\n\tba.SetBit(2)\n\n\tother.SetBit(1)\n\n\tif !ba.Intersects(other) {\n\t\tt.Errorf(`Is intersecting.`)\n\t}\n\n\tother.SetBit(5)\n\n\tif ba.Intersects(other) {\n\t\tt.Errorf(`Is not intersecting.`)\n\t}\n\n\tother = newBitArray(ba.Capacity() + 1)\n\tother.SetBit(1)\n\n\tif ba.Intersects(other) {\n\t\tt.Errorf(`Is not intersecting.`)\n\t}\n}\n\nfunc BenchmarkIntersects(b *testing.B) {\n\tba := newBitArray(162432)\n\tother := newBitArray(ba.Capacity())\n\n\tba.SetBit(159999)\n\tother.SetBit(159999)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tba.Intersects(other)\n\t}\n}\n\nfunc TestComplement(t *testing.T) {\n\tba := newBitArray(10)\n\n\tba.SetBit(5)\n\n\tba.complement()\n\n\tif ok, _ := ba.GetBit(5); ok {\n\t\tt.Errorf(`Expected clear.`)\n\t}\n\n\tif ok, _ := ba.GetBit(4); !ok {\n\t\tt.Errorf(`Expected set.`)\n\t}\n}\n\nfunc BenchmarkComplement(b *testing.B) {\n\tba := newBitArray(160000)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tba.complement()\n\t}\n}\n\nfunc TestSetHighestLowest(t *testing.T) {\n\tba := newBitArray(10)\n\n\tassert.False(t, ba.anyset)\n\tassert.Equal(t, uint64(0), ba.lowest)\n\tassert.Equal(t, uint64(0), ba.highest)\n\n\tba.SetBit(5)\n\n\tassert.True(t, ba.anyset)\n\tassert.Equal(t, uint64(5), ba.lowest)\n\tassert.Equal(t, uint64(5), ba.highest)\n\n\tba.SetBit(8)\n\tassert.Equal(t, uint64(5), ba.lowest)\n\tassert.Equal(t, uint64(8), ba.highest)\n}\n\nfunc TestGetBitAtCapacity(t *testing.T) {\n\tba := newBitArray(s * 2)\n\t_, err := ba.GetBit(s * 2)\n\tassert.Error(t, err)\n}\n\nfunc TestSetBitAtCapacity(t *testing.T) {\n\tba := newBitArray(s * 2)\n\terr := ba.SetBit(s * 2)\n\tassert.Error(t, err)\n}\n\nfunc TestClearBitAtCapacity(t *testing.T) {\n\tba := newBitArray(s * 2)\n\terr := ba.ClearBit(s * 2)\n\tassert.Error(t, err)\n}\n\nfunc TestClearHighestLowest(t *testing.T) {\n\tba := newBitArray(10)\n\n\tba.SetBit(5)\n\tba.ClearBit(5)\n\n\tassert.False(t, ba.anyset)\n\tassert.Equal(t, uint64(0), ba.lowest)\n\tassert.Equal(t, uint64(0), ba.highest)\n\n\tba.SetBit(3)\n\tba.SetBit(5)\n\tba.SetBit(7)\n\n\tba.ClearBit(7)\n\tassert.True(t, ba.anyset)\n\tassert.Equal(t, uint64(5), ba.highest)\n\tassert.Equal(t, uint64(3), ba.lowest)\n\n\tba.SetBit(7)\n\tba.ClearBit(3)\n\tassert.True(t, ba.anyset)\n\tassert.Equal(t, uint64(5), ba.lowest)\n\tassert.Equal(t, uint64(7), ba.highest)\n\n\tba.ClearBit(7)\n\tassert.True(t, ba.anyset)\n\tassert.Equal(t, uint64(5), ba.lowest)\n\tassert.Equal(t, uint64(5), ba.highest)\n\n\tba.ClearBit(5)\n\tassert.False(t, ba.anyset)\n\tassert.Equal(t, uint64(0), ba.lowest)\n\tassert.Equal(t, uint64(0), ba.highest)\n}\n\nfunc TestComplementResetsBounds(t *testing.T) {\n\tba := newBitArray(5)\n\n\tba.complement()\n\tassert.True(t, ba.anyset)\n\tassert.Equal(t, uint64(0), ba.lowest)\n\tassert.Equal(t, uint64(s-1), ba.highest)\n}\n\nfunc TestBitArrayIntersectsSparse(t *testing.T) {\n\tba := newBitArray(s * 2)\n\tcba := newSparseBitArray()\n\n\tassert.True(t, ba.Intersects(cba))\n\n\tcba.SetBit(5)\n\tassert.False(t, ba.Intersects(cba))\n\n\tba.SetBit(5)\n\tassert.True(t, ba.Intersects(cba))\n\n\tcba.SetBit(s + 1)\n\tassert.False(t, ba.Intersects(cba))\n\n\tba.SetBit(s + 1)\n\tassert.True(t, ba.Intersects(cba))\n}\n\nfunc TestBitArrayEqualsSparse(t *testing.T) {\n\tba := newBitArray(s * 2)\n\tcba := newSparseBitArray()\n\n\tassert.True(t, ba.Equals(cba))\n\n\tba.SetBit(5)\n\tassert.False(t, ba.Equals(cba))\n\n\tcba.SetBit(5)\n\tassert.True(t, ba.Equals(cba))\n\n\tba.SetBit(s + 1)\n\tassert.False(t, ba.Equals(cba))\n\n\tcba.SetBit(s + 1)\n\tassert.True(t, ba.Equals(cba))\n}\n\nfunc TestConstructorSetBitArray(t *testing.T) {\n\tba := newBitArray(8, true)\n\n\tresult, err := ba.GetBit(7)\n\tassert.Nil(t, err)\n\tassert.True(t, result)\n\tassert.Equal(t, s-1, ba.highest)\n\tassert.Equal(t, uint64(0), ba.lowest)\n\tassert.True(t, ba.anyset)\n}\n\nfunc TestCopyBitArray(t *testing.T) {\n\tba := newBitArray(10)\n\tba.SetBit(5)\n\tba.SetBit(1)\n\n\tresult := ba.copy().(*bitArray)\n\tassert.Equal(t, ba.anyset, result.anyset)\n\tassert.Equal(t, ba.lowest, result.lowest)\n\tassert.Equal(t, ba.highest, result.highest)\n\tassert.Equal(t, ba.blocks, result.blocks)\n}\n\nfunc BenchmarkDenseIntersectsCompressed(b *testing.B) {\n\tnumBits := uint64(162432)\n\tba := newBitArray(numBits)\n\tother := newSparseBitArray()\n\n\tfor i := uint64(0); i < numBits; i++ {\n\t\tba.SetBit(i)\n\t\tother.SetBit(i)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tba.intersectsSparseBitArray(other)\n\t}\n}\n\nfunc TestBitArrayToNums(t *testing.T) {\n\tba := newBitArray(s * 2)\n\n\tba.SetBit(s - 1)\n\tba.SetBit(s + 1)\n\n\texpected := []uint64{s - 1, s + 1}\n\n\tresult := ba.ToNums()\n\n\tassert.Equal(t, expected, result)\n}\n\nfunc BenchmarkBitArrayToNums(b *testing.B) {\n\tnumItems := uint64(1000)\n\tba := newBitArray(numItems)\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tba.SetBit(i)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tba.ToNums()\n\t}\n}\n" }, { "path": "bitarray/bitmap.go", "content": "/*\nCopyright (c) 2016, Theodore Butler\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/\n\n// Package bitmap contains bitmaps of length 32 and 64 for tracking bool\n// values without the need for arrays or hashing.\npackage bitarray\n\n// Bitmap32 tracks 32 bool values within a uint32\ntype Bitmap32 uint32\n\n// SetBit returns a Bitmap32 with the bit at the given position set to 1\nfunc (b Bitmap32) SetBit(pos uint) Bitmap32 {\n\treturn b | (1 << pos)\n}\n\n// ClearBit returns a Bitmap32 with the bit at the given position set to 0\nfunc (b Bitmap32) ClearBit(pos uint) Bitmap32 {\n\treturn b & ^(1 << pos)\n}\n\n// GetBit returns true if the bit at the given position in the Bitmap32 is 1\nfunc (b Bitmap32) GetBit(pos uint) bool {\n\treturn (b & (1 << pos)) != 0\n}\n\n// PopCount returns the amount of bits set to 1 in the Bitmap32\nfunc (b Bitmap32) PopCount() int {\n\t// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel\n\tb -= (b >> 1) & 0x55555555\n\tb = (b>>2)&0x33333333 + b&0x33333333\n\tb += b >> 4\n\tb &= 0x0f0f0f0f\n\tb *= 0x01010101\n\treturn int(byte(b >> 24))\n}\n\n// Bitmap64 tracks 64 bool values within a uint64\ntype Bitmap64 uint64\n\n// SetBit returns a Bitmap64 with the bit at the given position set to 1\nfunc (b Bitmap64) SetBit(pos uint) Bitmap64 {\n\treturn b | (1 << pos)\n}\n\n// ClearBit returns a Bitmap64 with the bit at the given position set to 0\nfunc (b Bitmap64) ClearBit(pos uint) Bitmap64 {\n\treturn b & ^(1 << pos)\n}\n\n// GetBit returns true if the bit at the given position in the Bitmap64 is 1\nfunc (b Bitmap64) GetBit(pos uint) bool {\n\treturn (b & (1 << pos)) != 0\n}\n\n// PopCount returns the amount of bits set to 1 in the Bitmap64\nfunc (b Bitmap64) PopCount() int {\n\t// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel\n\tb -= (b >> 1) & 0x5555555555555555\n\tb = (b>>2)&0x3333333333333333 + b&0x3333333333333333\n\tb += b >> 4\n\tb &= 0x0f0f0f0f0f0f0f0f\n\tb *= 0x0101010101010101\n\treturn int(byte(b >> 56))\n}\n" }, { "path": "bitarray/bitmap_test.go", "content": "/*\nCopyright (c) 2016, Theodore Butler\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/\n\npackage bitarray\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestBitmap32_PopCount(t *testing.T) {\n\tb := []uint32{\n\t\tuint32(0x55555555), // 0x55555555 = 01010101 01010101 01010101 01010101\n\t\tuint32(0x33333333), // 0x33333333 = 00110011 00110011 00110011 00110011\n\t\tuint32(0x0F0F0F0F), // 0x0F0F0F0F = 00001111 00001111 00001111 00001111\n\t\tuint32(0x00FF00FF), // 0x00FF00FF = 00000000 11111111 00000000 11111111\n\t\tuint32(0x0000FFFF), // 0x0000FFFF = 00000000 00000000 11111111 11111111\n\t}\n\tfor _, x := range b {\n\t\tassert.Equal(t, 16, Bitmap32(x).PopCount())\n\t}\n}\n\nfunc TestBitmap64_PopCount(t *testing.T) {\n\tb := []uint64{\n\t\tuint64(0x5555555555555555),\n\t\tuint64(0x3333333333333333),\n\t\tuint64(0x0F0F0F0F0F0F0F0F),\n\t\tuint64(0x00FF00FF00FF00FF),\n\t\tuint64(0x0000FFFF0000FFFF),\n\t}\n\tfor _, x := range b {\n\t\tassert.Equal(t, 32, Bitmap64(x).PopCount())\n\t}\n}\n\nfunc TestBitmap32_SetBit(t *testing.T) {\n\tm := Bitmap32(0)\n\tassert.Equal(t, Bitmap32(0x4), m.SetBit(2))\n}\n\nfunc TestBitmap32_ClearBit(t *testing.T) {\n\tm := Bitmap32(0x4)\n\tassert.Equal(t, Bitmap32(0), m.ClearBit(2))\n}\n\nfunc TestBitmap32_zGetBit(t *testing.T) {\n\tm := Bitmap32(0x55555555)\n\tassert.Equal(t, true, m.GetBit(2))\n}\n\nfunc TestBitmap64_SetBit(t *testing.T) {\n\tm := Bitmap64(0)\n\tassert.Equal(t, Bitmap64(0x4), m.SetBit(2))\n}\n\nfunc TestBitmap64_ClearBit(t *testing.T) {\n\tm := Bitmap64(0x4)\n\tassert.Equal(t, Bitmap64(0), m.ClearBit(2))\n}\n\nfunc TestBitmap64_GetBit(t *testing.T) {\n\tm := Bitmap64(0x55555555)\n\tassert.Equal(t, true, m.GetBit(2))\n}\n\nfunc BenchmarkBitmap32_PopCount(b *testing.B) {\n\tm := Bitmap32(0x33333333)\n\tb.ResetTimer()\n\tfor i := b.N; i > 0; i-- {\n\t\tm.PopCount()\n\t}\n}\n\nfunc BenchmarkBitmap64_PopCount(b *testing.B) {\n\tm := Bitmap64(0x3333333333333333)\n\tb.ResetTimer()\n\tfor i := b.N; i > 0; i-- {\n\t\tm.PopCount()\n\t}\n}\n" }, { "path": "bitarray/block.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nimport (\n\t\"fmt\"\n\t\"unsafe\"\n)\n\n// block defines how we split apart the bit array. This also determines the size\n// of s. This can be changed to any unsigned integer type: uint8, uint16,\n// uint32, and so on.\ntype block uint64\n\n// s denotes the size of any element in the block array.\n// For a block of uint64, s will be equal to 64\n// For a block of uint32, s will be equal to 32\n// and so on...\nconst s = uint64(unsafe.Sizeof(block(0)) * 8)\n\n// maximumBlock represents a block of all 1s and is used in the constructors.\nconst maximumBlock = block(0) | ^block(0)\n\nfunc (b block) toNums(offset uint64, nums *[]uint64) {\n\tfor i := uint64(0); i < s; i++ {\n\t\tif b&block(1< 0 {\n\t\t\t*nums = append(*nums, i+offset)\n\t\t}\n\t}\n}\n\nfunc (b block) findLeftPosition() uint64 {\n\tfor i := s - 1; i < s; i-- {\n\t\ttest := block(1 << i)\n\t\tif b&test == test {\n\t\t\treturn i\n\t\t}\n\t}\n\n\treturn s\n}\n\nfunc (b block) findRightPosition() uint64 {\n\tfor i := uint64(0); i < s; i++ {\n\t\ttest := block(1 << i)\n\t\tif b&test == test {\n\t\t\treturn i\n\t\t}\n\t}\n\n\treturn s\n}\n\nfunc (b block) insert(position uint64) block {\n\treturn b | block(1< selfValue:\n\t\t\t// Here, the sba array has blocks that the other array doesn't\n\t\t\t// have. In this case, we just copy exactly the sba array values\n\t\t\tindices = append(indices, selfValue)\n\t\t\tblocks = append(blocks, sba.blocks[selfIndex])\n\n\t\t\t// This is the exact logical inverse of the above case.\n\t\t\tselfIndex++\n\n\t\tdefault:\n\t\t\t// Here, our indices match for both `sba` and `other`.\n\t\t\t// Time to do the bitwise AND operation and add a block\n\t\t\t// to our result list if the block has values in it.\n\t\t\tresultBlock = sba.blocks[selfIndex].nand(other.blocks[otherIndex])\n\t\t\tif resultBlock > 0 {\n\t\t\t\tindices = append(indices, selfValue)\n\t\t\t\tblocks = append(blocks, resultBlock)\n\t\t\t}\n\t\t\tselfIndex++\n\t\t\totherIndex++\n\t\t}\n\t}\n\n\treturn &sparseBitArray{\n\t\tindices: indices,\n\t\tblocks: blocks,\n\t}\n}\n\nfunc nandSparseWithDenseBitArray(sba *sparseBitArray, other *bitArray) BitArray {\n\t// Since nand is non-commutative, the resulting array should be sparse,\n\t// and the same length or less than the sparse array\n\tindices := make(uintSlice, 0, len(sba.indices))\n\tblocks := make(blocks, 0, len(sba.indices))\n\n\tvar resultBlock block\n\n\t// Loop through the sparse array and match it with the dense array.\n\tfor selfIndex, selfValue := range sba.indices {\n\t\tif selfValue >= uint64(len(other.blocks)) {\n\t\t\t// Since the dense array is exhausted, just copy over the data\n\t\t\t// from the sparse array\n\t\t\tresultBlock = sba.blocks[selfIndex]\n\t\t\tindices = append(indices, selfValue)\n\t\t\tblocks = append(blocks, resultBlock)\n\t\t\tcontinue\n\t\t}\n\n\t\tresultBlock = sba.blocks[selfIndex].nand(other.blocks[selfValue])\n\t\tif resultBlock > 0 {\n\t\t\tindices = append(indices, selfValue)\n\t\t\tblocks = append(blocks, resultBlock)\n\t\t}\n\t}\n\n\treturn &sparseBitArray{\n\t\tindices: indices,\n\t\tblocks: blocks,\n\t}\n}\n\nfunc nandDenseWithSparseBitArray(sba *bitArray, other *sparseBitArray) BitArray {\n\t// Since nand is non-commutative, the resulting array should be dense,\n\t// and the same length or less than the dense array\n\ttmp := sba.copy()\n\tret := tmp.(*bitArray)\n\n\t// Loop through the other array and match it with the sba array.\n\tfor otherIndex, otherValue := range other.indices {\n\t\tif otherValue >= uint64(len(ret.blocks)) {\n\t\t\tbreak\n\t\t}\n\n\t\tret.blocks[otherValue] = sba.blocks[otherValue].nand(other.blocks[otherIndex])\n\t}\n\n\tret.setLowest()\n\tret.setHighest()\n\n\treturn ret\n}\n\nfunc nandDenseWithDenseBitArray(dba, other *bitArray) BitArray {\n\tmin := uint64(len(dba.blocks))\n\n\tba := newBitArray(min * s)\n\n\tfor i := uint64(0); i < min; i++ {\n\t\tba.blocks[i] = dba.blocks[i].nand(other.blocks[i])\n\t}\n\n\tba.setLowest()\n\tba.setHighest()\n\n\treturn ba\n}\n" }, { "path": "bitarray/nand_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestNandSparseWithSparseBitArray(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\t// bits for which only one of the arrays is set\n\tsba.SetBit(3)\n\tsba.SetBit(280)\n\tother.SetBit(9)\n\tother.SetBit(100)\n\tsba.SetBit(1000)\n\tother.SetBit(1001)\n\n\t// bits for which both arrays are set\n\tsba.SetBit(1)\n\tother.SetBit(1)\n\tsba.SetBit(2680)\n\tother.SetBit(2680)\n\tsba.SetBit(30)\n\tother.SetBit(30)\n\n\tba := nandSparseWithSparseBitArray(sba, other)\n\n\t// Bits in both\n\tcheckBit(t, ba, 1, false)\n\tcheckBit(t, ba, 30, false)\n\tcheckBit(t, ba, 2680, false)\n\n\t// Bits in sba but not other\n\tcheckBit(t, ba, 3, true)\n\tcheckBit(t, ba, 280, true)\n\tcheckBit(t, ba, 1000, true)\n\n\t// Bits in other but not sba\n\tcheckBit(t, ba, 9, false)\n\tcheckBit(t, ba, 100, false)\n\tcheckBit(t, ba, 2, false)\n\n\tnums := ba.ToNums()\n\tassert.Equal(t, []uint64{3, 280, 1000}, nums)\n}\n\nfunc TestNandSparseWithDenseBitArray(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newBitArray(300)\n\n\tother.SetBit(1)\n\tsba.SetBit(1)\n\tother.SetBit(150)\n\tsba.SetBit(150)\n\tsba.SetBit(155)\n\tother.SetBit(156)\n\tsba.SetBit(300)\n\tother.SetBit(300)\n\n\tba := nandSparseWithDenseBitArray(sba, other)\n\n\t// Bits in both\n\tcheckBit(t, ba, 1, false)\n\tcheckBit(t, ba, 150, false)\n\tcheckBit(t, ba, 300, false)\n\n\t// Bits in sba but not other\n\tcheckBit(t, ba, 155, true)\n\n\t// Bits in other but not sba\n\tcheckBit(t, ba, 156, false)\n\n\tnums := ba.ToNums()\n\tassert.Equal(t, []uint64{155}, nums)\n}\n\nfunc TestNandDenseWithSparseBitArray(t *testing.T) {\n\tsba := newBitArray(300)\n\tother := newSparseBitArray()\n\n\tother.SetBit(1)\n\tsba.SetBit(1)\n\tother.SetBit(150)\n\tsba.SetBit(150)\n\tsba.SetBit(155)\n\tother.SetBit(156)\n\tsba.SetBit(300)\n\tother.SetBit(300)\n\n\tba := nandDenseWithSparseBitArray(sba, other)\n\n\t// Bits in both\n\tcheckBit(t, ba, 1, false)\n\tcheckBit(t, ba, 150, false)\n\tcheckBit(t, ba, 300, false)\n\n\t// Bits in sba but not other\n\tcheckBit(t, ba, 155, true)\n\n\t// Bits in other but not sba\n\tcheckBit(t, ba, 156, false)\n\n\tnums := ba.ToNums()\n\tassert.Equal(t, []uint64{155}, nums)\n}\n\nfunc TestNandSparseWithSmallerDenseBitArray(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newBitArray(512)\n\n\tother.SetBit(1)\n\tsba.SetBit(1)\n\tother.SetBit(150)\n\tsba.SetBit(150)\n\tsba.SetBit(155)\n\tsba.SetBit(500)\n\n\tother.SetBit(128)\n\tsba.SetBit(1500)\n\tsba.SetBit(1200)\n\n\tba := nandSparseWithDenseBitArray(sba, other)\n\n\t// Bits in both\n\tcheckBit(t, ba, 1, false)\n\tcheckBit(t, ba, 150, false)\n\n\t// Bits in sba but not other\n\tcheckBit(t, ba, 155, true)\n\tcheckBit(t, ba, 500, true)\n\tcheckBit(t, ba, 1200, true)\n\tcheckBit(t, ba, 1500, true)\n\n\t// Bits in other but not sba\n\tcheckBit(t, ba, 128, false)\n\n\tnums := ba.ToNums()\n\tassert.Equal(t, []uint64{155, 500, 1200, 1500}, nums)\n}\n\nfunc TestNandDenseWithDenseBitArray(t *testing.T) {\n\tdba := newBitArray(1000)\n\tother := newBitArray(2000)\n\n\tdba.SetBit(1)\n\tother.SetBit(18)\n\tdba.SetBit(222)\n\tother.SetBit(222)\n\tother.SetBit(1501)\n\n\tba := nandDenseWithDenseBitArray(dba, other)\n\n\t// Bits in both\n\tcheckBit(t, ba, 222, false)\n\n\t// Bits in dba and not other\n\tcheckBit(t, ba, 1, true)\n\n\t// Bits in other\n\tcheckBit(t, ba, 18, false)\n\n\t// Bits in neither\n\tcheckBit(t, ba, 0, false)\n\tcheckBit(t, ba, 3, false)\n\n\t// check that the ba is the minimum of the size of `dba` and `other`\n\t// (dense bitarrays return an error on an out-of-bounds access)\n\t_, err := ba.GetBit(1500)\n\tassert.Equal(t, OutOfRangeError(1500), err)\n\t_, err = ba.GetBit(1501)\n\tassert.Equal(t, OutOfRangeError(1501), err)\n\n\tnums := ba.ToNums()\n\tassert.Equal(t, []uint64{1}, nums)\n}\n\nfunc TestNandSparseWithEmptySparse(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tsba.SetBit(5)\n\n\tba := nandSparseWithSparseBitArray(sba, other)\n\n\tcheckBit(t, ba, 0, false)\n\tcheckBit(t, ba, 5, true)\n\tcheckBit(t, ba, 100, false)\n}\n\nfunc TestNandSparseWithEmptyDense(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newBitArray(1000)\n\n\tsba.SetBit(5)\n\tba := nandSparseWithDenseBitArray(sba, other)\n\tcheckBit(t, ba, 5, true)\n\n\tsba.Reset()\n\tother.SetBit(5)\n\n\tba = nandSparseWithDenseBitArray(sba, other)\n\tcheckBit(t, ba, 5, false)\n}\n\nfunc TestNandDenseWithEmptyDense(t *testing.T) {\n\tdba := newBitArray(1000)\n\tother := newBitArray(1000)\n\n\tdba.SetBit(5)\n\tba := nandDenseWithDenseBitArray(dba, other)\n\tcheckBit(t, ba, 5, true)\n\n\tdba.Reset()\n\tother.SetBit(5)\n\tba = nandDenseWithDenseBitArray(dba, other)\n\tcheckBit(t, ba, 5, false)\n}\n\nfunc BenchmarkNandSparseWithSparse(b *testing.B) {\n\tnumItems := uint64(160000)\n\tsba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tfor i := uint64(0); i < numItems; i += s {\n\t\tif i%200 == 0 {\n\t\t\tsba.SetBit(i)\n\t\t} else if i%300 == 0 {\n\t\t\tother.SetBit(i)\n\t\t}\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tnandSparseWithSparseBitArray(sba, other)\n\t}\n}\n\nfunc BenchmarkNandSparseWithDense(b *testing.B) {\n\tnumItems := uint64(160000)\n\tsba := newSparseBitArray()\n\tother := newBitArray(numItems)\n\n\tfor i := uint64(0); i < numItems; i += s {\n\t\tif i%2 == 0 {\n\t\t\tsba.SetBit(i)\n\t\t} else if i%3 == 0 {\n\t\t\tother.SetBit(i)\n\t\t}\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tnandSparseWithDenseBitArray(sba, other)\n\t}\n}\n\nfunc BenchmarkNandDenseWithSparse(b *testing.B) {\n\tnumItems := uint64(160000)\n\tba := newBitArray(numItems)\n\tother := newSparseBitArray()\n\n\tfor i := uint64(0); i < numItems; i += s {\n\t\tif i%2 == 0 {\n\t\t\tba.SetBit(i)\n\t\t} else if i%3 == 0 {\n\t\t\tother.SetBit(i)\n\t\t}\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tnandDenseWithSparseBitArray(ba, other)\n\t}\n}\n\nfunc BenchmarkNandDenseWithDense(b *testing.B) {\n\tnumItems := uint64(160000)\n\tdba := newBitArray(numItems)\n\tother := newBitArray(numItems)\n\n\tfor i := uint64(0); i < numItems; i += s {\n\t\tif i%2 == 0 {\n\t\t\tdba.SetBit(i)\n\t\t} else if i%3 == 0 {\n\t\t\tother.SetBit(i)\n\t\t}\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tnandDenseWithDenseBitArray(dba, other)\n\t}\n}\n" }, { "path": "bitarray/or.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nfunc orSparseWithSparseBitArray(sba *sparseBitArray,\n\tother *sparseBitArray) BitArray {\n\n\tif len(other.indices) == 0 {\n\t\treturn sba.copy()\n\t}\n\n\tif len(sba.indices) == 0 {\n\t\treturn other.copy()\n\t}\n\n\tmax := maxInt64(int64(len(sba.indices)), int64(len(other.indices)))\n\tindices := make(uintSlice, 0, max)\n\tblocks := make(blocks, 0, max)\n\n\tselfIndex := 0\n\totherIndex := 0\n\tfor {\n\t\t// last comparison was a real or, we are both exhausted now\n\t\tif selfIndex == len(sba.indices) && otherIndex == len(other.indices) {\n\t\t\tbreak\n\t\t} else if selfIndex == len(sba.indices) {\n\t\t\tindices = append(indices, other.indices[otherIndex:]...)\n\t\t\tblocks = append(blocks, other.blocks[otherIndex:]...)\n\t\t\tbreak\n\t\t} else if otherIndex == len(other.indices) {\n\t\t\tindices = append(indices, sba.indices[selfIndex:]...)\n\t\t\tblocks = append(blocks, sba.blocks[selfIndex:]...)\n\t\t\tbreak\n\t\t}\n\n\t\tselfValue := sba.indices[selfIndex]\n\t\totherValue := other.indices[otherIndex]\n\n\t\tswitch diff := int(otherValue) - int(selfValue); {\n\t\tcase diff > 0:\n\t\t\tindices = append(indices, selfValue)\n\t\t\tblocks = append(blocks, sba.blocks[selfIndex])\n\t\t\tselfIndex++\n\t\tcase diff < 0:\n\t\t\tindices = append(indices, otherValue)\n\t\t\tblocks = append(blocks, other.blocks[otherIndex])\n\t\t\totherIndex++\n\t\tdefault:\n\t\t\tindices = append(indices, otherValue)\n\t\t\tblocks = append(blocks, sba.blocks[selfIndex].or(other.blocks[otherIndex]))\n\t\t\tselfIndex++\n\t\t\totherIndex++\n\t\t}\n\t}\n\n\treturn &sparseBitArray{\n\t\tindices: indices,\n\t\tblocks: blocks,\n\t}\n}\n\nfunc orSparseWithDenseBitArray(sba *sparseBitArray, other *bitArray) BitArray {\n\tif other.Capacity() == 0 || !other.anyset {\n\t\treturn sba.copy()\n\t}\n\n\tif sba.Capacity() == 0 {\n\t\treturn other.copy()\n\t}\n\n\tmax := maxUint64(uint64(sba.Capacity()), uint64(other.Capacity()))\n\n\tba := newBitArray(max * s)\n\tselfIndex := 0\n\totherIndex := 0\n\tfor {\n\t\tif selfIndex == len(sba.indices) && otherIndex == len(other.blocks) {\n\t\t\tbreak\n\t\t} else if selfIndex == len(sba.indices) {\n\t\t\tcopy(ba.blocks[otherIndex:], other.blocks[otherIndex:])\n\t\t\tbreak\n\t\t} else if otherIndex == len(other.blocks) {\n\t\t\tfor i, value := range sba.indices[selfIndex:] {\n\t\t\t\tba.blocks[value] = sba.blocks[i+selfIndex]\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\tselfValue := sba.indices[selfIndex]\n\t\tif selfValue == uint64(otherIndex) {\n\t\t\tba.blocks[otherIndex] = sba.blocks[selfIndex].or(other.blocks[otherIndex])\n\t\t\tselfIndex++\n\t\t\totherIndex++\n\t\t\tcontinue\n\t\t}\n\n\t\tba.blocks[otherIndex] = other.blocks[otherIndex]\n\t\totherIndex++\n\t}\n\n\tba.setHighest()\n\tba.setLowest()\n\n\treturn ba\n}\n\nfunc orDenseWithDenseBitArray(dba *bitArray, other *bitArray) BitArray {\n\tif dba.Capacity() == 0 || !dba.anyset {\n\t\treturn other.copy()\n\t}\n\n\tif other.Capacity() == 0 || !other.anyset {\n\t\treturn dba.copy()\n\t}\n\n\tmax := maxUint64(uint64(len(dba.blocks)), uint64(len(other.blocks)))\n\n\tba := newBitArray(max * s)\n\n\tfor i := uint64(0); i < max; i++ {\n\t\tif i == uint64(len(dba.blocks)) {\n\t\t\tcopy(ba.blocks[i:], other.blocks[i:])\n\t\t\tbreak\n\t\t}\n\n\t\tif i == uint64(len(other.blocks)) {\n\t\t\tcopy(ba.blocks[i:], dba.blocks[i:])\n\t\t\tbreak\n\t\t}\n\n\t\tba.blocks[i] = dba.blocks[i].or(other.blocks[i])\n\t}\n\n\tba.setLowest()\n\tba.setHighest()\n\n\treturn ba\n}\n" }, { "path": "bitarray/or_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestOrSparseWithSparseBitArray(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tctx := false\n\tfor i := uint64(0); i < 1000; i += s {\n\t\tif ctx {\n\t\t\tsba.SetBit(i)\n\t\t} else {\n\t\t\tother.SetBit(i)\n\t\t}\n\n\t\tctx = !ctx\n\t}\n\n\tsba.SetBit(s - 1)\n\tother.SetBit(s - 1)\n\n\tresult := orSparseWithSparseBitArray(sba, other)\n\n\tfor i := uint64(0); i < 1000; i += s {\n\t\tok, err := result.GetBit(i)\n\t\tassert.Nil(t, err)\n\t\tassert.True(t, ok)\n\t}\n\n\tok, err := result.GetBit(s - 1)\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n\n\tok, err = result.GetBit(s - 2)\n\tassert.Nil(t, err)\n\tassert.False(t, ok)\n\n\tother.SetBit(2000)\n\tresult = orSparseWithSparseBitArray(sba, other)\n\n\tok, err = result.GetBit(2000)\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n\n\tsba.SetBit(2000)\n\tresult = orSparseWithSparseBitArray(sba, other)\n\n\tok, err = result.GetBit(2000)\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n}\n\nfunc BenchmarkOrSparseWithSparse(b *testing.B) {\n\tnumItems := uint64(160000)\n\tsba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tctx := false\n\tfor i := uint64(0); i < numItems; i += s {\n\t\tif ctx {\n\t\t\tsba.SetBit(i)\n\t\t} else {\n\t\t\tother.SetBit(i)\n\t\t}\n\n\t\tctx = !ctx\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\torSparseWithSparseBitArray(sba, other)\n\t}\n}\n\nfunc TestOrSparseWithDenseBitArray(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newBitArray(2000)\n\n\tctx := false\n\tfor i := uint64(0); i < 1000; i += s {\n\t\tif ctx {\n\t\t\tsba.SetBit(i)\n\t\t} else {\n\t\t\tother.SetBit(i)\n\t\t}\n\n\t\tctx = !ctx\n\t}\n\n\tother.SetBit(1500)\n\tother.SetBit(s - 1)\n\tsba.SetBit(s - 1)\n\n\tresult := orSparseWithDenseBitArray(sba, other)\n\n\tfor i := uint64(0); i < 1000; i += s {\n\t\tok, err := result.GetBit(i)\n\t\tassert.Nil(t, err)\n\t\tassert.True(t, ok)\n\t}\n\n\tok, err := result.GetBit(1500)\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n\n\tok, err = result.GetBit(s - 1)\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n\n\tok, err = result.GetBit(s - 2)\n\tassert.Nil(t, err)\n\tassert.False(t, ok)\n\n\tsba.SetBit(2500)\n\tresult = orSparseWithDenseBitArray(sba, other)\n\n\tok, err = result.GetBit(2500)\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n}\n\nfunc BenchmarkOrSparseWithDense(b *testing.B) {\n\tnumItems := uint64(160000)\n\tsba := newSparseBitArray()\n\tother := newBitArray(numItems)\n\n\tctx := false\n\tfor i := uint64(0); i < numItems; i += s {\n\t\tif ctx {\n\t\t\tsba.SetBit(i)\n\t\t} else {\n\t\t\tother.SetBit(i)\n\t\t}\n\n\t\tctx = !ctx\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\torSparseWithDenseBitArray(sba, other)\n\t}\n}\n\nfunc TestOrDenseWithDenseBitArray(t *testing.T) {\n\tdba := newBitArray(1000)\n\tother := newBitArray(2000)\n\n\tctx := false\n\tfor i := uint64(0); i < 1000; i += s {\n\t\tif ctx {\n\t\t\tdba.SetBit(i)\n\t\t} else {\n\t\t\tother.SetBit(i)\n\t\t}\n\n\t\tctx = !ctx\n\t}\n\n\tother.SetBit(1500)\n\tother.SetBit(s - 1)\n\tdba.SetBit(s - 1)\n\n\tresult := orDenseWithDenseBitArray(dba, other)\n\n\tfor i := uint64(0); i < 1000; i += s {\n\t\tok, err := result.GetBit(i)\n\t\tassert.Nil(t, err)\n\t\tassert.True(t, ok)\n\t}\n\n\tok, err := result.GetBit(s - 1)\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n\n\tok, err = result.GetBit(1500)\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n\n\tok, err = result.GetBit(1700)\n\tassert.Nil(t, err)\n\tassert.False(t, ok)\n}\n\nfunc BenchmarkOrDenseWithDense(b *testing.B) {\n\tnumItems := uint64(160000)\n\tdba := newBitArray(numItems)\n\tother := newBitArray(numItems)\n\n\tctx := false\n\tfor i := uint64(0); i < numItems; i += s {\n\t\tif ctx {\n\t\t\tdba.SetBit(i)\n\t\t} else {\n\t\t\tother.SetBit(i)\n\t\t}\n\n\t\tctx = !ctx\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\torDenseWithDenseBitArray(dba, other)\n\t}\n}\n\nfunc TestOrSparseWithEmptySparse(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tsba.SetBit(5)\n\n\tresult := orSparseWithSparseBitArray(sba, other)\n\tassert.Equal(t, sba, result)\n\n\tsba.Reset()\n\tother.SetBit(5)\n\n\tresult = orSparseWithSparseBitArray(sba, other)\n\tassert.Equal(t, other, result)\n}\n\nfunc TestOrSparseWithEmptyDense(t *testing.T) {\n\tsba := newSparseBitArray()\n\tother := newBitArray(1000)\n\n\tsba.SetBit(5)\n\tresult := orSparseWithDenseBitArray(sba, other)\n\tassert.Equal(t, sba, result)\n\n\tsba.Reset()\n\tother.SetBit(5)\n\n\tresult = orSparseWithDenseBitArray(sba, other)\n\tassert.Equal(t, other, result)\n}\n\nfunc TestOrDenseWithEmptyDense(t *testing.T) {\n\tdba := newBitArray(1000)\n\tother := newBitArray(1000)\n\n\tdba.SetBit(5)\n\tresult := orDenseWithDenseBitArray(dba, other)\n\tassert.Equal(t, dba, result)\n\n\tdba.Reset()\n\tother.SetBit(5)\n\tresult = orDenseWithDenseBitArray(dba, other)\n\tassert.Equal(t, other, result)\n}\n" }, { "path": "bitarray/sparse_bitarray.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nimport (\n\t\"math/bits\"\n\t\"sort\"\n)\n\n// uintSlice is an alias for a slice of ints. Len, Swap, and Less\n// are exported to fulfill an interface needed for the search\n// function in the sort library.\ntype uintSlice []uint64\n\n// Len returns the length of the slice.\nfunc (u uintSlice) Len() int64 {\n\treturn int64(len(u))\n}\n\n// Swap swaps values in this slice at the positions given.\nfunc (u uintSlice) Swap(i, j int64) {\n\tu[i], u[j] = u[j], u[i]\n}\n\n// Less returns a bool indicating if the value at position i is\n// less than position j.\nfunc (u uintSlice) Less(i, j int64) bool {\n\treturn u[i] < u[j]\n}\n\nfunc (u uintSlice) search(x uint64) int64 {\n\treturn int64(sort.Search(len(u), func(i int) bool { return uint64(u[i]) >= x }))\n}\n\nfunc (u *uintSlice) insert(x uint64) (int64, bool) {\n\ti := u.search(x)\n\n\tif i == int64(len(*u)) {\n\t\t*u = append(*u, x)\n\t\treturn i, true\n\t}\n\n\tif (*u)[i] == x {\n\t\treturn i, false\n\t}\n\n\t*u = append(*u, 0)\n\tcopy((*u)[i+1:], (*u)[i:])\n\t(*u)[i] = x\n\treturn i, true\n}\n\nfunc (u *uintSlice) deleteAtIndex(i int64) {\n\tcopy((*u)[i:], (*u)[i+1:])\n\t(*u)[len(*u)-1] = 0\n\t*u = (*u)[:len(*u)-1]\n}\n\nfunc (u uintSlice) get(x uint64) int64 {\n\ti := u.search(x)\n\tif i == int64(len(u)) {\n\t\treturn -1\n\t}\n\n\tif u[i] == x {\n\t\treturn i\n\t}\n\n\treturn -1\n}\n\ntype blocks []block\n\nfunc (b *blocks) insert(index int64) {\n\tif index == int64(len(*b)) {\n\t\t*b = append(*b, block(0))\n\t\treturn\n\t}\n\n\t*b = append(*b, block(0))\n\tcopy((*b)[index+1:], (*b)[index:])\n\t(*b)[index] = block(0)\n}\n\nfunc (b *blocks) deleteAtIndex(i int64) {\n\tcopy((*b)[i:], (*b)[i+1:])\n\t(*b)[len(*b)-1] = block(0)\n\t*b = (*b)[:len(*b)-1]\n}\n\ntype sparseBitArray struct {\n\tblocks blocks\n\tindices uintSlice\n}\n\n// SetBit sets the bit at the given position.\nfunc (sba *sparseBitArray) SetBit(k uint64) error {\n\tindex, position := getIndexAndRemainder(k)\n\ti, inserted := sba.indices.insert(index)\n\n\tif inserted {\n\t\tsba.blocks.insert(i)\n\t}\n\tsba.blocks[i] = sba.blocks[i].insert(position)\n\treturn nil\n}\n\n// GetBit gets the bit at the given position.\nfunc (sba *sparseBitArray) GetBit(k uint64) (bool, error) {\n\tindex, position := getIndexAndRemainder(k)\n\ti := sba.indices.get(index)\n\tif i == -1 {\n\t\treturn false, nil\n\t}\n\n\treturn sba.blocks[i].get(position), nil\n}\n\n// GetSetBits gets the position of bits set in the array.\nfunc (sba *sparseBitArray) GetSetBits(from uint64, buffer []uint64) []uint64 {\n\tfromBlockIndex, fromOffset := getIndexAndRemainder(from)\n\n\tfromBlockLocation := sba.indices.search(fromBlockIndex)\n\tif int(fromBlockLocation) == len(sba.indices) {\n\t\treturn buffer[:0]\n\t}\n\n\treturn getSetBitsInBlocks(\n\t\tfromBlockIndex,\n\t\tfromOffset,\n\t\tsba.blocks[fromBlockLocation:],\n\t\tsba.indices[fromBlockLocation:],\n\t\tbuffer,\n\t)\n}\n\n// ToNums converts this sparse bitarray to a list of numbers contained\n// within it.\nfunc (sba *sparseBitArray) ToNums() []uint64 {\n\tif len(sba.indices) == 0 {\n\t\treturn nil\n\t}\n\n\tdiff := uint64(len(sba.indices)) * s\n\tnums := make([]uint64, 0, diff/4)\n\n\tfor i, offset := range sba.indices {\n\t\tsba.blocks[i].toNums(offset*s, &nums)\n\t}\n\n\treturn nums\n}\n\n// ClearBit clears the bit at the given position.\nfunc (sba *sparseBitArray) ClearBit(k uint64) error {\n\tindex, position := getIndexAndRemainder(k)\n\ti := sba.indices.get(index)\n\tif i == -1 {\n\t\treturn nil\n\t}\n\n\tsba.blocks[i] = sba.blocks[i].remove(position)\n\tif sba.blocks[i] == 0 {\n\t\tsba.blocks.deleteAtIndex(i)\n\t\tsba.indices.deleteAtIndex(i)\n\t}\n\n\treturn nil\n}\n\n// Reset erases all values from this bitarray.\nfunc (sba *sparseBitArray) Reset() {\n\tsba.blocks = sba.blocks[:0]\n\tsba.indices = sba.indices[:0]\n}\n\n// Blocks returns an iterator to iterator of this bitarray's blocks.\nfunc (sba *sparseBitArray) Blocks() Iterator {\n\treturn newCompressedBitArrayIterator(sba)\n}\n\n// Capacity returns the value of the highest possible *seen* value\n// in this sparse bitarray.\nfunc (sba *sparseBitArray) Capacity() uint64 {\n\tif len(sba.indices) == 0 {\n\t\treturn 0\n\t}\n\n\treturn (sba.indices[len(sba.indices)-1] + 1) * s\n}\n\n// Equals returns a bool indicating if the provided bit array\n// equals this bitarray.\nfunc (sba *sparseBitArray) Equals(other BitArray) bool {\n\tif other.Capacity() == 0 && sba.Capacity() > 0 {\n\t\treturn false\n\t}\n\n\tvar selfIndex uint64\n\tfor iter := other.Blocks(); iter.Next(); {\n\t\totherIndex, otherBlock := iter.Value()\n\t\tif len(sba.indices) == 0 {\n\t\t\tif otherBlock > 0 {\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tcontinue\n\t\t}\n\n\t\tif selfIndex >= uint64(len(sba.indices)) {\n\t\t\treturn false\n\t\t}\n\n\t\tif otherIndex < sba.indices[selfIndex] {\n\t\t\tif otherBlock > 0 {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tif otherIndex > sba.indices[selfIndex] {\n\t\t\treturn false\n\t\t}\n\n\t\tif !sba.blocks[selfIndex].equals(otherBlock) {\n\t\t\treturn false\n\t\t}\n\n\t\tselfIndex++\n\t}\n\n\treturn true\n}\n\n// Count returns the number of set bits in this array.\nfunc (sba *sparseBitArray) Count() int {\n\tcount := 0\n\tfor _, block := range sba.blocks {\n\t\tcount += bits.OnesCount64(uint64(block))\n\t}\n\treturn count\n}\n\n// Or will perform a bitwise or operation with the provided bitarray and\n// return a new result bitarray.\nfunc (sba *sparseBitArray) Or(other BitArray) BitArray {\n\tif ba, ok := other.(*sparseBitArray); ok {\n\t\treturn orSparseWithSparseBitArray(sba, ba)\n\t}\n\n\treturn orSparseWithDenseBitArray(sba, other.(*bitArray))\n}\n\n// And will perform a bitwise and operation with the provided bitarray and\n// return a new result bitarray.\nfunc (sba *sparseBitArray) And(other BitArray) BitArray {\n\tif ba, ok := other.(*sparseBitArray); ok {\n\t\treturn andSparseWithSparseBitArray(sba, ba)\n\t}\n\n\treturn andSparseWithDenseBitArray(sba, other.(*bitArray))\n}\n\n// Nand will return the result of doing a bitwise and not of the bit array\n// with the other bit array on each block.\nfunc (sba *sparseBitArray) Nand(other BitArray) BitArray {\n\tif ba, ok := other.(*sparseBitArray); ok {\n\t\treturn nandSparseWithSparseBitArray(sba, ba)\n\t}\n\n\treturn nandSparseWithDenseBitArray(sba, other.(*bitArray))\n}\n\nfunc (sba *sparseBitArray) IsEmpty() bool {\n\t// This works because the and, nand and delete functions only\n\t// keep values that have a non-zero block.\n\treturn len(sba.indices) == 0\n}\n\nfunc (sba *sparseBitArray) copy() *sparseBitArray {\n\tblocks := make(blocks, len(sba.blocks))\n\tcopy(blocks, sba.blocks)\n\tindices := make(uintSlice, len(sba.indices))\n\tcopy(indices, sba.indices)\n\treturn &sparseBitArray{\n\t\tblocks: blocks,\n\t\tindices: indices,\n\t}\n}\n\n// Intersects returns a bool indicating if the provided bit array\n// intersects with this bitarray.\nfunc (sba *sparseBitArray) Intersects(other BitArray) bool {\n\tif other.Capacity() == 0 {\n\t\treturn true\n\t}\n\n\tvar selfIndex int64\n\tfor iter := other.Blocks(); iter.Next(); {\n\t\totherI, otherBlock := iter.Value()\n\t\tif len(sba.indices) == 0 {\n\t\t\tif otherBlock > 0 {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\t// here we grab where the block should live in ourselves\n\t\ti := uintSlice(sba.indices[selfIndex:]).search(otherI)\n\t\t// this is a block we don't have, doesn't intersect\n\t\tif i == int64(len(sba.indices)) {\n\t\t\treturn false\n\t\t}\n\n\t\tif sba.indices[i] != otherI {\n\t\t\treturn false\n\t\t}\n\n\t\tif !sba.blocks[i].intersects(otherBlock) {\n\t\t\treturn false\n\t\t}\n\n\t\tselfIndex = i\n\t}\n\n\treturn true\n}\n\nfunc (sba *sparseBitArray) IntersectsBetween(other BitArray, start, stop uint64) bool {\n\treturn true\n}\n\nfunc newSparseBitArray() *sparseBitArray {\n\treturn &sparseBitArray{}\n}\n\n// NewSparseBitArray will create a bit array that consumes a great\n// deal less memory at the expense of longer sets and gets.\nfunc NewSparseBitArray() BitArray {\n\treturn newSparseBitArray()\n}\n" }, { "path": "bitarray/sparse_bitarray_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestGetCompressedBit(t *testing.T) {\n\tba := newSparseBitArray()\n\n\tresult, err := ba.GetBit(5)\n\tassert.Nil(t, err)\n\tassert.False(t, result)\n}\n\nfunc BenchmarkGetCompressedBit(b *testing.B) {\n\tnumItems := 1000\n\tba := newSparseBitArray()\n\n\tfor i := 0; i < numItems; i++ {\n\t\tba.SetBit(uint64(i))\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tba.GetBit(s)\n\t}\n}\n\nfunc TestGetSetCompressedBit(t *testing.T) {\n\tba := newSparseBitArray()\n\n\tba.SetBit(5)\n\n\tresult, err := ba.GetBit(5)\n\tassert.Nil(t, err)\n\tassert.True(t, result)\n\tresult, err = ba.GetBit(7)\n\tassert.Nil(t, err)\n\tassert.False(t, result)\n\n\tba.SetBit(s * 2)\n\tresult, _ = ba.GetBit(s * 2)\n\tassert.True(t, result)\n\tresult, _ = ba.GetBit(s*2 + 1)\n\tassert.False(t, result)\n}\n\nfunc BenchmarkSetCompressedBit(b *testing.B) {\n\tnumItems := 1000\n\tba := newSparseBitArray()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfor j := 0; j < numItems; j++ {\n\t\t\tba.SetBit(uint64(j))\n\t\t}\n\t}\n}\n\nfunc TestGetSetCompressedBits(t *testing.T) {\n\tba := newSparseBitArray()\n\tbuf := make([]uint64, 0, 5)\n\n\trequire.NoError(t, ba.SetBit(1))\n\trequire.NoError(t, ba.SetBit(4))\n\trequire.NoError(t, ba.SetBit(8))\n\trequire.NoError(t, ba.SetBit(63))\n\trequire.NoError(t, ba.SetBit(64))\n\trequire.NoError(t, ba.SetBit(200))\n\trequire.NoError(t, ba.SetBit(1000))\n\n\tassert.Equal(t, []uint64(nil), ba.GetSetBits(0, nil))\n\tassert.Equal(t, []uint64{}, ba.GetSetBits(0, []uint64{}))\n\n\tassert.Equal(t, []uint64{1, 4, 8, 63, 64}, ba.GetSetBits(0, buf))\n\tassert.Equal(t, []uint64{63, 64, 200, 1000}, ba.GetSetBits(10, buf))\n\tassert.Equal(t, []uint64{63, 64, 200, 1000}, ba.GetSetBits(63, buf))\n\tassert.Equal(t, []uint64{200, 1000}, ba.GetSetBits(128, buf))\n\n\trequire.NoError(t, ba.ClearBit(4))\n\trequire.NoError(t, ba.ClearBit(64))\n\tassert.Equal(t, []uint64{1, 8, 63, 200, 1000}, ba.GetSetBits(0, buf))\n\tassert.Empty(t, ba.GetSetBits(1001, buf))\n\n\tba.Reset()\n\tassert.Empty(t, ba.GetSetBits(0, buf))\n}\n\nfunc BenchmarkGetSetCompressedBits(b *testing.B) {\n\tba := newSparseBitArray()\n\tfor i := uint64(0); i < 168000; i++ {\n\t\tif i%13 == 0 || i%5 == 0 {\n\t\t\trequire.NoError(b, ba.SetBit(i))\n\t\t}\n\t}\n\n\tbuf := make([]uint64, 0, ba.Capacity())\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tba.GetSetBits(0, buf)\n\t}\n}\n\nfunc TestCompressedCount(t *testing.T) {\n\tba := newSparseBitArray()\n\tassert.Equal(t, 0, ba.Count())\n\n\trequire.NoError(t, ba.SetBit(0))\n\tassert.Equal(t, 1, ba.Count())\n\n\trequire.NoError(t, ba.SetBit(40))\n\trequire.NoError(t, ba.SetBit(64))\n\trequire.NoError(t, ba.SetBit(100))\n\trequire.NoError(t, ba.SetBit(200))\n\trequire.NoError(t, ba.SetBit(469))\n\trequire.NoError(t, ba.SetBit(500))\n\tassert.Equal(t, 7, ba.Count())\n\n\trequire.NoError(t, ba.ClearBit(200))\n\tassert.Equal(t, 6, ba.Count())\n\n\tba.Reset()\n\tassert.Equal(t, 0, ba.Count())\n}\n\nfunc TestClearCompressedBit(t *testing.T) {\n\tba := newSparseBitArray()\n\tba.SetBit(5)\n\tba.ClearBit(5)\n\n\tresult, err := ba.GetBit(5)\n\tassert.Nil(t, err)\n\tassert.False(t, result)\n\tassert.Len(t, ba.blocks, 0)\n\tassert.Len(t, ba.indices, 0)\n\n\tba.SetBit(s * 2)\n\tba.ClearBit(s * 2)\n\n\tresult, _ = ba.GetBit(s * 2)\n\tassert.False(t, result)\n\tassert.Len(t, ba.indices, 0)\n\tassert.Len(t, ba.blocks, 0)\n}\n\nfunc BenchmarkClearCompressedBit(b *testing.B) {\n\tnumItems := 1000\n\tba := newSparseBitArray()\n\tfor i := 0; i < numItems; i++ {\n\t\tba.SetBit(uint64(i))\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tba.ClearBit(uint64(i))\n\t}\n}\n\nfunc TestClearCompressedBitArray(t *testing.T) {\n\tba := newSparseBitArray()\n\tba.SetBit(5)\n\tba.SetBit(s * 2)\n\n\tresult, err := ba.GetBit(5)\n\tassert.Nil(t, err)\n\tassert.True(t, result)\n\tresult, _ = ba.GetBit(s * 2)\n\tassert.True(t, result)\n\n\tba.Reset()\n\n\tresult, err = ba.GetBit(5)\n\tassert.Nil(t, err)\n\tassert.False(t, result)\n\tresult, _ = ba.GetBit(s * 2)\n\tassert.False(t, result)\n}\n\nfunc TestCompressedEquals(t *testing.T) {\n\tba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tassert.True(t, ba.Equals(other))\n\n\tba.SetBit(5)\n\tassert.False(t, ba.Equals(other))\n\n\tother.SetBit(5)\n\tassert.True(t, ba.Equals(other))\n\n\tba.ClearBit(5)\n\tassert.False(t, ba.Equals(other))\n}\n\nfunc TestCompressedIntersects(t *testing.T) {\n\tba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tassert.True(t, ba.Intersects(other))\n\n\tother.SetBit(5)\n\n\tassert.False(t, ba.Intersects(other))\n\tassert.True(t, other.Intersects(ba))\n\n\tba.SetBit(5)\n\n\tassert.True(t, ba.Intersects(other))\n\tassert.True(t, other.Intersects(ba))\n\n\tother.SetBit(10)\n\n\tassert.False(t, ba.Intersects(other))\n\tassert.True(t, other.Intersects(ba))\n}\n\nfunc TestLongCompressedIntersects(t *testing.T) {\n\tba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tba.SetBit(5)\n\tother.SetBit(5)\n\n\tassert.True(t, ba.Intersects(other))\n\n\tother.SetBit(s * 2)\n\n\tassert.False(t, ba.Intersects(other))\n\tassert.True(t, other.Intersects(ba))\n\n\tba.SetBit(s * 2)\n\n\tassert.True(t, ba.Intersects(other))\n\tassert.True(t, other.Intersects(ba))\n\n\tother.SetBit(s*2 + 1)\n\n\tassert.False(t, ba.Intersects(other))\n\tassert.True(t, other.Intersects(ba))\n}\n\nfunc BenchmarkCompressedIntersects(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\tba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tba.SetBit(i)\n\t\tother.SetBit(i)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tba.Intersects(other)\n\t}\n}\n\nfunc TestSparseIntersectsBitArray(t *testing.T) {\n\tcba := newSparseBitArray()\n\tba := newBitArray(s * 2)\n\n\tassert.True(t, cba.Intersects(ba))\n\tba.SetBit(5)\n\n\tassert.False(t, cba.Intersects(ba))\n\tcba.SetBit(5)\n\n\tassert.True(t, cba.Intersects(ba))\n\tcba.SetBit(10)\n\n\tassert.True(t, cba.Intersects(ba))\n\tba.SetBit(s + 1)\n\n\tassert.False(t, cba.Intersects(ba))\n\tcba.SetBit(s + 1)\n\n\tassert.True(t, cba.Intersects(ba))\n\tcba.SetBit(s * 3)\n\n\tassert.True(t, cba.Intersects(ba))\n}\n\nfunc TestSparseEqualsBitArray(t *testing.T) {\n\tcba := newSparseBitArray()\n\tba := newBitArray(s * 2)\n\n\tassert.True(t, cba.Equals(ba))\n\n\tba.SetBit(5)\n\tassert.False(t, cba.Equals(ba))\n\n\tcba.SetBit(5)\n\tassert.True(t, cba.Equals(ba))\n\n\tba.SetBit(s + 1)\n\tassert.False(t, cba.Equals(ba))\n\n\tcba.SetBit(s + 1)\n\tassert.True(t, cba.Equals(ba))\n}\n\nfunc BenchmarkCompressedEquals(b *testing.B) {\n\tnumItems := uint64(1000)\n\tcba := newSparseBitArray()\n\tother := newSparseBitArray()\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tcba.SetBit(i)\n\t\tother.SetBit(i)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tcba.Equals(other)\n\t}\n}\n\nfunc TestInsertPreviousBlockInSparse(t *testing.T) {\n\tsba := newSparseBitArray()\n\n\tsba.SetBit(s * 2)\n\tsba.SetBit(s - 1)\n\n\tresult, err := sba.GetBit(s - 1)\n\tassert.Nil(t, err)\n\tassert.True(t, result)\n}\n\nfunc TestSparseBitArrayToNums(t *testing.T) {\n\tsba := newSparseBitArray()\n\n\tsba.SetBit(s - 1)\n\tsba.SetBit(s + 1)\n\n\texpected := []uint64{s - 1, s + 1}\n\n\tresults := sba.ToNums()\n\tassert.Equal(t, expected, results)\n}\n\nfunc BenchmarkSparseBitArrayToNums(b *testing.B) {\n\tnumItems := uint64(1000)\n\tsba := newSparseBitArray()\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tsba.SetBit(i)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tsba.ToNums()\n\t}\n}\n" }, { "path": "bitarray/util.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage bitarray\n\n// maxInt64 returns the highest integer in the provided list of int64s\nfunc maxInt64(ints ...int64) int64 {\n\tmaxInt := ints[0]\n\tfor i := 1; i < len(ints); i++ {\n\t\tif ints[i] > maxInt {\n\t\t\tmaxInt = ints[i]\n\t\t}\n\t}\n\n\treturn maxInt\n}\n\n// maxUint64 returns the highest integer in the provided list of uint64s\nfunc maxUint64(ints ...uint64) uint64 {\n\tmaxInt := ints[0]\n\tfor i := 1; i < len(ints); i++ {\n\t\tif ints[i] > maxInt {\n\t\t\tmaxInt = ints[i]\n\t\t}\n\t}\n\n\treturn maxInt\n}\n\n// minUint64 returns the lowest integer in the provided list of int32s\nfunc minUint64(ints ...uint64) uint64 {\n\tminInt := ints[0]\n\tfor i := 1; i < len(ints); i++ {\n\t\tif ints[i] < minInt {\n\t\t\tminInt = ints[i]\n\t\t}\n\t}\n\n\treturn minInt\n}\n" }, { "path": "btree/_link/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage link\n\n// Keys is a typed list of Key interfaces.\ntype Keys []Key\n\ntype Key interface {\n\t// Compare should return an int indicating how this key relates\n\t// to the provided key. -1 will indicate less than, 0 will indicate\n\t// equality, and 1 will indicate greater than. Duplicate keys\n\t// are allowed, but duplicate IDs are not.\n\tCompare(Key) int\n}\n" }, { "path": "btree/_link/key.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage link\n\nimport \"sort\"\n\nfunc (keys Keys) search(key Key) int {\n\treturn sort.Search(len(keys), func(i int) bool {\n\t\treturn keys[i].Compare(key) >= 0\n\t})\n}\n\nfunc (keys *Keys) insert(key Key) Key {\n\ti := keys.search(key)\n\treturn keys.insertAt(key, i)\n}\n\nfunc (keys *Keys) insertAt(key Key, i int) Key {\n\tif i == len(*keys) {\n\t\t*keys = append(*keys, key)\n\t\treturn nil\n\t}\n\n\tif (*keys)[i].Compare(key) == 0 { //overwrite case\n\t\toldKey := (*keys)[i]\n\t\t(*keys)[i] = key\n\t\treturn oldKey\n\t}\n\n\t*keys = append(*keys, nil)\n\tcopy((*keys)[i+1:], (*keys)[i:])\n\t(*keys)[i] = key\n\treturn nil\n}\n\nfunc (keys *Keys) split() (Key, Keys, Keys) {\n\ti := (len(*keys) / 2) - 1\n\tmiddle := (*keys)[i]\n\n\tleft, right := keys.splitAt(i)\n\treturn middle, left, right\n}\n\nfunc (keys *Keys) splitAt(i int) (Keys, Keys) {\n\tright := make(Keys, len(*keys)-i-1, cap(*keys))\n\tcopy(right, (*keys)[i+1:])\n\tfor j := i + 1; j < len(*keys); j++ {\n\t\t(*keys)[j] = nil\n\t}\n\t*keys = (*keys)[:i+1]\n\n\treturn *keys, right\n}\n\nfunc (keys Keys) last() Key {\n\treturn keys[len(keys)-1]\n}\n\nfunc (keys Keys) first() Key {\n\treturn keys[0]\n}\n\nfunc (keys Keys) needsSplit() bool {\n\treturn cap(keys) == len(keys)\n}\n\nfunc (keys Keys) reverse() Keys {\n\treversed := make(Keys, len(keys))\n\tfor i := len(keys) - 1; i >= 0; i-- {\n\t\treversed[len(keys)-1-i] = keys[i]\n\t}\n\n\treturn reversed\n}\n\nfunc chunkKeys(keys Keys, numParts int64) []Keys {\n\tparts := make([]Keys, numParts)\n\tfor i := int64(0); i < numParts; i++ {\n\t\tparts[i] = keys[i*int64(len(keys))/numParts : (i+1)*int64(len(keys))/numParts]\n\t}\n\treturn parts\n}\n" }, { "path": "btree/_link/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage link\n\ntype mockKey uint64\n\nfunc (mk mockKey) Compare(other Key) int {\n\totherK := other.(mockKey)\n\tif mk < otherK {\n\t\treturn -1\n\t}\n\n\tif mk > otherK {\n\t\treturn 1\n\t}\n\n\treturn 0\n}\n" }, { "path": "btree/_link/node.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage link\n\nimport (\n\t\"log\"\n\t\"sync\"\n)\n\nfunc search(parent *node, key Key) Key {\n\tparent = getParent(parent, nil, key)\n\tparent.lock.RLock()\n\tparent = moveRight(parent, key, false)\n\tdefer parent.lock.RUnlock()\n\n\ti := parent.search(key)\n\tif i == len(parent.keys) {\n\t\treturn nil\n\t}\n\n\treturn parent.keys[i]\n}\n\nfunc getParent(parent *node, stack *nodes, key Key) *node {\n\tvar n *node\n\tfor parent != nil && !parent.isLeaf {\n\t\tparent.lock.RLock()\n\t\tparent = moveRight(parent, key, false) // if this happens on the root this should always just return\n\t\tn = parent.searchNode(key)\n\t\tif stack != nil {\n\t\t\tstack.push(parent)\n\t\t}\n\n\t\tparent.lock.RUnlock()\n\t\tparent = n\n\t}\n\n\treturn parent\n}\n\nfunc insert(tree *blink, parent *node, stack *nodes, key Key) Key {\n\tparent = getParent(parent, stack, key)\n\n\tparent.lock.Lock()\n\tparent = moveRight(parent, key, true)\n\n\tresult := parent.insert(key)\n\tif result != nil { // overwrite\n\t\tparent.lock.Unlock()\n\t\treturn result\n\t}\n\n\tif !parent.needsSplit() {\n\t\tparent.lock.Unlock()\n\t\treturn nil\n\t}\n\n\tsplit(tree, parent, stack)\n\n\treturn nil\n}\n\nfunc split(tree *blink, n *node, stack *nodes) {\n\tvar l, r *node\n\tvar k Key\n\tvar parent *node\n\tfor n.needsSplit() {\n\t\tk, l, r = n.split()\n\t\tparent = stack.pop()\n\t\tif parent == nil {\n\t\t\ttree.lock.Lock()\n\t\t\tif tree.root == nil || tree.root == n {\n\t\t\t\tparent = newNode(false, make(Keys, 0, tree.ary), make(nodes, 0, tree.ary+1))\n\t\t\t\tparent.maxSeen = r.max()\n\t\t\t\tparent.keys.insert(k)\n\t\t\t\tparent.nodes.push(l)\n\t\t\t\tparent.nodes.push(r)\n\t\t\t\ttree.root = parent\n\t\t\t\tn.lock.Unlock()\n\t\t\t\ttree.lock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tparent = tree.root\n\t\t\ttree.lock.Unlock()\n\t\t}\n\n\t\tparent.lock.Lock()\n\t\tparent = moveRight(parent, r.key(), true)\n\t\ti := parent.search(k)\n\t\tparent.keys.insertAt(k, i)\n\t\tparent.nodes[i] = l\n\t\tparent.nodes.insertAt(r, i+1)\n\n\t\tn.lock.Unlock()\n\t\tn = parent\n\t}\n\n\tn.lock.Unlock()\n}\n\nfunc moveRight(n *node, key Key, getLock bool) *node {\n\tvar right *node\n\tfor {\n\t\tif len(n.keys) == 0 || n.right == nil { // this is either the node or the rightmost node\n\t\t\treturn n\n\t\t}\n\t\tif key.Compare(n.max()) < 1 {\n\t\t\treturn n\n\t\t}\n\n\t\tif getLock {\n\t\t\tn.right.lock.Lock()\n\t\t\tright = n.right\n\t\t\tn.lock.Unlock()\n\t\t} else {\n\t\t\tn.right.lock.RLock()\n\t\t\tright = n.right\n\t\t\tn.lock.RUnlock()\n\t\t}\n\t\tn = right\n\t}\n}\n\ntype nodes []*node\n\nfunc (ns *nodes) reset() {\n\tfor i := range *ns {\n\t\t(*ns)[i] = nil\n\t}\n\n\t*ns = (*ns)[:0]\n}\n\nfunc (ns *nodes) push(n *node) {\n\t*ns = append(*ns, n)\n}\n\nfunc (ns *nodes) pop() *node {\n\tif len(*ns) == 0 {\n\t\treturn nil\n\t}\n\n\tn := (*ns)[len(*ns)-1]\n\t(*ns)[len(*ns)-1] = nil\n\t*ns = (*ns)[:len(*ns)-1]\n\treturn n\n}\n\nfunc (ns *nodes) insertAt(n *node, i int) {\n\tif i == len(*ns) {\n\t\t*ns = append(*ns, n)\n\t\treturn\n\t}\n\n\t*ns = append(*ns, nil)\n\tcopy((*ns)[i+1:], (*ns)[i:])\n\t(*ns)[i] = n\n}\n\nfunc (ns *nodes) splitAt(i int) (nodes, nodes) {\n\tlength := len(*ns) - i\n\tright := make(nodes, length, cap(*ns))\n\tcopy(right, (*ns)[i+1:])\n\tfor j := i + 1; j < len(*ns); j++ {\n\t\t(*ns)[j] = nil\n\t}\n\t*ns = (*ns)[:i+1]\n\treturn *ns, right\n}\n\ntype node struct {\n\tkeys Keys\n\tnodes nodes\n\tright *node\n\tlock sync.RWMutex\n\tisLeaf bool\n\tmaxSeen Key\n}\n\nfunc (n *node) key() Key {\n\treturn n.keys.last()\n}\n\nfunc (n *node) insert(key Key) Key {\n\tif !n.isLeaf {\n\t\tpanic(`Can't only insert key in an internal node.`)\n\t}\n\n\toverwritten := n.keys.insert(key)\n\treturn overwritten\n}\n\nfunc (n *node) insertNode(other *node) {\n\tkey := other.key()\n\ti := n.keys.search(key)\n\tn.keys.insertAt(key, i)\n\tn.nodes.insertAt(other, i)\n}\n\nfunc (n *node) needsSplit() bool {\n\treturn n.keys.needsSplit()\n}\n\nfunc (n *node) max() Key {\n\tif n.isLeaf {\n\t\treturn n.keys.last()\n\t}\n\n\treturn n.maxSeen\n}\n\nfunc (n *node) splitLeaf() (Key, *node, *node) {\n\ti := (len(n.keys) / 2)\n\tkey := n.keys[i]\n\t_, rightKeys := n.keys.splitAt(i)\n\tnn := &node{\n\t\tkeys: rightKeys,\n\t\tright: n.right,\n\t\tisLeaf: true,\n\t}\n\tn.right = nn\n\treturn key, n, nn\n}\n\nfunc (n *node) splitInternal() (Key, *node, *node) {\n\ti := (len(n.keys) / 2)\n\tkey := n.keys[i]\n\n\trightKeys := make(Keys, len(n.keys)-1-i, cap(n.keys))\n\trightNodes := make(nodes, len(rightKeys)+1, cap(n.nodes))\n\n\tcopy(rightKeys, n.keys[i+1:])\n\tcopy(rightNodes, n.nodes[i+1:])\n\n\t// for garbage collection\n\tfor j := i + 1; j < len(n.nodes); j++ {\n\t\tif j != len(n.keys) {\n\t\t\tn.keys[j] = nil\n\t\t}\n\t\tn.nodes[j] = nil\n\t}\n\n\tnn := newNode(false, rightKeys, rightNodes)\n\tnn.maxSeen = n.max()\n\n\tn.maxSeen = key\n\tn.keys = n.keys[:i]\n\tn.nodes = n.nodes[:i+1]\n\tn.right = nn\n\n\treturn key, n, nn\n}\n\nfunc (n *node) split() (Key, *node, *node) {\n\tif n.isLeaf {\n\t\treturn n.splitLeaf()\n\t}\n\n\treturn n.splitInternal()\n}\n\nfunc (n *node) search(key Key) int {\n\treturn n.keys.search(key)\n}\n\nfunc (n *node) searchNode(key Key) *node {\n\ti := n.search(key)\n\n\treturn n.nodes[i]\n}\n\nfunc (n *node) print(output *log.Logger) {\n\toutput.Printf(`NODE: %+v, %p`, n, n)\n\tif !n.isLeaf {\n\t\tfor _, n := range n.nodes {\n\t\t\tif n == nil {\n\t\t\t\toutput.Println(`NIL NODE`)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tn.print(output)\n\t\t}\n\t}\n}\n\nfunc newNode(isLeaf bool, keys Keys, ns nodes) *node {\n\treturn &node{\n\t\tisLeaf: isLeaf,\n\t\tkeys: keys,\n\t\tnodes: ns,\n\t}\n}\n" }, { "path": "btree/_link/node_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage link\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc newTestNode(isLeaf bool, ary int) *node {\n\treturn &node{\n\t\tisLeaf: isLeaf,\n\t\tkeys: make(Keys, 0, ary),\n\t\tnodes: make(nodes, 0, ary+1),\n\t}\n}\n\nfunc checkTree(t testing.TB, tree *blink) bool {\n\tif tree.root == nil {\n\t\treturn true\n\t}\n\n\treturn checkNode(t, tree.root)\n}\n\nfunc checkNode(t testing.TB, n *node) bool {\n\tif len(n.keys) == 0 {\n\t\tassert.Len(t, n.nodes, 0)\n\t\treturn false\n\t}\n\n\tif n.isLeaf {\n\t\tassert.Len(t, n.nodes, 0)\n\t\treturn false\n\t}\n\n\tif !assert.Len(t, n.nodes, len(n.keys)+1) {\n\t\treturn false\n\t}\n\n\tfor i := 0; i < len(n.keys); i++ {\n\t\tif !assert.True(t, n.keys[i].Compare(n.nodes[i].key()) >= 0) {\n\t\t\tt.Logf(`N: %+v %p, n.keys[i]: %+v, n.nodes[i]: %+v`, n, n, n.keys[i], n.nodes[i])\n\t\t\treturn false\n\t\t}\n\t}\n\n\tif !assert.True(t, n.nodes[len(n.nodes)-1].key().Compare(n.keys.last()) > 0) {\n\t\tt.Logf(`m: %+v, %p, n.nodes[len(n.nodes)-1].key(): %+v, n.keys.last(): %+v`, n, n, n.nodes[len(n.nodes)-1].key(), n.keys.last())\n\t\treturn false\n\t}\n\tfor _, child := range n.nodes {\n\t\tif !assert.NotNil(t, child) {\n\t\t\treturn false\n\t\t}\n\t\tif !checkNode(t, child) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc TestSplitInternalNodeOddAry(t *testing.T) {\n\tparent := newTestNode(false, 3)\n\tn1 := newTestNode(true, 3)\n\tn1.keys.insert(mockKey(1))\n\tn2 := newTestNode(true, 3)\n\tn2.keys.insert(mockKey(5))\n\tn3 := newTestNode(true, 3)\n\tn3.keys.insert(mockKey(10))\n\tn4 := newTestNode(true, 3)\n\tn4.keys.insert(mockKey(15))\n\n\tparent.nodes = nodes{n1, n2, n3, n4}\n\tparent.keys = Keys{mockKey(5), mockKey(10), mockKey(15)}\n\n\tkey, l, r := parent.split()\n\tassert.Equal(t, mockKey(10), key)\n\tassert.Equal(t, Keys{mockKey(5)}, l.keys)\n\tassert.Equal(t, Keys{mockKey(15)}, r.keys)\n\n\tassert.Equal(t, nodes{n1, n2}, l.nodes)\n\tassert.Equal(t, nodes{n3, n4}, r.nodes)\n\tassert.Equal(t, l.right, r)\n\tassert.False(t, l.isLeaf)\n\tassert.False(t, r.isLeaf)\n}\n\nfunc TestSplitInternalNodeEvenAry(t *testing.T) {\n\tparent := newTestNode(false, 4)\n\tn1 := newTestNode(true, 4)\n\tn1.keys.insert(mockKey(1))\n\tn2 := newTestNode(true, 4)\n\tn2.keys.insert(mockKey(5))\n\tn3 := newTestNode(true, 4)\n\tn3.keys.insert(mockKey(10))\n\tn4 := newTestNode(true, 4)\n\tn4.keys.insert(mockKey(15))\n\tn5 := newTestNode(true, 4)\n\tn5.keys.insert(mockKey(20))\n\n\tparent.nodes = nodes{n1, n2, n3, n4, n5}\n\tparent.keys = Keys{mockKey(5), mockKey(10), mockKey(15), mockKey(20)}\n\n\tkey, l, r := parent.split()\n\tassert.Equal(t, mockKey(15), key)\n\tassert.Equal(t, Keys{mockKey(5), mockKey(10)}, l.keys)\n\tassert.Equal(t, Keys{mockKey(20)}, r.keys)\n\n\tassert.Equal(t, nodes{n1, n2, n3}, l.nodes)\n\tassert.Equal(t, nodes{n4, n5}, r.nodes)\n\tassert.Equal(t, l.right, r)\n\tassert.False(t, l.isLeaf)\n\tassert.False(t, r.isLeaf)\n}\n\nfunc TestSplitLeafNodeOddAry(t *testing.T) {\n\tparent := newTestNode(true, 3)\n\tk1 := mockKey(5)\n\tk2 := mockKey(15)\n\tk3 := mockKey(20)\n\n\tparent.keys = Keys{k1, k2, k3}\n\tkey, l, r := parent.split()\n\tassert.Equal(t, k2, key)\n\tassert.Equal(t, Keys{k1, k2}, l.keys)\n\tassert.Equal(t, Keys{k3}, r.keys)\n\tassert.True(t, l.isLeaf)\n\tassert.True(t, r.isLeaf)\n\tassert.Equal(t, r, l.right)\n}\n\nfunc TestSplitLeafNodeEvenAry(t *testing.T) {\n\tparent := newTestNode(true, 4)\n\tk1 := mockKey(5)\n\tk2 := mockKey(15)\n\tk3 := mockKey(20)\n\tk4 := mockKey(25)\n\n\tparent.keys = Keys{k1, k2, k3, k4}\n\tkey, l, r := parent.split()\n\tassert.Equal(t, k3, key)\n\tassert.Equal(t, Keys{k1, k2, k3}, l.keys)\n\tassert.Equal(t, Keys{k4}, r.keys)\n\tassert.True(t, l.isLeaf)\n\tassert.True(t, r.isLeaf)\n\tassert.Equal(t, r, l.right)\n}\n" }, { "path": "btree/_link/tree.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nThis is a b-link tree in progress from the following paper:\nhttp://www.csd.uoc.gr/~hy460/pdf/p650-lehman.pdf\n\nThis is still a work in progress and the CRUD methods on the tree\nneed to be parallelized. Until this is complete, there is no\nconstructor method for this package.\n\nTime complexities:\nSpace: O(n)\nSearch: O(log n)\nInsert: O(log n)\nDelete: O(log n)\n\nCurrent benchmarks with 16 ary:\nBenchmarkSimpleAdd-8\t \t1000000\t 1455 ns/op\nBenchmarkGet-8\t \t\t\t2000000\t 704 ns/op\n\nB-link was chosen after examining this paper:\nhttp://www.vldb.org/journal/VLDBJ2/P361.pdf\n*/\n\npackage link\n\nimport (\n\t\"log\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\n// numberOfItemsBeforeMultithread defines the number of items that have\n// to be called with a method before we multithread.\nconst numberOfItemsBeforeMultithread = 10\n\ntype blink struct {\n\troot *node\n\tlock sync.RWMutex\n\tnumber, ary, numRoutines uint64\n}\n\nfunc (blink *blink) insert(key Key, stack *nodes) Key {\n\tvar parent *node\n\tblink.lock.Lock()\n\tif blink.root == nil {\n\t\tblink.root = newNode(\n\t\t\ttrue, make(Keys, 0, blink.ary), make(nodes, 0, blink.ary+1),\n\t\t)\n\t\tblink.root.keys = make(Keys, 0, blink.ary)\n\t\tblink.root.isLeaf = true\n\t}\n\tparent = blink.root\n\tblink.lock.Unlock()\n\n\tresult := insert(blink, parent, stack, key)\n\tif result == nil {\n\t\tatomic.AddUint64(&blink.number, 1)\n\t\treturn nil\n\t}\n\n\treturn result\n}\n\nfunc (blink *blink) multithreadedInsert(keys Keys) Keys {\n\tchunks := chunkKeys(keys, int64(blink.numRoutines))\n\toverwritten := make(Keys, len(keys))\n\tvar offset uint64\n\tvar wg sync.WaitGroup\n\twg.Add(len(chunks))\n\n\tfor _, chunk := range chunks {\n\t\tgo func(chunk Keys, offset uint64) {\n\t\t\tdefer wg.Done()\n\t\t\tstack := make(nodes, 0, blink.ary)\n\n\t\t\tfor i := 0; i < len(chunk); i++ {\n\t\t\t\tresult := blink.insert(chunk[i], &stack)\n\t\t\t\tstack.reset()\n\t\t\t\toverwritten[offset+uint64(i)] = result\n\t\t\t}\n\t\t}(chunk, offset)\n\t\toffset += uint64(len(chunk))\n\t}\n\n\twg.Wait()\n\n\treturn overwritten\n}\n\n// Insert will insert the provided keys into the b-tree and return\n// a list of keys overwritten, if any. Each insert is an O(log n)\n// operation.\nfunc (blink *blink) Insert(keys ...Key) Keys {\n\tif len(keys) > numberOfItemsBeforeMultithread {\n\t\treturn blink.multithreadedInsert(keys)\n\t}\n\toverwritten := make(Keys, 0, len(keys))\n\tstack := make(nodes, 0, blink.ary)\n\tfor _, k := range keys {\n\t\toverwritten = append(overwritten, blink.insert(k, &stack))\n\t\tstack.reset()\n\t}\n\n\treturn overwritten\n}\n\n// Len returns the number of items in this b-link tree.\nfunc (blink *blink) Len() uint64 {\n\treturn atomic.LoadUint64(&blink.number)\n}\n\nfunc (blink *blink) get(key Key) Key {\n\tvar parent *node\n\tblink.lock.RLock()\n\tparent = blink.root\n\tblink.lock.RUnlock()\n\tk := search(parent, key)\n\tif k == nil {\n\t\treturn nil\n\t}\n\n\tif k.Compare(key) == 0 {\n\t\treturn k\n\t}\n\n\treturn nil\n}\n\n// Get will retrieve the keys if they exist in this tree. If not,\n// a nil is returned in the proper place in the list of keys. Each\n// lookup is O(log n) time complexity.\nfunc (blink *blink) Get(keys ...Key) Keys {\n\tfound := make(Keys, 0, len(keys))\n\tfor _, k := range keys {\n\t\tfound = append(found, blink.get(k))\n\t}\n\n\treturn found\n}\n\nfunc (blink *blink) print(output *log.Logger) {\n\toutput.Println(`PRINTING B-LINK`)\n\tif blink.root == nil {\n\t\treturn\n\t}\n\n\tblink.root.print(output)\n}\n\nfunc newTree(ary, numRoutines uint64) *blink {\n\treturn &blink{ary: ary, numRoutines: numRoutines}\n}\n" }, { "path": "btree/_link/tree_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage link\n\nimport (\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc getConsoleLogger() *log.Logger {\n\treturn log.New(os.Stderr, \"\", log.LstdFlags)\n}\n\nfunc generateRandomKeys(num int) Keys {\n\tkeys := make(Keys, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tkeys = append(keys, mockKey(uint64(rand.Uint32()%uint32(100))))\n\t}\n\n\treturn keys\n}\n\nfunc generateKeys(num int) Keys {\n\tkeys := make(Keys, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tkeys = append(keys, mockKey(uint64(i)))\n\t}\n\n\treturn keys\n}\n\nfunc TestSimpleInsert(t *testing.T) {\n\tk1 := mockKey(5)\n\n\ttree := newTree(8, 1)\n\tresult := tree.Insert(k1)\n\tassert.Equal(t, Keys{nil}, result)\n\tassert.Equal(t, uint64(1), tree.Len())\n\tif !assert.Equal(t, Keys{k1}, tree.Get(k1)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n}\n\nfunc TestMultipleInsert(t *testing.T) {\n\tk1 := mockKey(10)\n\tk2 := mockKey(5)\n\ttree := newTree(8, 1)\n\n\tresult := tree.Insert(k1, k2)\n\tassert.Equal(t, Keys{nil, nil}, result)\n\tassert.Equal(t, uint64(2), tree.Len())\n\tassert.Equal(t, Keys{k1, k2}, tree.Get(k1, k2))\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitOddAryReverseOrder(t *testing.T) {\n\tk1, k2, k3 := mockKey(15), mockKey(10), mockKey(5)\n\ttree := newTree(3, 1)\n\n\tresult := tree.Insert(k1, k2, k3)\n\tassert.Equal(t, Keys{nil, nil, nil}, result)\n\tassert.Equal(t, uint64(3), tree.Len())\n\tif !assert.Equal(t, Keys{k1, k2, k3}, tree.Get(k1, k2, k3)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitOddAry(t *testing.T) {\n\tk1, k2, k3 := mockKey(5), mockKey(10), mockKey(15)\n\ttree := newTree(3, 1)\n\n\tresult := tree.Insert(k1, k2, k3)\n\tassert.Equal(t, Keys{nil, nil, nil}, result)\n\tassert.Equal(t, uint64(3), tree.Len())\n\tif !assert.Equal(t, Keys{k1, k2, k3}, tree.Get(k1, k2, k3)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitOddAryRandomOrder(t *testing.T) {\n\tk1, k2, k3 := mockKey(10), mockKey(5), mockKey(15)\n\ttree := newTree(3, 1)\n\n\tresult := tree.Insert(k1, k2, k3)\n\tassert.Equal(t, Keys{nil, nil, nil}, result)\n\tassert.Equal(t, uint64(3), tree.Len())\n\tif !assert.Equal(t, Keys{k1, k2, k3}, tree.Get(k1, k2, k3)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitEvenAryReverseOrder(t *testing.T) {\n\tk1, k2, k3, k4 := mockKey(20), mockKey(15), mockKey(10), mockKey(5)\n\ttree := newTree(4, 1)\n\n\tresult := tree.Insert(k1, k2, k3, k4)\n\tassert.Equal(t, Keys{nil, nil, nil, nil}, result)\n\tassert.Equal(t, uint64(4), tree.Len())\n\n\tif !assert.Equal(t, Keys{k1, k2, k3, k4}, tree.Get(k1, k2, k3, k4)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitEvenAry(t *testing.T) {\n\tk1, k2, k3, k4 := mockKey(5), mockKey(10), mockKey(15), mockKey(20)\n\ttree := newTree(4, 1)\n\n\tresult := tree.Insert(k1, k2, k3, k4)\n\tassert.Equal(t, Keys{nil, nil, nil, nil}, result)\n\tassert.Equal(t, uint64(4), tree.Len())\n\tif !assert.Equal(t, Keys{k1, k2, k3, k4}, tree.Get(k1, k2, k3, k4)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitEvenAryRandomOrder(t *testing.T) {\n\tk1, k2, k3, k4 := mockKey(10), mockKey(15), mockKey(20), mockKey(5)\n\ttree := newTree(4, 1)\n\n\tresult := tree.Insert(k1, k2, k3, k4)\n\tassert.Equal(t, Keys{nil, nil, nil, nil}, result)\n\tassert.Equal(t, uint64(4), tree.Len())\n\tif !assert.Equal(t, Keys{k1, k2, k3, k4}, tree.Get(k1, k2, k3, k4)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitEvenAryMultiThreaded(t *testing.T) {\n\tkeys := generateRandomKeys(16)\n\ttree := newTree(16, 8)\n\n\tresult := tree.Insert(keys...)\n\tassert.Len(t, result, len(keys))\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesCascadingSplitsOddAry(t *testing.T) {\n\tkeys := generateRandomKeys(1600)\n\ttree := newTree(9, 8)\n\n\tresult := tree.Insert(keys...)\n\tassert.Len(t, result, len(keys)) // about all we can assert, random may produce duplicates\n\tcheckTree(t, tree)\n\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesCascadingSplitsOddAryReverseOrder(t *testing.T) {\n\tkeys := generateKeys(30000)\n\ttree := newTree(17, 8)\n\n\treversed := keys.reverse()\n\n\tresult := tree.Insert(reversed...)\n\n\tassert.Len(t, result, len(keys)) // about all we can assert, random may produce duplicates\n\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\t//tree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesCascadingSplitsEvenAry(t *testing.T) {\n\tkeys := generateRandomKeys(200)\n\ttree := newTree(12, 8)\n\n\tresult := tree.Insert(keys...)\n\tassert.Len(t, result, len(keys)) // about all we can assert, random may produce duplicates\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n}\n\nfunc TestOverwriteOddAry(t *testing.T) {\n\tkeys := generateRandomKeys(15)\n\ttree := newTree(3, 8)\n\tduplicate := mockKey(uint64(keys[0].(mockKey)))\n\n\tresult := tree.Insert(keys...)\n\tassert.Len(t, result, len(keys))\n\toldLength := tree.Len()\n\n\tresult = tree.Insert(duplicate)\n\tassert.Equal(t, Keys{keys[0]}, result)\n\tassert.Equal(t, oldLength, tree.Len())\n}\n\nfunc TestOverwriteEvenAry(t *testing.T) {\n\tkeys := generateRandomKeys(15)\n\ttree := newTree(12, 8)\n\tduplicate := mockKey(uint64(keys[0].(mockKey)))\n\n\tresult := tree.Insert(keys...)\n\tassert.Len(t, result, len(keys))\n\toldLength := tree.Len()\n\n\tresult = tree.Insert(duplicate)\n\tassert.Equal(t, Keys{keys[0]}, result)\n\tassert.Equal(t, oldLength, tree.Len())\n}\n\nfunc BenchmarkSimpleAdd(b *testing.B) {\n\tnumItems := 1000\n\tkeys := generateRandomKeys(numItems)\n\ttree := newTree(16, 8)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(keys[i%numItems])\n\t}\n}\n\nfunc BenchmarkGet(b *testing.B) {\n\tnumItems := 1000\n\tkeys := generateRandomKeys(numItems)\n\ttree := newTree(16, 4)\n\ttree.Insert(keys...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Get(keys[i%numItems])\n\t}\n}\n\nfunc BenchmarkBulkAdd(b *testing.B) {\n\tnumItems := b.N\n\tkeys := generateRandomKeys(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree := newTree(64, 1)\n\t\ttree.Insert(keys...)\n\t}\n}\n" }, { "path": "btree/immutable/add.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\nimport (\n\t\"runtime\"\n\t\"sort\"\n\t\"sync\"\n\n\tterr \"github.com/Workiva/go-datastructures/threadsafe/err\"\n)\n\nfunc (t *Tr) AddItems(its ...*Item) ([]*Item, error) {\n\tif len(its) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tkeys := make(Keys, 0, len(its))\n\tfor _, item := range its {\n\t\tkeys = append(keys, &Key{Value: item.Value, Payload: item.Payload})\n\t}\n\n\toverwrittens, err := t.add(keys)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn overwrittens.toItems(), nil\n}\n\nfunc (t *Tr) add(keys Keys) (Keys, error) {\n\tif t.Root == nil {\n\t\tn := t.createRoot()\n\t\tt.Root = n.ID\n\t\tt.context.addNode(n)\n\t}\n\n\tnodes, err := t.determinePaths(keys)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar overwrittens Keys\n\n\tvar wg sync.WaitGroup\n\twg.Add(len(nodes))\n\tvar treeLock sync.Mutex\n\tlocalOverwrittens := make([]Keys, len(nodes))\n\ttree := make(map[string]*path, runtime.NumCPU())\n\tlerr := terr.New()\n\n\ti := 0\n\tfor id, bundles := range nodes {\n\t\tgo func(i int, id string, bundles []*nodeBundle) {\n\t\t\tdefer wg.Done()\n\t\t\tif len(bundles) == 0 {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tn, err := t.contextOrCachedNode(ID(id), true)\n\t\t\tif err != nil {\n\t\t\t\tlerr.Set(err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif !t.context.nodeExists(n.ID) {\n\t\t\t\tn = n.copy()\n\t\t\t\tt.context.addNode(n)\n\t\t\t}\n\n\t\t\toverwrittens, err := insertLastDimension(t, n, bundles)\n\n\t\t\tif err != nil {\n\t\t\t\tlerr.Set(err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlocalOverwrittens[i] = overwrittens\n\t\t\tpath := bundles[0].path\n\t\t\ttreeLock.Lock()\n\t\t\ttree[string(n.ID)] = path\n\t\t\ttreeLock.Unlock()\n\t\t}(i, id, bundles)\n\t\ti++\n\t}\n\n\twg.Wait()\n\n\tif lerr.Get() != nil {\n\t\treturn nil, lerr.Get()\n\t}\n\n\tt.walkupInsert(tree)\n\n\tfor _, chunk := range localOverwrittens {\n\t\toverwrittens = append(overwrittens, chunk...)\n\t}\n\n\tt.Count += len(keys) - len(overwrittens)\n\n\treturn overwrittens, nil\n}\n\nfunc (t *Tr) determinePaths(keys Keys) (map[string][]*nodeBundle, error) {\n\tchunks := splitKeys(keys, runtime.NumCPU())\n\tvar wg sync.WaitGroup\n\twg.Add(len(chunks))\n\tchunkPaths := make([]map[interface{}]*nodeBundle, len(chunks))\n\tlerr := terr.New()\n\n\tfor i := range chunks {\n\t\tgo func(i int) {\n\t\t\tdefer wg.Done()\n\t\t\tkeys := chunks[i]\n\t\t\tif len(keys) == 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tmp := make(map[interface{}]*nodeBundle, len(keys))\n\t\t\tfor _, key := range keys {\n\t\t\t\tpath, err := t.iterativeFind(\n\t\t\t\t\tkey.Value, t.Root,\n\t\t\t\t)\n\n\t\t\t\tif err != nil {\n\t\t\t\t\tlerr.Set(err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tmp[key.Value] = &nodeBundle{path: path, k: key}\n\t\t\t}\n\t\t\tchunkPaths[i] = mp\n\t\t}(i)\n\t}\n\n\twg.Wait()\n\n\tif lerr.Get() != nil {\n\t\treturn nil, lerr.Get()\n\t}\n\n\tnodes := make(map[string][]*nodeBundle, 10)\n\tfor _, chunk := range chunkPaths {\n\t\tfor _, pb := range chunk {\n\t\t\tnodes[string(pb.path.peek().n.ID)] = append(nodes[string(pb.path.pop().n.ID)], pb)\n\t\t}\n\t}\n\n\treturn nodes, nil\n}\n\nfunc insertByMerge(comparator Comparator, n *Node, bundles []*nodeBundle) (Keys, error) {\n\tpositions := make(map[interface{}]int, len(n.ChildValues))\n\toverwrittens := make(Keys, 0, 10)\n\n\tfor i, value := range n.ChildValues {\n\t\tpositions[value] = i\n\t}\n\n\tfor _, bundle := range bundles {\n\t\tif i, ok := positions[bundle.k.Value]; ok {\n\t\t\toverwrittens = append(overwrittens, n.ChildKeys[i])\n\t\t\tn.ChildKeys[i] = bundle.k\n\t\t} else {\n\t\t\tn.ChildValues = append(n.ChildValues, bundle.k.Value)\n\t\t\tn.ChildKeys = append(n.ChildKeys, bundle.k)\n\t\t}\n\t}\n\n\tnsw := &nodeSortWrapper{\n\t\tvalues: n.ChildValues,\n\t\tkeys: n.ChildKeys,\n\t\tcomparator: comparator,\n\t}\n\n\tsort.Sort(nsw)\n\n\tfor i := 0; i < len(nsw.values); i++ {\n\t\tif nsw.values[i] != nil {\n\t\t\tnsw.values = nsw.values[i:]\n\t\t\tnsw.keys = nsw.keys[i:]\n\t\t\tbreak\n\t\t}\n\n\t\tnsw.keys[i] = nil\n\t}\n\n\tn.ChildValues = nsw.values\n\tn.ChildKeys = nsw.keys\n\treturn overwrittens, nil\n}\n\nfunc insertLastDimension(t *Tr, n *Node, bundles []*nodeBundle) (Keys, error) {\n\tif n.IsLeaf && len(bundles) >= n.lenValues()/16 { // Found through empirical testing, it appears that the memmoves are more sensitive when dealing with interface{}'s.\n\t\treturn insertByMerge(t.config.Comparator, n, bundles)\n\t}\n\n\toverwrittens := make(Keys, 0, len(bundles))\n\tfor _, bundle := range bundles {\n\t\toverwritten := n.insert(t.config.Comparator, bundle.k)\n\t\tif overwritten != nil {\n\t\t\toverwrittens = append(overwrittens, overwritten)\n\t\t}\n\t}\n\n\treturn overwrittens, nil\n}\n\nfunc (t *Tr) iterativeSplit(n *Node) Keys {\n\tkeys := make(Keys, 0, 10)\n\tfor n.needsSplit(t.config.NodeWidth) {\n\t\tleftValue, leftNode := n.splitAt(t.config.NodeWidth / 2)\n\t\tt.context.addNode(leftNode)\n\t\tkeys = append(keys, &Key{UUID: leftNode.ID, Value: leftValue})\n\t}\n\n\treturn keys\n}\n\n// walkupInsert walks up nodes during the insertion process and adds\n// any new keys due to splits. Each layer of the tree can have insertions\n// performed in parallel as splits are local changes.\nfunc (t *Tr) walkupInsert(nodes map[string]*path) error {\n\tmapping := make(map[string]*Node, len(nodes))\n\n\tfor len(nodes) > 0 {\n\t\tsplitNodes := make(map[string]Keys)\n\t\tnewNodes := make(map[string]*path)\n\t\tfor id, path := range nodes {\n\t\t\tnode := t.context.getNode(ID(id))\n\n\t\t\tparentPath := path.pop()\n\t\t\tif parentPath == nil {\n\t\t\t\tt.Root = node.ID\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tparent := parentPath.n\n\t\t\tnewNode := mapping[string(parent.ID)]\n\t\t\tif newNode == nil {\n\t\t\t\tif !t.context.nodeExists(parent.ID) {\n\t\t\t\t\tcp := parent.copy()\n\t\t\t\t\tif string(t.Root) == string(parent.ID) {\n\t\t\t\t\t\tt.Root = cp.ID\n\t\t\t\t\t}\n\n\t\t\t\t\tt.context.addNode(cp)\n\t\t\t\t\tmapping[string(parent.ID)] = cp\n\t\t\t\t\tparent = cp\n\t\t\t\t} else {\n\t\t\t\t\tnewNode = t.context.getNode(parent.ID)\n\t\t\t\t\tmapping[string(parent.ID)] = newNode\n\t\t\t\t\tparent = newNode\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tparent = newNode\n\t\t\t}\n\n\t\t\ti := parentPath.i\n\n\t\t\tparent.replaceKeyAt(&Key{UUID: node.ID}, i)\n\t\t\tsplitNodes[string(parent.ID)] = append(splitNodes[string(parent.ID)], t.iterativeSplit(node)...)\n\t\t\tnewNodes[string(parent.ID)] = path\n\t\t}\n\n\t\tvar wg sync.WaitGroup\n\t\twg.Add(len(splitNodes))\n\t\tlerr := terr.New()\n\n\t\tfor id, keys := range splitNodes {\n\t\t\tgo func(id ID, keys Keys) {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tn, err := t.contextOrCachedNode(id, true)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlerr.Set(err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tfor _, key := range keys {\n\t\t\t\t\tn.insert(t.config.Comparator, key)\n\t\t\t\t}\n\t\t\t}(ID(id), keys)\n\t\t}\n\n\t\twg.Wait()\n\n\t\tif lerr.Get() != nil {\n\t\t\treturn lerr.Get()\n\t\t}\n\n\t\tnodes = newNodes\n\t}\n\n\tn := t.context.getNode(t.Root)\n\tfor n.needsSplit(t.config.NodeWidth) {\n\t\troot := newNode()\n\t\tt.Root = root.ID\n\t\tt.context.addNode(root)\n\t\troot.appendChild(&Key{UUID: n.ID})\n\t\tkeys := t.iterativeSplit(n)\n\t\tfor _, key := range keys {\n\t\t\troot.insert(t.config.Comparator, key)\n\t\t}\n\t\tn = root\n\t}\n\n\treturn nil\n}\n" }, { "path": "btree/immutable/cacher.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/Workiva/go-datastructures/futures\"\n)\n\n// cacher provides a convenient construct for retrieving,\n// storing, and caching nodes; basically wrapper persister with a caching layer.\n// This ensures that we don't have to constantly\n// run to the persister to fetch nodes we are using over and over again.\n// TODO: this should probably evict items from the cache if the cache gets\n// too full.\ntype cacher struct {\n\tlock sync.Mutex\n\tcache map[string]*futures.Future\n\tpersister Persister\n}\n\nfunc (c *cacher) asyncLoadNode(t *Tr, key ID, completer chan interface{}) {\n\tn, err := c.loadNode(t, key)\n\tif err != nil {\n\t\tcompleter <- err\n\t\treturn\n\t}\n\n\tif n == nil {\n\t\tcompleter <- ErrNodeNotFound\n\t\treturn\n\t}\n\n\tcompleter <- n\n}\n\n// clear deletes all items from the cache.\nfunc (c *cacher) clear() {\n\tc.lock.Lock()\n\tdefer c.lock.Unlock()\n\n\tc.cache = make(map[string]*futures.Future, 10)\n}\n\n// deleteFromCache will remove the provided ID from the cache. This\n// is a threadsafe operation.\nfunc (c *cacher) deleteFromCache(id ID) {\n\tc.lock.Lock()\n\tdefer c.lock.Unlock()\n\n\tdelete(c.cache, string(id))\n}\n\nfunc (c *cacher) loadNode(t *Tr, key ID) (*Node, error) {\n\titems, err := c.persister.Load(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tn, err := nodeFromBytes(t, items[0].Payload)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn n, nil\n}\n\n// getNode will return a Node matching the provided id. An error is returned\n// if the cacher could not go to the persister or the node could not be found.\n// All found nodes are cached so subsequent calls should be faster than\n// the initial. This blocks until the node is loaded, but is also threadsafe.\nfunc (c *cacher) getNode(t *Tr, key ID, useCache bool) (*Node, error) {\n\tif !useCache {\n\t\treturn c.loadNode(t, key)\n\t}\n\n\tc.lock.Lock()\n\tfuture, ok := c.cache[string(key)]\n\tif ok {\n\t\tc.lock.Unlock()\n\t\tifc, err := future.GetResult()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\treturn ifc.(*Node), nil\n\t}\n\n\tcompleter := make(chan interface{}, 1)\n\tfuture = futures.New(completer, 30*time.Second)\n\tc.cache[string(key)] = future\n\tc.lock.Unlock()\n\n\tgo c.asyncLoadNode(t, key, completer)\n\n\tifc, err := future.GetResult()\n\tif err != nil {\n\t\tc.deleteFromCache(key)\n\t\treturn nil, err\n\t}\n\n\tif err, ok := ifc.(error); ok {\n\t\tc.deleteFromCache(key)\n\t\treturn nil, err\n\t}\n\n\treturn ifc.(*Node), nil\n}\n\n// newCacher is the constructor for a cacher that caches nodes for\n// an indefinite period of time.\nfunc newCacher(persister Persister) *cacher {\n\treturn &cacher{\n\t\tpersister: persister,\n\t\tcache: make(map[string]*futures.Future, 10),\n\t}\n}\n" }, { "path": "btree/immutable/config.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\n// Config defines all the parameters available to the UB-tree.\n// Of most important are nodewidth and the persister to be used\n// during commit phase.\ntype Config struct {\n\t// NodeWidth defines the branching factor of the tree. Any node\n\t// wider than this value will get split and when the width of a node\n\t// falls to less than half this value the node gets merged. This\n\t// ensures optimal performance while running to the key value store.\n\tNodeWidth int\n\t// Perister defines the key value store that the tree can use to\n\t// save and load nodes.\n\tPersister Persister\n\t// Comparator is the function used to determine ordering.\n\tComparator Comparator `msg:\"-\"`\n}\n\n// DefaultConfig returns a configuration with the persister set. All other\n// fields are set to smart defaults for persistence.\nfunc DefaultConfig(persister Persister, comparator Comparator) Config {\n\treturn Config{\n\t\tNodeWidth: 10000,\n\t\tPersister: persister,\n\t\tComparator: comparator,\n\t}\n}\n" }, { "path": "btree/immutable/delete.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\nimport \"bytes\"\n\nfunc (t *Tr) DeleteItems(values ...interface{}) ([]*Item, error) {\n\tif len(values) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tkeys := make(Keys, 0, len(values))\n\terr := t.Apply(func(item *Item) {\n\t\tkeys = append(keys, &Key{Value: item.Value, Payload: item.Payload})\n\t}, values...)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// we need to sort the keys to ensure that a multidelete\n\t// distributes deletes across a single node correctly\n\tkeys = keys.sort(t.config.Comparator)\n\n\terr = t.delete(keys)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tt.Count -= len(keys)\n\n\treturn keys.toItems(), nil\n}\n\nfunc (t *Tr) delete(keys Keys) error {\n\tif len(keys) == 0 {\n\t\treturn nil\n\t}\n\n\ttoDelete := make([]*Key, 0, len(keys))\n\n\tfor i := 0; i < len(keys); {\n\t\tkey := keys[i]\n\t\tmapping := make(map[string]*Node, 10)\n\t\tpath, err := t.iterativeFind(key.Value, t.Root)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tpb := path.peek()\n\t\tnode := pb.n\n\t\tisRoot := bytes.Compare(node.ID, t.Root) == 0\n\t\tif !t.context.nodeExists(node.ID) {\n\t\t\tcp := node.copy()\n\t\t\tt.context.addNode(cp)\n\t\t\tmapping[string(node.ID)] = cp\n\t\t\tnode = cp\n\t\t}\n\t\tbase := node\n\n\t\ttoDelete = append(toDelete, key)\n\t\tfor j := i + 1; j <= len(keys); j++ {\n\t\t\ti = j\n\t\t\tif j == len(keys) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tneighbor := keys[j]\n\t\t\tif t.config.Comparator(neighbor.Value, node.lastValue()) <= 0 {\n\t\t\t\ttoDelete = append(toDelete, neighbor)\n\t\t\t} else {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif len(toDelete) > len(node.ChildValues)/4 {\n\t\t\tnode.multiDelete(t.config.Comparator, toDelete...)\n\t\t} else {\n\t\t\tfor _, k := range toDelete {\n\t\t\t\tnode.delete(t.config.Comparator, k)\n\t\t\t}\n\t\t}\n\n\t\ttoDelete = toDelete[:0]\n\t\tif isRoot {\n\t\t\tt.Root = node.ID\n\t\t\tcontinue\n\t\t}\n\n\t\tfor pb.prev != nil {\n\t\t\tparentBundle := pb.prev\n\t\t\tparent := parentBundle.n\n\t\t\tisRoot := bytes.Compare(parent.ID, t.Root) == 0\n\t\t\tif !t.context.nodeExists(parent.ID) {\n\t\t\t\tcp := parent.copy()\n\t\t\t\tt.context.addNode(cp)\n\t\t\t\tmapping[string(parent.ID)] = cp\n\t\t\t\tparent = cp\n\t\t\t} else {\n\t\t\t\tmapping[string(parent.ID)] = parent\n\t\t\t}\n\n\t\t\tif isRoot {\n\t\t\t\tt.Root = parent.ID\n\t\t\t}\n\n\t\t\ti := pb.prev.i\n\t\t\tparent.replaceKeyAt(&Key{UUID: node.ID}, i)\n\t\t\tnode = parent\n\t\t\tpb = pb.prev\n\t\t}\n\n\t\tpath.pop()\n\t\terr = t.walkupDelete(key, base, path, mapping)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tn := t.context.getNode(t.Root)\n\tif n.lenValues() == 0 {\n\t\tt.Root = nil\n\t}\n\n\treturn nil\n}\n\n// walkupDelete is similar to walkupInsert but is only done one at a time.\n// This is because deletes can cause borrowing or merging with neighbors which makes\n// the changes non-local.\n// TODO: come up with a good way to parallelize this.\nfunc (t *Tr) walkupDelete(key *Key, node *Node, path *path, mapping map[string]*Node) error {\n\tneedsMerged := t.config.NodeWidth / 2\n\tif needsMerged < 1 {\n\t\tneedsMerged = 1\n\t}\n\tif node.lenValues() >= needsMerged {\n\t\treturn nil\n\t}\n\n\tif string(node.ID) == string(t.Root) {\n\t\tif node.lenKeys() == 1 {\n\t\t\tid := node.keyAt(0)\n\t\t\tt.Root = id.UUID\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tvar getSibling = func(parent *Node, i int) (*Node, error) {\n\t\tkey := parent.keyAt(i)\n\t\tn, err := t.contextOrCachedNode(key.UUID, true)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif !t.context.nodeExists(n.ID) {\n\t\t\tcp := t.copyNode(n)\n\t\t\tmapping[string(n.ID)] = cp\n\t\t\tparent.replaceKeyAt(&Key{UUID: cp.ID}, i)\n\t\t\tn = cp\n\t\t}\n\n\t\treturn n, nil\n\t}\n\n\tparentBundle := path.pop()\n\tparent := mapping[string(parentBundle.n.ID)]\n\n\t_, i := parent.searchKey(t.config.Comparator, key.Value)\n\tsiblingPosition := i\n\tif i == parent.lenValues() {\n\t\tsiblingPosition--\n\t} else {\n\t\tsiblingPosition++\n\t}\n\n\tsibling, err := getSibling(parent, siblingPosition)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tprepend := false\n\t// thing are just easier if we make this swap so we can grok\n\t// left to right always assuming node is on the left and sibling\n\t// is on the right\n\tif siblingPosition < i {\n\t\tnode, sibling = sibling, node\n\t\tprepend = true\n\t}\n\n\t// first case, can we just borrow? if so, simply shift values from one node\n\t// to the other. Once done, replace the parent value with the middle value\n\t// shifted and return.\n\tif (sibling.lenValues()+node.lenValues())/2 >= needsMerged {\n\t\tif i == parent.lenValues() {\n\t\t\ti--\n\t\t}\n\n\t\tvar key *Key\n\t\tvar value interface{}\n\t\tfor node.lenValues() < needsMerged || sibling.lenValues() < needsMerged {\n\t\t\tif prepend {\n\t\t\t\tcorrectedValue, key := node.popValue(), node.popKey()\n\t\t\t\tif node.IsLeaf {\n\t\t\t\t\tsibling.prependValue(correctedValue)\n\t\t\t\t\tsibling.prependKey(key)\n\t\t\t\t\tparent.replaceValueAt(i, node.lastValue())\n\t\t\t\t} else {\n\t\t\t\t\tparentValue := parent.valueAt(i)\n\t\t\t\t\tsibling.prependKey(key)\n\t\t\t\t\tsibling.prependValue(parentValue)\n\t\t\t\t\tparent.replaceValueAt(i, correctedValue)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tvalue, key = sibling.popFirstValue(), sibling.popFirstKey()\n\t\t\t\tcorrectedValue := value\n\t\t\t\tif !node.IsLeaf {\n\t\t\t\t\tcorrectedValue = parent.valueAt(i)\n\t\t\t\t}\n\t\t\t\tnode.appendValue(correctedValue)\n\t\t\t\tnode.appendChild(key)\n\t\t\t\tparent.replaceValueAt(i, value)\n\t\t\t}\n\t\t}\n\n\t\treturn nil\n\t}\n\n\t// the harder case, we need to merge with sibling, pull a value down\n\t// from the parent, and recurse on this function\n\n\t// easier case, merge the nodes and delete value and child from parent\n\tif node.IsLeaf {\n\t\tnode.append(sibling)\n\t\tif prepend {\n\t\t\tparent.deleteKeyAt(i)\n\t\t} else {\n\t\t\tparent.deleteKeyAt(i + 1)\n\t\t}\n\n\t\tif i == parent.lenValues() {\n\t\t\ti--\n\t\t}\n\n\t\tparent.deleteValueAt(i)\n\t\treturn t.walkupDelete(key, parent, path, mapping)\n\t}\n\n\t// harder case, need to pull a value down from the parent, insert\n\t// value into the left node, append the nodes, and then delete\n\t// the value from the parent\n\n\tvalueIndex := i\n\tif i == parent.lenValues() {\n\t\tvalueIndex--\n\t}\n\n\tparentValue := parent.valueAt(valueIndex)\n\tnode.appendValue(parentValue)\n\tnode.append(sibling)\n\tparent.deleteKeyAt(i)\n\tparent.deleteValueAt(valueIndex)\n\tparent.replaceKeyAt(&Key{UUID: node.ID}, valueIndex)\n\treturn t.walkupDelete(key, parent, path, mapping)\n}\n" }, { "path": "btree/immutable/error.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\nimport \"errors\"\n\n// ErrNodeNotFound is returned when the cacher could not find a node.\nvar ErrNodeNotFound = errors.New(`node not found`)\n\n// ErrTreeNotFound is returned when a tree with the provided key could\n// not be loaded.\nvar ErrTreeNotFound = errors.New(`tree not found`)\n" }, { "path": "btree/immutable/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage btree provides a very specific set implementation for k/v\nlookup. This is based on a B PALM tree as described here:\nhttp://irvcvcs01.intel-research.net/publications/palm.pdf\n\nThis tree is best interacted with in batches. Insertions and deletions\nare optimized for dealing with large amounts of data.\n\nFuture work includes:\n\n1) Optimization\n2) Range scans\n\nUsage:\n\nrt := New(config)\nmutable := rt.AsMutable()\n... operations\n\nrt, err := mutable.Commit() // saves all mutated nodes\n\n.. rt reading/operations\n\nOnce a mutable has been committed, its further operations are undefined.\n*/\npackage btree\n\n// Tree describes the common functionality of both the read-only and mutable\n// forms of a btree.\ntype Tree interface {\n\t// Apply takes a range and applies the provided function to every value\n\t// in that range in order. If a key could not be found, it is\n\t// skipped.\n\tApply(fn func(item *Item), keys ...interface{}) error\n\t// ID returns the identifier for this tree.\n\tID() ID\n\t// Len returns the number of items in the tree.\n\tLen() int\n}\n\n// ReadableTree represents the operations that can be performed on a read-only\n// version of the tree. All reads of the readable tree are threadsafe and\n// an indefinite number of mutable trees can be created from a single readable\n// tree with the caveat that no mutable trees reflect any mutations to any other\n// mutable tree.\ntype ReadableTree interface {\n\tTree\n\t// AsMutable returns a mutable version of this tree. The mutable version\n\t// has common mutations and you can create as many mutable versions of this\n\t// tree as you'd like. However, the returned mutable is not threadsafe.\n\tAsMutable() MutableTree\n}\n\n// MutableTree represents a mutable version of the btree. This interface\n// is not threadsafe.\ntype MutableTree interface {\n\tTree\n\t// Commit commits all mutated nodes to persistence and returns a\n\t// read-only version of this tree. An error is returned if nodes\n\t// could not be committed to persistence.\n\tCommit() (ReadableTree, error)\n\t// AddItems adds the provided items to the btree. Any existing items\n\t// are overwritten. An error is returned if the tree could not be\n\t// traversed due to an error in the persistence layer.\n\tAddItems(items ...*Item) ([]*Item, error)\n\t// DeleteItems removes all provided keys and returns them.\n\t// An error is returned if the tree could not be traversed.\n\tDeleteItems(keys ...interface{}) ([]*Item, error)\n}\n\n// Comparator is used to determine ordering in the tree. If item1\n// is less than item2, a negative number should be returned and\n// vice versa. If equal, 0 should be returned.\ntype Comparator func(item1, item2 interface{}) int\n\n// Payload is very basic and simply contains a key and a payload.\ntype Payload struct {\n\tKey []byte\n\tPayload []byte\n}\n\n// Perister describes the interface of the different implementations.\n// Given that we expect that datastrutures are immutable, we never\n// have the need to delete.\ntype Persister interface {\n\tSave(items ...*Payload) error\n\tLoad(keys ...[]byte) ([]*Payload, error)\n}\n" }, { "path": "btree/immutable/item.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\ntype Item struct {\n\tValue interface{}\n\tPayload []byte\n}\n\ntype items []*Item\n\nfunc (its items) split(numParts int) []items {\n\tparts := make([]items, numParts)\n\tfor i := int64(0); i < int64(numParts); i++ {\n\t\tparts[i] = its[i*int64(len(its))/int64(numParts) : (i+1)*int64(len(its))/int64(numParts)]\n\t}\n\treturn parts\n}\n" }, { "path": "btree/immutable/node.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n//go:generate msgp -tests=false -io=false\n\npackage btree\n\nimport (\n\t\"crypto/rand\"\n\t\"io\"\n\t\"sort\"\n)\n\nfunc newID() []byte {\n\tid := make([]byte, 16)\n\t_, err := io.ReadFull(rand.Reader, id)\n\tif err != nil {\n\t\t// This can't happen unless the system is badly\n\t\t// configured, like /dev/urandom isn't readable.\n\t\tpanic(\"reading random: \" + err.Error())\n\t}\n\treturn id\n}\n\n// ID exists because i'm tired of writing []byte\ntype ID []byte\n\n// Key a convenience struct that holds both an id and a value. Internally,\n// this is how we reference items in nodes but consumers interface with\n// the tree using row/col/id.\ntype Key struct {\n\tUUID ID `msg:\"u\"`\n\tValue interface{} `msg:\"v\"`\n\tPayload []byte `msg:\"p\"`\n}\n\n// ID returns the unique identifier.\nfunc (k Key) ID() []byte {\n\treturn k.UUID[:16] // to maintain backwards compatibility\n}\n\nfunc (k Key) ToItem() *Item {\n\treturn &Item{\n\t\tValue: k.Value,\n\t\tPayload: k.Payload,\n\t}\n}\n\ntype Keys []*Key\n\nfunc (k Keys) toItems() items {\n\titems := make(items, 0, len(k))\n\tfor _, key := range k {\n\t\titems = append(items, key.ToItem())\n\t}\n\n\treturn items\n}\n\nfunc (k Keys) sort(comparator Comparator) Keys {\n\treturn (&keySortWrapper{comparator, k}).sort()\n}\n\ntype keySortWrapper struct {\n\tcomparator Comparator\n\tkeys Keys\n}\n\nfunc (sw *keySortWrapper) Len() int {\n\treturn len(sw.keys)\n}\n\nfunc (sw *keySortWrapper) Swap(i, j int) {\n\tsw.keys[i], sw.keys[j] = sw.keys[j], sw.keys[i]\n}\n\nfunc (sw *keySortWrapper) Less(i, j int) bool {\n\treturn sw.comparator(sw.keys[i].Value, sw.keys[j].Value) < 0\n}\n\nfunc (sw *keySortWrapper) sort() Keys {\n\tsort.Sort(sw)\n\treturn sw.keys\n}\n\nfunc splitKeys(keys Keys, numParts int) []Keys {\n\tparts := make([]Keys, numParts)\n\tfor i := int64(0); i < int64(numParts); i++ {\n\t\tparts[i] = keys[i*int64(len(keys))/int64(numParts) : (i+1)*int64(len(keys))/int64(numParts)]\n\t}\n\treturn parts\n}\n\n// Node represents either a leaf node or an internal node. These\n// are the value containers. This is exported because code generation\n// requires it. Only exported fields are required to be persisted. We\n// use msgpack for optimal performance.\ntype Node struct {\n\t// ID is the unique UUID that addresses this singular node.\n\tID ID `msg:\"id\"`\n\t// IsLeaf is a bool indicating if this is a leaf node as opposed\n\t// to an internal node. The primary difference between these nodes\n\t// is that leaf nodes have an equal number of values and IDs while\n\t// internal nodes have n+1 ids.\n\tIsLeaf bool `msg:\"il\"`\n\t// ChildValues is only a temporary field that is used to house all\n\t// values for serialization purposes.\n\tChildValues []interface{} `msg:\"cv\"`\n\t// ChildKeys is similar to child values but holds the IDs of children.\n\tChildKeys Keys `msg:\"ck\"`\n}\n\n// copy makes a deep copy of this node. Required before any mutation.\nfunc (n *Node) copy() *Node {\n\tcpValues := make([]interface{}, len(n.ChildValues))\n\tcopy(cpValues, n.ChildValues)\n\tcpKeys := make(Keys, len(n.ChildKeys))\n\tcopy(cpKeys, n.ChildKeys)\n\n\treturn &Node{\n\t\tID: newID(),\n\t\tIsLeaf: n.IsLeaf,\n\t\tChildValues: cpValues,\n\t\tChildKeys: cpKeys,\n\t}\n}\n\n// searchKey returns the key associated with the provided value. If the\n// provided value is greater than the highest value in this node and this\n// node is an internal node, this method returns the last ID and an index\n// equal to lenValues.\nfunc (n *Node) searchKey(comparator Comparator, value interface{}) (*Key, int) {\n\ti := n.search(comparator, value)\n\n\tif n.IsLeaf && i == len(n.ChildValues) { // not found\n\t\treturn nil, i\n\t}\n\n\tif n.IsLeaf { // equal number of ids and values\n\t\treturn n.ChildKeys[i], i\n\t}\n\n\tif i == len(n.ChildValues) { // we need to go to the farthest node to the write\n\t\treturn n.ChildKeys[len(n.ChildKeys)-1], i\n\t}\n\n\treturn n.ChildKeys[i], i\n}\n\n// insert adds the provided key to this node and returns any ID that has\n// been overwritten. This method should only be called on leaf nodes.\nfunc (n *Node) insert(comparator Comparator, key *Key) *Key {\n\tvar overwrittenKey *Key\n\ti := n.search(comparator, key.Value)\n\tif i == len(n.ChildValues) {\n\t\tn.ChildValues = append(n.ChildValues, key.Value)\n\t} else {\n\t\tif n.ChildValues[i] == key.Value {\n\t\t\toverwrittenKey = n.ChildKeys[i]\n\t\t\tn.ChildKeys[i] = key\n\t\t\treturn overwrittenKey\n\t\t} else {\n\t\t\tn.ChildValues = append(n.ChildValues, 0)\n\t\t\tcopy(n.ChildValues[i+1:], n.ChildValues[i:])\n\t\t\tn.ChildValues[i] = key.Value\n\t\t}\n\t}\n\n\tif n.IsLeaf && i == len(n.ChildKeys) {\n\t\tn.ChildKeys = append(n.ChildKeys, key)\n\t} else {\n\t\tn.ChildKeys = append(n.ChildKeys, nil)\n\t\tcopy(n.ChildKeys[i+1:], n.ChildKeys[i:])\n\t\tn.ChildKeys[i] = key\n\t}\n\n\treturn overwrittenKey\n}\n\n// delete removes the provided key from the node and returns any key that\n// was deleted. Returns nil of the key could not be found.\nfunc (n *Node) delete(comparator Comparator, key *Key) *Key {\n\ti := n.search(comparator, key.Value)\n\tif i == len(n.ChildValues) {\n\t\treturn nil\n\t}\n\n\tn.deleteValueAt(i)\n\tn.deleteKeyAt(i)\n\n\treturn key\n}\n\nfunc (n *Node) multiDelete(comparator Comparator, keys ...*Key) {\n\tindices := make([]int, 0, len(keys))\n\tfor _, k := range keys {\n\t\ti := n.search(comparator, k.Value)\n\t\tif i < len(n.ChildValues) {\n\t\t\tindices = append(indices, i)\n\t\t}\n\t}\n\n\tfor _, i := range indices {\n\t\tn.ChildValues[i] = nil\n\t\tn.ChildKeys[i] = nil\n\t}\n\n\tif len(indices) == len(n.ChildValues) {\n\t\tn.ChildKeys = n.ChildKeys[:0]\n\t\tn.ChildValues = n.ChildValues[:0]\n\t\treturn\n\t}\n\n\t// get the indices in the correct order for the next stage\n\t// which is removing the nils\n\tsort.Ints(indices)\n\n\t// iterate through the list moving all values up to overwrite the\n\t// nils and place all nils at the \"back\"\n\tfor i, j := range indices {\n\t\tindex := j - i // correct for previous copies\n\t\tcopy(n.ChildValues[index:], n.ChildValues[index+1:])\n\t\tcopy(n.ChildKeys[index:], n.ChildKeys[index+1:])\n\t}\n\n\tn.ChildValues = n.ChildValues[:len(n.ChildValues)-len(indices)]\n\tn.ChildKeys = n.ChildKeys[:len(n.ChildKeys)-len(indices)]\n}\n\n// replaceKeyAt replaces the key at index i with the provided id. This does\n// not do any bounds checking.\nfunc (n *Node) replaceKeyAt(key *Key, i int) {\n\tn.ChildKeys[i] = key\n}\n\n// flatten returns a flattened list of values and IDs. Useful for serialization.\nfunc (n *Node) flatten() ([]interface{}, Keys) {\n\treturn n.ChildValues, n.ChildKeys\n}\n\n// iter returns an iterator that will iterate through the provided Morton\n// numbers as they exist in this node.\nfunc (n *Node) iter(comparator Comparator, start, stop interface{}) iterator {\n\tpointer := n.search(comparator, start)\n\tpointer--\n\treturn &sliceIterator{\n\t\tstop: stop,\n\t\tn: n,\n\t\tpointer: pointer,\n\t\tcomparator: comparator,\n\t}\n}\n\nfunc (n *Node) valueAt(i int) interface{} {\n\treturn n.ChildValues[i]\n}\n\nfunc (n *Node) keyAt(i int) *Key {\n\treturn n.ChildKeys[i]\n}\n\nfunc (n *Node) needsSplit(max int) bool {\n\treturn n.lenValues() > max\n}\n\nfunc (n *Node) lastValue() interface{} {\n\treturn n.ChildValues[len(n.ChildValues)-1]\n}\n\nfunc (n *Node) firstValue() interface{} {\n\treturn n.ChildValues[0]\n}\n\nfunc (n *Node) append(other *Node) {\n\tn.ChildValues = append(n.ChildValues, other.ChildValues...)\n\tn.ChildKeys = append(n.ChildKeys, other.ChildKeys...)\n}\n\nfunc (n *Node) replaceValueAt(i int, value interface{}) {\n\tn.ChildValues[i] = value\n}\n\nfunc (n *Node) deleteValueAt(i int) {\n\tcopy(n.ChildValues[i:], n.ChildValues[i+1:])\n\tn.ChildValues[len(n.ChildValues)-1] = 0 // or the zero value of T\n\tn.ChildValues = n.ChildValues[:len(n.ChildValues)-1]\n}\n\nfunc (n *Node) deleteKeyAt(i int) {\n\tcopy(n.ChildKeys[i:], n.ChildKeys[i+1:])\n\tn.ChildKeys[len(n.ChildKeys)-1] = nil // or the zero value of T\n\tn.ChildKeys = n.ChildKeys[:len(n.ChildKeys)-1]\n}\n\nfunc (n *Node) splitLeafAt(i int) (interface{}, *Node) {\n\tleft := newNode()\n\tleft.IsLeaf = n.IsLeaf\n\tleft.ID = newID()\n\n\tvalue := n.ChildValues[i]\n\tleftValues := make([]interface{}, i+1)\n\tcopy(leftValues, n.ChildValues[:i+1])\n\tn.ChildValues = n.ChildValues[i+1:]\n\tleftKeys := make(Keys, i+1)\n\tcopy(leftKeys, n.ChildKeys[:i+1])\n\tfor j := 0; j <= i; j++ {\n\t\tn.ChildKeys[j] = nil\n\t}\n\tn.ChildKeys = n.ChildKeys[i+1:]\n\tleft.ChildValues = leftValues\n\tleft.ChildKeys = leftKeys\n\treturn value, left\n}\n\n// splitInternalAt is a method that generates a new set of children\n// for an internal node and returns the new set and the value that\n// separates them.\nfunc (n *Node) splitInternalAt(i int) (interface{}, *Node) {\n\tleft := newNode()\n\tleft.IsLeaf = n.IsLeaf\n\tleft.ID = newID()\n\tvalue := n.ChildValues[i]\n\tleftValues := make([]interface{}, i)\n\tcopy(leftValues, n.ChildValues[:i])\n\tn.ChildValues = n.ChildValues[i+1:]\n\tleftKeys := make(Keys, i+1)\n\tcopy(leftKeys, n.ChildKeys[:i+1])\n\tfor j := 0; j <= i; j++ {\n\t\tn.ChildKeys[j] = nil\n\t}\n\tn.ChildKeys = n.ChildKeys[i+1:]\n\tleft.ChildKeys = leftKeys\n\tleft.ChildValues = leftValues\n\treturn value, left\n}\n\n// splitAt breaks this node into two parts and conceptually\n// returns the left part\nfunc (n *Node) splitAt(i int) (interface{}, *Node) {\n\tif n.IsLeaf {\n\t\treturn n.splitLeafAt(i)\n\t}\n\n\treturn n.splitInternalAt(i)\n}\n\nfunc (n *Node) lenKeys() int {\n\treturn len(n.ChildKeys)\n}\n\nfunc (n *Node) lenValues() int {\n\treturn len(n.ChildValues)\n}\n\nfunc (n *Node) appendChild(key *Key) {\n\tn.ChildKeys = append(n.ChildKeys, key)\n}\n\nfunc (n *Node) appendValue(value interface{}) {\n\tn.ChildValues = append(n.ChildValues, value)\n}\n\nfunc (n *Node) popFirstKey() *Key {\n\tkey := n.ChildKeys[0]\n\tn.deleteKeyAt(0)\n\treturn key\n}\n\nfunc (n *Node) popFirstValue() interface{} {\n\tvalue := n.ChildValues[0]\n\tn.deleteValueAt(0)\n\treturn value\n}\n\nfunc (n *Node) popKey() *Key {\n\tkey := n.ChildKeys[len(n.ChildKeys)-1]\n\tn.deleteKeyAt(len(n.ChildKeys) - 1)\n\treturn key\n}\n\nfunc (n *Node) popValue() interface{} {\n\tvalue := n.ChildValues[len(n.ChildValues)-1]\n\tn.deleteValueAt(len(n.ChildValues) - 1)\n\treturn value\n}\n\nfunc (n *Node) prependKey(key *Key) {\n\tn.ChildKeys = append(n.ChildKeys, nil)\n\tcopy(n.ChildKeys[1:], n.ChildKeys)\n\tn.ChildKeys[0] = key\n}\n\nfunc (n *Node) prependValue(value interface{}) {\n\tn.ChildValues = append(n.ChildValues, nil)\n\tcopy(n.ChildValues[1:], n.ChildValues)\n\tn.ChildValues[0] = value\n}\n\nfunc (n *Node) search(comparator Comparator, value interface{}) int {\n\treturn sort.Search(len(n.ChildValues), func(i int) bool {\n\t\treturn comparator(n.ChildValues[i], value) >= 0\n\t})\n}\n\n// nodeFromBytes returns a new node struct deserialized from the provided\n// bytes. An error is returned for any deserialization errors.\nfunc nodeFromBytes(t *Tr, data []byte) (*Node, error) {\n\tn := &Node{}\n\t_, err := n.UnmarshalMsg(data)\n\tif err != nil {\n\t\tpanic(err)\n\t\treturn nil, err\n\t}\n\n\treturn n, nil\n}\n\n// newNode returns a node with a random id and empty values and children.\n// IsLeaf is false by default.\nfunc newNode() *Node {\n\treturn &Node{\n\t\tID: newID(),\n\t}\n}\n\ntype sliceIterator struct {\n\tstop interface{}\n\tn *Node\n\tpointer int\n\tcomparator Comparator\n}\n\nfunc (s *sliceIterator) next() bool {\n\ts.pointer++\n\tif s.n.IsLeaf {\n\t\treturn s.pointer < len(s.n.ChildValues) && s.comparator(s.stop, s.n.ChildValues[s.pointer]) >= 0\n\t} else {\n\t\tif s.pointer >= len(s.n.ChildKeys) {\n\t\t\treturn false\n\t\t}\n\t\tif s.pointer == len(s.n.ChildValues) {\n\t\t\treturn true\n\t\t}\n\n\t\tif s.comparator(s.stop, s.n.ChildValues[s.pointer]) < 0 {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc (s *sliceIterator) value() (*Key, int) {\n\treturn s.n.ChildKeys[s.pointer], s.pointer\n}\n\ntype iterator interface {\n\tnext() bool\n\tvalue() (*Key, int)\n}\n\ntype nodeBundle struct {\n\tpath *path\n\tk *Key\n}\n\ntype nodeSortWrapper struct {\n\tvalues []interface{}\n\tkeys Keys\n\tcomparator Comparator\n}\n\nfunc (n *nodeSortWrapper) Len() int {\n\treturn len(n.values)\n}\n\nfunc (n *nodeSortWrapper) Swap(i, j int) {\n\tn.values[i], n.values[j] = n.values[j], n.values[i]\n\tn.keys[i], n.keys[j] = n.keys[j], n.keys[i]\n}\n\nfunc (n *nodeSortWrapper) Less(i, j int) bool {\n\treturn n.comparator(n.values[i], n.values[j]) < 0\n}\n\nfunc splitValues(values []interface{}, numParts int) [][]interface{} {\n\tparts := make([][]interface{}, numParts)\n\tfor i := int64(0); i < int64(numParts); i++ {\n\t\tparts[i] = values[i*int64(len(values))/int64(numParts) : (i+1)*int64(len(values))/int64(numParts)]\n\t}\n\treturn parts\n}\n" }, { "path": "btree/immutable/node_gen.go", "content": "package btree\n\n// NOTE: THIS FILE WAS PRODUCED BY THE\n// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)\n// DO NOT EDIT\n\nimport (\n\t\"github.com/tinylib/msgp/msgp\"\n)\n\n// MarshalMsg implements msgp.Marshaler\nfunc (z ID) MarshalMsg(b []byte) (o []byte, err error) {\n\to = msgp.Require(b, z.Msgsize())\n\to = msgp.AppendBytes(o, []byte(z))\n\treturn\n}\n\n// UnmarshalMsg implements msgp.Unmarshaler\nfunc (z *ID) UnmarshalMsg(bts []byte) (o []byte, err error) {\n\t{\n\t\tvar tmp []byte\n\t\ttmp, bts, err = msgp.ReadBytesBytes(bts, []byte((*z)))\n\t\t(*z) = ID(tmp)\n\t}\n\tif err != nil {\n\t\treturn\n\t}\n\to = bts\n\treturn\n}\n\nfunc (z ID) Msgsize() (s int) {\n\ts = msgp.BytesPrefixSize + len([]byte(z))\n\treturn\n}\n\n// MarshalMsg implements msgp.Marshaler\nfunc (z *Key) MarshalMsg(b []byte) (o []byte, err error) {\n\to = msgp.Require(b, z.Msgsize())\n\t// map header, size 3\n\t// string \"u\"\n\to = append(o, 0x83, 0xa1, 0x75)\n\to = msgp.AppendBytes(o, []byte(z.UUID))\n\t// string \"v\"\n\to = append(o, 0xa1, 0x76)\n\to, err = msgp.AppendIntf(o, z.Value)\n\tif err != nil {\n\t\treturn\n\t}\n\t// string \"p\"\n\to = append(o, 0xa1, 0x70)\n\to = msgp.AppendBytes(o, z.Payload)\n\treturn\n}\n\n// UnmarshalMsg implements msgp.Unmarshaler\nfunc (z *Key) UnmarshalMsg(bts []byte) (o []byte, err error) {\n\tvar field []byte\n\t_ = field\n\tvar isz uint32\n\tisz, bts, err = msgp.ReadMapHeaderBytes(bts)\n\tif err != nil {\n\t\treturn\n\t}\n\tfor isz > 0 {\n\t\tisz--\n\t\tfield, bts, err = msgp.ReadMapKeyZC(bts)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tswitch msgp.UnsafeString(field) {\n\t\tcase \"u\":\n\t\t\t{\n\t\t\t\tvar tmp []byte\n\t\t\t\ttmp, bts, err = msgp.ReadBytesBytes(bts, []byte(z.UUID))\n\t\t\t\tz.UUID = ID(tmp)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase \"v\":\n\t\t\tz.Value, bts, err = msgp.ReadIntfBytes(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase \"p\":\n\t\t\tz.Payload, bts, err = msgp.ReadBytesBytes(bts, z.Payload)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tdefault:\n\t\t\tbts, err = msgp.Skip(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\to = bts\n\treturn\n}\n\nfunc (z *Key) Msgsize() (s int) {\n\ts = 1 + 2 + msgp.BytesPrefixSize + len([]byte(z.UUID)) + 2 + msgp.GuessSize(z.Value) + 2 + msgp.BytesPrefixSize + len(z.Payload)\n\treturn\n}\n\n// MarshalMsg implements msgp.Marshaler\nfunc (z Keys) MarshalMsg(b []byte) (o []byte, err error) {\n\to = msgp.Require(b, z.Msgsize())\n\to = msgp.AppendArrayHeader(o, uint32(len(z)))\n\tfor xvk := range z {\n\t\tif z[xvk] == nil {\n\t\t\to = msgp.AppendNil(o)\n\t\t} else {\n\t\t\to, err = z[xvk].MarshalMsg(o)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n\n// UnmarshalMsg implements msgp.Unmarshaler\nfunc (z *Keys) UnmarshalMsg(bts []byte) (o []byte, err error) {\n\tvar xsz uint32\n\txsz, bts, err = msgp.ReadArrayHeaderBytes(bts)\n\tif err != nil {\n\t\treturn\n\t}\n\tif cap((*z)) >= int(xsz) {\n\t\t(*z) = (*z)[:xsz]\n\t} else {\n\t\t(*z) = make(Keys, xsz)\n\t}\n\tfor bzg := range *z {\n\t\tif msgp.IsNil(bts) {\n\t\t\tbts, err = msgp.ReadNilBytes(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\t(*z)[bzg] = nil\n\t\t} else {\n\t\t\tif (*z)[bzg] == nil {\n\t\t\t\t(*z)[bzg] = new(Key)\n\t\t\t}\n\t\t\tbts, err = (*z)[bzg].UnmarshalMsg(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\to = bts\n\treturn\n}\n\nfunc (z Keys) Msgsize() (s int) {\n\ts = msgp.ArrayHeaderSize\n\tfor bai := range z {\n\t\tif z[bai] == nil {\n\t\t\ts += msgp.NilSize\n\t\t} else {\n\t\t\ts += z[bai].Msgsize()\n\t\t}\n\t}\n\treturn\n}\n\n// MarshalMsg implements msgp.Marshaler\nfunc (z *Node) MarshalMsg(b []byte) (o []byte, err error) {\n\to = msgp.Require(b, z.Msgsize())\n\t// map header, size 4\n\t// string \"id\"\n\to = append(o, 0x84, 0xa2, 0x69, 0x64)\n\to = msgp.AppendBytes(o, []byte(z.ID))\n\t// string \"il\"\n\to = append(o, 0xa2, 0x69, 0x6c)\n\to = msgp.AppendBool(o, z.IsLeaf)\n\t// string \"cv\"\n\to = append(o, 0xa2, 0x63, 0x76)\n\to = msgp.AppendArrayHeader(o, uint32(len(z.ChildValues)))\n\tfor cmr := range z.ChildValues {\n\t\to, err = msgp.AppendIntf(o, z.ChildValues[cmr])\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\t// string \"ck\"\n\to = append(o, 0xa2, 0x63, 0x6b)\n\to = msgp.AppendArrayHeader(o, uint32(len(z.ChildKeys)))\n\tfor ajw := range z.ChildKeys {\n\t\tif z.ChildKeys[ajw] == nil {\n\t\t\to = msgp.AppendNil(o)\n\t\t} else {\n\t\t\to, err = z.ChildKeys[ajw].MarshalMsg(o)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n\n// UnmarshalMsg implements msgp.Unmarshaler\nfunc (z *Node) UnmarshalMsg(bts []byte) (o []byte, err error) {\n\tvar field []byte\n\t_ = field\n\tvar isz uint32\n\tisz, bts, err = msgp.ReadMapHeaderBytes(bts)\n\tif err != nil {\n\t\treturn\n\t}\n\tfor isz > 0 {\n\t\tisz--\n\t\tfield, bts, err = msgp.ReadMapKeyZC(bts)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tswitch msgp.UnsafeString(field) {\n\t\tcase \"id\":\n\t\t\t{\n\t\t\t\tvar tmp []byte\n\t\t\t\ttmp, bts, err = msgp.ReadBytesBytes(bts, []byte(z.ID))\n\t\t\t\tz.ID = ID(tmp)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase \"il\":\n\t\t\tz.IsLeaf, bts, err = msgp.ReadBoolBytes(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase \"cv\":\n\t\t\tvar xsz uint32\n\t\t\txsz, bts, err = msgp.ReadArrayHeaderBytes(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif cap(z.ChildValues) >= int(xsz) {\n\t\t\t\tz.ChildValues = z.ChildValues[:xsz]\n\t\t\t} else {\n\t\t\t\tz.ChildValues = make([]interface{}, xsz)\n\t\t\t}\n\t\t\tfor cmr := range z.ChildValues {\n\t\t\t\tz.ChildValues[cmr], bts, err = msgp.ReadIntfBytes(bts)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\tcase \"ck\":\n\t\t\tvar xsz uint32\n\t\t\txsz, bts, err = msgp.ReadArrayHeaderBytes(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif cap(z.ChildKeys) >= int(xsz) {\n\t\t\t\tz.ChildKeys = z.ChildKeys[:xsz]\n\t\t\t} else {\n\t\t\t\tz.ChildKeys = make(Keys, xsz)\n\t\t\t}\n\t\t\tfor ajw := range z.ChildKeys {\n\t\t\t\tif msgp.IsNil(bts) {\n\t\t\t\t\tbts, err = msgp.ReadNilBytes(bts)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tz.ChildKeys[ajw] = nil\n\t\t\t\t} else {\n\t\t\t\t\tif z.ChildKeys[ajw] == nil {\n\t\t\t\t\t\tz.ChildKeys[ajw] = new(Key)\n\t\t\t\t\t}\n\t\t\t\t\tbts, err = z.ChildKeys[ajw].UnmarshalMsg(bts)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tbts, err = msgp.Skip(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\to = bts\n\treturn\n}\n\nfunc (z *Node) Msgsize() (s int) {\n\ts = 1 + 3 + msgp.BytesPrefixSize + len([]byte(z.ID)) + 3 + msgp.BoolSize + 3 + msgp.ArrayHeaderSize\n\tfor cmr := range z.ChildValues {\n\t\ts += msgp.GuessSize(z.ChildValues[cmr])\n\t}\n\ts += 3 + msgp.ArrayHeaderSize\n\tfor ajw := range z.ChildKeys {\n\t\tif z.ChildKeys[ajw] == nil {\n\t\t\ts += msgp.NilSize\n\t\t} else {\n\t\t\ts += z.ChildKeys[ajw].Msgsize()\n\t\t}\n\t}\n\treturn\n}\n" }, { "path": "btree/immutable/path.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\n/*\nThis file contains logic pertaining to keeping track of the path followed\nto find a particular node while descending the tree.\n*/\n\ntype pathBundle struct {\n\t// i defines the child index of the n.\n\ti int\n\tn *Node\n\tprev *pathBundle\n}\n\n// path is simply a linked list of pathBundles. We only ever\n// go in one direction and there's no need to search so a linked list\n// makes sense.\ntype path struct {\n\thead *pathBundle\n\ttail *pathBundle\n}\n\nfunc (p *path) append(pb *pathBundle) {\n\tif p.head == nil {\n\t\tp.head = pb\n\t\tp.tail = pb\n\t\treturn\n\t}\n\n\tpb.prev = p.tail\n\tp.tail = pb\n}\n\n// pop removes the last item from the path. Note that it also nils\n// out the returned pathBundle's prev field. Returns nil if no items\n// remain.\nfunc (p *path) pop() *pathBundle {\n\tif pb := p.tail; pb != nil {\n\t\tp.tail = pb.prev\n\t\tpb.prev = nil\n\t\treturn pb\n\t}\n\n\treturn nil\n}\n\nfunc (p *path) peek() *pathBundle {\n\treturn p.tail\n}\n" }, { "path": "btree/immutable/query.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\nimport (\n\t\"runtime\"\n\t\"sync\"\n\n\tterr \"github.com/Workiva/go-datastructures/threadsafe/err\"\n)\n\nfunc (t *Tr) Apply(fn func(item *Item), keys ...interface{}) error {\n\tif t.Root == nil || len(keys) == 0 {\n\t\treturn nil\n\t}\n\n\tpositions := make(map[interface{}]int, len(keys))\n\tfor i, key := range keys {\n\t\tpositions[key] = i\n\t}\n\n\tchunks := splitValues(keys, runtime.NumCPU())\n\tvar wg sync.WaitGroup\n\twg.Add(len(chunks))\n\tlerr := terr.New()\n\tresult := make(Keys, len(keys))\n\n\tfor i := 0; i < len(chunks); i++ {\n\t\tgo func(i int) {\n\t\t\tdefer wg.Done()\n\n\t\t\tchunk := chunks[i]\n\t\t\tif len(chunk) == 0 {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfor _, value := range chunk {\n\t\t\t\tn, _, err := t.iterativeFindWithoutPath(value, t.Root)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlerr.Set(err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tif n == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tk, _ := n.searchKey(t.config.Comparator, value)\n\t\t\t\tif k != nil && t.config.Comparator(k.Value, value) == 0 {\n\t\t\t\t\tresult[positions[value]] = k\n\t\t\t\t}\n\t\t\t}\n\t\t}(i)\n\t}\n\n\twg.Wait()\n\n\tif lerr.Get() != nil {\n\t\treturn lerr.Get()\n\t}\n\n\tfor _, k := range result {\n\t\tif k == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\titem := k.ToItem()\n\t\tfn(item)\n\t}\n\n\treturn nil\n}\n\n// filter performs an after fetch filtering of the values in the provided node.\n// Due to the nature of the UB-Tree, we may get results in the node that\n// aren't in the provided range. The returned list of keys is not necessarily\n// in the correct row-major order.\nfunc (t *Tr) filter(start, stop interface{}, n *Node, fn func(key *Key) bool) bool {\n\tfor iter := n.iter(t.config.Comparator, start, stop); iter.next(); {\n\t\tid, _ := iter.value()\n\t\tif !fn(id) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc (t *Tr) iter(start, stop interface{}, fn func(*Key) bool) error {\n\tif len(t.Root) == 0 {\n\t\treturn nil\n\t}\n\n\tcur := start\n\tseen := make(map[string]struct{}, 10)\n\n\tfor t.config.Comparator(stop, cur) > 0 {\n\t\tn, highestValue, err := t.iterativeFindWithoutPath(cur, t.Root)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif n == nil && highestValue == nil {\n\t\t\tbreak\n\t\t} else if n != nil {\n\t\t\tif _, ok := seen[string(n.ID)]; ok {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif !t.filter(cur, stop, n, fn) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tcur = n.lastValue()\n\t\tseen[string(n.ID)] = struct{}{}\n\t}\n\n\treturn nil\n}\n\n// iterativeFind searches for the node with the provided value. This\n// is an iterative function and returns an error if there was a problem\n// with persistence.\nfunc (t *Tr) iterativeFind(value interface{}, id ID) (*path, error) {\n\tif len(id) == 0 { // can't find a matching node\n\t\treturn nil, nil\n\t}\n\n\tpath := &path{}\n\tvar n *Node\n\tvar err error\n\tvar i int\n\tvar key *Key\n\n\tfor {\n\t\tn, err = t.contextOrCachedNode(id, t.mutable)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tkey, i = n.searchKey(t.config.Comparator, value)\n\n\t\tpb := &pathBundle{i: i, n: n}\n\t\tpath.append(pb)\n\t\tif n.IsLeaf {\n\t\t\treturn path, nil\n\t\t}\n\t\tid = key.ID()\n\t}\n\n\treturn path, nil\n}\n\nfunc (t *Tr) iterativeFindWithoutPath(value interface{}, id ID) (*Node, interface{}, error) {\n\tvar n *Node\n\tvar err error\n\tvar i int\n\tvar key *Key\n\tvar highestValue interface{}\n\n\tfor {\n\t\tn, err = t.contextOrCachedNode(id, t.mutable)\n\t\tif err != nil {\n\t\t\treturn nil, highestValue, err\n\t\t}\n\n\t\tif n.IsLeaf {\n\t\t\tif t.config.Comparator(n.lastValue(), value) < 0 {\n\t\t\t\treturn nil, highestValue, nil\n\t\t\t}\n\t\t\thighestValue = n.lastValue()\n\t\t\treturn n, highestValue, nil\n\t\t}\n\n\t\tkey, i = n.searchKey(t.config.Comparator, value)\n\t\tif i < n.lenValues() {\n\t\t\thighestValue = n.valueAt(i)\n\t\t}\n\t\tid = key.ID()\n\t}\n\n\treturn n, highestValue, nil\n}\n" }, { "path": "btree/immutable/rt.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n//go:generate msgp -tests=false -io=false\n\npackage btree\n\nimport \"sync\"\n\n// context is used to keep track of the nodes in this mutable\n// that have been created. This is basically any node that had\n// to be touched to perform mutations. Further mutations will visit\n// this context first so we don't have to constantly copy if\n// we don't need to.\ntype context struct {\n\tlock sync.RWMutex\n\tseenNodes map[string]*Node\n}\n\nfunc (c *context) nodeExists(id ID) bool {\n\tc.lock.RLock()\n\tdefer c.lock.RUnlock()\n\t_, ok := c.seenNodes[string(id)]\n\treturn ok\n}\n\nfunc (c *context) addNode(n *Node) {\n\tc.lock.Lock()\n\tdefer c.lock.Unlock()\n\tc.seenNodes[string(n.ID)] = n\n}\n\nfunc (c *context) getNode(id ID) *Node {\n\tc.lock.RLock()\n\tdefer c.lock.RUnlock()\n\treturn c.seenNodes[string(id)]\n}\n\nfunc newContext() *context {\n\treturn &context{\n\t\tseenNodes: make(map[string]*Node, 10),\n\t}\n}\n\n// Tr itself is exported so that the code generated for serialization/deserialization\n// works on Tr. Exported fields on Tr are those fields that need to be\n// serialized.\ntype Tr struct {\n\tUUID ID `msg:\"u\"`\n\tCount int `msg:\"c\"`\n\tconfig Config\n\tRoot ID `msg:\"r\"`\n\tcacher *cacher\n\tcontext *context\n\tNodeWidth int `msg:\"nw\"`\n\tmutable bool\n}\n\nfunc (t *Tr) createRoot() *Node {\n\tn := newNode()\n\tn.IsLeaf = true\n\treturn n\n}\n\n// contextOrCachedNode is a convenience function for either fetching\n// a node from the context or persistence.\nfunc (t *Tr) contextOrCachedNode(id ID, cache bool) (*Node, error) {\n\tif t.context != nil {\n\t\tn := t.context.getNode(id)\n\t\tif n != nil {\n\t\t\treturn n, nil\n\t\t}\n\t}\n\n\treturn t.cacher.getNode(t, id, cache)\n}\n\nfunc (t *Tr) ID() ID {\n\treturn t.UUID\n}\n\n// toBytes encodes this tree into a byte array. Panics if unable\n// as this error has to be fixed in code.\nfunc (t *Tr) toBytes() []byte {\n\tbuf, err := t.MarshalMsg(nil)\n\tif err != nil {\n\t\tpanic(`unable to encode tree`)\n\t}\n\n\treturn buf\n}\n\n// reset is called on a tree to empty the context and clear the cache.\nfunc (t *Tr) reset() {\n\tt.cacher.clear()\n\tt.context = nil\n}\n\n// commit will gather up all created nodes and serialize them into\n// items that can be persisted.\nfunc (t *Tr) commit() []*Payload {\n\titems := make([]*Payload, 0, len(t.context.seenNodes))\n\tfor _, n := range t.context.seenNodes {\n\t\tn.ChildValues, n.ChildKeys = n.flatten()\n\t\tbuf, err := n.MarshalMsg(nil)\n\t\tif err != nil {\n\t\t\tpanic(`unable to encode node`)\n\t\t}\n\n\t\tn.ChildValues, n.ChildKeys = nil, nil\n\t\titem := &Payload{n.ID, buf}\n\t\titems = append(items, item)\n\t}\n\n\treturn items\n}\n\nfunc (t *Tr) copyNode(n *Node) *Node {\n\tif t.context.nodeExists(n.ID) {\n\t\treturn n\n\t}\n\n\tcp := n.copy()\n\tt.context.addNode(cp)\n\treturn cp\n}\n\nfunc (t *Tr) Len() int {\n\treturn t.Count\n}\n\nfunc (t *Tr) AsMutable() MutableTree {\n\treturn &Tr{\n\t\tCount: t.Count,\n\t\tUUID: newID(),\n\t\tRoot: t.Root,\n\t\tconfig: t.config,\n\t\tcacher: t.cacher,\n\t\tcontext: newContext(),\n\t\tNodeWidth: t.NodeWidth,\n\t\tmutable: true,\n\t}\n}\n\nfunc (t *Tr) Commit() (ReadableTree, error) {\n\tt.NodeWidth = t.config.NodeWidth\n\titems := make([]*Payload, 0, len(t.context.seenNodes))\n\titems = append(items, t.commit()...)\n\n\t// save self\n\titems = append(items, &Payload{t.ID(), t.toBytes()})\n\n\terr := t.config.Persister.Save(items...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tt.reset()\n\tt.context = nil\n\treturn t, nil\n}\n\nfunc treeFromBytes(p Persister, data []byte, comparator Comparator) (*Tr, error) {\n\tt := &Tr{}\n\t_, err := t.UnmarshalMsg(data)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tcfg := DefaultConfig(p, comparator)\n\tif t.NodeWidth > 0 {\n\t\tcfg.NodeWidth = t.NodeWidth\n\t}\n\tt.config = cfg\n\tt.cacher = newCacher(cfg.Persister)\n\n\treturn t, nil\n}\n\nfunc newTree(cfg Config) *Tr {\n\treturn &Tr{\n\t\tconfig: cfg,\n\t\tUUID: newID(),\n\t\tcacher: newCacher(cfg.Persister),\n\t}\n}\n\n// New creates a new ReadableTree using the provided config.\nfunc New(cfg Config) ReadableTree {\n\treturn newTree(cfg)\n}\n\n// Load returns a ReadableTree from persistence. The provided\n// config should contain a persister that can be used for this purpose.\n// An error is returned if the tree could not be found or an error\n// occurred in the persistence layer.\nfunc Load(p Persister, id []byte, comparator Comparator) (ReadableTree, error) {\n\titems, err := p.Load(id)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(items) == 0 || items[0] == nil {\n\t\treturn nil, ErrTreeNotFound\n\t}\n\n\trt, err := treeFromBytes(p, items[0].Payload, comparator)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rt, nil\n}\n" }, { "path": "btree/immutable/rt_gen.go", "content": "package btree\n\n// NOTE: THIS FILE WAS PRODUCED BY THE\n// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)\n// DO NOT EDIT\n\nimport (\n\t\"github.com/tinylib/msgp/msgp\"\n)\n\n// MarshalMsg implements msgp.Marshaler\nfunc (z *Tr) MarshalMsg(b []byte) (o []byte, err error) {\n\to = msgp.Require(b, z.Msgsize())\n\t// map header, size 4\n\t// string \"u\"\n\to = append(o, 0x84, 0xa1, 0x75)\n\to, err = z.UUID.MarshalMsg(o)\n\tif err != nil {\n\t\treturn\n\t}\n\t// string \"c\"\n\to = append(o, 0xa1, 0x63)\n\to = msgp.AppendInt(o, z.Count)\n\t// string \"r\"\n\to = append(o, 0xa1, 0x72)\n\to, err = z.Root.MarshalMsg(o)\n\tif err != nil {\n\t\treturn\n\t}\n\t// string \"nw\"\n\to = append(o, 0xa2, 0x6e, 0x77)\n\to = msgp.AppendInt(o, z.NodeWidth)\n\treturn\n}\n\n// UnmarshalMsg implements msgp.Unmarshaler\nfunc (z *Tr) UnmarshalMsg(bts []byte) (o []byte, err error) {\n\tvar field []byte\n\t_ = field\n\tvar isz uint32\n\tisz, bts, err = msgp.ReadMapHeaderBytes(bts)\n\tif err != nil {\n\t\treturn\n\t}\n\tfor isz > 0 {\n\t\tisz--\n\t\tfield, bts, err = msgp.ReadMapKeyZC(bts)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tswitch msgp.UnsafeString(field) {\n\t\tcase \"u\":\n\t\t\tbts, err = z.UUID.UnmarshalMsg(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase \"c\":\n\t\t\tz.Count, bts, err = msgp.ReadIntBytes(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase \"r\":\n\t\t\tbts, err = z.Root.UnmarshalMsg(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tcase \"nw\":\n\t\t\tz.NodeWidth, bts, err = msgp.ReadIntBytes(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\tdefault:\n\t\t\tbts, err = msgp.Skip(bts)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\to = bts\n\treturn\n}\n\nfunc (z *Tr) Msgsize() (s int) {\n\ts = 1 + 2 + z.UUID.Msgsize() + 2 + msgp.IntSize + 2 + z.Root.Msgsize() + 3 + msgp.IntSize\n\treturn\n}\n" }, { "path": "btree/immutable/rt_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage btree\n\nimport (\n\t\"log\"\n\t\"math/rand\"\n\t\"sort\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\ntype ephemeral struct {\n\tmp map[string]*Payload\n\tlock sync.RWMutex\n}\n\nfunc (e *ephemeral) Save(items ...*Payload) error {\n\te.lock.Lock()\n\tdefer e.lock.Unlock()\n\n\tif len(items) == 0 {\n\t\treturn nil\n\t}\n\n\tfor _, item := range items {\n\t\te.mp[string(item.Key)] = item\n\t}\n\n\treturn nil\n}\n\nfunc (e *ephemeral) Load(keys ...[]byte) ([]*Payload, error) {\n\te.lock.RLock()\n\tdefer e.lock.RUnlock()\n\n\tif len(keys) == 0 {\n\t\treturn nil, nil\n\t}\n\n\titems := make([]*Payload, 0, len(keys))\n\tfor _, k := range keys {\n\t\titems = append(items, e.mp[string(k)])\n\t}\n\n\treturn items, nil\n}\n\nconst (\n\tmaxValue = int64(100000)\n)\n\nfunc init() {\n\trand.Seed(time.Now().Unix())\n}\n\ntype valueSortWrapper struct {\n\tcomparator Comparator\n\tvalues []interface{}\n}\n\nfunc (v *valueSortWrapper) Len() int {\n\treturn len(v.values)\n}\n\nfunc (v *valueSortWrapper) Swap(i, j int) {\n\tv.values[i], v.values[j] = v.values[j], v.values[i]\n}\n\nfunc (v *valueSortWrapper) Less(i, j int) bool {\n\treturn v.comparator(v.values[i], v.values[j]) < 0\n}\n\nfunc (v *valueSortWrapper) sort() {\n\tsort.Sort(v)\n}\n\nfunc reverse(items items) items {\n\tfor i := 0; i < len(items)/2; i++ {\n\t\titems[i], items[len(items)-1-i] = items[len(items)-1-i], items[i]\n\t}\n\n\treturn items\n}\n\nvar comparator = func(item1, item2 interface{}) int {\n\tint1, int2 := item1.(int64), item2.(int64)\n\tif int1 < int2 {\n\t\treturn -1\n\t}\n\n\tif int1 > int2 {\n\t\treturn 1\n\t}\n\n\treturn 0\n}\n\n// orderedItems is going to contain our \"master\" copy of items in\n// sorted order. Because the operations on a flat list are well\n// understood, we can use this type to do generative type testing and\n// confirm the results.\ntype orderedItems []*Item\n\nfunc (o orderedItems) Len() int {\n\treturn len(o)\n}\n\nfunc (o orderedItems) Swap(i, j int) {\n\to[i], o[j] = o[j], o[i]\n}\n\nfunc (o orderedItems) Less(i, j int) bool {\n\treturn comparator(o[i].Value, o[j].Value) < 0\n}\n\nfunc (o orderedItems) equal(item1, item2 *Item) bool {\n\treturn comparator(item1.Value, item2.Value) == 0\n}\n\nfunc (o orderedItems) copy() orderedItems {\n\tcp := make(orderedItems, len(o))\n\tcopy(cp, o)\n\treturn cp\n}\n\nfunc (o orderedItems) search(value interface{}) int {\n\treturn sort.Search(len(o), func(i int) bool {\n\t\treturn comparator(o[i].Value, value) >= 0\n\t})\n}\n\nfunc (o orderedItems) add(item *Item) orderedItems {\n\tcp := make(orderedItems, len(o))\n\tcopy(cp, o)\n\ti := cp.search(item.Value)\n\tif i < len(o) && o.equal(o[i], item) {\n\t\tcp[i] = item\n\t\treturn cp\n\t}\n\n\tif i == len(cp) {\n\t\tcp = append(cp, item)\n\t\treturn cp\n\t}\n\n\tcp = append(cp, nil)\n\tcopy(cp[i+1:], cp[i:])\n\tcp[i] = item\n\treturn cp\n}\n\nfunc (o orderedItems) delete(item *Item) orderedItems {\n\ti := o.search(item.Value)\n\tif i == len(o) {\n\t\treturn o\n\t}\n\n\tif !o.equal(o[i], item) {\n\t\treturn o\n\t}\n\n\tcp := make(orderedItems, len(o))\n\tcopy(cp, o)\n\n\tcopy(cp[i:], cp[i+1:])\n\tcp[len(cp)-1] = nil // or the zero value of T\n\tcp = cp[:len(cp)-1]\n\treturn cp\n}\n\nfunc (o orderedItems) toItems() items {\n\tcp := make(items, 0, len(o))\n\tfor _, item := range o {\n\t\tcp = append(cp, item)\n\t}\n\n\treturn cp\n}\n\nfunc (o orderedItems) query(start, stop interface{}) items {\n\titems := make(items, 0, len(o))\n\n\tfor i := o.search(start); i < len(o); i++ {\n\t\tif comparator(o[i], stop) > 0 {\n\t\t\tbreak\n\t\t}\n\n\t\titems = append(items, o[i])\n\t}\n\n\treturn items\n}\n\nfunc generateRandomQuery() (interface{}, interface{}) {\n\tstart := int64(rand.Intn(int(maxValue)))\n\toffset := int64(rand.Intn(100))\n\treturn start, start + offset\n}\n\nfunc newItem(value interface{}) *Item {\n\treturn &Item{\n\t\tValue: value,\n\t\tPayload: newID(),\n\t}\n}\n\nfunc newEphemeral() Persister {\n\treturn &ephemeral{\n\t\tmp: make(map[string]*Payload),\n\t}\n}\n\ntype delayedPersister struct {\n\tPersister\n}\n\nfunc (d *delayedPersister) Load(keys ...[]byte) ([]*Payload, error) {\n\ttime.Sleep(5 * time.Millisecond)\n\treturn d.Persister.Load(keys...)\n}\n\nfunc newDelayed() Persister {\n\treturn &delayedPersister{newEphemeral()}\n}\n\nfunc defaultConfig() Config {\n\treturn Config{\n\t\tNodeWidth: 10, // easy number to test with\n\t\tPersister: newEphemeral(),\n\t\tComparator: comparator,\n\t}\n}\n\nfunc generateRandomItem() *Item {\n\treturn newItem(int64(rand.Intn(int(maxValue))))\n}\n\n// generateRandomItems will generate a list of random items with\n// no duplicates.\nfunc generateRandomItems(num int) items {\n\titems := make(items, 0, num)\n\tmp := make(map[interface{}]struct{}, num)\n\tfor len(items) < num {\n\t\tc := generateRandomItem()\n\t\tif _, ok := mp[c.Value]; ok {\n\t\t\tcontinue\n\t\t}\n\t\tmp[c.Value] = struct{}{}\n\t\titems = append(items, c)\n\t}\n\n\treturn items\n}\n\n// generateLinearItems is similar to random item generation except that\n// items are returned in sorted order.\nfunc generateLinearItems(num int) items {\n\titems := make(items, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tc := newItem(int64(i))\n\t\titems = append(items, c)\n\t}\n\n\treturn items\n}\n\nfunc toOrdered(items items) orderedItems {\n\toc := make(orderedItems, 0, len(items))\n\tfor _, item := range items {\n\t\toc = oc.add(item)\n\t}\n\n\treturn oc\n}\n\n// the following 3 methods are in the _test file as they are only used\n// in a testing environment.\nfunc (t *Tr) toList(values ...interface{}) (items, error) {\n\titems := make(items, 0, t.Count)\n\terr := t.Apply(func(item *Item) {\n\t\titems = append(items, item)\n\t}, values...)\n\n\treturn items, err\n}\n\nfunc (t *Tr) pprint(id ID) {\n\tn, _ := t.contextOrCachedNode(id, true)\n\tif n == nil {\n\t\tlog.Printf(`NODE: %+v`, n)\n\t\treturn\n\t}\n\tlog.Printf(`NODE: %+v, LEN(ids): %+v, LEN(values): %+v`, n, n.lenKeys(), n.lenValues())\n\tfor i, key := range n.ChildKeys {\n\t\tchild, _ := t.contextOrCachedNode(key.ID(), true)\n\t\tif child == nil {\n\t\t\tcontinue\n\t\t}\n\t\tlog.Printf(`CHILD %d: %+v`, i, child)\n\t}\n\n\tfor _, key := range n.ChildKeys {\n\t\tchild, _ := t.contextOrCachedNode(key.ID(), true)\n\t\tif child == nil {\n\t\t\tcontinue\n\t\t}\n\t\tt.pprint(key.ID())\n\t}\n}\n\nfunc (t *Tr) verify(id ID, tb testing.TB) (interface{}, interface{}) {\n\tn, err := t.contextOrCachedNode(id, true)\n\trequire.NoError(tb, err)\n\n\tcp := n.copy() // copy the values and sort them, ensure node values are sorted\n\tcpValues := cp.ChildValues\n\n\t(&valueSortWrapper{comparator: comparator, values: cpValues}).sort()\n\tassert.Equal(tb, cpValues, n.ChildValues)\n\n\tif !assert.False(tb, n.needsSplit(t.config.NodeWidth)) {\n\t\ttb.Logf(`NODE NEEDS SPLIT: NODE: %+v`, n)\n\t}\n\tif string(t.Root) != string(n.ID) {\n\t\tassert.True(tb, n.lenValues() >= t.config.NodeWidth/2)\n\t}\n\n\tif n.IsLeaf {\n\t\tassert.Equal(tb, n.lenValues(), n.lenKeys()) // assert lens are equal\n\t\treturn n.firstValue(), n.lastValue() // return last value\n\t} else {\n\t\tfor _, key := range n.ChildKeys {\n\t\t\tassert.Empty(tb, key.Payload)\n\t\t}\n\t}\n\n\tfor i, key := range n.ChildKeys {\n\t\tmin, max := t.verify(key.ID(), tb)\n\t\tif i == 0 {\n\t\t\tassert.True(tb, comparator(max, n.valueAt(i)) <= 0)\n\t\t} else if i == n.lenValues() {\n\t\t\tassert.True(tb, comparator(min, n.lastValue()) > 0)\n\t\t} else {\n\t\t\tassert.True(tb, comparator(max, n.valueAt(i)) <= 0)\n\t\t\tassert.True(tb, comparator(min, n.valueAt(i-1)) > 0)\n\t\t}\n\t}\n\n\treturn n.firstValue(), n.lastValue()\n}\n\nfunc itemsToValues(items ...*Item) []interface{} {\n\tvalues := make([]interface{}, 0, len(items))\n\tfor _, item := range items {\n\t\tvalues = append(values, item.Value)\n\t}\n\n\treturn values\n}\n\nfunc TestNodeSplit(t *testing.T) {\n\tnumber := 100\n\titems := generateLinearItems(number)\n\tcfg := defaultConfig()\n\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\t_, err := mutable.AddItems(items...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, number, mutable.Len())\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\n\tresult, err := mutable.(*Tr).toList(itemsToValues(items[5:10]...)...)\n\trequire.NoError(t, err)\n\tif !assert.Equal(t, items[5:10], result) {\n\t\tmutable.(*Tr).pprint(mutable.(*Tr).Root)\n\t\tfor i, c := range items[5:10] {\n\t\t\tt.Logf(`EXPECTED: %+v, RESULT: %+v`, c, result[i])\n\t\t}\n\t\tt.FailNow()\n\t}\n\n\tmutable = rt.AsMutable()\n\tfor _, c := range items {\n\t\t_, err := mutable.AddItems(c)\n\t\trequire.NoError(t, err)\n\t}\n\n\tresult, err = mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, items, result)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\n\trt, err = mutable.Commit()\n\trequire.NoError(t, err)\n\trt, err = Load(cfg.Persister, rt.ID(), comparator)\n\n\tresult, err = mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, items, result)\n\trt.(*Tr).verify(rt.(*Tr).Root, t)\n}\n\nfunc TestReverseNodeSplit(t *testing.T) {\n\tnumber := 400\n\titems := generateLinearItems(number)\n\n\treversed := make([]*Item, len(items))\n\tcopy(reversed, items)\n\treversed = reverse(reversed)\n\trt := New(defaultConfig())\n\tmutable := rt.AsMutable()\n\t_, err := mutable.AddItems(reversed...)\n\trequire.NoError(t, err)\n\n\tresult, err := mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tif !assert.Equal(t, items, result) {\n\t\tfor _, c := range result {\n\t\t\tt.Logf(`RESULT: %+v`, c)\n\t\t}\n\t}\n\n\tmutable = rt.AsMutable()\n\tfor _, c := range reversed {\n\t\t_, err := mutable.AddItems(c)\n\t\trequire.NoError(t, err)\n\t}\n\n\tresult, err = mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, items, result)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n}\n\nfunc TestDuplicate(t *testing.T) {\n\titem1 := newItem(int64(1))\n\titem2 := newItem(int64(1))\n\n\trt := New(defaultConfig())\n\tmutable := rt.AsMutable()\n\t_, err := mutable.AddItems(item1)\n\trequire.NoError(t, err)\n\t_, err = mutable.AddItems(item2)\n\trequire.NoError(t, err)\n\n\tassert.Equal(t, 1, mutable.Len())\n\tresult, err := mutable.(*Tr).toList(int64(1))\n\trequire.NoError(t, err)\n\n\tassert.Equal(t, items{item2}, result)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n}\n\nfunc TestCommit(t *testing.T) {\n\titems := generateRandomItems(5)\n\trt := New(defaultConfig())\n\tmutable := rt.AsMutable()\n\t_, err := mutable.AddItems(items...)\n\trequire.Nil(t, err)\n\n\trt, err = mutable.Commit()\n\trequire.NoError(t, err)\n\texpected := toOrdered(items).toItems()\n\tresult, err := rt.(*Tr).toList(itemsToValues(expected...)...)\n\trequire.NoError(t, err)\n\tif !assert.Equal(t, expected, result) {\n\t\trequire.Equal(t, len(expected), len(result))\n\t\tfor i, c := range expected {\n\t\t\tif !assert.Equal(t, c, result[i]) {\n\t\t\t\tt.Logf(`EXPECTED: %+v, RESULT: %+v`, c, result[i])\n\t\t\t}\n\t\t}\n\t}\n\n\trt.(*Tr).verify(rt.(*Tr).Root, t)\n}\n\nfunc TestRandom(t *testing.T) {\n\titems := generateRandomItems(1000)\n\trt := New(defaultConfig())\n\tmutable := rt.AsMutable()\n\t_, err := mutable.AddItems(items...)\n\trequire.Nil(t, err)\n\n\trequire.NoError(t, err)\n\texpected := toOrdered(items).toItems()\n\tresult, err := mutable.(*Tr).toList(itemsToValues(expected...)...)\n\tif !assert.Equal(t, expected, result) {\n\t\tassert.Equal(t, len(expected), len(result))\n\t\tfor i, c := range expected {\n\t\t\tassert.Equal(t, c, result[i])\n\t\t}\n\t}\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n}\n\nfunc TestLoad(t *testing.T) {\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateRandomItems(1000)\n\t_, err := mutable.AddItems(items...)\n\trequire.NoError(t, err)\n\n\tid := mutable.ID()\n\t_, err = mutable.Commit()\n\trequire.NoError(t, err)\n\n\trt, err = Load(cfg.Persister, id, comparator)\n\trequire.NoError(t, err)\n\tsort.Sort(orderedItems(items))\n\tresult, err := rt.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, items, result)\n\trt.(*Tr).verify(rt.(*Tr).Root, t)\n}\n\nfunc TestDeleteFromRoot(t *testing.T) {\n\tnumber := 5\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateLinearItems(number)\n\n\tmutable.AddItems(items...)\n\tmutable.DeleteItems(items[0].Value, items[1].Value, items[2].Value)\n\n\tresult, err := mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.Nil(t, err)\n\tassert.Equal(t, items[3:], result)\n\tassert.Equal(t, 2, mutable.Len())\n\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n}\n\nfunc TestDeleteAllFromRoot(t *testing.T) {\n\tnum := 5\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateLinearItems(num)\n\n\tmutable.AddItems(items...)\n\tmutable.DeleteItems(itemsToValues(items...)...)\n\n\tresult, err := mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.Nil(t, err)\n\tassert.Empty(t, result)\n\tassert.Equal(t, 0, mutable.Len())\n}\n\nfunc TestDeleteAfterSplitIncreasing(t *testing.T) {\n\tnum := 11\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateLinearItems(num)\n\n\tmutable.AddItems(items...)\n\tfor i := 0; i < num-1; i++ {\n\t\tmutable.DeleteItems(itemsToValues(items[i])...)\n\t\tresult, err := mutable.(*Tr).toList(itemsToValues(items...)...)\n\t\trequire.Nil(t, err)\n\t\tassert.Equal(t, items[i+1:], result)\n\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\t}\n}\n\nfunc TestDeleteMultipleLevelsRandomlyBulk(t *testing.T) {\n\tnum := 200\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateRandomItems(num)\n\tmutable.AddItems(items...)\n\tmutable.DeleteItems(itemsToValues(items[:100]...)...)\n\tresult, _ := mutable.(*Tr).toList(itemsToValues(items...)...)\n\tassert.Len(t, result, 100)\n}\n\nfunc TestDeleteAfterSplitDecreasing(t *testing.T) {\n\tnum := 11\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateLinearItems(num)\n\n\tmutable.AddItems(items...)\n\tfor i := num - 1; i >= 0; i-- {\n\t\tmutable.DeleteItems(itemsToValues(items[i])...)\n\t\tresult, err := mutable.(*Tr).toList(itemsToValues(items...)...)\n\t\trequire.Nil(t, err)\n\t\tassert.Equal(t, items[:i], result)\n\t\tif i > 0 {\n\t\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\t\t}\n\t}\n}\n\nfunc TestDeleteMultipleLevels(t *testing.T) {\n\tnum := 20\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateRandomItems(num)\n\tmutable.AddItems(items...)\n\tordered := toOrdered(items)\n\n\tfor i, c := range ordered {\n\t\t_, err := mutable.DeleteItems(c.Value)\n\t\trequire.NoError(t, err)\n\t\tresult, err := mutable.(*Tr).toList(itemsToValues(ordered...)...)\n\t\trequire.NoError(t, err)\n\t\tif !assert.Equal(t, ordered[i+1:].toItems(), result) {\n\t\t\tlog.Printf(`LEN EXPECTED: %+v, RESULT: %+v`, len(ordered[i+1:]), len(result))\n\t\t\tmutable.(*Tr).pprint(mutable.(*Tr).Root)\n\t\t\tassert.Equal(t, len(ordered[i+1:]), len(result))\n\t\t\tfor i, c := range ordered[i+1:] {\n\t\t\t\tlog.Printf(`EXPECTED: %+v`, c)\n\t\t\t\tif i < len(result) {\n\t\t\t\t\tlog.Printf(`RECEIVED: %+v`, result[i])\n\t\t\t\t}\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tif len(ordered[i+1:]) > 0 {\n\t\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\t\t}\n\t}\n\n\tassert.Nil(t, mutable.(*Tr).Root)\n}\n\nfunc TestDeleteMultipleLevelsRandomly(t *testing.T) {\n\tnum := 200\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateRandomItems(num)\n\tmutable.AddItems(items...)\n\tordered := toOrdered(items)\n\n\tfor _, c := range items {\n\t\t_, err := mutable.DeleteItems(c.Value)\n\t\trequire.NoError(t, err)\n\t\tordered = ordered.delete(c)\n\n\t\tresult, err := mutable.(*Tr).toList(itemsToValues(ordered...)...)\n\t\trequire.NoError(t, err)\n\t\tassert.Equal(t, ordered.toItems(), result)\n\t\tif len(ordered) > 0 {\n\t\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\t\t}\n\t}\n\n\tassert.Nil(t, mutable.(*Tr).Root)\n}\n\nfunc TestDeleteMultipleLevelsWithCommit(t *testing.T) {\n\tnum := 20\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateRandomItems(num)\n\tmutable.AddItems(items...)\n\trt, _ = mutable.Commit()\n\n\trt, _ = Load(cfg.Persister, rt.ID(), comparator)\n\tresult, err := rt.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, items, result)\n\tmutable = rt.AsMutable()\n\n\tfor _, c := range items[:10] {\n\t\t_, err := mutable.DeleteItems(c.Value)\n\t\trequire.Nil(t, err)\n\t}\n\n\tresult, err = mutable.(*Tr).toList(itemsToValues(items[10:]...)...)\n\trequire.Nil(t, err)\n\tassert.Equal(t, items[10:], result)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\n\tresult, err = rt.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, items, result)\n\trt.(*Tr).verify(rt.(*Tr).Root, t)\n}\n\nfunc TestCommitAfterDelete(t *testing.T) {\n\tnum := 15\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\tmutable := rt.AsMutable()\n\titems := generateRandomItems(num)\n\tmutable.AddItems(items...)\n\tfor _, c := range items[:5] {\n\t\tmutable.DeleteItems(c.Value)\n\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\t}\n\n\trt, err := mutable.Commit()\n\trequire.Nil(t, err)\n\tresult, err := rt.(*Tr).toList(itemsToValues(items...)...)\n\trequire.Nil(t, err)\n\tassert.Equal(t, items[5:], result)\n\trt.(*Tr).verify(rt.(*Tr).Root, t)\n\n}\n\nfunc TestSecondCommitSplitsRoot(t *testing.T) {\n\tnumber := 15\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\titems := generateLinearItems(number)\n\n\tmutable := rt.AsMutable()\n\tmutable.AddItems(items[:10]...)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\trt, _ = mutable.Commit()\n\trt.(*Tr).verify(rt.(*Tr).Root, t)\n\tmutable = rt.AsMutable()\n\tmutable.AddItems(items[10:]...)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\n\tresult, err := mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.Nil(t, err)\n\n\tif !assert.Equal(t, items, result) {\n\t\tfor i, c := range items {\n\t\t\tlog.Printf(`EXPECTED: %+v, RECEIVED: %+v`, c, result[i])\n\t\t}\n\t}\n}\n\nfunc TestSecondCommitMultipleSplits(t *testing.T) {\n\tnum := 50\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\titems := generateRandomItems(num)\n\n\tmutable := rt.AsMutable()\n\tmutable.AddItems(items[:25]...)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\trt, err := mutable.Commit()\n\trt.(*Tr).verify(rt.(*Tr).Root, t)\n\n\tresult, err := rt.(*Tr).toList(itemsToValues(items...)...)\n\trequire.Nil(t, err)\n\tassert.Equal(t, items[:25], result)\n\n\tmutable = rt.AsMutable()\n\tmutable.AddItems(items[25:]...)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\n\tsort.Sort(orderedItems(items))\n\tresult, err = mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.Nil(t, err)\n\tif !assert.Equal(t, items, result) {\n\t\tmutable.(*Tr).pprint(mutable.(*Tr).Root)\n\t}\n}\n\nfunc TestLargeAdd(t *testing.T) {\n\tcfg := defaultConfig()\n\tnumber := cfg.NodeWidth * 5\n\trt := New(cfg)\n\titems := generateLinearItems(number)\n\n\tmutable := rt.AsMutable()\n\t_, err := mutable.AddItems(items...)\n\trequire.NoError(t, err)\n\n\tid := mutable.ID()\n\tresult, err := mutable.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, items, result)\n\n\t_, err = mutable.Commit()\n\trequire.NoError(t, err)\n\n\trt, err = Load(cfg.Persister, id, comparator)\n\trequire.NoError(t, err)\n\tresult, err = rt.(*Tr).toList(itemsToValues(items...)...)\n\trequire.NoError(t, err)\n\tassert.Equal(t, items, result)\n}\n\nfunc TestNodeInfiniteLoop(t *testing.T) {\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\titems := generateLinearItems(3)\n\n\tmutable := rt.AsMutable()\n\t_, err := mutable.AddItems(items...)\n\trequire.NoError(t, err)\n\n\tresult, err := mutable.DeleteItems(items[1].Value, items[2].Value)\n\trequire.NoError(t, err)\n\tassert.Len(t, result, 2)\n}\n\n// all remaining tests are generative in nature to catch things\n// I can't think of.\n\nfunc TestGenerativeAdds(t *testing.T) {\n\tif testing.Short() {\n\t\tt.Skipf(`skipping generative add`)\n\t\treturn\n\t}\n\n\tnumber := 100\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\toc := make(orderedItems, 0)\n\tfor i := 0; i < number; i++ {\n\t\tnum := int(rand.Int31n(100))\n\t\tif num == 0 {\n\t\t\tnum++\n\t\t}\n\n\t\titems := generateRandomItems(num)\n\t\tmutated := oc.copy()\n\t\tfor _, c := range items {\n\t\t\tmutated = mutated.add(c)\n\t\t}\n\n\t\tmutable := rt.AsMutable()\n\t\t_, err := mutable.AddItems(items...)\n\t\trequire.Nil(t, err)\n\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\n\t\trtMutated, err := mutable.Commit()\n\t\trequire.Nil(t, err)\n\t\trtMutated.(*Tr).verify(rtMutated.(*Tr).Root, t)\n\n\t\tresult, err := rtMutated.(*Tr).toList(itemsToValues(mutated.toItems()...)...)\n\t\trequire.Nil(t, err)\n\t\tif !assert.Equal(t, mutated.toItems(), result) {\n\t\t\trtMutated.(*Tr).pprint(rtMutated.(*Tr).Root)\n\t\t\tif len(mutated) == len(result) {\n\t\t\t\tfor i, c := range mutated.toItems() {\n\t\t\t\t\tlog.Printf(`EXPECTED: %+v, RECEIVED: %+v`, c, result[i])\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tassert.Equal(t, len(mutated), rtMutated.Len())\n\n\t\tresult, err = rt.(*Tr).toList(itemsToValues(oc.toItems()...)...)\n\t\trequire.Nil(t, err)\n\t\tassert.Equal(t, oc.toItems(), result)\n\n\t\toc = mutated\n\t\trt = rtMutated\n\t}\n}\n\nfunc TestGenerativeDeletes(t *testing.T) {\n\tif testing.Short() {\n\t\tt.Skipf(`skipping generative delete`)\n\t\treturn\n\t}\n\n\tnumber := 100\n\tvar err error\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\toc := toOrdered(generateRandomItems(1000))\n\tmutable := rt.AsMutable()\n\tmutable.AddItems(oc.toItems()...)\n\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\trt, err = mutable.Commit()\n\trequire.NoError(t, err)\n\tfor i := 0; i < number; i++ {\n\t\tmutable = rt.AsMutable()\n\t\tindex := rand.Intn(len(oc))\n\t\tc := oc[index]\n\t\tmutated := oc.delete(c)\n\n\t\tresult, err := rt.(*Tr).toList(itemsToValues(oc.toItems()...)...)\n\t\trequire.NoError(t, err)\n\t\tassert.Equal(t, oc.toItems(), result)\n\t\tassert.Equal(t, len(oc), rt.Len())\n\n\t\t_, err = mutable.DeleteItems(c.Value)\n\t\trequire.NoError(t, err)\n\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\t\tresult, err = mutable.(*Tr).toList(itemsToValues(mutated.toItems()...)...)\n\t\trequire.NoError(t, err)\n\t\tassert.Equal(t, len(mutated), len(result))\n\t\trequire.Equal(t, mutated.toItems(), result)\n\t\toc = mutated\n\t\trt, err = mutable.Commit()\n\t\trequire.NoError(t, err)\n\t}\n}\n\nfunc TestGenerativeOperations(t *testing.T) {\n\tif testing.Short() {\n\t\tt.Skipf(`skipping generative operations`)\n\t\treturn\n\t}\n\n\tnumber := 100\n\tcfg := defaultConfig()\n\trt := New(cfg)\n\n\t// seed the tree\n\titems := generateRandomItems(1000)\n\toc := toOrdered(items)\n\n\tmutable := rt.AsMutable()\n\tmutable.AddItems(items...)\n\n\tresult, err := mutable.(*Tr).toList(itemsToValues(oc.toItems()...)...)\n\trequire.NoError(t, err)\n\trequire.Equal(t, oc.toItems(), result)\n\n\trt, err = mutable.Commit()\n\trequire.NoError(t, err)\n\n\tfor i := 0; i < number; i++ {\n\t\tmutable = rt.AsMutable()\n\t\tif rand.Float64() < .5 && len(oc) > 0 {\n\t\t\tc := oc[rand.Intn(len(oc))]\n\t\t\toc = oc.delete(c)\n\t\t\t_, err = mutable.DeleteItems(c.Value)\n\t\t\trequire.NoError(t, err)\n\t\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\t\t\tresult, err := mutable.(*Tr).toList(itemsToValues(oc.toItems()...)...)\n\t\t\trequire.NoError(t, err)\n\t\t\trequire.Equal(t, oc.toItems(), result)\n\t\t\tassert.Equal(t, len(oc), mutable.Len())\n\t\t} else {\n\t\t\tc := generateRandomItem()\n\t\t\toc = oc.add(c)\n\t\t\t_, err = mutable.AddItems(c)\n\t\t\trequire.NoError(t, err)\n\t\t\tmutable.(*Tr).verify(mutable.(*Tr).Root, t)\n\t\t\tresult, err = mutable.(*Tr).toList(itemsToValues(oc.toItems()...)...)\n\t\t\trequire.NoError(t, err)\n\t\t\trequire.Equal(t, oc.toItems(), result)\n\t\t\tassert.Equal(t, len(oc), mutable.Len())\n\t\t}\n\n\t\trt, err = mutable.Commit()\n\t\trequire.NoError(t, err)\n\t}\n}\n\nfunc BenchmarkGetitems(b *testing.B) {\n\tnumber := 100\n\tcfg := defaultConfig()\n\tcfg.Persister = newDelayed()\n\trt := New(cfg)\n\n\titems := generateRandomItems(number)\n\tmutable := rt.AsMutable()\n\t_, err := mutable.AddItems(items...)\n\trequire.NoError(b, err)\n\n\trt, err = mutable.Commit()\n\trequire.NoError(b, err)\n\tid := rt.ID()\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trt, err = Load(cfg.Persister, id, comparator)\n\t\trequire.NoError(b, err)\n\t\t_, err = rt.(*Tr).toList(itemsToValues(items...)...)\n\t\trequire.NoError(b, err)\n\t}\n}\n\nfunc BenchmarkBulkAdd(b *testing.B) {\n\tnumber := 1000000\n\titems := generateLinearItems(number)\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\ttr := New(defaultConfig())\n\t\tmutable := tr.AsMutable()\n\t\tmutable.AddItems(items...)\n\t}\n}\n" }, { "path": "btree/palm/action.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage palm\n\nimport (\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/Workiva/go-datastructures/common\"\n)\n\ntype actions []action\n\ntype action interface {\n\toperation() operation\n\tkeys() common.Comparators\n\tcomplete()\n\taddNode(int64, *node)\n\tnodes() []*node\n}\n\ntype getAction struct {\n\tresult common.Comparators\n\tcompleter *sync.WaitGroup\n}\n\nfunc (ga *getAction) complete() {\n\tga.completer.Done()\n}\n\nfunc (ga *getAction) operation() operation {\n\treturn get\n}\n\nfunc (ga *getAction) keys() common.Comparators {\n\treturn ga.result\n}\n\nfunc (ga *getAction) addNode(i int64, n *node) {\n\treturn // not necessary for gets\n}\n\nfunc (ga *getAction) nodes() []*node {\n\treturn nil\n}\n\nfunc newGetAction(keys common.Comparators) *getAction {\n\tresult := make(common.Comparators, len(keys))\n\tcopy(result, keys) // don't want to mutate passed in keys\n\tga := &getAction{\n\t\tresult: result,\n\t\tcompleter: new(sync.WaitGroup),\n\t}\n\tga.completer.Add(1)\n\treturn ga\n}\n\ntype insertAction struct {\n\tresult common.Comparators\n\tcompleter *sync.WaitGroup\n\tns []*node\n}\n\nfunc (ia *insertAction) complete() {\n\tia.completer.Done()\n}\n\nfunc (ia *insertAction) operation() operation {\n\treturn add\n}\n\nfunc (ia *insertAction) keys() common.Comparators {\n\treturn ia.result\n}\n\nfunc (ia *insertAction) addNode(i int64, n *node) {\n\tia.ns[i] = n\n}\n\nfunc (ia *insertAction) nodes() []*node {\n\treturn ia.ns\n}\n\nfunc newInsertAction(keys common.Comparators) *insertAction {\n\tresult := make(common.Comparators, len(keys))\n\tcopy(result, keys)\n\tia := &insertAction{\n\t\tresult: result,\n\t\tcompleter: new(sync.WaitGroup),\n\t\tns: make([]*node, len(keys)),\n\t}\n\tia.completer.Add(1)\n\treturn ia\n}\n\ntype removeAction struct {\n\t*insertAction\n}\n\nfunc (ra *removeAction) operation() operation {\n\treturn remove\n}\n\nfunc newRemoveAction(keys common.Comparators) *removeAction {\n\treturn &removeAction{\n\t\tnewInsertAction(keys),\n\t}\n}\n\ntype applyAction struct {\n\tfn func(common.Comparator) bool\n\tstart, stop common.Comparator\n\tcompleter *sync.WaitGroup\n}\n\nfunc (aa *applyAction) operation() operation {\n\treturn apply\n}\n\nfunc (aa *applyAction) nodes() []*node {\n\treturn nil\n}\n\nfunc (aa *applyAction) addNode(i int64, n *node) {}\n\nfunc (aa *applyAction) keys() common.Comparators {\n\treturn nil\n}\n\nfunc (aa *applyAction) complete() {\n\taa.completer.Done()\n}\n\nfunc newApplyAction(fn func(common.Comparator) bool, start, stop common.Comparator) *applyAction {\n\taa := &applyAction{\n\t\tfn: fn,\n\t\tstart: start,\n\t\tstop: stop,\n\t\tcompleter: new(sync.WaitGroup),\n\t}\n\taa.completer.Add(1)\n\treturn aa\n}\n\nfunc minUint64(choices ...uint64) uint64 {\n\tmin := choices[0]\n\tfor i := 1; i < len(choices); i++ {\n\t\tif choices[i] < min {\n\t\t\tmin = choices[i]\n\t\t}\n\t}\n\n\treturn min\n}\n\ntype interfaces []interface{}\n\nfunc executeInterfacesInParallel(ifs interfaces, fn func(interface{})) {\n\tif len(ifs) == 0 {\n\t\treturn\n\t}\n\n\tdone := int64(-1)\n\tnumCPU := uint64(runtime.NumCPU())\n\tif numCPU > 1 {\n\t\tnumCPU--\n\t}\n\n\tnumCPU = minUint64(numCPU, uint64(len(ifs)))\n\n\tvar wg sync.WaitGroup\n\twg.Add(int(numCPU))\n\n\tfor i := uint64(0); i < numCPU; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\tfor {\n\t\t\t\ti := atomic.AddInt64(&done, 1)\n\t\t\t\tif i >= int64(len(ifs)) {\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tfn(ifs[i])\n\t\t\t}\n\t\t}()\n\t}\n\n\twg.Wait()\n}\n\nfunc executeInterfacesInSerial(ifs interfaces, fn func(interface{})) {\n\tif len(ifs) == 0 {\n\t\treturn\n\t}\n\n\tfor _, ifc := range ifs {\n\t\tfn(ifc)\n\t}\n}\n" }, { "path": "btree/palm/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage palm implements parallel architecture-friendly latch-free\nmodifications (PALM). Details can be found here:\n\nhttp://cs.unc.edu/~sewall/palm.pdf\n\nThe primary purpose of the tree is to efficiently batch operations\nin such a way that locks are not required. This is most beneficial\nfor in-memory indices. Otherwise, the operations have typical B-tree\ntime complexities.\n\nYou primarily see the benefits of multithreading in availability and\nbulk operations.\n\nBenchmarks:\n\nBenchmarkReadAndWrites-8\t \t\t\t3000\t 483140 ns/op\nBenchmarkSimultaneousReadsAndWrites-8\t 300\t 4418123 ns/op\nBenchmarkBulkAdd-8\t \t\t\t\t\t 300\t 5569750 ns/op\nBenchmarkAdd-8\t \t\t\t\t\t\t 500000\t\t 2478 ns/op\nBenchmarkBulkAddToExisting-8\t \t\t 100\t 20552674 ns/op\nBenchmarkGet-8\t \t\t\t\t\t\t 2000000\t 629 ns/op\nBenchmarkBulkGet-8\t \t\t\t\t\t5000\t 223249 ns/op\nBenchmarkDelete-8\t \t\t\t\t\t 500000\t 2421 ns/op\nBenchmarkBulkDelete-8\t \t\t\t\t 500\t 2790461 ns/op\nBenchmarkFindQuery-8\t \t\t\t\t 1000000\t 1166 ns/op\nBenchmarkExecuteQuery-8\t \t\t\t\t 10000\t 1290732 ns/op\n\n*/\npackage palm\n\nimport \"github.com/Workiva/go-datastructures/common\"\n\n// BTree is the interface returned from this package's constructor.\ntype BTree interface {\n\t// Insert will insert the provided keys into the tree.\n\tInsert(...common.Comparator)\n\t// Delete will remove the provided keys from the tree. If no\n\t// matching key is found, this is a no-op.\n\tDelete(...common.Comparator)\n\t// Get will return a key matching the associated provided\n\t// key if it exists.\n\tGet(...common.Comparator) common.Comparators\n\t// Len returns the number of items in the tree.\n\tLen() uint64\n\t// Query will return a list of Comparators that fall within the\n\t// provided start and stop Comparators. Start is inclusive while\n\t// stop is exclusive, ie [start, stop).\n\tQuery(start, stop common.Comparator) common.Comparators\n\t// Dispose will clean up any resources used by this tree. This\n\t// must be called to prevent a memory leak.\n\tDispose()\n}\n" }, { "path": "btree/palm/key.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage palm\n\nimport \"github.com/Workiva/go-datastructures/common\"\n\nfunc reverseKeys(cmps common.Comparators) common.Comparators {\n\treversed := make(common.Comparators, len(cmps))\n\tfor i := len(cmps) - 1; i >= 0; i-- {\n\t\treversed[len(cmps)-1-i] = cmps[i]\n\t}\n\n\treturn reversed\n}\n\nfunc chunkKeys(keys common.Comparators, numParts int64) []common.Comparators {\n\tparts := make([]common.Comparators, numParts)\n\tfor i := int64(0); i < numParts; i++ {\n\t\tparts[i] = keys[i*int64(len(keys))/numParts : (i+1)*int64(len(keys))/numParts]\n\t}\n\treturn parts\n}\n" }, { "path": "btree/palm/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\npackage palm\n\nimport \"github.com/Workiva/go-datastructures/common\"\n\ntype mockKey int\n\nfunc (mk mockKey) Compare(other common.Comparator) int {\n\totherKey := other.(mockKey)\n\n\tif mk == otherKey {\n\t\treturn 0\n\t}\n\n\tif mk > otherKey {\n\t\treturn 1\n\t}\n\n\treturn -1\n}\n" }, { "path": "btree/palm/node.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage palm\n\nimport (\n\t\"log\"\n\t\"sort\"\n\n\t\"github.com/Workiva/go-datastructures/common\"\n)\n\nfunc getParent(parent *node, key common.Comparator) *node {\n\tvar n *node\n\tfor parent != nil && !parent.isLeaf {\n\t\tn = parent.searchNode(key)\n\t\tparent = n\n\t}\n\n\treturn parent\n}\n\ntype nodes struct {\n\tlist []*node\n}\n\nfunc (ns *nodes) push(n *node) {\n\tns.list = append(ns.list, n)\n}\n\nfunc (ns *nodes) splitAt(i, capacity uint64) (*nodes, *nodes) {\n\ti++\n\tright := make([]*node, uint64(len(ns.list))-i, capacity)\n\tcopy(right, ns.list[i:])\n\tfor j := i; j < uint64(len(ns.list)); j++ {\n\t\tns.list[j] = nil\n\t}\n\tns.list = ns.list[:i]\n\treturn ns, &nodes{list: right}\n}\n\nfunc (ns *nodes) byPosition(pos uint64) *node {\n\tif pos >= uint64(len(ns.list)) {\n\t\treturn nil\n\t}\n\n\treturn ns.list[pos]\n}\n\nfunc (ns *nodes) insertAt(i uint64, n *node) {\n\tns.list = append(ns.list, nil)\n\tcopy(ns.list[i+1:], ns.list[i:])\n\tns.list[i] = n\n}\n\nfunc (ns *nodes) replaceAt(i uint64, n *node) {\n\tns.list[i] = n\n}\n\nfunc (ns *nodes) len() uint64 {\n\treturn uint64(len(ns.list))\n}\n\nfunc newNodes(size uint64) *nodes {\n\treturn &nodes{\n\t\tlist: make([]*node, 0, size),\n\t}\n}\n\ntype keys struct {\n\tlist common.Comparators\n}\n\nfunc (ks *keys) splitAt(i, capacity uint64) (*keys, *keys) {\n\ti++\n\tright := make(common.Comparators, uint64(len(ks.list))-i, capacity)\n\tcopy(right, ks.list[i:])\n\tfor j := i; j < uint64(len(ks.list)); j++ {\n\t\tks.list[j] = nil\n\t}\n\tks.list = ks.list[:i]\n\treturn ks, &keys{list: right}\n}\n\nfunc (ks *keys) len() uint64 {\n\treturn uint64(len(ks.list))\n}\n\nfunc (ks *keys) byPosition(i uint64) common.Comparator {\n\tif i >= uint64(len(ks.list)) {\n\t\treturn nil\n\t}\n\treturn ks.list[i]\n}\n\nfunc (ks *keys) delete(k common.Comparator) common.Comparator {\n\ti := ks.search(k)\n\tif i >= uint64(len(ks.list)) {\n\t\treturn nil\n\t}\n\n\tif ks.list[i].Compare(k) != 0 {\n\t\treturn nil\n\t}\n\told := ks.list[i]\n\n\tcopy(ks.list[i:], ks.list[i+1:])\n\tks.list[len(ks.list)-1] = nil // GC\n\tks.list = ks.list[:len(ks.list)-1]\n\treturn old\n}\n\nfunc (ks *keys) search(key common.Comparator) uint64 {\n\ti := sort.Search(len(ks.list), func(i int) bool {\n\t\treturn ks.list[i].Compare(key) > -1\n\t})\n\n\treturn uint64(i)\n}\n\nfunc (ks *keys) insert(key common.Comparator) (common.Comparator, uint64) {\n\ti := ks.search(key)\n\tif i == uint64(len(ks.list)) {\n\t\tks.list = append(ks.list, key)\n\t\treturn nil, i\n\t}\n\n\tvar old common.Comparator\n\tif ks.list[i].Compare(key) == 0 {\n\t\told = ks.list[i]\n\t\tks.list[i] = key\n\t} else {\n\t\tks.insertAt(i, key)\n\t}\n\n\treturn old, i\n}\n\nfunc (ks *keys) last() common.Comparator {\n\treturn ks.list[len(ks.list)-1]\n}\n\nfunc (ks *keys) insertAt(i uint64, k common.Comparator) {\n\tks.list = append(ks.list, nil)\n\tcopy(ks.list[i+1:], ks.list[i:])\n\tks.list[i] = k\n}\n\nfunc (ks *keys) withPosition(k common.Comparator) (common.Comparator, uint64) {\n\ti := ks.search(k)\n\tif i == uint64(len(ks.list)) {\n\t\treturn nil, i\n\t}\n\tif ks.list[i].Compare(k) == 0 {\n\t\treturn ks.list[i], i\n\t}\n\n\treturn nil, i\n}\n\nfunc newKeys(size uint64) *keys {\n\treturn &keys{\n\t\tlist: make(common.Comparators, 0, size),\n\t}\n}\n\ntype node struct {\n\tkeys *keys\n\tnodes *nodes\n\tisLeaf bool\n\tparent, right *node\n}\n\nfunc (n *node) needsSplit(ary uint64) bool {\n\treturn n.keys.len() >= ary\n}\n\nfunc (n *node) splitLeaf(i, capacity uint64) (common.Comparator, *node, *node) {\n\tkey := n.keys.byPosition(i)\n\t_, rightKeys := n.keys.splitAt(i, capacity)\n\tnn := &node{\n\t\tkeys: rightKeys,\n\t\tnodes: newNodes(uint64(cap(n.nodes.list))),\n\t\tisLeaf: true,\n\t\tright: n.right,\n\t}\n\tn.right = nn\n\treturn key, n, nn\n}\n\nfunc (n *node) splitInternal(i, capacity uint64) (common.Comparator, *node, *node) {\n\tkey := n.keys.byPosition(i)\n\tn.keys.delete(key)\n\n\t_, rightKeys := n.keys.splitAt(i-1, capacity)\n\t_, rightNodes := n.nodes.splitAt(i, capacity)\n\n\tnn := newNode(false, rightKeys, rightNodes)\n\tfor _, n := range rightNodes.list {\n\t\tn.parent = nn\n\t}\n\n\treturn key, n, nn\n}\n\nfunc (n *node) split(i, capacity uint64) (common.Comparator, *node, *node) {\n\tif n.isLeaf {\n\t\treturn n.splitLeaf(i, capacity)\n\t}\n\n\treturn n.splitInternal(i, capacity)\n}\n\nfunc (n *node) search(key common.Comparator) uint64 {\n\treturn n.keys.search(key)\n}\n\nfunc (n *node) searchNode(key common.Comparator) *node {\n\ti := n.search(key)\n\n\treturn n.nodes.byPosition(uint64(i))\n}\n\nfunc (n *node) key() common.Comparator {\n\treturn n.keys.last()\n}\n\nfunc (n *node) print(output *log.Logger) {\n\toutput.Printf(`NODE: %+v, %p`, n, n)\n\tfor _, k := range n.keys.list {\n\t\toutput.Printf(`KEY: %+v`, k)\n\t}\n\tif !n.isLeaf {\n\t\tfor _, n := range n.nodes.list {\n\t\t\tif n == nil {\n\t\t\t\toutput.Println(`NIL NODE`)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tn.print(output)\n\t\t}\n\t}\n}\n\n// Compare is required by the skip.Entry interface but nodes are always\n// added by position so while this method is required it doesn't\n// need to return anything useful.\nfunc (n *node) Compare(e common.Comparator) int {\n\treturn 0\n}\n\nfunc newNode(isLeaf bool, keys *keys, ns *nodes) *node {\n\treturn &node{\n\t\tisLeaf: isLeaf,\n\t\tkeys: keys,\n\t\tnodes: ns,\n\t}\n}\n" }, { "path": "btree/palm/tree.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage palm\n\nimport (\n\t\"log\"\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/Workiva/go-datastructures/common\"\n\t\"github.com/Workiva/go-datastructures/queue\"\n)\n\ntype operation int\n\nconst (\n\tget operation = iota\n\tadd\n\tremove\n\tapply\n)\n\nconst multiThreadAt = 400 // number of keys before we multithread lookups\n\ntype keyBundle struct {\n\tkey common.Comparator\n\tleft, right *node\n}\n\nfunc (kb *keyBundle) dispose(ptree *ptree) {\n\tif ptree.kbRing.Len() == ptree.kbRing.Cap() {\n\t\treturn\n\t}\n\tkb.key, kb.left, kb.right = nil, nil, nil\n\tptree.kbRing.Put(kb)\n}\n\ntype ptree struct {\n\troot *node\n\t_padding0 [8]uint64\n\tnumber uint64\n\t_padding1 [8]uint64\n\tary, bufferSize uint64\n\tactions *queue.RingBuffer\n\tcache []interface{}\n\tbuffer0 [8]uint64\n\tdisposed uint64\n\tbuffer1 [8]uint64\n\trunning uint64\n\t_padding2 [8]uint64\n\tkbRing *queue.RingBuffer\n\tdisposeChannel chan bool\n\tmpChannel chan map[*node][]*keyBundle\n}\n\nfunc (ptree *ptree) checkAndRun(action action) {\n\tif ptree.actions.Len() > 0 {\n\t\tif action != nil {\n\t\t\tptree.actions.Put(action)\n\t\t}\n\t\tif atomic.CompareAndSwapUint64(&ptree.running, 0, 1) {\n\t\t\tvar a interface{}\n\t\t\tvar err error\n\t\t\tfor ptree.actions.Len() > 0 {\n\t\t\t\ta, err = ptree.actions.Get()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tptree.cache = append(ptree.cache, a)\n\t\t\t\tif uint64(len(ptree.cache)) >= ptree.bufferSize {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tgo ptree.operationRunner(ptree.cache, true)\n\t\t}\n\t} else if action != nil {\n\t\tif atomic.CompareAndSwapUint64(&ptree.running, 0, 1) {\n\t\t\tswitch action.operation() {\n\t\t\tcase get:\n\t\t\t\tptree.read(action)\n\t\t\t\taction.complete()\n\t\t\t\tptree.reset()\n\t\t\tcase add, remove:\n\t\t\t\tif len(action.keys()) > multiThreadAt {\n\t\t\t\t\tptree.operationRunner(interfaces{action}, true)\n\t\t\t\t} else {\n\t\t\t\t\tptree.operationRunner(interfaces{action}, false)\n\t\t\t\t}\n\t\t\tcase apply:\n\t\t\t\tq := action.(*applyAction)\n\t\t\t\tn := getParent(ptree.root, q.start)\n\t\t\t\tptree.apply(n, q)\n\t\t\t\tq.complete()\n\t\t\t\tptree.reset()\n\t\t\t}\n\t\t} else {\n\t\t\tptree.actions.Put(action)\n\t\t\tptree.checkAndRun(nil)\n\t\t}\n\t}\n}\n\nfunc (ptree *ptree) init(bufferSize, ary uint64) {\n\tptree.bufferSize = bufferSize\n\tptree.ary = ary\n\tptree.cache = make([]interface{}, 0, bufferSize)\n\tptree.root = newNode(true, newKeys(ary), newNodes(ary))\n\tptree.actions = queue.NewRingBuffer(ptree.bufferSize)\n\tptree.kbRing = queue.NewRingBuffer(1024)\n\tfor i := uint64(0); i < ptree.kbRing.Cap(); i++ {\n\t\tptree.kbRing.Put(&keyBundle{})\n\t}\n\tptree.disposeChannel = make(chan bool)\n\tptree.mpChannel = make(chan map[*node][]*keyBundle, 1024)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo ptree.disposer(&wg)\n\twg.Wait()\n}\n\nfunc (ptree *ptree) newKeyBundle(key common.Comparator) *keyBundle {\n\tif ptree.kbRing.Len() == 0 {\n\t\treturn &keyBundle{key: key}\n\t}\n\tifc, err := ptree.kbRing.Get()\n\tif err != nil {\n\t\treturn nil\n\t}\n\tkb := ifc.(*keyBundle)\n\tkb.key = key\n\treturn kb\n}\n\nfunc (ptree *ptree) operationRunner(xns interfaces, threaded bool) {\n\twriteOperations, deleteOperations, toComplete := ptree.fetchKeys(xns, threaded)\n\tptree.recursiveMutate(writeOperations, deleteOperations, false, threaded)\n\tfor _, a := range toComplete {\n\t\ta.complete()\n\t}\n\n\tptree.reset()\n}\n\nfunc (ptree *ptree) read(action action) {\n\tfor i, k := range action.keys() {\n\t\tn := getParent(ptree.root, k)\n\t\tif n == nil {\n\t\t\taction.keys()[i] = nil\n\t\t} else {\n\t\t\tkey, _ := n.keys.withPosition(k)\n\t\t\tif key == nil {\n\t\t\t\taction.keys()[i] = nil\n\t\t\t} else {\n\t\t\t\taction.keys()[i] = key\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (ptree *ptree) fetchKeys(xns interfaces, inParallel bool) (map[*node][]*keyBundle, map[*node][]*keyBundle, actions) {\n\tif inParallel {\n\t\tptree.fetchKeysInParallel(xns)\n\t} else {\n\t\tptree.fetchKeysInSerial(xns)\n\t}\n\n\twriteOperations := make(map[*node][]*keyBundle)\n\tdeleteOperations := make(map[*node][]*keyBundle)\n\ttoComplete := make(actions, 0, len(xns)/2)\n\tfor _, ifc := range xns {\n\t\taction := ifc.(action)\n\t\tswitch action.operation() {\n\t\tcase add:\n\t\t\tfor i, n := range action.nodes() {\n\t\t\t\twriteOperations[n] = append(writeOperations[n], ptree.newKeyBundle(action.keys()[i]))\n\t\t\t}\n\t\t\ttoComplete = append(toComplete, action)\n\t\tcase remove:\n\t\t\tfor i, n := range action.nodes() {\n\t\t\t\tdeleteOperations[n] = append(deleteOperations[n], ptree.newKeyBundle(action.keys()[i]))\n\t\t\t}\n\t\t\ttoComplete = append(toComplete, action)\n\t\tcase get, apply:\n\t\t\taction.complete()\n\t\t}\n\t}\n\n\treturn writeOperations, deleteOperations, toComplete\n}\n\nfunc (ptree *ptree) apply(n *node, aa *applyAction) {\n\ti := n.search(aa.start)\n\tif i == n.keys.len() { // nothing to apply against\n\t\treturn\n\t}\n\n\tvar k common.Comparator\n\tfor n != nil {\n\t\tfor j := i; j < n.keys.len(); j++ {\n\t\t\tk = n.keys.byPosition(j)\n\t\t\tif aa.stop.Compare(k) < 1 || !aa.fn(k) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t\tn = n.right\n\t\ti = 0\n\t}\n}\n\nfunc (ptree *ptree) disposer(wg *sync.WaitGroup) {\n\twg.Done()\n\n\tfor {\n\t\tselect {\n\t\tcase mp := <-ptree.mpChannel:\n\t\t\tptree.cleanMap(mp)\n\t\tcase <-ptree.disposeChannel:\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (ptree *ptree) fetchKeysInSerial(xns interfaces) {\n\tfor _, ifc := range xns {\n\t\taction := ifc.(action)\n\t\tfor i, key := range action.keys() {\n\t\t\tn := getParent(ptree.root, key)\n\t\t\tswitch action.operation() {\n\t\t\tcase add, remove:\n\t\t\t\taction.addNode(int64(i), n)\n\t\t\tcase get:\n\t\t\t\tif n == nil {\n\t\t\t\t\taction.keys()[i] = nil\n\t\t\t\t} else {\n\t\t\t\t\tk, _ := n.keys.withPosition(key)\n\t\t\t\t\tif k == nil {\n\t\t\t\t\t\taction.keys()[i] = nil\n\t\t\t\t\t} else {\n\t\t\t\t\t\taction.keys()[i] = k\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tcase apply:\n\t\t\t\tq := action.(*applyAction)\n\t\t\t\tptree.apply(n, q)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (ptree *ptree) reset() {\n\tfor i := range ptree.cache {\n\t\tptree.cache[i] = nil\n\t}\n\n\tptree.cache = ptree.cache[:0]\n\tatomic.StoreUint64(&ptree.running, 0)\n\tptree.checkAndRun(nil)\n}\n\nfunc (ptree *ptree) fetchKeysInParallel(xns []interface{}) {\n\tvar forCache struct {\n\t\ti int64\n\t\tbuffer [8]uint64 // different cache lines\n\t\tjs []int64\n\t}\n\n\tfor j := 0; j < len(xns); j++ {\n\t\tforCache.js = append(forCache.js, -1)\n\t}\n\tnumCPU := runtime.NumCPU()\n\tif numCPU > 1 {\n\t\tnumCPU--\n\t}\n\tvar wg sync.WaitGroup\n\twg.Add(numCPU)\n\n\tfor k := 0; k < numCPU; k++ {\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tindex := atomic.LoadInt64(&forCache.i)\n\t\t\t\tif index >= int64(len(xns)) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\taction := xns[index].(action)\n\n\t\t\t\tj := atomic.AddInt64(&forCache.js[index], 1)\n\t\t\t\tif j > int64(len(action.keys())) { // someone else is updating i\n\t\t\t\t\tcontinue\n\t\t\t\t} else if j == int64(len(action.keys())) {\n\t\t\t\t\tatomic.StoreInt64(&forCache.i, index+1)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tn := getParent(ptree.root, action.keys()[j])\n\t\t\t\tswitch action.operation() {\n\t\t\t\tcase add, remove:\n\t\t\t\t\taction.addNode(j, n)\n\t\t\t\tcase get:\n\t\t\t\t\tif n == nil {\n\t\t\t\t\t\taction.keys()[j] = nil\n\t\t\t\t\t} else {\n\t\t\t\t\t\tk, _ := n.keys.withPosition(action.keys()[j])\n\t\t\t\t\t\tif k == nil {\n\t\t\t\t\t\t\taction.keys()[j] = nil\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\taction.keys()[j] = k\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\tcase apply:\n\t\t\t\t\tq := action.(*applyAction)\n\t\t\t\t\tptree.apply(n, q)\n\t\t\t\t}\n\t\t\t}\n\t\t\twg.Done()\n\t\t}()\n\t}\n\twg.Wait()\n}\n\nfunc (ptree *ptree) splitNode(n, parent *node, nodes *[]*node, keys *common.Comparators) {\n\tif !n.needsSplit(ptree.ary) {\n\t\treturn\n\t}\n\n\tlength := n.keys.len()\n\tsplitAt := ptree.ary - 1\n\n\tfor i := splitAt; i < length; i += splitAt {\n\t\toffset := length - i\n\t\tk, left, right := n.split(offset, ptree.ary)\n\t\tleft.right = right\n\t\t*keys = append(*keys, k)\n\t\t*nodes = append(*nodes, left, right)\n\t\tleft.parent = parent\n\t\tright.parent = parent\n\t}\n}\n\nfunc (ptree *ptree) applyNode(n *node, adds, deletes []*keyBundle) {\n\tfor _, kb := range deletes {\n\t\tif n.keys.len() == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tdeleted := n.keys.delete(kb.key)\n\t\tif deleted != nil {\n\t\t\tatomic.AddUint64(&ptree.number, ^uint64(0))\n\t\t}\n\t}\n\n\tfor _, kb := range adds {\n\t\tif n.keys.len() == 0 {\n\t\t\toldKey, _ := n.keys.insert(kb.key)\n\t\t\tif n.isLeaf && oldKey == nil {\n\t\t\t\tatomic.AddUint64(&ptree.number, 1)\n\t\t\t}\n\t\t\tif kb.left != nil {\n\t\t\t\tn.nodes.push(kb.left)\n\t\t\t\tn.nodes.push(kb.right)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\toldKey, index := n.keys.insert(kb.key)\n\t\tif n.isLeaf && oldKey == nil {\n\t\t\tatomic.AddUint64(&ptree.number, 1)\n\t\t}\n\t\tif kb.left != nil {\n\t\t\tn.nodes.replaceAt(index, kb.left)\n\t\t\tn.nodes.insertAt(index+1, kb.right)\n\t\t}\n\t}\n}\n\nfunc (ptree *ptree) cleanMap(op map[*node][]*keyBundle) {\n\tfor _, bundles := range op {\n\t\tfor _, kb := range bundles {\n\t\t\tkb.dispose(ptree)\n\t\t}\n\t}\n}\n\nfunc (ptree *ptree) recursiveMutate(adds, deletes map[*node][]*keyBundle, setRoot, inParallel bool) {\n\tif len(adds) == 0 && len(deletes) == 0 {\n\t\treturn\n\t}\n\n\tif setRoot && len(adds) > 1 {\n\t\tpanic(`SHOULD ONLY HAVE ONE ROOT`)\n\t}\n\n\tifs := make(interfaces, 0, len(adds))\n\tfor n := range adds {\n\t\tif n.parent == nil {\n\t\t\tsetRoot = true\n\t\t}\n\t\tifs = append(ifs, n)\n\t}\n\n\tfor n := range deletes {\n\t\tif n.parent == nil {\n\t\t\tsetRoot = true\n\t\t}\n\n\t\tif _, ok := adds[n]; !ok {\n\t\t\tifs = append(ifs, n)\n\t\t}\n\t}\n\n\tvar dummyRoot *node\n\tif setRoot {\n\t\tdummyRoot = &node{\n\t\t\tkeys: newKeys(ptree.ary),\n\t\t\tnodes: newNodes(ptree.ary),\n\t\t}\n\t}\n\n\tvar write sync.Mutex\n\tnextLayerWrite := make(map[*node][]*keyBundle)\n\tnextLayerDelete := make(map[*node][]*keyBundle)\n\n\tvar mutate func(interfaces, func(interface{}))\n\tif inParallel {\n\t\tmutate = executeInterfacesInParallel\n\t} else {\n\t\tmutate = executeInterfacesInSerial\n\t}\n\n\tmutate(ifs, func(ifc interface{}) {\n\t\tn := ifc.(*node)\n\t\tadds := adds[n]\n\t\tdeletes := deletes[n]\n\n\t\tif len(adds) == 0 && len(deletes) == 0 {\n\t\t\treturn\n\t\t}\n\n\t\tif setRoot {\n\t\t\tptree.root = n\n\t\t}\n\n\t\tparent := n.parent\n\t\tif parent == nil {\n\t\t\tparent = dummyRoot\n\t\t\tsetRoot = true\n\t\t}\n\n\t\tptree.applyNode(n, adds, deletes)\n\n\t\tif n.needsSplit(ptree.ary) {\n\t\t\tkeys := make(common.Comparators, 0, n.keys.len())\n\t\t\tnodes := make([]*node, 0, n.nodes.len())\n\t\t\tptree.splitNode(n, parent, &nodes, &keys)\n\t\t\twrite.Lock()\n\t\t\tfor i, k := range keys {\n\t\t\t\tkb := ptree.newKeyBundle(k)\n\t\t\t\tkb.left = nodes[i*2]\n\t\t\t\tkb.right = nodes[i*2+1]\n\t\t\t\tnextLayerWrite[parent] = append(nextLayerWrite[parent], kb)\n\t\t\t}\n\t\t\twrite.Unlock()\n\t\t}\n\t})\n\n\tptree.mpChannel <- adds\n\tptree.mpChannel <- deletes\n\n\tptree.recursiveMutate(nextLayerWrite, nextLayerDelete, setRoot, inParallel)\n}\n\n// Insert will add the provided keys to the tree.\nfunc (ptree *ptree) Insert(keys ...common.Comparator) {\n\tia := newInsertAction(keys)\n\tptree.checkAndRun(ia)\n\tia.completer.Wait()\n}\n\n// Delete will remove the provided keys from the tree. If no\n// matching key is found, this is a no-op.\nfunc (ptree *ptree) Delete(keys ...common.Comparator) {\n\tra := newRemoveAction(keys)\n\tptree.checkAndRun(ra)\n\tra.completer.Wait()\n}\n\n// Get will retrieve a list of keys from the provided keys.\nfunc (ptree *ptree) Get(keys ...common.Comparator) common.Comparators {\n\tga := newGetAction(keys)\n\tptree.checkAndRun(ga)\n\tga.completer.Wait()\n\treturn ga.result\n}\n\n// Len returns the number of items in the tree.\nfunc (ptree *ptree) Len() uint64 {\n\treturn atomic.LoadUint64(&ptree.number)\n}\n\n// Query will return a list of Comparators that fall within the\n// provided start and stop Comparators. Start is inclusive while\n// stop is exclusive, ie [start, stop).\nfunc (ptree *ptree) Query(start, stop common.Comparator) common.Comparators {\n\tcmps := make(common.Comparators, 0, 32)\n\taa := newApplyAction(func(cmp common.Comparator) bool {\n\t\tcmps = append(cmps, cmp)\n\t\treturn true\n\t}, start, stop)\n\tptree.checkAndRun(aa)\n\taa.completer.Wait()\n\treturn cmps\n}\n\n// Dispose will clean up any resources used by this tree. This\n// must be called to prevent a memory leak.\nfunc (ptree *ptree) Dispose() {\n\tif atomic.LoadUint64(&ptree.disposed) == 1 {\n\t\treturn\n\t}\n\tptree.actions.Dispose()\n\tatomic.StoreUint64(&ptree.disposed, 1)\n\tclose(ptree.disposeChannel)\n}\n\nfunc (ptree *ptree) print(output *log.Logger) {\n\tprintln(`PRINTING TREE`)\n\tif ptree.root == nil {\n\t\treturn\n\t}\n\n\tptree.root.print(output)\n}\n\nfunc newTree(bufferSize, ary uint64) *ptree {\n\tptree := &ptree{}\n\tptree.init(bufferSize, ary)\n\treturn ptree\n}\n\n// New will allocate, initialize, and return a new B-Tree based\n// on PALM principles. This type of tree is suited for in-memory\n// indices in a multi-threaded environment.\nfunc New(bufferSize, ary uint64) BTree {\n\treturn newTree(bufferSize, ary)\n}\n" }, { "path": "btree/palm/tree_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage palm\n\nimport (\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/Workiva/go-datastructures/common\"\n)\n\nfunc checkTree(t testing.TB, tree *ptree) bool {\n\treturn true\n\n\tif tree.root == nil {\n\t\treturn true\n\t}\n\n\treturn checkNode(t, tree.root)\n}\n\nfunc checkNode(t testing.TB, n *node) bool {\n\tif n.keys.len() == 0 {\n\t\tassert.Equal(t, uint64(0), n.nodes.len())\n\t\treturn false\n\t}\n\n\tif n.isLeaf {\n\t\tassert.Equal(t, uint64(0), n.nodes.len())\n\t\treturn false\n\t}\n\n\tif !assert.Equal(t, n.keys.len()+1, n.nodes.len()) {\n\t\treturn false\n\t}\n\n\tfor i, k := range n.keys.list {\n\t\tnd := n.nodes.list[i]\n\t\tif !assert.NotNil(t, nd) {\n\t\t\treturn false\n\t\t}\n\n\t\tif !assert.True(t, k.Compare(nd.key()) >= 0) {\n\t\t\tt.Logf(`N: %+v %p, n.keys[i]: %+v, n.nodes[i]: %+v`, n, n, k, nd)\n\t\t\treturn false\n\t\t}\n\t}\n\n\tk := n.keys.last()\n\tnd := n.nodes.byPosition(n.nodes.len() - 1)\n\tif !assert.True(t, k.Compare(nd.key()) < 0) {\n\t\tt.Logf(`m: %+v, %p, n.nodes[len(n.nodes)-1].key(): %+v, n.keys.last(): %+v`, n, n, nd, k)\n\t\treturn false\n\t}\n\tfor _, child := range n.nodes.list {\n\t\tif !assert.NotNil(t, child) {\n\t\t\treturn false\n\t\t}\n\t\tif !checkNode(t, child) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc getConsoleLogger() *log.Logger {\n\treturn log.New(os.Stderr, \"\", log.LstdFlags)\n}\n\nfunc generateRandomKeys(num int) common.Comparators {\n\tkeys := make(common.Comparators, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tm := rand.Int()\n\t\tkeys = append(keys, mockKey(m%50))\n\t}\n\treturn keys\n}\n\nfunc generateKeys(num int) common.Comparators {\n\tkeys := make(common.Comparators, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tkeys = append(keys, mockKey(i))\n\t}\n\n\treturn keys\n}\n\nfunc TestSimpleInsert(t *testing.T) {\n\ttree := newTree(16, 16)\n\tdefer tree.Dispose()\n\tm1 := mockKey(1)\n\n\ttree.Insert(m1)\n\tassert.Equal(t, common.Comparators{m1}, tree.Get(m1))\n\tassert.Equal(t, uint64(1), tree.Len())\n\tcheckTree(t, tree)\n}\n\nfunc TestSimpleDelete(t *testing.T) {\n\ttree := newTree(8, 8)\n\tdefer tree.Dispose()\n\tm1 := mockKey(1)\n\ttree.Insert(m1)\n\n\ttree.Delete(m1)\n\tassert.Equal(t, uint64(0), tree.Len())\n\tassert.Equal(t, common.Comparators{nil}, tree.Get(m1))\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleAdd(t *testing.T) {\n\ttree := newTree(16, 16)\n\tdefer tree.Dispose()\n\tm1 := mockKey(1)\n\tm2 := mockKey(10)\n\n\ttree.Insert(m1, m2)\n\tif !assert.Equal(t, common.Comparators{m1, m2}, tree.Get(m1, m2)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tassert.Equal(t, uint64(2), tree.Len())\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleDelete(t *testing.T) {\n\ttree := newTree(16, 16)\n\tdefer tree.Dispose()\n\tm1 := mockKey(1)\n\tm2 := mockKey(10)\n\ttree.Insert(m1, m2)\n\n\ttree.Delete(m1, m2)\n\tassert.Equal(t, uint64(0), tree.Len())\n\tassert.Equal(t, common.Comparators{nil, nil}, tree.Get(m1, m2))\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitOddAryReverseOrder(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\tkeys := generateKeys(100)\n\treversed := reverseKeys(keys)\n\n\ttree.Insert(reversed...)\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleDeleteOddAryReverseOrder(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\tkeys := generateKeys(100)\n\treversed := reverseKeys(keys)\n\ttree.Insert(reversed...)\n\tassert.Equal(t, uint64(100), tree.Len())\n\n\ttree.Delete(reversed...)\n\tassert.Equal(t, uint64(0), tree.Len())\n\tfor _, k := range reversed {\n\t\tassert.Equal(t, common.Comparators{nil}, tree.Get(k))\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitOddAry(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\tkeys := generateKeys(100)\n\n\ttree.Insert(keys...)\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitOddAryRandomOrder(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\tkeys := generateRandomKeys(10)\n\n\ttree.Insert(keys...)\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleBulkInsertOddAry(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\tkeys1 := generateRandomKeys(100)\n\tkeys2 := generateRandomKeys(100)\n\n\ttree.Insert(keys1...)\n\n\tif !assert.Equal(t, keys1, tree.Get(keys1...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\n\ttree.Insert(keys2...)\n\n\tif !assert.Equal(t, keys2, tree.Get(keys2...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleBulkInsertEvenAry(t *testing.T) {\n\ttree := newTree(4, 4)\n\tdefer tree.Dispose()\n\tkeys1 := generateRandomKeys(100)\n\tkeys2 := generateRandomKeys(100)\n\n\ttree.Insert(keys1...)\n\ttree.Insert(keys2...)\n\n\tif !assert.Equal(t, keys1, tree.Get(keys1...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\n\tif !assert.Equal(t, keys2, tree.Get(keys2...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitEvenAryReverseOrder(t *testing.T) {\n\ttree := newTree(4, 4)\n\tdefer tree.Dispose()\n\tkeys := generateKeys(100)\n\treversed := reverseKeys(keys)\n\n\ttree.Insert(reversed...)\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitEvenAry(t *testing.T) {\n\ttree := newTree(4, 4)\n\tdefer tree.Dispose()\n\tkeys := generateKeys(100)\n\n\ttree.Insert(keys...)\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestMultipleInsertCausesSplitEvenAryRandomOrder(t *testing.T) {\n\ttree := newTree(4, 4)\n\tdefer tree.Dispose()\n\tkeys := generateRandomKeys(100)\n\n\ttree.Insert(keys...)\n\tif !assert.Equal(t, keys, tree.Get(keys...)) {\n\t\ttree.print(getConsoleLogger())\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestInsertOverwrite(t *testing.T) {\n\ttree := newTree(4, 4)\n\tdefer tree.Dispose()\n\tkeys := generateKeys(10)\n\tduplicate := mockKey(0)\n\ttree.Insert(keys...)\n\n\ttree.Insert(duplicate)\n\tassert.Equal(t, common.Comparators{duplicate}, tree.Get(duplicate))\n\tcheckTree(t, tree)\n}\n\nfunc TestSimultaneousReadsAndWrites(t *testing.T) {\n\tnumLoops := 3\n\tkeys := make([]common.Comparators, 0, numLoops)\n\tfor i := 0; i < numLoops; i++ {\n\t\tkeys = append(keys, generateRandomKeys(10))\n\t}\n\n\ttree := newTree(16, 16)\n\tdefer tree.Dispose()\n\tvar wg sync.WaitGroup\n\twg.Add(numLoops)\n\tfor i := 0; i < numLoops; i++ {\n\t\tgo func(i int) {\n\t\t\ttree.Insert(keys[i]...)\n\t\t\ttree.Get(keys[i]...)\n\t\t\twg.Done()\n\t\t}(i)\n\t}\n\n\twg.Wait()\n\n\tfor i := 0; i < numLoops; i++ {\n\t\tassert.Equal(t, keys[i], tree.Get(keys[i]...))\n\t}\n\tcheckTree(t, tree)\n}\n\nfunc TestInsertAndDelete(t *testing.T) {\n\ttree := newTree(1024, 1024)\n\tdefer tree.Dispose()\n\n\tkeys := generateKeys(100)\n\tkeys1 := keys[:50]\n\tkeys2 := keys[50:]\n\ttree.Insert(keys1...)\n\tassert.Equal(t, uint64(len(keys1)), tree.Len())\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\n\tgo func() {\n\t\ttree.Insert(keys2...)\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\ttree.Delete(keys1...)\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\n\tassert.Equal(t, uint64(len(keys2)), tree.Len())\n\tassert.Equal(t, keys2, tree.Get(keys2...))\n}\n\nfunc TestInsertAndDeletesWithSplits(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\n\tkeys := generateKeys(100)\n\tkeys1 := keys[:50]\n\tkeys2 := keys[50:]\n\ttree.Insert(keys1...)\n\tassert.Equal(t, uint64(len(keys1)), tree.Len())\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\n\tgo func() {\n\t\ttree.Insert(keys2...)\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\ttree.Delete(keys1...)\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\n\tassert.Equal(t, uint64(len(keys2)), tree.Len())\n\tassert.Equal(t, keys2, tree.Get(keys2...))\n}\n\nfunc TestSimpleQuery(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\tm1 := mockKey(1)\n\ttree.Insert(m1)\n\n\tresult := tree.Query(mockKey(0), mockKey(5))\n\tassert.Equal(t, common.Comparators{m1}, result)\n\n\tresult = tree.Query(mockKey(0), mockKey(1))\n\tassert.Len(t, result, 0)\n\n\tresult = tree.Query(mockKey(2), mockKey(10))\n\tassert.Len(t, result, 0)\n\n\tresult = tree.Query(mockKey(1), mockKey(10))\n\tassert.Equal(t, common.Comparators{m1}, result)\n}\n\nfunc TestMultipleQuery(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\tm1 := mockKey(1)\n\tm2 := mockKey(5)\n\ttree.Insert(m1, m2)\n\n\tresult := tree.Query(mockKey(0), mockKey(10))\n\tassert.Equal(t, common.Comparators{m1, m2}, result)\n\n\tresult = tree.Query(mockKey(1), mockKey(5))\n\tassert.Equal(t, common.Comparators{m1}, result)\n\n\tresult = tree.Query(mockKey(6), mockKey(10))\n\tassert.Len(t, result, 0)\n\n\tresult = tree.Query(mockKey(5), mockKey(10))\n\tassert.Equal(t, common.Comparators{m2}, result)\n}\n\nfunc TestCrossNodeQuery(t *testing.T) {\n\ttree := newTree(3, 3)\n\tdefer tree.Dispose()\n\tkeys := generateKeys(100)\n\ttree.Insert(keys...)\n\n\tresult := tree.Query(mockKey(0), mockKey(len(keys)))\n\tif !assert.Equal(t, keys, result) {\n\t\ttree.print(getConsoleLogger())\n\t}\n}\n\nfunc BenchmarkReadAndWrites(b *testing.B) {\n\tnumItems := 1000\n\tkeys := make([]common.Comparators, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tkeys = append(keys, generateRandomKeys(numItems))\n\t}\n\n\ttree := newTree(8, 8)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(keys[i]...)\n\t\ttree.Get(keys[i]...)\n\t}\n}\n\nfunc BenchmarkSimultaneousReadsAndWrites(b *testing.B) {\n\tnumItems := 10000\n\tnumRoutines := 8\n\tkeys := generateRandomKeys(numItems)\n\tchunks := chunkKeys(keys, int64(numRoutines))\n\n\ttrees := make([]*ptree, 0, numItems)\n\tfor i := 0; i < b.N; i++ {\n\t\ttrees = append(trees, newTree(8, 8))\n\t}\n\n\tvar wg sync.WaitGroup\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\twg.Add(numRoutines)\n\t\tfor j := 0; j < numRoutines; j++ {\n\t\t\tgo func(i, j int) {\n\t\t\t\ttrees[i].Insert(chunks[j]...)\n\t\t\t\ttrees[i].Get(chunks[j]...)\n\t\t\t\twg.Done()\n\t\t\t}(i, j)\n\t\t}\n\n\t\twg.Wait()\n\t}\n}\n\nfunc BenchmarkBulkAdd(b *testing.B) {\n\tnumItems := 10000\n\tkeys := generateRandomKeys(numItems)\n\ttrees := make([]*ptree, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\ttrees = append(trees, newTree(8, 8))\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttrees[i].Insert(keys...)\n\t}\n}\n\nfunc BenchmarkAdd(b *testing.B) {\n\tnumItems := b.N\n\tkeys := generateRandomKeys(numItems)\n\ttree := newTree(8, 8) // writes will be amortized over node splits\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(keys[i%numItems])\n\t}\n}\n\nfunc BenchmarkBulkAddToExisting(b *testing.B) {\n\tnumItems := 100000\n\tkeySet := make([]common.Comparators, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tkeySet = append(keySet, generateRandomKeys(numItems))\n\t}\n\n\ttree := newTree(8, 8)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(keySet[i]...)\n\t}\n}\n\nfunc BenchmarkGet(b *testing.B) {\n\tnumItems := 10000\n\tkeys := generateRandomKeys(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(keys...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Get(keys[i%numItems])\n\t}\n}\n\nfunc BenchmarkBulkGet(b *testing.B) {\n\tnumItems := b.N\n\tkeys := generateRandomKeys(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(keys...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Get(keys...)\n\t}\n}\n\nfunc BenchmarkDelete(b *testing.B) {\n\tnumItems := b.N\n\tkeys := generateRandomKeys(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(keys...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Delete(keys[i%numItems])\n\t}\n}\n\nfunc BenchmarkBulkDelete(b *testing.B) {\n\tnumItems := 10000\n\tkeys := generateRandomKeys(numItems)\n\ttrees := make([]*ptree, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\ttree := newTree(8, 8)\n\t\ttree.Insert(keys...)\n\t\ttrees = append(trees, tree)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttrees[i].Delete(keys...)\n\t}\n}\n\nfunc BenchmarkFindQuery(b *testing.B) {\n\tnumItems := b.N\n\tkeys := generateKeys(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(keys...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Query(mockKey(numItems/2), mockKey(numItems/2+1))\n\t}\n}\n\nfunc BenchmarkExecuteQuery(b *testing.B) {\n\tnumItems := b.N\n\tkeys := generateKeys(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(keys...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Query(mockKey(0), mockKey(numItems))\n\t}\n}\n" }, { "path": "btree/plus/btree.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage btree/plus implements the ubiquitous B+ tree. As of this writing,\nthe tree is not quite finished. The delete-node merge functionality needs\nto be added. There are also some performance improvements that can be\nmade, with some possible concurrency mechanisms.\n\nThis is a mutable b-tree so it is not threadsafe.\n\nPerformance characteristics:\nSpace: O(n)\nInsert: O(log n)\nSearch: O(log n)\n\nBenchmarkIteration-8\t \t10000\t \t\t \t109347 ns/op\nBenchmarkInsert-8\t \t\t3000000\t \t\t608 ns/op\nBenchmarkGet-8\t \t\t\t3000000\t \t\t627 ns/op\n*/\npackage plus\n\nfunc keySearch(keys keys, key Key) int {\n\tlow, high := 0, len(keys)-1\n\tvar mid int\n\tfor low <= high {\n\t\tmid = (high + low) / 2\n\t\tswitch keys[mid].Compare(key) {\n\t\tcase 1:\n\t\t\tlow = mid + 1\n\t\tcase -1:\n\t\t\thigh = mid - 1\n\t\tcase 0:\n\t\t\treturn mid\n\t\t}\n\t}\n\treturn low\n}\n\ntype btree struct {\n\troot node\n\tnodeSize, number uint64\n}\n\nfunc (tree *btree) insert(key Key) {\n\tif tree.root == nil {\n\t\tn := newLeafNode(tree.nodeSize)\n\t\tn.insert(tree, key)\n\t\ttree.number = 1\n\t\treturn\n\t}\n\n\tresult := tree.root.insert(tree, key)\n\tif result {\n\t\ttree.number++\n\t}\n\n\tif tree.root.needsSplit(tree.nodeSize) {\n\t\ttree.root = split(tree, nil, tree.root)\n\t}\n}\n\n// Insert will insert the provided keys into the btree. This is an\n// O(m*log n) operation where m is the number of keys to be inserted\n// and n is the number of items in the tree.\nfunc (tree *btree) Insert(keys ...Key) {\n\tfor _, key := range keys {\n\t\ttree.insert(key)\n\t}\n}\n\n// Iter returns an iterator that can be used to traverse the b-tree\n// starting from the specified key or its successor.\nfunc (tree *btree) Iter(key Key) Iterator {\n\tif tree.root == nil {\n\t\treturn nilIterator()\n\t}\n\n\treturn tree.root.find(key)\n}\n\nfunc (tree *btree) get(key Key) Key {\n\titer := tree.root.find(key)\n\tif !iter.Next() {\n\t\treturn nil\n\t}\n\n\tif iter.Value().Compare(key) == 0 {\n\t\treturn iter.Value()\n\t}\n\n\treturn nil\n}\n\n// Get will retrieve any keys matching the provided keys in the tree.\n// Returns nil in any place of a key that couldn't be found. Each lookup\n// is an O(log n) operation.\nfunc (tree *btree) Get(keys ...Key) Keys {\n\tresults := make(Keys, 0, len(keys))\n\tfor _, k := range keys {\n\t\tresults = append(results, tree.get(k))\n\t}\n\n\treturn results\n}\n\n// Len returns the number of items in this tree.\nfunc (tree *btree) Len() uint64 {\n\treturn tree.number\n}\n\nfunc newBTree(nodeSize uint64) *btree {\n\treturn &btree{\n\t\tnodeSize: nodeSize,\n\t\troot: newLeafNode(nodeSize),\n\t}\n}\n" }, { "path": "btree/plus/btree_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage plus\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestSearchKeys(t *testing.T) {\n\tkeys := keys{newMockKey(1), newMockKey(2), newMockKey(4)}\n\n\ttestKey := newMockKey(5)\n\tassert.Equal(t, 3, keySearch(keys, testKey))\n\n\ttestKey = newMockKey(2)\n\tassert.Equal(t, 1, keySearch(keys, testKey))\n\n\ttestKey = newMockKey(0)\n\tassert.Equal(t, 0, keySearch(keys, testKey))\n\n\ttestKey = newMockKey(3)\n\tassert.Equal(t, 2, keySearch(keys, testKey))\n\n\tassert.Equal(t, 0, keySearch(nil, testKey))\n}\n\nfunc TestTreeInsert2_3_4(t *testing.T) {\n\ttree := newBTree(3)\n\tkeys := constructMockKeys(4)\n\n\ttree.Insert(keys...)\n\n\tassert.Len(t, tree.root.(*inode).keys, 2)\n\tassert.Len(t, tree.root.(*inode).nodes, 3)\n\tassert.IsType(t, &inode{}, tree.root)\n}\n\nfunc TestTreeInsert3_4_5(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\ttree.Insert(keys...)\n\n\tassert.Len(t, tree.root.(*inode).keys, 1)\n\tassert.Len(t, tree.root.(*inode).nodes, 2)\n\tassert.IsType(t, &inode{}, tree.root)\n}\n\nfunc TestTreeInsertQuery2_3_4(t *testing.T) {\n\ttree := newBTree(3)\n\tkeys := constructMockKeys(4)\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInsertQuery3_4_5(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInsertReverseOrder2_3_4(t *testing.T) {\n\ttree := newBTree(3)\n\tkeys := constructMockKeys(4)\n\tkeys.reverse()\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\tkeys.reverse() // we want to fetch things in the correct\n\t// ascending order\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInsertReverseOrder3_4_5(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\tkeys.reverse()\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\tkeys.reverse() // we want to fetch things in the correct\n\t// ascending order\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInsert3_4_5_WithEndDuplicate(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\ttree.Insert(keys...)\n\tduplicate := newMockKey(4)\n\ttree.Insert(duplicate)\n\tkeys[4] = duplicate\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInsert3_4_5_WithMiddleDuplicate(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\ttree.Insert(keys...)\n\tduplicate := newMockKey(2)\n\ttree.Insert(duplicate)\n\tkeys[2] = duplicate\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInsert3_4_5WithEarlyDuplicate(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\ttree.Insert(keys...)\n\tduplicate := newMockKey(0)\n\ttree.Insert(duplicate)\n\tkeys[0] = duplicate\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInsert3_4_5WithDuplicateID(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\tkey := newMockKey(2)\n\ttree.Insert(keys...)\n\ttree.Insert(key)\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInsert3_4_5MiddleQuery(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(2))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys[2:], result)\n}\n\nfunc TestTreeInsert3_4_5LateQuery(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(4))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys[4:], result)\n}\n\nfunc TestTreeInsert3_4_5AfterQuery(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(5)\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(5))\n\tresult := iter.exhaust()\n\n\tassert.Len(t, result, 0)\n}\n\nfunc TestTreeInternalNodeSplit(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(10)\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInternalNodeSplitReverseOrder(t *testing.T) {\n\ttree := newBTree(4)\n\tkeys := constructMockKeys(10)\n\tkeys.reverse()\n\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\tkeys.reverse()\n\n\tassert.Equal(t, keys, result)\n}\n\nfunc TestTreeInternalNodeSplitRandomOrder(t *testing.T) {\n\tids := []int{6, 2, 9, 0, 3, 4, 7, 1, 8, 5}\n\tkeys := make(keys, 0, len(ids))\n\n\tfor _, id := range ids {\n\t\tkeys = append(keys, newMockKey(id))\n\t}\n\n\ttree := newBTree(4)\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(0))\n\tresult := iter.exhaust()\n\n\tassert.Len(t, result, 10)\n\tfor i, key := range result {\n\t\tassert.Equal(t, newMockKey(i), key)\n\t}\n}\n\nfunc TestTreeRandomOrderQuery(t *testing.T) {\n\tids := []int{6, 2, 9, 0, 3, 4, 7, 1, 8, 5}\n\tkeys := make(keys, 0, len(ids))\n\n\tfor _, id := range ids {\n\t\tkeys = append(keys, newMockKey(id))\n\t}\n\n\ttree := newBTree(4)\n\ttree.Insert(keys...)\n\n\titer := tree.Iter(newMockKey(4))\n\tresult := iter.exhaust()\n\n\tassert.Len(t, result, 6)\n\tfor i, key := range result {\n\t\tassert.Equal(t, newMockKey(i+4), key)\n\t}\n}\n\nfunc TestTreeGet(t *testing.T) {\n\tkeys := constructRandomMockKeys(100)\n\ttree := newBTree(64)\n\ttree.Insert(keys...)\n\n\tassert.Equal(t, uint64(100), tree.Len())\n\tfromTree := tree.Get(keys...)\n\tfor _, key := range keys {\n\t\tassert.Contains(t, fromTree, key)\n\t}\n}\n\nfunc TestTreeGetNotFound(t *testing.T) {\n\tkeys := constructMockKeys(5)\n\ttree := newBTree(64)\n\ttree.Insert(keys...)\n\n\tassert.Equal(t, Keys{nil}, tree.Get(newMockKey(20)))\n}\n\nfunc TestGetExactMatchesOnly(t *testing.T) {\n\tk1 := newMockKey(0)\n\tk2 := newMockKey(5)\n\ttree := newBTree(64)\n\ttree.Insert(k1, k2)\n\n\tassert.Equal(t, Keys{nil}, tree.Get(newMockKey(3)))\n}\n\nfunc BenchmarkIteration(b *testing.B) {\n\tnumItems := 1000\n\tary := uint64(16)\n\n\tkeys := constructMockKeys(numItems)\n\ttree := newBTree(ary)\n\ttree.Insert(keys...)\n\n\tsearchKey := newMockKey(0)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\titer := tree.Iter(searchKey)\n\t\titer.exhaust()\n\t}\n}\n\nfunc BenchmarkInsert(b *testing.B) {\n\tnumItems := b.N\n\tary := uint64(16)\n\n\tkeys := constructMockKeys(numItems)\n\ttree := newBTree(ary)\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(keys[i%numItems])\n\t}\n}\n\nfunc BenchmarkBulkAdd(b *testing.B) {\n\tnumItems := 10000\n\tkeys := constructRandomMockKeys(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree := newBTree(1024)\n\t\ttree.Insert(keys...)\n\t}\n}\n\nfunc BenchmarkGet(b *testing.B) {\n\tnumItems := b.N\n\tary := uint64(16)\n\n\tkeys := constructMockKeys(numItems)\n\ttree := newBTree(ary)\n\ttree.Insert(keys...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Get(keys[i%numItems])\n\t}\n}\n\nfunc BenchmarkBulkAddToExisting(b *testing.B) {\n\tnumItems := 100000\n\tkeySet := make([]keys, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tkeySet = append(keySet, constructRandomMockKeys(numItems))\n\t}\n\n\ttree := newBTree(1024)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(keySet[i]...)\n\t}\n}\n\nfunc BenchmarkReadAndWrites(b *testing.B) {\n\tnumItems := 1000\n\tks := make([]keys, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tks = append(ks, constructRandomMockKeys(numItems))\n\t}\n\n\ttree := newBTree(16)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(ks[i]...)\n\t\ttree.Get(ks[i]...)\n\t}\n}\n\nfunc BenchmarkSimultaneousReadsAndWrites(b *testing.B) {\n\tnumItems := 10000\n\tnumRoutines := 8\n\tkeys := constructRandomMockKeys(numItems)\n\tchunks := chunkKeys(keys, int64(numRoutines))\n\n\ttrees := make([]*btree, 0, numItems)\n\tfor i := 0; i < b.N; i++ {\n\t\ttrees = append(trees, newBTree(8))\n\t}\n\n\tvar wg sync.WaitGroup\n\tvar lock sync.Mutex\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\twg.Add(numRoutines)\n\t\tfor j := 0; j < numRoutines; j++ {\n\t\t\tgo func(i, j int) {\n\t\t\t\tlock.Lock()\n\t\t\t\ttrees[i].Insert(chunks[j]...)\n\t\t\t\ttrees[i].Get(chunks[j]...)\n\t\t\t\tlock.Unlock()\n\t\t\t\twg.Done()\n\t\t\t}(i, j)\n\t\t}\n\n\t\twg.Wait()\n\t}\n}\n" }, { "path": "btree/plus/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage plus\n\n// Keys is a typed list of Key interfaces.\ntype Keys []Key\n\ntype Key interface {\n\t// Compare should return an int indicating how this key relates\n\t// to the provided key. -1 will indicate less than, 0 will indicate\n\t// equality, and 1 will indicate greater than. Duplicate keys\n\t// are allowed, but duplicate IDs are not.\n\tCompare(Key) int\n}\n\n// Iterator will be called with matching keys until either false is\n// returned or we run out of keys to iterate.\ntype Iterator interface {\n\t// Next will move the iterator to the next position and return\n\t// a bool indicating if there is a value.\n\tNext() bool\n\t// Value returns a Key at the associated iterator position. Returns\n\t// nil if the iterator is exhausted or has never been nexted.\n\tValue() Key\n\t// exhaust is an internal helper method to iterate this iterator\n\t// until exhausted and returns the resulting list of keys.\n\texhaust() keys\n}\n" }, { "path": "btree/plus/iterator.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage plus\n\nconst iteratorExhausted = -2\n\ntype iterator struct {\n\tnode *lnode\n\tindex int\n}\n\nfunc (iter *iterator) Next() bool {\n\tif iter.index == iteratorExhausted {\n\t\treturn false\n\t}\n\n\titer.index++\n\tif iter.index >= len(iter.node.keys) {\n\t\titer.node = iter.node.pointer\n\t\tif iter.node == nil {\n\t\t\titer.index = iteratorExhausted\n\t\t\treturn false\n\t\t}\n\t\titer.index = 0\n\t}\n\n\treturn true\n}\n\nfunc (iter *iterator) Value() Key {\n\tif iter.index == iteratorExhausted ||\n\t\titer.index < 0 || iter.index >= len(iter.node.keys) {\n\n\t\treturn nil\n\t}\n\n\treturn iter.node.keys[iter.index]\n}\n\n// exhaust is a test function that's not exported\nfunc (iter *iterator) exhaust() keys {\n\tkeys := make(keys, 0, 10)\n\tfor iter := iter; iter.Next(); {\n\t\tkeys = append(keys, iter.Value())\n\t}\n\n\treturn keys\n}\n\nfunc nilIterator() *iterator {\n\treturn &iterator{\n\t\tindex: iteratorExhausted,\n\t}\n}\n" }, { "path": "btree/plus/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage plus\n\nfunc chunkKeys(ks keys, numParts int64) []keys {\n\tparts := make([]keys, numParts)\n\tfor i := int64(0); i < numParts; i++ {\n\t\tparts[i] = ks[i*int64(len(ks))/numParts : (i+1)*int64(len(ks))/numParts]\n\t}\n\treturn parts\n}\n\ntype mockKey struct {\n\tvalue int\n}\n\nfunc (mk *mockKey) Compare(other Key) int {\n\tkey := other.(*mockKey)\n\tif key.value == mk.value {\n\t\treturn 0\n\t}\n\tif key.value > mk.value {\n\t\treturn 1\n\t}\n\n\treturn -1\n}\n\nfunc newMockKey(value int) *mockKey {\n\treturn &mockKey{value}\n}\n" }, { "path": "btree/plus/node.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage plus\n\nfunc split(tree *btree, parent, child node) node {\n\tif !child.needsSplit(tree.nodeSize) {\n\t\treturn parent\n\t}\n\n\tkey, left, right := child.split()\n\tif parent == nil {\n\t\tin := newInternalNode(tree.nodeSize)\n\t\tin.keys = append(in.keys, key)\n\t\tin.nodes = append(in.nodes, left)\n\t\tin.nodes = append(in.nodes, right)\n\t\treturn in\n\t}\n\n\tp := parent.(*inode)\n\ti := p.search(key)\n\t// we want to ensure if the children are leaves we set\n\t// the left node's left sibling to point to left\n\tif cr, ok := left.(*lnode); ok {\n\t\tif i > 0 {\n\t\t\tp.nodes[i-1].(*lnode).pointer = cr\n\t\t}\n\t}\n\tp.keys.insertAt(i, key)\n\tp.nodes[i] = left\n\tp.nodes.insertAt(i+1, right)\n\n\treturn parent\n}\n\ntype node interface {\n\tinsert(tree *btree, key Key) bool\n\tneedsSplit(nodeSize uint64) bool\n\t// key is the median key while left and right nodes\n\t// represent the left and right nodes respectively\n\tsplit() (Key, node, node)\n\tsearch(key Key) int\n\tfind(key Key) *iterator\n}\n\ntype nodes []node\n\nfunc (nodes *nodes) insertAt(i int, node node) {\n\tif i == len(*nodes) {\n\t\t*nodes = append(*nodes, node)\n\t\treturn\n\t}\n\n\t*nodes = append(*nodes, nil)\n\tcopy((*nodes)[i+1:], (*nodes)[i:])\n\t(*nodes)[i] = node\n}\n\nfunc (ns nodes) splitAt(i int) (nodes, nodes) {\n\tleft := make(nodes, i, cap(ns))\n\tright := make(nodes, len(ns)-i, cap(ns))\n\tcopy(left, ns[:i])\n\tcopy(right, ns[i:])\n\treturn left, right\n}\n\ntype inode struct {\n\tkeys keys\n\tnodes nodes\n}\n\nfunc (node *inode) search(key Key) int {\n\treturn node.keys.search(key)\n}\n\nfunc (node *inode) find(key Key) *iterator {\n\ti := node.search(key)\n\tif i == len(node.keys) {\n\t\treturn node.nodes[len(node.nodes)-1].find(key)\n\t}\n\n\tfound := node.keys[i]\n\tswitch found.Compare(key) {\n\tcase 0, 1:\n\t\treturn node.nodes[i+1].find(key)\n\tdefault:\n\t\treturn node.nodes[i].find(key)\n\t}\n}\n\nfunc (n *inode) insert(tree *btree, key Key) bool {\n\ti := n.search(key)\n\tvar child node\n\tif i == len(n.keys) { // we want the last child node in this case\n\t\tchild = n.nodes[len(n.nodes)-1]\n\t} else {\n\t\tmatch := n.keys[i]\n\t\tswitch match.Compare(key) {\n\t\tcase 1, 0:\n\t\t\tchild = n.nodes[i+1]\n\t\tdefault:\n\t\t\tchild = n.nodes[i]\n\t\t}\n\t}\n\n\tresult := child.insert(tree, key)\n\tif !result { // no change of state occurred\n\t\treturn result\n\t}\n\n\tif child.needsSplit(tree.nodeSize) {\n\t\tsplit(tree, n, child)\n\t}\n\n\treturn result\n}\n\nfunc (n *inode) needsSplit(nodeSize uint64) bool {\n\treturn uint64(len(n.keys)) >= nodeSize\n}\n\nfunc (n *inode) split() (Key, node, node) {\n\tif len(n.keys) < 3 {\n\t\treturn nil, nil, nil\n\t}\n\n\ti := len(n.keys) / 2\n\tkey := n.keys[i]\n\n\tourKeys := make(keys, len(n.keys)-i-1, cap(n.keys))\n\totherKeys := make(keys, i, cap(n.keys))\n\tcopy(ourKeys, n.keys[i+1:])\n\tcopy(otherKeys, n.keys[:i])\n\tleft, right := n.nodes.splitAt(i + 1)\n\totherNode := &inode{\n\t\tkeys: otherKeys,\n\t\tnodes: left,\n\t}\n\tn.keys = ourKeys\n\tn.nodes = right\n\treturn key, otherNode, n\n}\n\nfunc newInternalNode(size uint64) *inode {\n\treturn &inode{\n\t\tkeys: make(keys, 0, size),\n\t\tnodes: make(nodes, 0, size+1),\n\t}\n}\n\ntype lnode struct {\n\t// points to the left leaf node is there is one\n\tpointer *lnode\n\tkeys keys\n}\n\nfunc (node *lnode) search(key Key) int {\n\treturn node.keys.search(key)\n}\n\nfunc (lnode *lnode) insert(tree *btree, key Key) bool {\n\ti := keySearch(lnode.keys, key)\n\tvar inserted bool\n\tif i == len(lnode.keys) { // simple append will do\n\t\tlnode.keys = append(lnode.keys, key)\n\t\tinserted = true\n\t} else {\n\t\tif lnode.keys[i].Compare(key) == 0 {\n\t\t\tlnode.keys[i] = key\n\t\t} else {\n\t\t\tlnode.keys.insertAt(i, key)\n\t\t\tinserted = true\n\t\t}\n\t}\n\n\tif !inserted {\n\t\treturn false\n\t}\n\n\treturn true\n}\n\nfunc (node *lnode) find(key Key) *iterator {\n\ti := node.search(key)\n\tif i == len(node.keys) {\n\t\tif node.pointer == nil {\n\t\t\treturn nilIterator()\n\t\t}\n\n\t\treturn &iterator{\n\t\t\tnode: node.pointer,\n\t\t\tindex: -1,\n\t\t}\n\t}\n\n\titer := &iterator{\n\t\tnode: node,\n\t\tindex: i - 1,\n\t}\n\treturn iter\n}\n\nfunc (node *lnode) split() (Key, node, node) {\n\tif len(node.keys) < 2 {\n\t\treturn nil, nil, nil\n\t}\n\ti := len(node.keys) / 2\n\tkey := node.keys[i]\n\totherKeys := make(keys, i, cap(node.keys))\n\tourKeys := make(keys, len(node.keys)-i, cap(node.keys))\n\t// we perform these copies so these slices don't all end up\n\t// pointing to the same underlying array which may make\n\t// for some very difficult to debug situations later.\n\tcopy(otherKeys, node.keys[:i])\n\tcopy(ourKeys, node.keys[i:])\n\n\t// this should release the original array for GC\n\tnode.keys = ourKeys\n\totherNode := &lnode{\n\t\tkeys: otherKeys,\n\t\tpointer: node,\n\t}\n\treturn key, otherNode, node\n}\n\nfunc (lnode *lnode) needsSplit(nodeSize uint64) bool {\n\treturn uint64(len(lnode.keys)) >= nodeSize\n}\n\nfunc newLeafNode(size uint64) *lnode {\n\treturn &lnode{\n\t\tkeys: make(keys, 0, size),\n\t}\n}\n\ntype keys []Key\n\nfunc (keys keys) search(key Key) int {\n\treturn keySearch(keys, key)\n}\n\nfunc (keys *keys) insertAt(i int, key Key) {\n\tif i == len(*keys) {\n\t\t*keys = append(*keys, key)\n\t\treturn\n\t}\n\n\t*keys = append(*keys, nil)\n\tcopy((*keys)[i+1:], (*keys)[i:])\n\t(*keys)[i] = key\n}\n\nfunc (keys keys) reverse() {\n\tfor i := 0; i < len(keys)/2; i++ {\n\t\tkeys[i], keys[len(keys)-i-1] = keys[len(keys)-i-1], keys[i]\n\t}\n}\n" }, { "path": "btree/plus/node_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage plus\n\nimport (\n\t\"math/rand\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc constructMockPayloads(num int) keys {\n\tkeys := make(keys, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tkeys = append(keys, newMockKey(i))\n\t}\n\n\treturn keys\n}\n\nfunc constructMockKeys(num int) keys {\n\tkeys := make(keys, 0, num)\n\n\tfor i := 0; i < num; i++ {\n\t\tkeys = append(keys, newMockKey(i))\n\t}\n\n\treturn keys\n}\n\nfunc constructRandomMockKeys(num int) keys {\n\tkeys := make(keys, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tkeys = append(keys, newMockKey(rand.Int()))\n\t}\n\n\treturn keys\n}\n\nfunc constructMockNodes(num int) nodes {\n\tnodes := make(nodes, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tkeys := make(keys, 0, num)\n\t\tfor j := 0; j < num; j++ {\n\t\t\tkeys = append(keys, newMockKey(j*i+j))\n\t\t}\n\n\t\tnode := &lnode{\n\t\t\tkeys: keys,\n\t\t}\n\t\tnodes = append(nodes, node)\n\t\tif i > 0 {\n\t\t\tnodes[i-1].(*lnode).pointer = node\n\t\t}\n\t}\n\n\treturn nodes\n}\n\nfunc constructMockInternalNode(nodes nodes) *inode {\n\tif len(nodes) < 2 {\n\t\treturn nil\n\t}\n\n\tkeys := make(keys, 0, len(nodes)-1)\n\tfor i := 1; i < len(nodes); i++ {\n\t\tkeys = append(keys, nodes[i].(*lnode).keys[0])\n\t}\n\n\tin := &inode{\n\t\tkeys: keys,\n\t\tnodes: nodes,\n\t}\n\treturn in\n}\n\nfunc TestLeafNodeInsert(t *testing.T) {\n\ttree := newBTree(3)\n\tn := newLeafNode(3)\n\tkey := newMockKey(3)\n\n\tn.insert(tree, key)\n\n\tassert.Len(t, n.keys, 1)\n\tassert.Nil(t, n.pointer)\n\tassert.Equal(t, n.keys[0], key)\n\tassert.Equal(t, 0, n.keys[0].Compare(key))\n}\n\nfunc TestDuplicateLeafNodeInsert(t *testing.T) {\n\ttree := newBTree(3)\n\tn := newLeafNode(3)\n\tk1 := newMockKey(3)\n\tk2 := newMockKey(3)\n\n\tassert.True(t, n.insert(tree, k1))\n\tassert.False(t, n.insert(tree, k2))\n\tassert.False(t, n.insert(tree, k1))\n}\n\nfunc TestMultipleLeafNodeInsert(t *testing.T) {\n\ttree := newBTree(3)\n\tn := newLeafNode(3)\n\n\tk1 := newMockKey(3)\n\tk2 := newMockKey(4)\n\n\tassert.True(t, n.insert(tree, k1))\n\tn.insert(tree, k2)\n\n\tif !assert.Len(t, n.keys, 2) {\n\t\treturn\n\t}\n\tassert.Nil(t, n.pointer)\n\tassert.Equal(t, k1, n.keys[0])\n\tassert.Equal(t, k2, n.keys[1])\n}\n\nfunc TestLeafNodeSplitEvenNumber(t *testing.T) {\n\tkeys := constructMockPayloads(4)\n\n\tnode := &lnode{\n\t\tkeys: keys,\n\t}\n\n\tkey, left, right := node.split()\n\tassert.Equal(t, keys[2], key)\n\tassert.Equal(t, left.(*lnode).keys, keys[:2])\n\tassert.Equal(t, right.(*lnode).keys, keys[2:])\n\tassert.Equal(t, left.(*lnode).pointer, right)\n}\n\nfunc TestLeafNodeSplitOddNumber(t *testing.T) {\n\tkeys := constructMockPayloads(3)\n\n\tnode := &lnode{\n\t\tkeys: keys,\n\t}\n\n\tkey, left, right := node.split()\n\tassert.Equal(t, keys[1], key)\n\tassert.Equal(t, left.(*lnode).keys, keys[:1])\n\tassert.Equal(t, right.(*lnode).keys, keys[1:])\n\tassert.Equal(t, left.(*lnode).pointer, right)\n}\n\nfunc TestTwoKeysLeafNodeSplit(t *testing.T) {\n\tkeys := constructMockPayloads(2)\n\n\tnode := &lnode{\n\t\tkeys: keys,\n\t}\n\n\tkey, left, right := node.split()\n\tassert.Equal(t, keys[1], key)\n\tassert.Equal(t, left.(*lnode).keys, keys[:1])\n\tassert.Equal(t, right.(*lnode).keys, keys[1:])\n\tassert.Equal(t, left.(*lnode).pointer, right)\n}\n\nfunc TestLessThanTwoKeysSplit(t *testing.T) {\n\tkeys := constructMockPayloads(1)\n\n\tnode := &lnode{\n\t\tkeys: keys,\n\t}\n\n\tkey, left, right := node.split()\n\tassert.Nil(t, key)\n\tassert.Nil(t, left)\n\tassert.Nil(t, right)\n}\n\nfunc TestInternalNodeSplit2_3_4(t *testing.T) {\n\tnodes := constructMockNodes(4)\n\tin := constructMockInternalNode(nodes)\n\n\tkey, left, right := in.split()\n\tassert.Equal(t, nodes[3].(*lnode).keys[0], key)\n\tassert.Len(t, left.(*inode).keys, 1)\n\tassert.Len(t, right.(*inode).keys, 1)\n\tassert.Equal(t, nodes[:2], left.(*inode).nodes)\n\tassert.Equal(t, nodes[2:], right.(*inode).nodes)\n}\n\nfunc TestInternalNodeSplit3_4_5(t *testing.T) {\n\tnodes := constructMockNodes(5)\n\tin := constructMockInternalNode(nodes)\n\n\tkey, left, right := in.split()\n\tassert.Equal(t, nodes[4].(*lnode).keys[0], key)\n\tassert.Len(t, left.(*inode).keys, 2)\n\tassert.Len(t, right.(*inode).keys, 1)\n\tassert.Equal(t, nodes[:3], left.(*inode).nodes)\n\tassert.Equal(t, nodes[3:], right.(*inode).nodes)\n}\n\nfunc TestInternalNodeLessThan3Keys(t *testing.T) {\n\tnodes := constructMockNodes(2)\n\tin := constructMockInternalNode(nodes)\n\n\tkey, left, right := in.split()\n\tassert.Nil(t, key)\n\tassert.Nil(t, left)\n\tassert.Nil(t, right)\n}\n" }, { "path": "cache/cache.go", "content": "package cache\n\nimport (\n\t\"container/list\"\n\t\"sync\"\n)\n\n// Cache is a bounded-size in-memory cache of sized items with a configurable eviction policy\ntype Cache interface {\n\t// Get retrieves items from the cache by key.\n\t// If an item for a particular key is not found, its position in the result will be nil.\n\tGet(keys ...string) []Item\n\n\t// Put adds an item to the cache.\n\tPut(key string, item Item)\n\n\t// Remove clears items with the given keys from the cache\n\tRemove(keys ...string)\n\n\t// Size returns the size of all items currently in the cache.\n\tSize() uint64\n}\n\n// Item is an item in a cache\ntype Item interface {\n\t// Size returns the item's size, in bytes\n\tSize() uint64\n}\n\n// A tuple tracking a cached item and a reference to its node in the eviction list\ntype cached struct {\n\titem Item\n\telement *list.Element\n}\n\n// Sets the provided list element on the cached item if it is not nil\nfunc (c *cached) setElementIfNotNil(element *list.Element) {\n\tif element != nil {\n\t\tc.element = element\n\t}\n}\n\n// Private cache implementation\ntype cache struct {\n\tsync.Mutex // Lock for synchronizing Get, Put, Remove\n\tcap uint64 // Capacity bound\n\tsize uint64 // Cumulative size\n\titems map[string]*cached // Map from keys to cached items\n\tkeyList *list.List // List of cached items in order of increasing evictability\n\trecordAdd func(key string) *list.Element // Function called to indicate that an item with the given key was added\n\trecordAccess func(key string) *list.Element // Function called to indicate that an item with the given key was accessed\n}\n\n// CacheOption configures a cache.\ntype CacheOption func(*cache)\n\n// Policy is a cache eviction policy for use with the EvictionPolicy CacheOption.\ntype Policy uint8\n\nconst (\n\t// LeastRecentlyAdded indicates a least-recently-added eviction policy.\n\tLeastRecentlyAdded Policy = iota\n\t// LeastRecentlyUsed indicates a least-recently-used eviction policy.\n\tLeastRecentlyUsed\n)\n\n// EvictionPolicy sets the eviction policy to be used to make room for new items.\n// If not provided, default is LeastRecentlyUsed.\nfunc EvictionPolicy(policy Policy) CacheOption {\n\treturn func(c *cache) {\n\t\tswitch policy {\n\t\tcase LeastRecentlyAdded:\n\t\t\tc.recordAccess = c.noop\n\t\t\tc.recordAdd = c.record\n\t\tcase LeastRecentlyUsed:\n\t\t\tc.recordAccess = c.record\n\t\t\tc.recordAdd = c.noop\n\t\t}\n\t}\n}\n\n// New returns a cache with the requested options configured.\n// The cache consumes memory bounded by a fixed capacity,\n// plus tracking overhead linear in the number of items.\nfunc New(capacity uint64, options ...CacheOption) Cache {\n\tc := &cache{\n\t\tcap: capacity,\n\t\tkeyList: list.New(),\n\t\titems: map[string]*cached{},\n\t}\n\t// Default LRU eviction policy\n\tEvictionPolicy(LeastRecentlyUsed)(c)\n\n\tfor _, option := range options {\n\t\toption(c)\n\t}\n\n\treturn c\n}\n\nfunc (c *cache) Get(keys ...string) []Item {\n\tc.Lock()\n\tdefer c.Unlock()\n\n\titems := make([]Item, len(keys))\n\tfor i, key := range keys {\n\t\tcached := c.items[key]\n\t\tif cached == nil {\n\t\t\titems[i] = nil\n\t\t} else {\n\t\t\tc.recordAccess(key)\n\t\t\titems[i] = cached.item\n\t\t}\n\t}\n\n\treturn items\n}\n\nfunc (c *cache) Put(key string, item Item) {\n\tc.Lock()\n\tdefer c.Unlock()\n\n\t// Remove the item currently with this key (if any)\n\tc.remove(key)\n\n\t// Make sure there's room to add this item\n\tc.ensureCapacity(item.Size())\n\n\t// Actually add the new item\n\tcached := &cached{item: item}\n\tcached.setElementIfNotNil(c.recordAdd(key))\n\tcached.setElementIfNotNil(c.recordAccess(key))\n\tc.items[key] = cached\n\tc.size += item.Size()\n}\n\nfunc (c *cache) Remove(keys ...string) {\n\tc.Lock()\n\tdefer c.Unlock()\n\n\tfor _, key := range keys {\n\t\tc.remove(key)\n\t}\n}\n\nfunc (c *cache) Size() uint64 {\n\tc.Lock()\n\tdefer c.Unlock()\n\n\treturn c.size\n}\n\n// Given the need to add some number of new bytes to the cache,\n// evict items according to the eviction policy until there is room.\n// The caller should hold the cache lock.\nfunc (c *cache) ensureCapacity(toAdd uint64) {\n\tmustRemove := int64(c.size+toAdd) - int64(c.cap)\n\tfor mustRemove > 0 {\n\t\tkey := c.keyList.Back().Value.(string)\n\t\tmustRemove -= int64(c.items[key].item.Size())\n\t\tc.remove(key)\n\t}\n}\n\n// Remove the item associated with the given key.\n// The caller should hold the cache lock.\nfunc (c *cache) remove(key string) {\n\tif cached, ok := c.items[key]; ok {\n\t\tdelete(c.items, key)\n\t\tc.size -= cached.item.Size()\n\t\tc.keyList.Remove(cached.element)\n\t}\n}\n\n// A no-op function that does nothing for the provided key\nfunc (c *cache) noop(string) *list.Element { return nil }\n\n// A function to record the given key and mark it as last to be evicted\nfunc (c *cache) record(key string) *list.Element {\n\tif item, ok := c.items[key]; ok {\n\t\tc.keyList.MoveToFront(item.element)\n\t\treturn item.element\n\t}\n\treturn c.keyList.PushFront(key)\n}\n" }, { "path": "cache/cache_test.go", "content": "package cache\n\nimport (\n\t\"container/list\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestEvictionPolicy(t *testing.T) {\n\tc := &cache{keyList: list.New()}\n\tEvictionPolicy(LeastRecentlyUsed)(c)\n\taccessed, added := c.recordAccess(\"foo\"), c.recordAdd(\"foo\")\n\tassert.NotNil(t, accessed)\n\tassert.Nil(t, added)\n\n\tc = &cache{keyList: list.New()}\n\tEvictionPolicy(LeastRecentlyAdded)(c)\n\taccessed, added = c.recordAccess(\"foo\"), c.recordAdd(\"foo\")\n\tassert.Nil(t, accessed)\n\tassert.NotNil(t, added)\n}\n\nfunc TestNew(t *testing.T) {\n\toptionApplied := false\n\toption := func(*cache) {\n\t\toptionApplied = true\n\t}\n\n\tc := New(314159, option).(*cache)\n\n\tassert.Equal(t, uint64(314159), c.cap)\n\tassert.Equal(t, uint64(0), c.size)\n\tassert.NotNil(t, c.items)\n\tassert.NotNil(t, c.keyList)\n\tassert.True(t, optionApplied)\n\n\taccessed, added := c.recordAccess(\"foo\"), c.recordAdd(\"foo\")\n\tassert.NotNil(t, accessed)\n\tassert.Nil(t, added)\n}\n\ntype testItem uint64\n\nfunc (ti testItem) Size() uint64 {\n\treturn uint64(ti)\n}\n\nfunc TestPutGetRemoveSize(t *testing.T) {\n\tkeys := []string{\"foo\", \"bar\", \"baz\"}\n\ttestCases := []struct {\n\t\tlabel string\n\t\tcache Cache\n\t\tuseCache func(c Cache)\n\t\texpectedSize uint64\n\t\texpectedItems []Item\n\t}{{\n\t\tlabel: \"Items added, key doesn't exist\",\n\t\tcache: New(10000),\n\t\tuseCache: func(c Cache) {\n\t\t\tc.Put(\"foo\", testItem(1))\n\t\t},\n\t\texpectedSize: 1,\n\t\texpectedItems: []Item{testItem(1), nil, nil},\n\t}, {\n\t\tlabel: \"Items added, key exists\",\n\t\tcache: New(10000),\n\t\tuseCache: func(c Cache) {\n\t\t\tc.Put(\"foo\", testItem(1))\n\t\t\tc.Put(\"foo\", testItem(10))\n\t\t},\n\t\texpectedSize: 10,\n\t\texpectedItems: []Item{testItem(10), nil, nil},\n\t}, {\n\t\tlabel: \"Items added, LRA eviction\",\n\t\tcache: New(2, EvictionPolicy(LeastRecentlyAdded)),\n\t\tuseCache: func(c Cache) {\n\t\t\tc.Put(\"foo\", testItem(1))\n\t\t\tc.Put(\"bar\", testItem(1))\n\t\t\tc.Get(\"foo\")\n\t\t\tc.Put(\"baz\", testItem(1))\n\t\t},\n\t\texpectedSize: 2,\n\t\texpectedItems: []Item{nil, testItem(1), testItem(1)},\n\t}, {\n\t\tlabel: \"Items added, LRU eviction\",\n\t\tcache: New(2, EvictionPolicy(LeastRecentlyUsed)),\n\t\tuseCache: func(c Cache) {\n\t\t\tc.Put(\"foo\", testItem(1))\n\t\t\tc.Put(\"bar\", testItem(1))\n\t\t\tc.Get(\"foo\")\n\t\t\tc.Put(\"baz\", testItem(1))\n\t\t},\n\t\texpectedSize: 2,\n\t\texpectedItems: []Item{testItem(1), nil, testItem(1)},\n\t}, {\n\t\tlabel: \"Items removed, key doesn't exist\",\n\t\tcache: New(10000),\n\t\tuseCache: func(c Cache) {\n\t\t\tc.Put(\"foo\", testItem(1))\n\t\t\tc.Remove(\"baz\")\n\t\t},\n\t\texpectedSize: 1,\n\t\texpectedItems: []Item{testItem(1), nil, nil},\n\t}, {\n\t\tlabel: \"Items removed, key exists\",\n\t\tcache: New(10000),\n\t\tuseCache: func(c Cache) {\n\t\t\tc.Put(\"foo\", testItem(1))\n\t\t\tc.Remove(\"foo\")\n\t\t},\n\t\texpectedSize: 0,\n\t\texpectedItems: []Item{nil, nil, nil},\n\t}}\n\n\tfor _, testCase := range testCases {\n\t\tt.Log(testCase.label)\n\t\ttestCase.useCache(testCase.cache)\n\t\tassert.Equal(t, testCase.expectedSize, testCase.cache.Size())\n\t\tassert.Equal(t, testCase.expectedItems, testCase.cache.Get(keys...))\n\t}\n}\n" }, { "path": "common/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage common\n\n// Comparator is a generic interface that represents items that can\n// be compared.\ntype Comparator interface {\n\t// Compare compares this interface with another. Returns a positive\n\t// number if this interface is greater, 0 if equal, negative number\n\t// if less.\n\tCompare(Comparator) int\n}\n\n// Comparators is a typed list of type Comparator.\ntype Comparators []Comparator\n" }, { "path": "datastructures.go", "content": "/*\nPackage datastructures exists solely to aid consumers of the go-datastructures\nlibrary when using dependency managers. Depman, for instance, will work\ncorrectly with any datastructure by simply importing this package instead of\neach subpackage individually.\n\nFor more information about the datastructures package, see the README at\n\n\thttp://github.com/Workiva/go-datastructures\n\n*/\npackage datastructures\n\nimport (\n\t_ \"github.com/Workiva/go-datastructures/augmentedtree\"\n\t_ \"github.com/Workiva/go-datastructures/bitarray\"\n\t_ \"github.com/Workiva/go-datastructures/btree/palm\"\n\t_ \"github.com/Workiva/go-datastructures/btree/plus\"\n\t_ \"github.com/Workiva/go-datastructures/fibheap\"\n\t_ \"github.com/Workiva/go-datastructures/futures\"\n\t_ \"github.com/Workiva/go-datastructures/hashmap/fastinteger\"\n\t_ \"github.com/Workiva/go-datastructures/numerics/optimization\"\n\t_ \"github.com/Workiva/go-datastructures/queue\"\n\t_ \"github.com/Workiva/go-datastructures/rangetree\"\n\t_ \"github.com/Workiva/go-datastructures/rangetree/skiplist\"\n\t_ \"github.com/Workiva/go-datastructures/set\"\n\t_ \"github.com/Workiva/go-datastructures/slice\"\n\t_ \"github.com/Workiva/go-datastructures/slice/skip\"\n\t_ \"github.com/Workiva/go-datastructures/sort\"\n\t_ \"github.com/Workiva/go-datastructures/threadsafe/err\"\n\t_ \"github.com/Workiva/go-datastructures/tree/avl\"\n\t_ \"github.com/Workiva/go-datastructures/trie/xfast\"\n\t_ \"github.com/Workiva/go-datastructures/trie/yfast\"\n)\n" }, { "path": "documentation.md", "content": "# Introducing go-datastructures\n\nThe goal of the go-datastructures library is to port implementations of some common datastructures to Go or to improve on some existing datastructures. These datastructures are designed to be re-used for anyone that needs them throughout the community. (and hopefully improved upon).\n\nGiven the commonality and popularity of these datastructures in other languages, it is hoped that by open sourcing this library we leverage a great deal of institutional knowledge to improve upon the Go-specific implementations.\n\n# Datastructures\n\n## Augmented Tree\n\nDesigned for determining intersections between ranges. For example, we can query the augmentedtree for any ranges that intersect with a cell, which can be represented as a range of size one (ie, Cell A1 can be represented as range A1:B2 where B2 is exclusive). In this way, we can walk through the graph looking for exact or approximate intersections.\n\nThe current implementation exists in n-dimensions, but is quickest when an n-dimensional query can be reduced in its first dimension. That is, queries only filtered in anything but the first dimension will be slowest.\n\nThe actual implementation is a top-down red-black binary search tree.\n\n### Future\n\nImplement a bottom-up version as well.\n\n## Bit Array\n\nAlso known as a bitmap, a bitarray is useful for comparing two sets of data that can be represented as an integer. It's useful because bitwise operations can compare a number of these integers at once instead of independently. For instance, the sets {1, 3, 5} and {3, 5, 7} can be intersected in a single clock cycle if these sets were represented in their associated bit array. Included in this package is the ability to convert a bitarray back to integers.\n\nThere are two implementations of bit arrays in this package, one is dense and the other borrows concepts from linear algebra's compressed row sparse matrix to represent bitarrays in much smaller spaces. Unfortunately, the sparse version has logarithmic insertions and existence checks but retains some speed advantages when checking for intersections.\n\nIncidentally, this is one of two things needed to build a native Go database.\n\n### Future\n\nImplement a dense but expandable bit array. Optimize the current package to utilize larger amounts of mechanical sympathy.\n\n## Futures\n\nWe ran into some cases where we wanted to indicate to a goroutine that an operation had started in another goroutine and to pause go routines until the initial routine had completed. You can do this with buffered channels, but it seems somewhat redundant to send the same result to a channel to ensure all waiting threads were alerted. Futures operate similarly to how ndb futures work in GAE and might be thought of as a \"broadcast\" channel.\n\n## Queue\n\nPretty self-explanatory, this package includes both a queue and a priority queue. Currently, waitgroups are used to orchestrate threads but with a proper constructor hint, this does end up being faster than channels when attempting to send data to a go routine. The other advantage over a channel is that the queue will return an error if you attempt to put to a queue that has had Dispose called on it instead of panicking like what would happen if you attempted to send to a closed channel. I believe this is closer to the Golang's stated design goals.\n\nSpeaking of Dispose, calling dispose on a queue will immediately return any waiting threads with an error.\n\n### Future\n\nWhen I get time, I'd like to implement a lockless ring buffer for further performance enhancements.\n\n## Range Tree\n\nThe range tree is a way to store n-dimensional points of data in a manner that permits logarithmic-complexity queries. These points are usually represented as points on a Cartesian graph represented by integers.\n\nThere are two implementations of a range tree in this package, one that is mutable and one that is immutable. The mutable version can be faster, but involves lock contention if the consumer needs to ensure threadsafety. The immutable version is a copy-on-write range tree that is optimized by only copying portions of the rangetree on write and is best written to in batches. Operations on the immutable version are slower, but it is safe to read and write from this version at the same time from different threads.\n\nAlthough rangetrees are often represented as BBSTs as described above, the n-dimensional nature of this rangetree actually made the design easier to implement as a sparse n-dimensional array.\n\n### Future\n\nUnite both implementations of the rangetree under the same interface. The implementations (especially the immutable one) could use some further performance optimizations.\n\n## Fibonacci Heap\n\nThe usual Fibonacci Heap with a floating-point priority key. Does a good job as a priority queue, especially for large n. Should be useful in writing an optimal solution for Dijkstra's and Prim's algorithms. (because of it's efficient decrease-key)\n\n### Future\n\nI'd like to add a value interface{} pointer that will be able to hold any user data attached to each node in the heap. Another thing would be writing a fast implementation of Dijkstra and Prim using this structure. And a third would be analysing thread-safety and coming up with a thread-safe variant.\n\n## Set\n\nNot much to say here. This is an unordered set back by a Go map. This particular version is threadsafe which does hinder performance a bit, although reads can happen simultaneously. \n\n### Future\n\nI'd like to experiment with a ground-up implementation of a hash map using the standard library's hash/fnv hashing function, which is a non-cryptographic hash that's proven to be very fast. I'd also like to experiment with a lockless hashmap.\n\n## Slice\n\nGolang's standard library \"sort\" includes a slice of ints that contain some sorting and searching functions. This is like that standard library package but with Int64s, which requires a new package as Go doesn't want us to have generics. I also added a method for inserting to the slice.\n\n## Threadsafe\n\nThis package just wraps some common interfaces with a lock to make them threadsafe. Golang would tell us to forget about locks and use channels (even though channels themselves are just fancy abstractions around locks as evidenced in their source code) but I found some situations where I wanted to protect some memory that was accessible from multiple goroutines where channels would be ugly, slow, and unnecessary. The only interface with an implemntation thusfar is error, which is useful if you need to indicate that an error was returned from logic running in any number of goroutines.\n\n# Going Forward\n\nThere is a PR into the datastructures repo that contains some pieces required for implementing a B+ tree. With a B+ tree and bitmap, the pieces are in place to write a native Go database. Going forward, I'd like to take these pieces, expand upon them, and implement a fast database in Go. \n\nAs always, any optimizations or bug fixes in any of this code would be greatly appreciated and encouraged :). These datastructures can and are the foundations of many programs and algorithms, even if they are abstracted away in different libraries which makes working with them a lot of fun and very informative.\n" }, { "path": "fibheap/Test Generator/EnqDecrKey.py", "content": "l = [-2901939070.965906, 4539462982.372177, -6222008480.049856,\n -1400427921.5968666, 9866088144.060883,\n -2943107648.529664, 8985474333.11443,\n 9204710651.257133, 5354113876.8447075,\n 8122228442.770859, -8121418938.303131,\n 538431208.3261185, 9913821013.519611,\n -8722989752.449871, -3091279426.694975,\n 7229910558.195713, -2908838839.99403,\n 2835257231.305996, 3922059795.3656673,\n -9298869735.322557]\n\nprint(l)\n\nl = sorted(l)\nprint(l)\na1 = l[19]\na2 = -8722989752.449871\nprint(str(a1) + \" -> \" + str(a2))\nl[19] = a2\n\nb1 = l[18]\nb2 = -9698869735.322557\nprint(str(b1) + \" -> \" + str(b2))\nl[18] = b2\n\nc1 = l[17]\nc2 = -9804710651.257133\nprint(str(c1) + \" -> \" + str(c2))\nl[17] = c2\n\nprint(sorted(l))\n" }, { "path": "fibheap/Test Generator/EnqDelete.py", "content": "l = [-2901939070.965906, 4539462982.372177, -6222008480.049856,\n -1400427921.5968666, 9866088144.060883, -2943107648.529664, 8985474333.11443,\n 9204710651.257133, 5354113876.8447075, 8122228442.770859, -8121418938.303131,\n 538431208.3261185, 9913821013.519611, -8722989752.449871, -3091279426.694975,\n 7229910558.195713, -2908838839.99403, 2835257231.305996, 3922059795.3656673,\n -9298869735.322557]\n\nprint(l)\n\nl = sorted(l)\nprint(l)\na1 = l[19]\na2 = 0\nprint(str(a1) + \" -> \" + str(a2))\nl[19] = a2\n\nb1 = l[18]\nb2 = 0\nprint(str(b1) + \" -> \" + str(b2))\nl[18] = b2\n\nc1 = l[17]\nc2 = 0\nprint(str(c1) + \" -> \" + str(c2))\nl[17] = c2\n\nprint(sorted(l))\n" }, { "path": "fibheap/Test Generator/EnqDeqMin.py", "content": "#!/usr/bin/python3\n\nimport random\n\nl = []\nfor i in range(20):\n l.append(random.uniform(-1E10, 1E10))\n\nprint(l)\nl = sorted(l)\nprint(l)\n" }, { "path": "fibheap/Test Generator/Merge.py", "content": "import random\n\nl1 = []\nl2 = []\nfor i in range(20):\n l1.append(random.uniform(-1E10, 1E10))\n l2.append(random.uniform(-1E10, 1E10))\n\n\nprint(l1)\nprint(l2)\n\nl = []\nl.extend(l1)\nl.extend(l2)\n\nprint(sorted(l))\n\n'''\n[6015943293.071386, -3878285748.0708866, 8674121166.062424, -1528465047.6118088,\n7584260716.494843, -373958476.80486107, -6367787695.054295, 6813992306.719868,\n5986097626.907181, 9011134545.052086, 7123644338.268343, 2646164210.08445,\n4407427446.995375, -888196668.2563229, 7973918726.985172, -6529216482.09644,\n6079069259.51853, -8415952427.784341, -6859960084.757652, -502409126.89040375]\n[9241165993.258648, -9423768405.578083, 3280085607.6687145, -5253703037.682413,\n3858507441.2785892, 9896256282.896187, -9439606732.236805, 3082628799.5320206,\n9453124863.59945, 9928066165.458393, 1135071669.4712334, 6380353457.986282,\n8329064041.853199, 2382910730.445751, -8478491750.445316, 9607469190.690144,\n5417691217.440792, -9698248424.421888, -3933774735.280322, -5984555343.381466]\n[-9698248424.421888, -9439606732.236805, -9423768405.578083, -8478491750.445316,\n-8415952427.784341, -6859960084.757652, -6529216482.09644, -6367787695.054295,\n-5984555343.381466, -5253703037.682413, -3933774735.280322, -3878285748.0708866,\n-1528465047.6118088, -888196668.2563229, -502409126.89040375,\n-373958476.80486107, 1135071669.4712334, 2382910730.445751, 2646164210.08445,\n3082628799.5320206, 3280085607.6687145, 3858507441.2785892, 4407427446.995375,\n5417691217.440792, 5986097626.907181, 6015943293.071386, 6079069259.51853,\n6380353457.986282, 6813992306.719868, 7123644338.268343, 7584260716.494843,\n7973918726.985172, 8329064041.853199, 8674121166.062424, 9011134545.052086,\n9241165993.258648, 9453124863.59945, 9607469190.690144, 9896256282.896187,\n9928066165.458393]\n'''\n" }, { "path": "fibheap/Test Generator/README.md", "content": "These are some Python helper scripts to help generate sample test arrays.\n" }, { "path": "fibheap/benchmarks.txt", "content": "BenchmarkFibHeap_Enqueue-4 \t10000000\t 280 ns/op\t 64 B/op\t 1 allocs/op\nBenchmarkFibHeap_DequeueMin-4 \t 100\t 16990302 ns/op\t16007168 B/op\t 2 allocs/op\nBenchmarkFibHeap_DecreaseKey-4 \t20000000\t 900 ns/op\t 122 B/op\t 3 allocs/op\nBenchmarkFibHeap_Delete-4 \t 100\t 19087592 ns/op\t16007168 B/op\t 2 allocs/op\nBenchmarkFibHeap_Merge-4 \t 3000000\t 482 ns/op\t 128 B/op\t 2 allocs/op\nPASS\ncoverage: 96.2% of statements\nok \t_/home/nikola/git/go-datastructures/fibheap\t37.206s\n" }, { "path": "fibheap/fibheap.go", "content": "/*\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nSpecial thanks to Keith Schwarz (htiek@cs.stanford.edu),\nwhose code and documentation have been used as a reference\nfor the algorithm implementation.\nJava implementation: http://www.keithschwarz.com/interesting/code/?dir=fibonacci-heap\nBinomial heaps: http://web.stanford.edu/class/archive/cs/cs166/cs166.1166/lectures/08/Slides08.pdf\nFibonacci heaps: http://web.stanford.edu/class/archive/cs/cs166/cs166.1166/lectures/09/Slides09.pdf\n*/\n\n/*Package fibheap is an implementation of a priority queue backed by a Fibonacci heap,\nas described by Fredman and Tarjan. Fibonacci heaps are interesting\ntheoretically because they have asymptotically good runtime guarantees\nfor many operations. In particular, insert, peek, and decrease-key all\nrun in amortized O(1) time. dequeueMin and delete each run in amortized\nO(lg n) time. This allows algorithms that rely heavily on decrease-key\nto gain significant performance boosts. For example, Dijkstra's algorithm\nfor single-source shortest paths can be shown to run in O(m + n lg n) using\na Fibonacci heap, compared to O(m lg n) using a standard binary or binomial\nheap.\n\nInternally, a Fibonacci heap is represented as a circular, doubly-linked\nlist of trees obeying the min-heap property. Each node stores pointers\nto its parent (if any) and some arbitrary child. Additionally, every\nnode stores its degree (the number of children it has) and whether it\nis a \"marked\" node. Finally, each Fibonacci heap stores a pointer to\nthe tree with the minimum value.\n\nTo insert a node into a Fibonacci heap, a singleton tree is created and\nmerged into the rest of the trees. The merge operation works by simply\nsplicing together the doubly-linked lists of the two trees, then updating\nthe min pointer to be the smaller of the minima of the two heaps. Peeking\nat the smallest element can therefore be accomplished by just looking at\nthe min element. All of these operations complete in O(1) time.\n\nThe tricky operations are dequeueMin and decreaseKey. dequeueMin works\nby removing the root of the tree containing the smallest element, then\nmerging its children with the topmost roots. Then, the roots are scanned\nand merged so that there is only one tree of each degree in the root list.\nThis works by maintaining a dynamic array of trees, each initially null,\npointing to the roots of trees of each dimension. The list is then scanned\nand this array is populated. Whenever a conflict is discovered, the\nappropriate trees are merged together until no more conflicts exist. The\nresulting trees are then put into the root list. A clever analysis using\nthe potential method can be used to show that the amortized cost of this\noperation is O(lg n), see \"Introduction to Algorithms, Second Edition\" by\nCormen, Rivest, Leiserson, and Stein for more details.\n\nThe other hard operation is decreaseKey, which works as follows. First, we\nupdate the key of the node to be the new value. If this leaves the node\nsmaller than its parent, we're done. Otherwise, we cut the node from its\nparent, add it as a root, and then mark its parent. If the parent was\nalready marked, we cut that node as well, recursively mark its parent,\nand continue this process. This can be shown to run in O(1) amortized time\nusing yet another clever potential function. Finally, given this function,\nwe can implement delete by decreasing a key to -\\infty, then calling\ndequeueMin to extract it.\n*/\npackage fibheap\n\nimport (\n\t\"fmt\"\n\t\"math\"\n)\n\n/******************************************\n ************** INTERFACE *****************\n ******************************************/\n\n// FloatingFibonacciHeap is an implementation of a fibonacci heap\n// with only floating-point priorities and no user data attached.\ntype FloatingFibonacciHeap struct {\n\tmin *Entry // The minimal element\n\tsize uint // Size of the heap\n}\n\n// Entry is the entry type that will be used\n// for each node of the Fibonacci heap\ntype Entry struct {\n\tdegree int\n\tmarked bool\n\tnext, prev, child, parent *Entry\n\t// Priority is the numerical priority of the node\n\tPriority float64\n}\n\n// EmptyHeapError fires when the heap is empty and an operation could\n// not be completed for that reason. Its string holds additional data.\ntype EmptyHeapError string\n\nfunc (e EmptyHeapError) Error() string {\n\treturn string(e)\n}\n\n// NilError fires when a heap or entry is nil and an operation could\n// not be completed for that reason. Its string holds additional data.\ntype NilError string\n\nfunc (e NilError) Error() string {\n\treturn string(e)\n}\n\n// NewFloatFibHeap creates a new, empty, Fibonacci heap object.\nfunc NewFloatFibHeap() FloatingFibonacciHeap { return FloatingFibonacciHeap{nil, 0} }\n\n// Enqueue adds and element to the heap\nfunc (heap *FloatingFibonacciHeap) Enqueue(priority float64) *Entry {\n\tsingleton := newEntry(priority)\n\n\t// Merge singleton list with heap\n\theap.min = mergeLists(heap.min, singleton)\n\theap.size++\n\treturn singleton\n}\n\n// Min returns the minimum element in the heap\nfunc (heap *FloatingFibonacciHeap) Min() (*Entry, error) {\n\tif heap.IsEmpty() {\n\t\treturn nil, EmptyHeapError(\"Trying to get minimum element of empty heap\")\n\t}\n\treturn heap.min, nil\n}\n\n// IsEmpty answers: is the heap empty?\nfunc (heap *FloatingFibonacciHeap) IsEmpty() bool {\n\treturn heap.size == 0\n}\n\n// Size gives the number of elements in the heap\nfunc (heap *FloatingFibonacciHeap) Size() uint {\n\treturn heap.size\n}\n\n// DequeueMin removes and returns the\n// minimal element in the heap\nfunc (heap *FloatingFibonacciHeap) DequeueMin() (*Entry, error) {\n\tif heap.IsEmpty() {\n\t\treturn nil, EmptyHeapError(\"Cannot dequeue minimum of empty heap\")\n\t}\n\n\theap.size--\n\n\t// Copy pointer. Will need it later.\n\tmin := heap.min\n\n\tif min.next == min { // This is the only root node\n\t\theap.min = nil\n\t} else { // There are more root nodes\n\t\theap.min.prev.next = heap.min.next\n\t\theap.min.next.prev = heap.min.prev\n\t\theap.min = heap.min.next // Arbitrary element of the root list\n\t}\n\n\tif min.child != nil {\n\t\t// Keep track of the first visited node\n\t\tcurr := min.child\n\t\tfor ok := true; ok; ok = (curr != min.child) {\n\t\t\tcurr.parent = nil\n\t\t\tcurr = curr.next\n\t\t}\n\t}\n\n\theap.min = mergeLists(heap.min, min.child)\n\n\tif heap.min == nil {\n\t\t// If there are no entries left, we're done.\n\t\treturn min, nil\n\t}\n\n\ttreeSlice := make([]*Entry, 0, heap.size)\n\ttoVisit := make([]*Entry, 0, heap.size)\n\n\tfor curr := heap.min; len(toVisit) == 0 || toVisit[0] != curr; curr = curr.next {\n\t\ttoVisit = append(toVisit, curr)\n\t}\n\n\tfor _, curr := range toVisit {\n\t\tfor {\n\t\t\tfor curr.degree >= len(treeSlice) {\n\t\t\t\ttreeSlice = append(treeSlice, nil)\n\t\t\t}\n\n\t\t\tif treeSlice[curr.degree] == nil {\n\t\t\t\ttreeSlice[curr.degree] = curr\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tother := treeSlice[curr.degree]\n\t\t\ttreeSlice[curr.degree] = nil\n\n\t\t\t// Determine which of two trees has the smaller root\n\t\t\tvar minT, maxT *Entry\n\t\t\tif other.Priority < curr.Priority {\n\t\t\t\tminT = other\n\t\t\t\tmaxT = curr\n\t\t\t} else {\n\t\t\t\tminT = curr\n\t\t\t\tmaxT = other\n\t\t\t}\n\n\t\t\t// Break max out of the root list,\n\t\t\t// then merge it into min's child list\n\t\t\tmaxT.next.prev = maxT.prev\n\t\t\tmaxT.prev.next = maxT.next\n\n\t\t\t// Make it a singleton so that we can merge it\n\t\t\tmaxT.prev = maxT\n\t\t\tmaxT.next = maxT\n\t\t\tminT.child = mergeLists(minT.child, maxT)\n\n\t\t\t// Reparent max appropriately\n\t\t\tmaxT.parent = minT\n\n\t\t\t// Clear max's mark, since it can now lose another child\n\t\t\tmaxT.marked = false\n\n\t\t\t// Increase min's degree. It has another child.\n\t\t\tminT.degree++\n\n\t\t\t// Continue merging this tree\n\t\t\tcurr = minT\n\t\t}\n\n\t\t/* Update the global min based on this node. Note that we compare\n\t\t * for <= instead of < here. That's because if we just did a\n\t\t * reparent operation that merged two different trees of equal\n\t\t * priority, we need to make sure that the min pointer points to\n\t\t * the root-level one.\n\t\t */\n\t\tif curr.Priority <= heap.min.Priority {\n\t\t\theap.min = curr\n\t\t}\n\t}\n\n\treturn min, nil\n}\n\n// DecreaseKey decreases the key of the given element, sets it to the new\n// given priority and returns the node if successfully set\nfunc (heap *FloatingFibonacciHeap) DecreaseKey(node *Entry, newPriority float64) (*Entry, error) {\n\n\tif heap.IsEmpty() {\n\t\treturn nil, EmptyHeapError(\"Cannot decrease key in an empty heap\")\n\t}\n\n\tif node == nil {\n\t\treturn nil, NilError(\"Cannot decrease key: given node is nil\")\n\t}\n\n\tif newPriority >= node.Priority {\n\t\treturn nil, fmt.Errorf(\"The given new priority: %v, is larger than or equal to the old: %v\",\n\t\t\tnewPriority, node.Priority)\n\t}\n\n\tdecreaseKeyUnchecked(heap, node, newPriority)\n\treturn node, nil\n}\n\n// Delete deletes the given element in the heap\nfunc (heap *FloatingFibonacciHeap) Delete(node *Entry) error {\n\n\tif heap.IsEmpty() {\n\t\treturn EmptyHeapError(\"Cannot delete element from an empty heap\")\n\t}\n\n\tif node == nil {\n\t\treturn NilError(\"Cannot delete node: given node is nil\")\n\t}\n\n\tdecreaseKeyUnchecked(heap, node, -math.MaxFloat64)\n\theap.DequeueMin()\n\treturn nil\n}\n\n// Merge returns a new Fibonacci heap that contains\n// all of the elements of the two heaps. Each of the input heaps is\n// destructively modified by having all its elements removed. You can\n// continue to use those heaps, but be aware that they will be empty\n// after this call completes.\nfunc (heap *FloatingFibonacciHeap) Merge(other *FloatingFibonacciHeap) (FloatingFibonacciHeap, error) {\n\n\tif heap == nil || other == nil {\n\t\treturn FloatingFibonacciHeap{}, NilError(\"One of the heaps to merge is nil. Cannot merge\")\n\t}\n\n\tresultSize := heap.size + other.size\n\n\tresultMin := mergeLists(heap.min, other.min)\n\n\theap.min = nil\n\tother.min = nil\n\theap.size = 0\n\tother.size = 0\n\n\treturn FloatingFibonacciHeap{resultMin, resultSize}, nil\n}\n\n/******************************************\n ************** END INTERFACE *************\n ******************************************/\n\n// ****************\n// HELPER FUNCTIONS\n// ****************\n\nfunc newEntry(priority float64) *Entry {\n\tresult := new(Entry)\n\tresult.degree = 0\n\tresult.marked = false\n\tresult.child = nil\n\tresult.parent = nil\n\tresult.next = result\n\tresult.prev = result\n\tresult.Priority = priority\n\treturn result\n}\n\nfunc mergeLists(one, two *Entry) *Entry {\n\tif one == nil && two == nil {\n\t\treturn nil\n\t} else if one != nil && two == nil {\n\t\treturn one\n\t} else if one == nil && two != nil {\n\t\treturn two\n\t}\n\t// Both trees non-null; actually do the merge.\n\toneNext := one.next\n\tone.next = two.next\n\tone.next.prev = one\n\ttwo.next = oneNext\n\ttwo.next.prev = two\n\n\tif one.Priority < two.Priority {\n\t\treturn one\n\t}\n\treturn two\n\n}\n\nfunc decreaseKeyUnchecked(heap *FloatingFibonacciHeap, node *Entry, priority float64) {\n\tnode.Priority = priority\n\n\tif node.parent != nil && node.Priority <= node.parent.Priority {\n\t\tcutNode(heap, node)\n\t}\n\n\tif node.Priority <= heap.min.Priority {\n\t\theap.min = node\n\t}\n}\n\nfunc cutNode(heap *FloatingFibonacciHeap, node *Entry) {\n\tnode.marked = false\n\n\tif node.parent == nil {\n\t\treturn\n\t}\n\n\t// Rewire siblings if it has any\n\tif node.next != node {\n\t\tnode.next.prev = node.prev\n\t\tnode.prev.next = node.next\n\t}\n\n\t// Rewrite pointer if this is the representative child node\n\tif node.parent.child == node {\n\t\tif node.next != node {\n\t\t\tnode.parent.child = node.next\n\t\t} else {\n\t\t\tnode.parent.child = nil\n\t\t}\n\t}\n\n\tnode.parent.degree--\n\n\tnode.prev = node\n\tnode.next = node\n\theap.min = mergeLists(heap.min, node)\n\n\t// cut parent recursively if marked\n\tif node.parent.marked {\n\t\tcutNode(heap, node.parent)\n\t} else {\n\t\tnode.parent.marked = true\n\t}\n\n\tnode.parent = nil\n}\n" }, { "path": "fibheap/fibheap_examples_test.go", "content": "package fibheap\n\n// Tests for the Fibonacci heap with floating point number priorities\n\nimport (\n\t\"fmt\"\n)\n\nconst SomeNumber float64 = 15.5\nconst SomeSmallerNumber float64 = -10.1\nconst SomeLargerNumber float64 = 112.211\n\nfunc ExampleFloatingFibonacciHeap_Enqueue() {\n\theap := NewFloatFibHeap()\n\t// The function returns a pointer\n\t// to the node that contains the new value\n\tnode := heap.Enqueue(SomeNumber)\n\tfmt.Println(node.Priority)\n\t// Output: 15.5\n}\n\nfunc ExampleFloatingFibonacciHeap_Min() {\n\theap := NewFloatFibHeap()\n\theap.Enqueue(SomeNumber)\n\theap.Enqueue(SomeLargerNumber)\n\tmin, _ := heap.Min()\n\tfmt.Println(min.Priority)\n\t// Output: 15.5\n}\n\nfunc ExampleFloatingFibonacciHeap_IsEmpty() {\n\theap := NewFloatFibHeap()\n\tfmt.Printf(\"Empty before insert? %v\\n\", heap.IsEmpty())\n\theap.Enqueue(SomeNumber)\n\tfmt.Printf(\"Empty after insert? %v\\n\", heap.IsEmpty())\n\t// Output:\n\t// Empty before insert? true\n\t// Empty after insert? false\n}\n\nfunc ExampleFloatingFibonacciHeap_Size() {\n\theap := NewFloatFibHeap()\n\tfmt.Printf(\"Size before insert: %v\\n\", heap.Size())\n\theap.Enqueue(SomeNumber)\n\tfmt.Printf(\"Size after insert: %v\\n\", heap.Size())\n\t// Output:\n\t// Size before insert: 0\n\t// Size after insert: 1\n}\n\nfunc ExampleFloatingFibonacciHeap_DequeueMin() {\n\theap := NewFloatFibHeap()\n\theap.Enqueue(SomeNumber)\n\tnode, _ := heap.DequeueMin()\n\tfmt.Printf(\"Dequeueing minimal element: %v\\n\", node.Priority)\n\t// Output:\n\t// Dequeueing minimal element: 15.5\n}\n\nfunc ExampleFloatingFibonacciHeap_DecreaseKey() {\n\theap := NewFloatFibHeap()\n\tnode := heap.Enqueue(SomeNumber)\n\tmin, _ := heap.Min()\n\tfmt.Printf(\"Minimal element before decreasing key: %v\\n\", min.Priority)\n\theap.DecreaseKey(node, SomeSmallerNumber)\n\tmin, _ = heap.Min()\n\tfmt.Printf(\"Minimal element after decreasing key: %v\\n\", min.Priority)\n\t// Output:\n\t// Minimal element before decreasing key: 15.5\n\t// Minimal element after decreasing key: -10.1\n}\n\nfunc ExampleFloatingFibonacciHeap_Delete() {\n\theap := NewFloatFibHeap()\n\tnode := heap.Enqueue(SomeNumber)\n\theap.Enqueue(SomeLargerNumber)\n\tmin, _ := heap.Min()\n\tfmt.Printf(\"Minimal element before deletion: %v\\n\", min.Priority)\n\theap.Delete(node)\n\tmin, _ = heap.Min()\n\tfmt.Printf(\"Minimal element after deletion: %v\\n\", min.Priority)\n\t// Output:\n\t// Minimal element before deletion: 15.5\n\t// Minimal element after deletion: 112.211\n}\n\nfunc ExampleFloatingFibonacciHeap_Merge() {\n\theap1 := NewFloatFibHeap()\n\theap2 := NewFloatFibHeap()\n\theap1.Enqueue(SomeNumber)\n\theap1.Enqueue(SomeLargerNumber)\n\theap2.Enqueue(SomeSmallerNumber)\n\tmin, _ := heap1.Min()\n\tfmt.Printf(\"Minimal element of heap 1: %v\\n\", min.Priority)\n\tmin, _ = heap2.Min()\n\tfmt.Printf(\"Minimal element of heap 2: %v\\n\", min.Priority)\n\theap, _ := heap1.Merge(&heap2)\n\tmin, _ = heap.Min()\n\tfmt.Printf(\"Minimal element of merged heap: %v\\n\", min.Priority)\n\t// Output:\n\t// Minimal element of heap 1: 15.5\n\t// Minimal element of heap 2: -10.1\n\t// Minimal element of merged heap: -10.1\n}\n" }, { "path": "fibheap/fibheap_single_example_test.go", "content": "package fibheap\n\n// Example usage of the Fibonacci heap\n\nimport (\n\t\"fmt\"\n)\n\nconst SomeNumberAround0 float64 = -0.001\nconst SomeLargerNumberAround15 float64 = 15.77\nconst SomeNumberAroundMinus1000 float64 = -1002.2001\nconst SomeNumberAroundMinus1003 float64 = -1003.4\n\nfunc Example() {\n\theap1 := NewFloatFibHeap()\n\tfmt.Println(\"Created heap 1.\")\n\tnodeh1_1 := heap1.Enqueue(SomeLargerNumberAround15)\n\tfmt.Printf(\"Heap 1 insert: %v\\n\", nodeh1_1.Priority)\n\n\theap2 := NewFloatFibHeap()\n\tfmt.Println(\"Created heap 2.\")\n\tfmt.Printf(\"Heap 2 is empty? %v\\n\", heap2.IsEmpty())\n\tnodeh2_1 := heap2.Enqueue(SomeNumberAroundMinus1000)\n\tfmt.Printf(\"Heap 2 insert: %v\\n\", nodeh2_1.Priority)\n\tnodeh2_2 := heap2.Enqueue(SomeNumberAround0)\n\tfmt.Printf(\"Heap 2 insert: %v\\n\", nodeh2_2.Priority)\n\tfmt.Printf(\"Heap 1 size: %v\\n\", heap1.Size())\n\tfmt.Printf(\"Heap 2 size: %v\\n\", heap2.Size())\n\tfmt.Printf(\"Heap 1 is empty? %v\\n\", heap1.IsEmpty())\n\tfmt.Printf(\"Heap 2 is empty? %v\\n\", heap2.IsEmpty())\n\n\tfmt.Printf(\"\\nMerge Heap 1 and Heap 2.\\n\")\n\tmergedHeap, _ := heap1.Merge(&heap2)\n\tfmt.Printf(\"Merged heap size: %v\\n\", mergedHeap.Size())\n\tfmt.Printf(\"Set node with priority %v to new priority %v\\n\", SomeNumberAroundMinus1000, SomeNumberAroundMinus1003)\n\n\tmergedHeap.DecreaseKey(nodeh2_1, SomeNumberAroundMinus1003)\n\tmin, _ := mergedHeap.DequeueMin()\n\tfmt.Printf(\"Dequeue minimum of merged heap: %v\\n\", min.Priority)\n\tfmt.Printf(\"Merged heap size: %v\\n\", mergedHeap.Size())\n\n\tfmt.Printf(\"Delete from merged heap: %v\\n\", SomeNumberAround0)\n\tmergedHeap.Delete(nodeh2_2)\n\tfmt.Printf(\"Merged heap size: %v\\n\", mergedHeap.Size())\n\n\tmin, _ = mergedHeap.DequeueMin()\n\tfmt.Printf(\"Extracting minimum of merged heap: %v\\n\", min.Priority)\n\tfmt.Printf(\"Merged heap size: %v\\n\", mergedHeap.Size())\n\tfmt.Printf(\"Merged heap is empty? %v\\n\", mergedHeap.IsEmpty())\n\n\t// Output:\n\t// Created heap 1.\n\t// Heap 1 insert: 15.77\n\t// Created heap 2.\n\t// Heap 2 is empty? true\n\t// Heap 2 insert: -1002.2001\n\t// Heap 2 insert: -0.001\n\t// Heap 1 size: 1\n\t// Heap 2 size: 2\n\t// Heap 1 is empty? false\n\t// Heap 2 is empty? false\n\t//\n\t// Merge Heap 1 and Heap 2.\n\t// Merged heap size: 3\n\t// Set node with priority -1002.2001 to new priority -1003.4\n\t// Dequeue minimum of merged heap: -1003.4\n\t// Merged heap size: 2\n\t// Delete from merged heap: -0.001\n\t// Merged heap size: 1\n\t// Extracting minimum of merged heap: 15.77\n\t// Merged heap size: 0\n\t// Merged heap is empty? true\n}\n" }, { "path": "fibheap/fibheap_test.go", "content": "package fibheap\n\n// Tests for the Fibonacci heap with floating point number priorities\n\nimport (\n\t\"testing\"\n\n\t\"math/rand\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\n// Go does not have constant arrays.\n// Settling for standard variables.\nvar NumberSequence1 = [...]float64{6145466173.743959, 1717075442.6908855, -9223106115.008125,\n\t6664774768.783949, -9185895273.675707, -2271628840.682966, -6843837387.469989,\n\t-3075112103.982916, -7315786187.596851, 9022422938.330479, 9230482598.051868,\n\t-2019031911.3141594, 4852342381.928253, 7767018098.497437, -5163143977.984332,\n\t7265142312.343864, -9974588724.261246, -4721177341.970384, 6608275091.590723,\n\t-2509051968.8908787, -2608600569.397663, 4602079812.256586, 4204221071.262924,\n\t2072073006.576254, -1375445006.5510921, 9753983872.378643, 3379810998.918478,\n\t-2120599284.15699, -9284902029.588614, 3804069225.763077, 4680667479.457649,\n\t3550845076.5165443, 689351033.7409191, -6170564101.460268, 5769309548.4711685,\n\t-7203959673.554039, -1542719821.5259266, 8314666872.8992195, 4582459708.761353,\n\t4558164249.709116, -409019759.7648945, 2050647646.0881348, 3337347280.2468243,\n\t8841975976.437397, -1540752999.8368673, 4548535015.628077, -7013783667.095476,\n\t2287926261.9939594, -2539231979.834078, -9359850979.452446, 5390795464.938633,\n\t-9969381716.563528, 3273172669.620493, -8839719143.511513, 9436856014.244781,\n\t9032693590.852093, 748366072.01511, -8165322713.346881, -9745450118.0132,\n\t-6554663739.562494, -8350123090.830288, 4767099194.408716, -741610722.9710865,\n\t978853190.937952, -4689006449.5764475, 6712607751.828266, 1834187952.9013042,\n\t8144068220.835762, 2649156704.6132507, 5206492575.513319, 2355676989.886942,\n\t6014313651.805082, 1559476573.9042358, -611075813.2161636, -3428570708.324188,\n\t3758297334.844446, -73880069.57582092, 7939090089.227123, -6135368824.336376,\n\t5680302744.840729, 7067968530.463007, -4736146992.716046, 6787733005.103142,\n\t8291261997.956814, -7976948033.245457, -2717662205.411746, 1753831326.4953232,\n\t3313929049.058649, -6798511690.417229, 4259620288.6441, -8795846089.203701,\n\t666087815.4947224, -3189108786.1266823, 6098522858.07811, 3670419236.2020073,\n\t-4904172359.7338295, 7081860835.300518, 4838004130.57917, -8403025837.455175,\n\t2858604246.067789, 9767232443.473625, 1853770486.2323227, 2111315124.8128128,\n\t-789990089.2266369, 3855299652.837984, -5262051498.344847, 5195097083.198868,\n\t-9453697711.29756, -144320772.42621613, -3280154832.042288, 4327603656.616592,\n\t-4916338352.631529, 177342499.89391518, -6863008836.282527, -4462732551.435464,\n\t563531299.3931465, 243815563.513546, -2177539298.657405, 9064363201.461056,\n\t7752407089.025448, 5072315736.623476, 1676308335.832735, 2368433225.444128,\n\t7191228067.770271, -7952866649.176966, 9029961422.270164, -3694580624.20329,\n\t2396384720.634838, 2919689806.6469193, 2516309466.887434, 5711191379.798178,\n\t-7111997035.1143055, -5887152915.558975, 7074496594.814234, 72399466.26899147,\n\t9162739770.93885, 545095642.1330223, 589248875.6552525, 5429718452.359911,\n\t2670541446.0850983, 7074768275.337322, -9376701618.064901, -719716639.8418808,\n\t5870465712.600103, 8906050348.824574, 5260686230.481573, 4525930216.3939705,\n\t-7558925556.569441, -3524217648.1943235, -8559543174.289785, -402353821.38601303,\n\t-2939238306.2766924, -8421788462.600799, 173509960.46243477, 2823962320.1096497,\n\t-2040044596.465724, 8093258879.034134, 1026657583.5726833, -5939324535.959578,\n\t1869187366.0910244, -8488159448.309237, -9162642241.327745, 9198652822.209103,\n\t9981219597.001732, 1245929264.1492062, 6333145610.418182, -5007933225.524759,\n\t-7507006648.70326, -8682109235.019928, 7572534048.487186, 9172777289.492256,\n\t-4374595711.753318, 7302929281.918972, 6813548014.888256, 7839035144.903576,\n\t-5126801855.122898, 6523728766.098036, -8063474434.226172, -1011764426.4069233,\n\t-5468146510.412097, -7725685149.169344, 5224407910.623154, 5337833362.662783,\n\t3878206583.8412895, -9990847539.012056, 2828249626.7454433, -8802730816.790993,\n\t-6223950138.847174, -5003095866.683969, 3701841328.9391365, -7438103512.551224,\n\t-1879515137.467103, -6931067459.813007, -3591253518.1452456, -3249229927.5027523,\n\t249923973.47061348, -7291235820.978601, -4073015010.864023, -3089932753.657503,\n\t8220825130.164364}\n\nconst Seq1FirstMinimum float64 = -9990847539.012056\nconst Seq1ThirdMinimum float64 = -9969381716.563528\nconst Seq1FifthMinimum float64 = -9453697711.29756\nconst Seq1LastMinimum float64 = 9981219597.001732\n\nvar NumberSequence2 = [...]float64{-2901939070.965906, 4539462982.372177, -6222008480.049856,\n\t-1400427921.5968666, 9866088144.060883, -2943107648.529664, 8985474333.11443,\n\t9204710651.257133, 5354113876.8447075, 8122228442.770859, -8121418938.303131,\n\t538431208.3261185, 9913821013.519611, -8722989752.449871, -3091279426.694975,\n\t7229910558.195713, -2908838839.99403, 2835257231.305996, 3922059795.3656673,\n\t-9298869735.322557}\n\nconst Seq2DecreaseKey1Orig float64 = 9913821013.519611\nconst Seq2DecreaseKey1Trgt float64 = -8722989752.449871\nconst Seq2DecreaseKey2Orig float64 = 9866088144.060883\nconst Seq2DecreaseKey2Trgt float64 = -9698869735.322557\nconst Seq2DecreaseKey3Orig float64 = 9204710651.257133\nconst Seq2DecreaseKey3Trgt float64 = -9804710651.257133\n\nvar NumberSequence2Sorted = [...]float64{-9804710651.257133, -9698869735.322557, -9298869735.322557,\n\t-8722989752.449871, -8722989752.449871, -8121418938.303131, -6222008480.049856,\n\t-3091279426.694975, -2943107648.529664, -2908838839.99403, -2901939070.965906,\n\t-1400427921.5968666, 538431208.3261185, 2835257231.305996, 3922059795.3656673,\n\t4539462982.372177, 5354113876.8447075, 7229910558.195713, 8122228442.770859,\n\t8985474333.11443}\n\nvar NumberSequence2Deleted3ElemSorted = [...]float64{-9298869735.322557, -8722989752.449871,\n\t-8121418938.303131, -6222008480.049856, -3091279426.694975, -2943107648.529664,\n\t-2908838839.99403, -2901939070.965906, -1400427921.5968666, 538431208.3261185,\n\t2835257231.305996, 3922059795.3656673, 4539462982.372177, 5354113876.8447075,\n\t7229910558.195713, 8122228442.770859, 8985474333.11443}\n\nvar NumberSequence3 = [...]float64{6015943293.071386, -3878285748.0708866, 8674121166.062424,\n\t-1528465047.6118088, 7584260716.494843, -373958476.80486107, -6367787695.054295,\n\t6813992306.719868, 5986097626.907181, 9011134545.052086, 7123644338.268343,\n\t2646164210.08445, 4407427446.995375, -888196668.2563229, 7973918726.985172,\n\t-6529216482.09644, 6079069259.51853, -8415952427.784341, -6859960084.757652,\n\t-502409126.89040375}\n\nvar NumberSequence4 = [...]float64{9241165993.258648, -9423768405.578083, 3280085607.6687145,\n\t-5253703037.682413, 3858507441.2785892, 9896256282.896187, -9439606732.236805,\n\t3082628799.5320206, 9453124863.59945, 9928066165.458393, 1135071669.4712334,\n\t6380353457.986282, 8329064041.853199, 2382910730.445751, -8478491750.445316,\n\t9607469190.690144, 5417691217.440792, -9698248424.421888, -3933774735.280322,\n\t-5984555343.381466}\n\nvar NumberSequenceMerged3And4Sorted = [...]float64{-9698248424.421888, -9439606732.236805,\n\t-9423768405.578083, -8478491750.445316, -8415952427.784341, -6859960084.757652,\n\t-6529216482.09644, -6367787695.054295, -5984555343.381466, -5253703037.682413,\n\t-3933774735.280322, -3878285748.0708866, -1528465047.6118088, -888196668.2563229,\n\t-502409126.89040375, -373958476.80486107, 1135071669.4712334, 2382910730.445751,\n\t2646164210.08445, 3082628799.5320206, 3280085607.6687145, 3858507441.2785892,\n\t4407427446.995375, 5417691217.440792, 5986097626.907181, 6015943293.071386,\n\t6079069259.51853, 6380353457.986282, 6813992306.719868, 7123644338.268343,\n\t7584260716.494843, 7973918726.985172, 8329064041.853199, 8674121166.062424,\n\t9011134545.052086, 9241165993.258648, 9453124863.59945, 9607469190.690144,\n\t9896256282.896187, 9928066165.458393}\n\nfunc TestEnqueueDequeueMin(t *testing.T) {\n\theap := NewFloatFibHeap()\n\tfor i := 0; i < len(NumberSequence1); i++ {\n\t\theap.Enqueue(NumberSequence1[i])\n\t}\n\n\tvar min *Entry\n\tvar err error\n\tfor heap.Size() > 0 {\n\t\tmin, err = heap.DequeueMin()\n\t\trequire.NoError(t, err)\n\t\tif heap.Size() == 199 {\n\t\t\tassert.Equal(t, Seq1FirstMinimum, min.Priority)\n\t\t}\n\t\tif heap.Size() == 197 {\n\t\t\tassert.Equal(t, Seq1ThirdMinimum, min.Priority)\n\t\t}\n\t\tif heap.Size() == 195 {\n\t\t\tassert.Equal(t, Seq1FifthMinimum, min.Priority)\n\t\t}\n\t\tif heap.Size() == 0 {\n\t\t\tassert.Equal(t, Seq1LastMinimum, min.Priority)\n\t\t}\n\t}\n}\n\nfunc TestFibHeap_Enqueue_Min(t *testing.T) {\n\theap := NewFloatFibHeap()\n\tfor i := 0; i < len(NumberSequence1); i++ {\n\t\theap.Enqueue(NumberSequence1[i])\n\t}\n\n\tmin, err := heap.Min()\n\trequire.NoError(t, err)\n\tassert.Equal(t, Seq1FirstMinimum, min.Priority)\n}\n\nfunc TestFibHeap_Min_EmptyHeap(t *testing.T) {\n\theap := NewFloatFibHeap()\n\n\theap.Enqueue(0)\n\tmin, err := heap.DequeueMin()\n\trequire.NoError(t, err)\n\n\t// Heap should be empty at this point\n\n\tmin, err = heap.Min()\n\n\tassert.EqualError(t, err, \"Trying to get minimum element of empty heap\")\n\tassert.Nil(t, min)\n}\n\nfunc TestFibHeap_DequeueMin_EmptyHeap(t *testing.T) {\n\theap := NewFloatFibHeap()\n\tmin, err := heap.DequeueMin()\n\n\tassert.IsType(t, EmptyHeapError(\"\"), err)\n\tassert.EqualError(t, err, \"Cannot dequeue minimum of empty heap\")\n\tassert.Nil(t, min)\n}\n\nfunc TestEnqueueDecreaseKey(t *testing.T) {\n\theap := NewFloatFibHeap()\n\tvar e1, e2, e3 *Entry\n\tfor i := 0; i < len(NumberSequence2); i++ {\n\t\tif NumberSequence2[i] == Seq2DecreaseKey1Orig {\n\t\t\te1 = heap.Enqueue(NumberSequence2[i])\n\t\t} else if NumberSequence2[i] == Seq2DecreaseKey2Orig {\n\t\t\te2 = heap.Enqueue(NumberSequence2[i])\n\t\t} else if NumberSequence2[i] == Seq2DecreaseKey3Orig {\n\t\t\te3 = heap.Enqueue(NumberSequence2[i])\n\t\t} else {\n\t\t\theap.Enqueue(NumberSequence2[i])\n\t\t}\n\t}\n\n\trequire.NotNil(t, e1)\n\trequire.NotNil(t, e2)\n\trequire.NotNil(t, e3)\n\n\t_, err := heap.DecreaseKey(e1, Seq2DecreaseKey1Trgt)\n\trequire.NoError(t, err)\n\t_, err = heap.DecreaseKey(e2, Seq2DecreaseKey2Trgt)\n\trequire.NoError(t, err)\n\t_, err = heap.DecreaseKey(e3, Seq2DecreaseKey3Trgt)\n\trequire.NoError(t, err)\n\n\tvar min *Entry\n\tfor i := 0; i < len(NumberSequence2Sorted); i++ {\n\t\tmin, err = heap.DequeueMin()\n\t\trequire.NoError(t, err)\n\t\tassert.Equal(t, NumberSequence2Sorted[i], min.Priority)\n\t}\n}\n\nfunc TestFibHeap_DecreaseKey_EmptyHeap(t *testing.T) {\n\theap := NewFloatFibHeap()\n\n\telem := heap.Enqueue(15)\n\theap.DequeueMin()\n\n\t// Heap should be empty at this point\n\tmin, err := heap.DecreaseKey(elem, 0)\n\n\tassert.IsType(t, EmptyHeapError(\"\"), err)\n\tassert.EqualError(t, err, \"Cannot decrease key in an empty heap\")\n\tassert.Nil(t, min)\n}\n\nfunc TestFibHeap_DecreaseKey_NilNode(t *testing.T) {\n\theap := NewFloatFibHeap()\n\theap.Enqueue(1)\n\tmin, err := heap.DecreaseKey(nil, 0)\n\n\tassert.IsType(t, NilError(\"\"), err)\n\tassert.EqualError(t, err, \"Cannot decrease key: given node is nil\")\n\tassert.Nil(t, min)\n}\n\nfunc TestFibHeap_DecreaseKey_LargerNewPriority(t *testing.T) {\n\theap := NewFloatFibHeap()\n\tnode := heap.Enqueue(1)\n\tmin, err := heap.DecreaseKey(node, 20)\n\n\tassert.EqualError(t, err, \"The given new priority: 20, is larger than or equal to the old: 1\")\n\tassert.Nil(t, min)\n}\n\nfunc TestEnqueueDelete(t *testing.T) {\n\theap := NewFloatFibHeap()\n\tvar e1, e2, e3 *Entry\n\tfor i := 0; i < len(NumberSequence2); i++ {\n\t\tif NumberSequence2[i] == Seq2DecreaseKey1Orig {\n\t\t\te1 = heap.Enqueue(NumberSequence2[i])\n\t\t} else if NumberSequence2[i] == Seq2DecreaseKey2Orig {\n\t\t\te2 = heap.Enqueue(NumberSequence2[i])\n\t\t} else if NumberSequence2[i] == Seq2DecreaseKey3Orig {\n\t\t\te3 = heap.Enqueue(NumberSequence2[i])\n\t\t} else {\n\t\t\theap.Enqueue(NumberSequence2[i])\n\t\t}\n\t}\n\n\tassert.NotNil(t, e1)\n\tassert.NotNil(t, e2)\n\tassert.NotNil(t, e3)\n\n\tvar err error\n\n\terr = heap.Delete(e1)\n\trequire.NoError(t, err)\n\terr = heap.Delete(e2)\n\trequire.NoError(t, err)\n\terr = heap.Delete(e3)\n\trequire.NoError(t, err)\n\n\tvar min *Entry\n\tfor i := 0; i < len(NumberSequence2Deleted3ElemSorted); i++ {\n\t\tmin, err = heap.DequeueMin()\n\t\trequire.NoError(t, err)\n\t\tassert.Equal(t, NumberSequence2Deleted3ElemSorted[i], min.Priority)\n\t}\n}\n\nfunc TestFibHeap_Delete_EmptyHeap(t *testing.T) {\n\theap := NewFloatFibHeap()\n\n\telem := heap.Enqueue(15)\n\theap.DequeueMin()\n\n\t// Heap should be empty at this point\n\terr := heap.Delete(elem)\n\tassert.IsType(t, EmptyHeapError(\"\"), err)\n\tassert.EqualError(t, err, \"Cannot delete element from an empty heap\")\n}\n\nfunc TestFibHeap_Delete_NilNode(t *testing.T) {\n\theap := NewFloatFibHeap()\n\theap.Enqueue(1)\n\terr := heap.Delete(nil)\n\tassert.IsType(t, NilError(\"\"), err)\n\tassert.EqualError(t, err, \"Cannot delete node: given node is nil\")\n}\n\nfunc TestMerge(t *testing.T) {\n\theap1 := NewFloatFibHeap()\n\tfor i := 0; i < len(NumberSequence3); i++ {\n\t\theap1.Enqueue(NumberSequence3[i])\n\t}\n\n\theap2 := NewFloatFibHeap()\n\tfor i := 0; i < len(NumberSequence4); i++ {\n\t\theap1.Enqueue(NumberSequence4[i])\n\t}\n\n\theap, err := heap1.Merge(&heap2)\n\trequire.NoError(t, err)\n\n\tvar min *Entry\n\tfor i := 0; i < len(NumberSequenceMerged3And4Sorted); i++ {\n\t\tmin, err = heap.DequeueMin()\n\t\trequire.NoError(t, err)\n\t\tassert.Equal(t, NumberSequenceMerged3And4Sorted[i], min.Priority)\n\t}\n}\n\nfunc TestFibHeap_Merge_NilHeap(t *testing.T) {\n\tvar heap FloatingFibonacciHeap\n\theap = NewFloatFibHeap()\n\tnewHeap, err := heap.Merge(nil)\n\tassert.IsType(t, NilError(\"\"), err)\n\tassert.EqualError(t, err, \"One of the heaps to merge is nil. Cannot merge\")\n\tassert.Equal(t, newHeap, FloatingFibonacciHeap{})\n}\n\n// ***************\n// BENCHMARK TESTS\n// ***************\n\n/*\nSince the e.g. Enqeue operation is constant time,\nwhen go benchmark increases N, the prep time\nwill increase linearly, but the actual operation\nwe want to measure will always take the same,\nconstant amount of time.\nThis means that on some machines, Go Bench\ncould try to exponentially increase N in order\nto decrease noise in the measurement, but it will\nget more and more noise. This can cause a system\nto run out of RAM. So be careful if you have a fast PC.\nI have removed the b.ResetTimer on constant-time\nfunctions to avoid this negative-feedback loop.\n*/\n\n// Runs in O(1) time\nfunc BenchmarkFibHeap_Enqueue(b *testing.B) {\n\n\theap := NewFloatFibHeap()\n\n\tfor i := 0; i < b.N; i++ {\n\t\theap.Enqueue(2 * 1E10 * (rand.Float64() - 0.5))\n\t}\n}\n\n// Runs in O(log(N)) time\nfunc BenchmarkFibHeap_DequeueMin(b *testing.B) {\n\n\theap := NewFloatFibHeap()\n\tN := 1000000\n\n\tslice := make([]float64, 0, N)\n\tfor i := 0; i < N; i++ {\n\t\tslice = append(slice, 2*1E10*(rand.Float64()-0.5))\n\t\theap.Enqueue(slice[i])\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\theap.DequeueMin()\n\t}\n}\n\n// Runs in O(1) amortized time\nfunc BenchmarkFibHeap_DecreaseKey(b *testing.B) {\n\theap := NewFloatFibHeap()\n\tN := 10000000\n\n\tsliceFlt := make([]float64, 0, N)\n\tsliceE := make([]*Entry, 0, N)\n\tfor i := 0; i < N; i++ {\n\t\tsliceFlt = append(sliceFlt, 2*1E10*(float64(i)-0.5))\n\t\tsliceE = append(sliceE, heap.Enqueue(sliceFlt[i]))\n\t}\n\n\tb.ResetTimer()\n\toffset := float64(2)\n\tfor i := 0; i < b.N; i++ {\n\t\t// Change offset if b.N larger than N\n\t\tif i%N == 0 && i > 0 {\n\t\t\toffset *= float64(i / N)\n\t\t}\n\t\t// Shift-decrease keys\n\t\theap.DecreaseKey(sliceE[i%N], sliceFlt[i%N]-offset)\n\t}\n}\n\n// Runs in O(log(N)) time\nfunc BenchmarkFibHeap_Delete(b *testing.B) {\n\theap := NewFloatFibHeap()\n\tN := 1000000\n\n\tsliceFlt := make([]float64, 0, N)\n\tsliceE := make([]*Entry, 0, N)\n\tfor i := 0; i < N; i++ {\n\t\tsliceFlt = append(sliceFlt, 2*1E10*(float64(i)-0.5))\n\t\tsliceE = append(sliceE, heap.Enqueue(sliceFlt[i]))\n\t}\n\n\t// Delete runs in log(N) time\n\t// so safe to reset timer here\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\terr := heap.Delete(sliceE[i])\n\t\tassert.NoError(b, err)\n\t}\n}\n\n// Runs in O(1) time\nfunc BenchmarkFibHeap_Merge(b *testing.B) {\n\theap1 := NewFloatFibHeap()\n\theap2 := NewFloatFibHeap()\n\n\tfor i := 0; i < b.N; i++ {\n\t\theap1.Enqueue(2 * 1E10 * (rand.Float64() - 0.5))\n\t\theap2.Enqueue(2 * 1E10 * (rand.Float64() - 0.5))\n\t\t_, err := heap1.Merge(&heap2)\n\t\tassert.NoError(b, err)\n\t}\n}\n" }, { "path": "futures/futures.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage futures is useful for broadcasting an identical message to a multitude\nof listeners as opposed to channels which will choose a listener at random\nif multiple listeners are listening to the same channel. The future will\nalso cache the result so any future interest will be immediately returned\nto the consumer.\n*/\npackage futures\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\t\"time\"\n)\n\n// Completer is a channel that the future expects to receive\n// a result on. The future only receives on this channel.\ntype Completer <-chan interface{}\n\n// Future represents an object that can be used to perform asynchronous\n// tasks. The constructor of the future will complete it, and listeners\n// will block on getresult until a result is received. This is different\n// from a channel in that the future is only completed once, and anyone\n// listening on the future will get the result, regardless of the number\n// of listeners.\ntype Future struct {\n\ttriggered bool // because item can technically be nil and still be valid\n\titem interface{}\n\terr error\n\tlock sync.Mutex\n\twg sync.WaitGroup\n}\n\n// GetResult will immediately fetch the result if it exists\n// or wait on the result until it is ready.\nfunc (f *Future) GetResult() (interface{}, error) {\n\tf.lock.Lock()\n\tif f.triggered {\n\t\tf.lock.Unlock()\n\t\treturn f.item, f.err\n\t}\n\tf.lock.Unlock()\n\n\tf.wg.Wait()\n\treturn f.item, f.err\n}\n\n// HasResult will return true iff the result exists\nfunc (f *Future) HasResult() bool {\n\tf.lock.Lock()\n\thasResult := f.triggered\n\tf.lock.Unlock()\n\treturn hasResult\n}\n\nfunc (f *Future) setItem(item interface{}, err error) {\n\tf.lock.Lock()\n\tf.triggered = true\n\tf.item = item\n\tf.err = err\n\tf.lock.Unlock()\n\tf.wg.Done()\n}\n\nfunc listenForResult(f *Future, ch Completer, timeout time.Duration, wg *sync.WaitGroup) {\n\twg.Done()\n\tt := time.NewTimer(timeout)\n\tselect {\n\tcase item := <-ch:\n\t\tf.setItem(item, nil)\n\t\tt.Stop() // we want to trigger GC of this timer as soon as it's no longer needed\n\tcase <-t.C:\n\t\tf.setItem(nil, fmt.Errorf(`timeout after %f seconds`, timeout.Seconds()))\n\t}\n}\n\n// New is the constructor to generate a new future. Pass the completed\n// item to the toComplete channel and any listeners will get\n// notified. If timeout is hit before toComplete is called,\n// any listeners will get passed an error.\nfunc New(completer Completer, timeout time.Duration) *Future {\n\tf := &Future{}\n\tf.wg.Add(1)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo listenForResult(f, completer, timeout, &wg)\n\twg.Wait()\n\treturn f\n}\n" }, { "path": "futures/futures_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage futures\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestWaitOnGetResult(t *testing.T) {\n\tcompleter := make(chan interface{})\n\tf := New(completer, time.Duration(30*time.Minute))\n\tvar result interface{}\n\tvar err error\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tresult, err = f.GetResult()\n\t\twg.Done()\n\t}()\n\n\tcompleter <- `test`\n\twg.Wait()\n\n\tassert.Nil(t, err)\n\tassert.Equal(t, `test`, result)\n\n\t// ensure we don't get paused on the next iteration.\n\tresult, err = f.GetResult()\n\n\tassert.Equal(t, `test`, result)\n\tassert.Nil(t, err)\n}\n\nfunc TestHasResult(t *testing.T) {\n\tcompleter := make(chan interface{})\n\tf := New(completer, time.Duration(30*time.Minute))\n\n\tassert.False(t, f.HasResult())\n\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tf.GetResult()\n\t\twg.Done()\n\t}()\n\n\tcompleter <- `test`\n\twg.Wait()\n\n\tassert.True(t, f.HasResult())\n}\n\nfunc TestTimeout(t *testing.T) {\n\tcompleter := make(chan interface{})\n\tf := New(completer, time.Duration(0))\n\n\tresult, err := f.GetResult()\n\n\tassert.Nil(t, result)\n\tassert.NotNil(t, err)\n}\n\nfunc BenchmarkFuture(b *testing.B) {\n\tcompleter := make(chan interface{})\n\ttimeout := time.Duration(30 * time.Minute)\n\tvar wg sync.WaitGroup\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\twg.Add(1)\n\t\tf := New(completer, timeout)\n\t\tgo func() {\n\t\t\tf.GetResult()\n\t\t\twg.Done()\n\t\t}()\n\n\t\tcompleter <- `test`\n\t\twg.Wait()\n\t}\n}\n" }, { "path": "futures/selectable.go", "content": "/*\nCopyright 2016 Workiva, LLC\nCopyright 2016 Sokolov Yura aka funny_falcon\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage futures\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\n// ErrFutureCanceled signals that futures in canceled by a call to `f.Cancel()`\nvar ErrFutureCanceled = errors.New(\"future canceled\")\n\n// Selectable is a future with channel exposed for external `select`.\n// Many simultaneous listeners may wait for result either with `f.Value()`\n// or by selecting/fetching from `f.WaitChan()`, which is closed when future\n// fulfilled.\n// Selectable contains sync.Mutex, so it is not movable/copyable.\ntype Selectable struct {\n\tm sync.Mutex\n\tval interface{}\n\terr error\n\twait chan struct{}\n\tfilled uint32\n}\n\n// NewSelectable returns new selectable future.\n// Note: this method is for backward compatibility.\n// You may allocate it directly on stack or embedding into larger structure\nfunc NewSelectable() *Selectable {\n\treturn &Selectable{}\n}\n\nfunc (f *Selectable) wchan() <-chan struct{} {\n\tf.m.Lock()\n\tif f.wait == nil {\n\t\tf.wait = make(chan struct{})\n\t}\n\tch := f.wait\n\tf.m.Unlock()\n\treturn ch\n}\n\n// WaitChan returns channel, which is closed when future is fulfilled.\nfunc (f *Selectable) WaitChan() <-chan struct{} {\n\tif atomic.LoadUint32(&f.filled) == 1 {\n\t\treturn closed\n\t}\n\treturn f.wchan()\n}\n\n// GetResult waits for future to be fulfilled and returns value or error,\n// whatever is set first\nfunc (f *Selectable) GetResult() (interface{}, error) {\n\tif atomic.LoadUint32(&f.filled) == 0 {\n\t\t<-f.wchan()\n\t}\n\treturn f.val, f.err\n}\n\n// Fill sets value for future, if it were not already fulfilled\n// Returns error, if it were already set to future.\nfunc (f *Selectable) Fill(v interface{}, e error) error {\n\tf.m.Lock()\n\tif f.filled == 0 {\n\t\tf.val = v\n\t\tf.err = e\n\t\tatomic.StoreUint32(&f.filled, 1)\n\t\tw := f.wait\n\t\tf.wait = closed\n\t\tif w != nil {\n\t\t\tclose(w)\n\t\t}\n\t}\n\tf.m.Unlock()\n\treturn f.err\n}\n\n// SetValue is alias for Fill(v, nil)\nfunc (f *Selectable) SetValue(v interface{}) error {\n\treturn f.Fill(v, nil)\n}\n\n// SetError is alias for Fill(nil, e)\nfunc (f *Selectable) SetError(e error) {\n\tf.Fill(nil, e)\n}\n\n// Cancel is alias for SetError(ErrFutureCanceled)\nfunc (f *Selectable) Cancel() {\n\tf.SetError(ErrFutureCanceled)\n}\n\nvar closed = make(chan struct{})\n\nfunc init() {\n\tclose(closed)\n}\n" }, { "path": "futures/selectable_test.go", "content": "/*\nCopyright 2016 Workiva, LLC\nCopyright 2016 Sokolov Yura aka funny_falcon\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage futures\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestSelectableGetResult(t *testing.T) {\n\tf := NewSelectable()\n\tvar result interface{}\n\tvar err error\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tresult, err = f.GetResult()\n\t\twg.Done()\n\t}()\n\n\tf.SetValue(`test`)\n\twg.Wait()\n\n\tassert.Nil(t, err)\n\tassert.Equal(t, `test`, result)\n\n\t// ensure we don't get paused on the next iteration.\n\tresult, err = f.GetResult()\n\n\tassert.Equal(t, `test`, result)\n\tassert.Nil(t, err)\n}\n\nfunc TestSelectableSetError(t *testing.T) {\n\tf := NewSelectable()\n\tselect {\n\tcase <-f.WaitChan():\n\tcase <-time.After(0):\n\t\tf.SetError(fmt.Errorf(\"timeout\"))\n\t}\n\n\tresult, err := f.GetResult()\n\n\tassert.Nil(t, result)\n\tassert.NotNil(t, err)\n}\n\nfunc BenchmarkSelectable(b *testing.B) {\n\ttimeout := time.After(30 * time.Minute)\n\tvar wg sync.WaitGroup\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\twg.Add(1)\n\t\tf := NewSelectable()\n\t\tgo func() {\n\t\t\tselect {\n\t\t\tcase <-f.WaitChan():\n\t\t\tcase <-timeout:\n\t\t\t\tf.SetError(fmt.Errorf(\"timeout\"))\n\t\t\t}\n\t\t\twg.Done()\n\t\t}()\n\n\t\tf.SetValue(`test`)\n\t\twg.Wait()\n\t}\n}\n" }, { "path": "go.mod", "content": "module github.com/Workiva/go-datastructures\n\ngo 1.15\n\nrequire (\n\tgithub.com/stretchr/testify v1.7.0\n\tgithub.com/tinylib/msgp v1.1.5\n)\n" }, { "path": "go.sum", "content": "github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=\ngithub.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=\ngithub.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0=\ngithub.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=\ngithub.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "graph/simple.go", "content": "/*\nCopyright 2017 Julian Griggs\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage graph provides graph implementations. Currently, this includes an\nundirected simple graph.\n*/\npackage graph\n\nimport (\n\t\"errors\"\n\t\"sync\"\n)\n\nvar (\n\t// ErrVertexNotFound is returned when an operation is requested on a\n\t// non-existent vertex.\n\tErrVertexNotFound = errors.New(\"vertex not found\")\n\n\t// ErrSelfLoop is returned when an operation tries to create a disallowed\n\t// self loop.\n\tErrSelfLoop = errors.New(\"self loops not permitted\")\n\n\t// ErrParallelEdge is returned when an operation tries to create a\n\t// disallowed parallel edge.\n\tErrParallelEdge = errors.New(\"parallel edges are not permitted\")\n)\n\n// SimpleGraph is a mutable, non-persistent undirected graph.\n// Parallel edges and self-loops are not permitted.\n// Additional description: https://en.wikipedia.org/wiki/Graph_(discrete_mathematics)#Simple_graph\ntype SimpleGraph struct {\n\tmutex sync.RWMutex\n\tadjacencyList map[interface{}]map[interface{}]struct{}\n\tv, e int\n}\n\n// V returns the number of vertices in the SimpleGraph\nfunc (g *SimpleGraph) V() int {\n\tg.mutex.RLock()\n\tdefer g.mutex.RUnlock()\n\n\treturn g.v\n}\n\n// E returns the number of edges in the SimpleGraph\nfunc (g *SimpleGraph) E() int {\n\tg.mutex.RLock()\n\tdefer g.mutex.RUnlock()\n\n\treturn g.e\n}\n\n// AddEdge will create an edge between vertices v and w\nfunc (g *SimpleGraph) AddEdge(v, w interface{}) error {\n\tg.mutex.Lock()\n\tdefer g.mutex.Unlock()\n\n\tif v == w {\n\t\treturn ErrSelfLoop\n\t}\n\n\tg.addVertex(v)\n\tg.addVertex(w)\n\n\tif _, ok := g.adjacencyList[v][w]; ok {\n\t\treturn ErrParallelEdge\n\t}\n\n\tg.adjacencyList[v][w] = struct{}{}\n\tg.adjacencyList[w][v] = struct{}{}\n\tg.e++\n\treturn nil\n}\n\n// Adj returns the list of all vertices connected to v\nfunc (g *SimpleGraph) Adj(v interface{}) ([]interface{}, error) {\n\tg.mutex.RLock()\n\tdefer g.mutex.RUnlock()\n\n\tdeg, err := g.Degree(v)\n\tif err != nil {\n\t\treturn nil, ErrVertexNotFound\n\t}\n\n\tadj := make([]interface{}, deg)\n\ti := 0\n\tfor key := range g.adjacencyList[v] {\n\t\tadj[i] = key\n\t\ti++\n\t}\n\treturn adj, nil\n}\n\n// Degree returns the number of vertices connected to v\nfunc (g *SimpleGraph) Degree(v interface{}) (int, error) {\n\tg.mutex.RLock()\n\tdefer g.mutex.RUnlock()\n\n\tval, ok := g.adjacencyList[v]\n\tif !ok {\n\t\treturn 0, ErrVertexNotFound\n\t}\n\treturn len(val), nil\n}\n\nfunc (g *SimpleGraph) addVertex(v interface{}) {\n\tmm, ok := g.adjacencyList[v]\n\tif !ok {\n\t\tmm = make(map[interface{}]struct{})\n\t\tg.adjacencyList[v] = mm\n\t\tg.v++\n\t}\n}\n\n// NewSimpleGraph creates and returns a SimpleGraph\nfunc NewSimpleGraph() *SimpleGraph {\n\treturn &SimpleGraph{\n\t\tadjacencyList: make(map[interface{}]map[interface{}]struct{}),\n\t\tv: 0,\n\t\te: 0,\n\t}\n}\n" }, { "path": "graph/simple_test.go", "content": "/*\nCopyright 2017 Julian Griggs\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage graph\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestV(t *testing.T) {\n\tassert := assert.New(t)\n\tsgraph := NewSimpleGraph()\n\tassert.Equal(0, sgraph.V())\n\n\tsgraph.AddEdge(\"A\", \"B\")\n\tassert.Equal(2, sgraph.V())\n\n\tsgraph.AddEdge(\"B\", \"C\")\n\tassert.Equal(3, sgraph.V())\n\n\tsgraph.AddEdge(\"A\", \"C\")\n\tassert.Equal(3, sgraph.V())\n\n\t// Parallel edges not allowed\n\tsgraph.AddEdge(\"A\", \"C\")\n\tassert.Equal(3, sgraph.V())\n\tsgraph.AddEdge(\"C\", \"A\")\n\tassert.Equal(3, sgraph.V())\n\n\t// Self loops not allowed\n\tsgraph.AddEdge(\"C\", \"C\")\n\tassert.Equal(3, sgraph.V())\n\tsgraph.AddEdge(\"D\", \"D\")\n\tassert.Equal(3, sgraph.V())\n}\n\nfunc TestE(t *testing.T) {\n\tassert := assert.New(t)\n\tsgraph := NewSimpleGraph()\n\n\tassert.Equal(0, sgraph.E())\n\n\tsgraph.AddEdge(\"A\", \"B\")\n\tassert.Equal(1, sgraph.E())\n\n\tsgraph.AddEdge(\"B\", \"C\")\n\tassert.Equal(2, sgraph.E())\n\n\tsgraph.AddEdge(\"A\", \"C\")\n\tassert.Equal(3, sgraph.E())\n\n\t// Parallel edges not allowed\n\tsgraph.AddEdge(\"A\", \"C\")\n\tassert.Equal(3, sgraph.E())\n\tsgraph.AddEdge(\"C\", \"A\")\n\tassert.Equal(3, sgraph.E())\n\n\t// Self loops not allowed so no edges added\n\tsgraph.AddEdge(\"C\", \"C\")\n\tassert.Equal(3, sgraph.E())\n\tsgraph.AddEdge(\"D\", \"D\")\n\tassert.Equal(3, sgraph.E())\n}\n\nfunc TestDegree(t *testing.T) {\n\tassert := assert.New(t)\n\tsgraph := NewSimpleGraph()\n\n\t// No edges added so degree is 0\n\tv, err := sgraph.Degree(\"A\")\n\tassert.Zero(v)\n\tassert.Error(err)\n\n\t// One edge added\n\tsgraph.AddEdge(\"A\", \"B\")\n\tv, err = sgraph.Degree(\"A\")\n\tassert.Equal(1, v)\n\tassert.Nil(err)\n\n\t// Self loops are not allowed\n\tsgraph.AddEdge(\"A\", \"A\")\n\tv, err = sgraph.Degree(\"A\")\n\tassert.Equal(1, v)\n\tassert.Nil(err)\n\n\t// Parallel edges are not allowed\n\tsgraph.AddEdge(\"A\", \"B\")\n\tv, err = sgraph.Degree(\"A\")\n\tassert.Equal(1, v)\n\tassert.Nil(err)\n\tsgraph.AddEdge(\"B\", \"A\")\n\tv, err = sgraph.Degree(\"A\")\n\tassert.Equal(1, v)\n\tassert.Nil(err)\n\n\tv, err = sgraph.Degree(\"B\")\n\tassert.Equal(1, v)\n\tassert.Nil(err)\n\n\tsgraph.AddEdge(\"C\", \"D\")\n\tsgraph.AddEdge(\"A\", \"C\")\n\tsgraph.AddEdge(\"E\", \"F\")\n\tsgraph.AddEdge(\"E\", \"G\")\n\tsgraph.AddEdge(\"H\", \"G\")\n\n\tv, err = sgraph.Degree(\"A\")\n\tassert.Equal(2, v)\n\tassert.Nil(err)\n\n\tv, err = sgraph.Degree(\"B\")\n\tassert.Equal(1, v)\n\tassert.Nil(err)\n\n\tv, err = sgraph.Degree(\"C\")\n\tassert.Equal(2, v)\n\tassert.Nil(err)\n\n\tv, err = sgraph.Degree(\"D\")\n\tassert.Equal(1, v)\n\tassert.Nil(err)\n\n\tv, err = sgraph.Degree(\"E\")\n\tassert.Equal(2, v)\n\tassert.Nil(err)\n\n\tv, err = sgraph.Degree(\"G\")\n\tassert.Equal(2, v)\n\tassert.Nil(err)\n}\n\nfunc TestAddEdge(t *testing.T) {\n\tassert := assert.New(t)\n\tsgraph := NewSimpleGraph()\n\n\terr := sgraph.AddEdge(\"A\", \"B\")\n\tassert.Nil(err)\n\n\terr = sgraph.AddEdge(\"A\", \"B\")\n\tassert.Error(err)\n\n\terr = sgraph.AddEdge(\"B\", \"A\")\n\tassert.Error(err)\n\n\terr = sgraph.AddEdge(\"A\", \"A\")\n\tassert.Error(err)\n\n\terr = sgraph.AddEdge(\"C\", \"C\")\n\tassert.Error(err)\n\n\terr = sgraph.AddEdge(\"B\", \"C\")\n\tassert.Nil(err)\n\n}\n\nfunc TestAdj(t *testing.T) {\n\tassert := assert.New(t)\n\tsgraph := NewSimpleGraph()\n\n\tv, err := sgraph.Adj(\"A\")\n\tassert.Zero(v)\n\tassert.Error(err)\n\n\t// Self loops not allowed\n\tsgraph.AddEdge(\"A\", \"A\")\n\tv, err = sgraph.Adj(\"A\")\n\tassert.Zero(v)\n\tassert.Error(err)\n\n\tsgraph.AddEdge(\"A\", \"B\")\n\tv, err = sgraph.Adj(\"A\")\n\tassert.Equal(1, len(v))\n\tassert.Nil(err)\n\tassert.Equal(\"B\", v[0])\n\n\tv, err = sgraph.Adj(\"B\")\n\tassert.Equal(1, len(v))\n\tassert.Nil(err)\n\tassert.Equal(\"A\", v[0])\n\n\t// Parallel Edges not allowed\n\tsgraph.AddEdge(\"A\", \"B\")\n\tsgraph.AddEdge(\"B\", \"A\")\n\tv, err = sgraph.Adj(\"B\")\n\tassert.Equal(1, len(v))\n\tassert.Nil(err)\n\tassert.Equal(\"A\", v[0])\n\n\tsgraph.AddEdge(\"C\", \"D\")\n\tsgraph.AddEdge(\"A\", \"C\")\n\tsgraph.AddEdge(\"E\", \"F\")\n\tsgraph.AddEdge(\"E\", \"G\")\n\tsgraph.AddEdge(\"H\", \"G\")\n\n\tv, err = sgraph.Adj(\"A\")\n\tassert.Equal(2, len(v))\n\tassert.Nil(err)\n\tassert.Contains(v, \"B\")\n\tassert.Contains(v, \"C\")\n\tassert.NotContains(v, \"A\")\n\tassert.NotContains(v, \"D\")\n\n\tv, err = sgraph.Adj(\"B\")\n\tassert.Equal(1, len(v))\n\tassert.Nil(err)\n\tassert.Contains(v, \"A\")\n\tassert.NotContains(v, \"B\")\n\tassert.NotContains(v, \"C\")\n\tassert.NotContains(v, \"D\")\n\n\tv, err = sgraph.Adj(\"C\")\n\tassert.Equal(2, len(v))\n\tassert.Nil(err)\n\tassert.Contains(v, \"A\")\n\tassert.Contains(v, \"D\")\n\tassert.NotContains(v, \"B\")\n\tassert.NotContains(v, \"C\")\n\n\tv, err = sgraph.Adj(\"E\")\n\tassert.Equal(2, len(v))\n\tassert.Nil(err)\n\tassert.Contains(v, \"F\")\n\tassert.Contains(v, \"G\")\n\tassert.NotContains(v, \"A\")\n\n\tv, err = sgraph.Adj(\"G\")\n\tassert.Equal(2, len(v))\n\tassert.Nil(err)\n\tassert.Contains(v, \"E\")\n\tassert.Contains(v, \"H\")\n\tassert.NotContains(v, \"A\")\n}\n" }, { "path": "hashmap/fastinteger/hash.go", "content": "package fastinteger\n\n// hash will convert the uint64 key into a hash based on Murmur3's 64-bit\n// integer finalizer.\n// Details here: https://code.google.com/p/smhasher/wiki/MurmurHash3\nfunc hash(key uint64) uint64 {\n\tkey ^= key >> 33\n\tkey *= 0xff51afd7ed558ccd\n\tkey ^= key >> 33\n\tkey *= 0xc4ceb9fe1a85ec53\n\tkey ^= key >> 33\n\treturn key\n}\n" }, { "path": "hashmap/fastinteger/hash_test.go", "content": "package fastinteger\n\nimport (\n\t\"encoding/binary\"\n\t\"hash/fnv\"\n\t\"math/rand\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestHash(t *testing.T) {\n\tkey := uint64(5)\n\th := hash(key)\n\n\tassert.NotEqual(t, key, h)\n}\n\nfunc BenchmarkHash(b *testing.B) {\n\tnumItems := 1000\n\tr := rand.New(rand.NewSource(time.Now().UnixNano()))\n\n\tkeys := make([]uint64, 0, numItems)\n\tfor i := 0; i < numItems; i++ {\n\t\tkey := uint64(r.Int63())\n\t\tkeys = append(keys, key)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, key := range keys {\n\t\t\thash(key)\n\t\t}\n\t}\n}\n\nfunc BenchmarkFnvHash(b *testing.B) {\n\tnumItems := 1000\n\tr := rand.New(rand.NewSource(time.Now().UnixNano()))\n\n\tkeys := make([]uint64, 0, numItems)\n\tfor i := 0; i < numItems; i++ {\n\t\tkey := uint64(r.Int63())\n\t\tkeys = append(keys, key)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, key := range keys {\n\t\t\thasher := fnv.New64()\n\t\t\tbinary.Write(hasher, binary.LittleEndian, key)\n\t\t\thasher.Sum64()\n\t\t}\n\t}\n}\n" }, { "path": "hashmap/fastinteger/hashmap.go", "content": "// Package fastinteger is designed to provide a very primitive\n// implementation of a hash map for unsigned integer keys and\n// values. It is designed to have existence checks and insertions\n// that are faster than Go's native implementation. Like Go's\n// native implementation, FastIntegerHashMap will dynamically\n// grow in size.\n//\n// Current benchmarks on identical machine against native Go implementation:\n// \t\tBenchmarkInsert-8\t 10000\t 131258 ns/op\n//\t\tBenchmarkGoMapInsert-8\t 10000\t 208787 ns/op\n//\t\tBenchmarkExists-8\t 100000\t 15820 ns/op\n//\t\tBenchmarkGoMapExists-8\t 100000\t 16394 ns/op\n//\t\tBenchmarkDelete-8\t 100000\t 17909 ns/op\n//\t\tBenchmarkGoDelete-8\t 30000\t 49376 ns/op\n// \t\tBenchmarkInsertWithExpand-8\t 20000\t 90301 ns/op\n//\t\tBenchmarkGoInsertWithExpand-8\t 10000\t 142088 ns/op\n//\n//\n// This performance could be further enhanced by using a\n// better probing technique.\npackage fastinteger\n\nconst ratio = .75 // ratio sets the capacity the hashmap has to be at before it expands\n\n// roundUp takes a uint64 greater than 0 and rounds it up to the next\n// power of 2.\nfunc roundUp(v uint64) uint64 {\n\tv--\n\tv |= v >> 1\n\tv |= v >> 2\n\tv |= v >> 4\n\tv |= v >> 8\n\tv |= v >> 16\n\tv |= v >> 32\n\tv++\n\treturn v\n}\n\ntype packet struct {\n\tkey, value uint64\n}\n\ntype packets []*packet\n\nfunc (packets packets) find(key uint64) uint64 {\n\th := hash(key)\n\ti := h & (uint64(len(packets)) - 1)\n\tfor packets[i] != nil && packets[i].key != key {\n\t\ti = (i + 1) & (uint64(len(packets)) - 1)\n\t}\n\n\treturn i\n}\n\nfunc (packets packets) set(packet *packet) {\n\ti := packets.find(packet.key)\n\tif packets[i] == nil {\n\t\tpackets[i] = packet\n\t\treturn\n\t}\n\n\tpackets[i].value = packet.value\n}\n\nfunc (packets packets) get(key uint64) (uint64, bool) {\n\ti := packets.find(key)\n\tif packets[i] == nil {\n\t\treturn 0, false\n\t}\n\n\treturn packets[i].value, true\n}\n\nfunc (packets packets) delete(key uint64) bool {\n\ti := packets.find(key)\n\tif packets[i] == nil {\n\t\treturn false\n\t}\n\tpackets[i] = nil\n\ti = (i + 1) & (uint64(len(packets)) - 1)\n\tfor packets[i] != nil {\n\t\tp := packets[i]\n\t\tpackets[i] = nil\n\t\tpackets.set(p)\n\t\ti = (i + 1) & (uint64(len(packets)) - 1)\n\t}\n\treturn true\n}\n\nfunc (packets packets) exists(key uint64) bool {\n\ti := packets.find(key)\n\treturn packets[i] != nil // technically, they can store nil\n}\n\n// FastIntegerHashMap is a simple hashmap to be used with\n// integer only keys. It supports few operations, and is designed\n// primarily for cases where the consumer needs a very simple\n// datastructure to set and check for existence of integer\n// keys over a sparse range.\ntype FastIntegerHashMap struct {\n\tcount uint64\n\tpackets packets\n}\n\n// rebuild is an expensive operation which requires us to iterate\n// over the current bucket and rehash the keys for insertion into\n// the new bucket. The new bucket is twice as large as the old\n// bucket by default.\nfunc (fi *FastIntegerHashMap) rebuild() {\n\tpackets := make(packets, roundUp(uint64(len(fi.packets))+1))\n\tfor _, packet := range fi.packets {\n\t\tif packet == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tpackets.set(packet)\n\t}\n\tfi.packets = packets\n}\n\n// Get returns an item from the map if it exists. Otherwise,\n// returns false for the second argument.\nfunc (fi *FastIntegerHashMap) Get(key uint64) (uint64, bool) {\n\treturn fi.packets.get(key)\n}\n\n// Set will set the provided key with the provided value.\nfunc (fi *FastIntegerHashMap) Set(key, value uint64) {\n\tif float64(fi.count+1)/float64(len(fi.packets)) > ratio {\n\t\tfi.rebuild()\n\t}\n\n\tfi.packets.set(&packet{key: key, value: value})\n\tfi.count++\n}\n\n// Exists will return a bool indicating if the provided key\n// exists in the map.\nfunc (fi *FastIntegerHashMap) Exists(key uint64) bool {\n\treturn fi.packets.exists(key)\n}\n\n// Delete will remove the provided key from the hashmap. If\n// the key cannot be found, this is a no-op.\nfunc (fi *FastIntegerHashMap) Delete(key uint64) {\n\tif fi.packets.delete(key) {\n\t\tfi.count--\n\t}\n}\n\n// Len returns the number of items in the hashmap.\nfunc (fi *FastIntegerHashMap) Len() uint64 {\n\treturn fi.count\n}\n\n// Cap returns the capacity of the hashmap.\nfunc (fi *FastIntegerHashMap) Cap() uint64 {\n\treturn uint64(len(fi.packets))\n}\n\n// New returns a new FastIntegerHashMap with a bucket size specified\n// by hint.\nfunc New(hint uint64) *FastIntegerHashMap {\n\tif hint == 0 {\n\t\thint = 16\n\t}\n\n\thint = roundUp(hint)\n\treturn &FastIntegerHashMap{\n\t\tcount: 0,\n\t\tpackets: make(packets, hint),\n\t}\n}\n" }, { "path": "hashmap/fastinteger/hashmap_test.go", "content": "package fastinteger\n\nimport (\n\t\"math/rand\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc generateKeys(num int) []uint64 {\n\tr := rand.New(rand.NewSource(time.Now().UnixNano()))\n\n\tkeys := make([]uint64, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tkey := uint64(r.Int63())\n\t\tkeys = append(keys, key)\n\t}\n\n\treturn keys\n}\n\nfunc TestRoundUp(t *testing.T) {\n\tresult := roundUp(21)\n\tassert.Equal(t, uint64(32), result)\n\n\tresult = roundUp(uint64(1<<31) - 234)\n\tassert.Equal(t, uint64(1<<31), result)\n\n\tresult = roundUp(uint64(1<<63) - 324)\n\tassert.Equal(t, uint64(1<<63), result)\n}\n\nfunc TestInsert(t *testing.T) {\n\thm := New(10)\n\n\thm.Set(5, 5)\n\n\tassert.True(t, hm.Exists(5))\n\tvalue, ok := hm.Get(5)\n\tassert.Equal(t, uint64(5), value)\n\tassert.True(t, ok)\n\tassert.Equal(t, uint64(16), hm.Cap())\n}\n\nfunc TestInsertOverwrite(t *testing.T) {\n\thm := New(10)\n\n\thm.Set(5, 5)\n\thm.Set(5, 10)\n\n\tassert.True(t, hm.Exists(5))\n\tvalue, ok := hm.Get(5)\n\tassert.Equal(t, uint64(10), value)\n\tassert.True(t, ok)\n}\n\nfunc TestGet(t *testing.T) {\n\thm := New(10)\n\n\tvalue, ok := hm.Get(5)\n\tassert.False(t, ok)\n\tassert.Equal(t, uint64(0), value)\n}\n\nfunc TestMultipleInserts(t *testing.T) {\n\thm := New(10)\n\n\thm.Set(5, 5)\n\thm.Set(6, 6)\n\n\tassert.True(t, hm.Exists(6))\n\tvalue, ok := hm.Get(6)\n\tassert.True(t, ok)\n\tassert.Equal(t, uint64(6), value)\n}\n\nfunc TestRebuild(t *testing.T) {\n\tnumItems := uint64(100)\n\n\thm := New(10)\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\thm.Set(i, i)\n\t}\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tvalue, _ := hm.Get(i)\n\t\tassert.Equal(t, i, value)\n\t}\n}\n\nfunc TestDelete(t *testing.T) {\n\thm := New(10)\n\n\thm.Set(5, 5)\n\thm.Set(6, 6)\n\n\thm.Delete(5)\n\n\tassert.Equal(t, uint64(1), hm.Len())\n\tassert.False(t, hm.Exists(5))\n\n\thm.Delete(6)\n\tassert.Equal(t, uint64(0), hm.Len())\n\tassert.False(t, hm.Exists(6))\n}\n\nfunc TestDeleteAll(t *testing.T) {\n\tnumItems := uint64(100)\n\n\thm := New(10)\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\thm.Set(i, i)\n\t}\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\thm.Delete(i)\n\t\tassert.False(t, hm.Exists(i))\n\t}\n}\n\nfunc TestDeleteCollision(t *testing.T) {\n\t// 1, 27, 42 all hash to the same value using our hash function % 32\n\tif hash(1)%32 != 12 || hash(27)%32 != 12 || hash(42)%32 != 12 {\n\t\tt.Error(\"test values don't hash to the same value\")\n\t}\n\n\tm := New(32)\n\tm.Set(1, 1)\n\tm.Set(27, 27)\n\tm.Set(42, 42)\n\n\tm.Delete(27)\n\tvalue, ok := m.Get(42)\n\tassert.True(t, ok)\n\tassert.Equal(t, uint64(42), value)\n}\n\nfunc BenchmarkInsert(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\tkeys := generateKeys(int(numItems))\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\thm := New(numItems * 2) // so we don't rebuild\n\t\tfor _, k := range keys {\n\t\t\thm.Set(k, k)\n\t\t}\n\t}\n}\n\nfunc BenchmarkGoMapInsert(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\tkeys := generateKeys(int(numItems))\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\thm := make(map[uint64]uint64, numItems*2) // so we don't rebuild\n\t\tfor _, k := range keys {\n\t\t\thm[k] = k\n\t\t}\n\t}\n}\n\nfunc BenchmarkExists(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\tkeys := generateKeys(int(numItems))\n\thm := New(numItems * 2) // so we don't rebuild\n\tfor _, key := range keys {\n\t\thm.Set(key, key)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, key := range keys {\n\t\t\thm.Exists(key)\n\t\t}\n\t}\n}\n\nfunc BenchmarkGoMapExists(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\tkeys := generateKeys(int(numItems))\n\thm := make(map[uint64]uint64, numItems*2) // so we don't rebuild\n\tfor _, key := range keys {\n\t\thm[key] = key\n\t}\n\n\tb.ResetTimer()\n\n\tvar ok bool\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, key := range keys {\n\t\t\t_, ok = hm[key] // or the compiler complains\n\t\t}\n\t}\n\n\tb.StopTimer()\n\tif ok { // or the compiler complains\n\t}\n}\n\nfunc BenchmarkDelete(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\thms := make([]*FastIntegerHashMap, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\thm := New(numItems * 2)\n\t\tfor j := uint64(0); j < numItems; j++ {\n\t\t\thm.Set(j, j)\n\t\t}\n\t\thms = append(hms, hm)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\thm := hms[i]\n\t\tfor j := uint64(0); j < numItems; j++ {\n\t\t\thm.Delete(j)\n\t\t}\n\t}\n}\n\nfunc BenchmarkGoDelete(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\thms := make([]map[uint64]uint64, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\thm := make(map[uint64]uint64, numItems*2)\n\t\tfor j := uint64(0); j < numItems; j++ {\n\t\t\thm[j] = j\n\t\t}\n\t\thms = append(hms, hm)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\thm := hms[i]\n\t\tfor j := uint64(0); j < numItems; j++ {\n\t\t\tdelete(hm, j)\n\t\t}\n\t}\n}\n\nfunc BenchmarkInsertWithExpand(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\thms := make([]*FastIntegerHashMap, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\thm := New(10)\n\t\thms = append(hms, hm)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\thm := hms[i]\n\t\tfor j := uint64(0); j < numItems; j++ {\n\t\t\thm.Set(j, j)\n\t\t}\n\t}\n}\n\nfunc BenchmarkGoInsertWithExpand(b *testing.B) {\n\tnumItems := uint64(1000)\n\n\thms := make([]map[uint64]uint64, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\thm := make(map[uint64]uint64, 10)\n\t\thms = append(hms, hm)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\thm := hms[i]\n\t\tfor j := uint64(0); j < numItems; j++ {\n\t\t\thm[j] = j\n\t\t}\n\t}\n}\n" }, { "path": "list/persistent.go", "content": "/*\nCopyright 2015 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage list provides list implementations. Currently, this includes a\npersistent, immutable linked list.\n*/\npackage list\n\nimport \"errors\"\n\nvar (\n\t// Empty is an empty PersistentList.\n\tEmpty PersistentList = &emptyList{}\n\n\t// ErrEmptyList is returned when an invalid operation is performed on an\n\t// empty list.\n\tErrEmptyList = errors.New(\"Empty list\")\n)\n\n// PersistentList is an immutable, persistent linked list.\ntype PersistentList interface {\n\t// Head returns the head of the list. The bool will be false if the list is\n\t// empty.\n\tHead() (interface{}, bool)\n\n\t// Tail returns the tail of the list. The bool will be false if the list is\n\t// empty.\n\tTail() (PersistentList, bool)\n\n\t// IsEmpty indicates if the list is empty.\n\tIsEmpty() bool\n\n\t// Length returns the number of items in the list.\n\tLength() uint\n\n\t// Add will add the item to the list, returning the new list.\n\tAdd(head interface{}) PersistentList\n\n\t// Insert will insert the item at the given position, returning the new\n\t// list or an error if the position is invalid.\n\tInsert(val interface{}, pos uint) (PersistentList, error)\n\n\t// Get returns the item at the given position or an error if the position\n\t// is invalid.\n\tGet(pos uint) (interface{}, bool)\n\n\t// Remove will remove the item at the given position, returning the new\n\t// list or an error if the position is invalid.\n\tRemove(pos uint) (PersistentList, error)\n\n\t// Find applies the predicate function to the list and returns the first\n\t// item which matches.\n\tFind(func(interface{}) bool) (interface{}, bool)\n\n\t// FindIndex applies the predicate function to the list and returns the\n\t// index of the first item which matches or -1 if there is no match.\n\tFindIndex(func(interface{}) bool) int\n\n\t// Map applies the function to each entry in the list and returns the\n\t// resulting slice.\n\tMap(func(interface{}) interface{}) []interface{}\n}\n\ntype emptyList struct{}\n\n// Head returns the head of the list. The bool will be false if the list is\n// empty.\nfunc (e *emptyList) Head() (interface{}, bool) {\n\treturn nil, false\n}\n\n// Tail returns the tail of the list. The bool will be false if the list is\n// empty.\nfunc (e *emptyList) Tail() (PersistentList, bool) {\n\treturn nil, false\n}\n\n// IsEmpty indicates if the list is empty.\nfunc (e *emptyList) IsEmpty() bool {\n\treturn true\n}\n\n// Length returns the number of items in the list.\nfunc (e *emptyList) Length() uint {\n\treturn 0\n}\n\n// Add will add the item to the list, returning the new list.\nfunc (e *emptyList) Add(head interface{}) PersistentList {\n\treturn &list{head, e}\n}\n\n// Insert will insert the item at the given position, returning the new list or\n// an error if the position is invalid.\nfunc (e *emptyList) Insert(val interface{}, pos uint) (PersistentList, error) {\n\tif pos == 0 {\n\t\treturn e.Add(val), nil\n\t}\n\treturn nil, ErrEmptyList\n}\n\n// Get returns the item at the given position or an error if the position is\n// invalid.\nfunc (e *emptyList) Get(pos uint) (interface{}, bool) {\n\treturn nil, false\n}\n\n// Remove will remove the item at the given position, returning the new list or\n// an error if the position is invalid.\nfunc (e *emptyList) Remove(pos uint) (PersistentList, error) {\n\treturn nil, ErrEmptyList\n}\n\n// Find applies the predicate function to the list and returns the first item\n// which matches.\nfunc (e *emptyList) Find(func(interface{}) bool) (interface{}, bool) {\n\treturn nil, false\n}\n\n// FindIndex applies the predicate function to the list and returns the index\n// of the first item which matches or -1 if there is no match.\nfunc (e *emptyList) FindIndex(func(interface{}) bool) int {\n\treturn -1\n}\n\n// Map applies the function to each entry in the list and returns the resulting\n// slice.\nfunc (e *emptyList) Map(func(interface{}) interface{}) []interface{} {\n\treturn nil\n}\n\ntype list struct {\n\thead interface{}\n\ttail PersistentList\n}\n\n// Head returns the head of the list. The bool will be false if the list is\n// empty.\nfunc (l *list) Head() (interface{}, bool) {\n\treturn l.head, true\n}\n\n// Tail returns the tail of the list. The bool will be false if the list is\n// empty.\nfunc (l *list) Tail() (PersistentList, bool) {\n\treturn l.tail, true\n}\n\n// IsEmpty indicates if the list is empty.\nfunc (l *list) IsEmpty() bool {\n\treturn false\n}\n\n// Length returns the number of items in the list.\nfunc (l *list) Length() uint {\n\tcurr := l\n\tlength := uint(0)\n\tfor {\n\t\tlength += 1\n\t\ttail, _ := curr.Tail()\n\t\tif tail.IsEmpty() {\n\t\t\treturn length\n\t\t}\n\t\tcurr = tail.(*list)\n\t}\n}\n\n// Add will add the item to the list, returning the new list.\nfunc (l *list) Add(head interface{}) PersistentList {\n\treturn &list{head, l}\n}\n\n// Insert will insert the item at the given position, returning the new list or\n// an error if the position is invalid.\nfunc (l *list) Insert(val interface{}, pos uint) (PersistentList, error) {\n\tif pos == 0 {\n\t\treturn l.Add(val), nil\n\t}\n\tnl, err := l.tail.Insert(val, pos-1)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn nl.Add(l.head), nil\n}\n\n// Get returns the item at the given position or an error if the position is\n// invalid.\nfunc (l *list) Get(pos uint) (interface{}, bool) {\n\tif pos == 0 {\n\t\treturn l.head, true\n\t}\n\treturn l.tail.Get(pos - 1)\n}\n\n// Remove will remove the item at the given position, returning the new list or\n// an error if the position is invalid.\nfunc (l *list) Remove(pos uint) (PersistentList, error) {\n\tif pos == 0 {\n\t\tnl, _ := l.Tail()\n\t\treturn nl, nil\n\t}\n\n\tnl, err := l.tail.Remove(pos - 1)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &list{l.head, nl}, nil\n}\n\n// Find applies the predicate function to the list and returns the first item\n// which matches.\nfunc (l *list) Find(pred func(interface{}) bool) (interface{}, bool) {\n\tif pred(l.head) {\n\t\treturn l.head, true\n\t}\n\treturn l.tail.Find(pred)\n}\n\n// FindIndex applies the predicate function to the list and returns the index\n// of the first item which matches or -1 if there is no match.\nfunc (l *list) FindIndex(pred func(interface{}) bool) int {\n\tcurr := l\n\tidx := 0\n\tfor {\n\t\tif pred(curr.head) {\n\t\t\treturn idx\n\t\t}\n\t\ttail, _ := curr.Tail()\n\t\tif tail.IsEmpty() {\n\t\t\treturn -1\n\t\t}\n\t\tcurr = tail.(*list)\n\t\tidx += 1\n\t}\n}\n\n// Map applies the function to each entry in the list and returns the resulting\n// slice.\nfunc (l *list) Map(f func(interface{}) interface{}) []interface{} {\n\treturn append(l.tail.Map(f), f(l.head))\n}\n" }, { "path": "list/persistent_test.go", "content": "/*\nCopyright 2015 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage list\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestEmptyList(t *testing.T) {\n\tassert := assert.New(t)\n\thead, ok := Empty.Head()\n\tassert.Nil(head)\n\tassert.False(ok)\n\n\ttail, ok := Empty.Tail()\n\tassert.Nil(tail)\n\tassert.False(ok)\n\n\tassert.True(Empty.IsEmpty())\n}\n\nfunc TestAdd(t *testing.T) {\n\tassert := assert.New(t)\n\tl1 := Empty.Add(1)\n\n\t// l1: [1]\n\tassert.False(l1.IsEmpty())\n\thead, ok := l1.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n\ttail, ok := l1.Tail()\n\tassert.True(ok)\n\tassert.Equal(Empty, tail)\n\n\tl1 = l1.Add(2)\n\n\t// l1: [2, 1]\n\thead, ok = l1.Head()\n\tassert.True(ok)\n\tassert.Equal(2, head)\n\ttail, ok = l1.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n\n\tl2, err := l1.Insert(\"a\", 1)\n\tassert.Nil(err)\n\n\t// l1: [2, 1]\n\t// l2: [2, \"a\", 1]\n\thead, ok = l1.Head()\n\tassert.True(ok)\n\tassert.Equal(2, head)\n\ttail, ok = l1.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n\n\thead, ok = l2.Head()\n\tassert.True(ok)\n\tassert.Equal(2, head)\n\ttail, ok = l2.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(\"a\", head)\n\ttail, ok = tail.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n}\n\nfunc TestInsertAndGet(t *testing.T) {\n\tassert := assert.New(t)\n\t_, err := Empty.Insert(1, 5)\n\tassert.Error(err)\n\n\tl, err := Empty.Insert(1, 0)\n\tassert.Nil(err)\n\n\t// [1]\n\titem, ok := l.Get(0)\n\tassert.True(ok)\n\tassert.Equal(1, item)\n\n\tl, err = l.Insert(2, 0)\n\tassert.Nil(err)\n\n\t// [2, 1]\n\titem, ok = l.Get(0)\n\tassert.True(ok)\n\tassert.Equal(2, item)\n\titem, ok = l.Get(1)\n\tassert.True(ok)\n\tassert.Equal(1, item)\n\n\t_, ok = l.Get(2)\n\tassert.False(ok)\n\n\tl, err = l.Insert(\"a\", 3)\n\tassert.Nil(l)\n\tassert.Error(err)\n}\n\nfunc TestRemove(t *testing.T) {\n\tassert := assert.New(t)\n\tl, err := Empty.Remove(0)\n\tassert.Nil(l)\n\tassert.Error(err)\n\n\tl = Empty.Add(1)\n\tl = l.Add(2)\n\tl = l.Add(3)\n\n\t// [3, 2, 1]\n\tl1, err := l.Remove(3)\n\tassert.Nil(l1)\n\tassert.Error(err)\n\n\tl2, err := l.Remove(0)\n\n\t// l: [3, 2, 1]\n\t// l2: [2, 1]\n\tassert.Nil(err)\n\thead, ok := l.Head()\n\tassert.True(ok)\n\tassert.Equal(3, head)\n\ttail, ok := l.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(2, head)\n\ttail, ok = tail.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n\n\tassert.Nil(err)\n\thead, ok = l2.Head()\n\tassert.True(ok)\n\tassert.Equal(2, head)\n\ttail, ok = l2.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n\n\tl2, err = l.Remove(1)\n\n\t// l: [3, 2, 1]\n\t// l2: [3, 1]\n\tassert.Nil(err)\n\thead, ok = l.Head()\n\tassert.True(ok)\n\tassert.Equal(3, head)\n\ttail, ok = l.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(2, head)\n\ttail, ok = tail.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n\n\tassert.Nil(err)\n\thead, ok = l2.Head()\n\tassert.True(ok)\n\tassert.Equal(3, head)\n\ttail, ok = l2.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n\n\tl2, err = l.Remove(2)\n\n\t// l: [3, 2, 1]\n\t// l2: [3, 2]\n\tassert.Nil(err)\n\thead, ok = l.Head()\n\tassert.True(ok)\n\tassert.Equal(3, head)\n\ttail, ok = l.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(2, head)\n\ttail, ok = tail.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(1, head)\n\n\tassert.Nil(err)\n\thead, ok = l2.Head()\n\tassert.True(ok)\n\tassert.Equal(3, head)\n\ttail, ok = l2.Tail()\n\tassert.True(ok)\n\thead, ok = tail.Head()\n\tassert.True(ok)\n\tassert.Equal(2, head)\n}\n\nfunc TestFind(t *testing.T) {\n\tassert := assert.New(t)\n\tpred := func(item interface{}) bool {\n\t\treturn item == 1\n\t}\n\n\tfound, ok := Empty.Find(pred)\n\tassert.Nil(found)\n\tassert.False(ok)\n\n\tl := Empty.Add(\"blah\").Add(\"bleh\")\n\n\tfound, ok = l.Find(pred)\n\tassert.Nil(found)\n\tassert.False(ok)\n\n\tl = l.Add(1).Add(\"foo\")\n\n\tfound, ok = l.Find(pred)\n\tassert.Equal(1, found)\n\tassert.True(ok)\n}\n\nfunc TestFindIndex(t *testing.T) {\n\tassert := assert.New(t)\n\tpred := func(item interface{}) bool {\n\t\treturn item == 1\n\t}\n\n\tidx := Empty.FindIndex(pred)\n\tassert.Equal(-1, idx)\n\n\tl := Empty.Add(\"blah\").Add(\"bleh\")\n\n\tidx = l.FindIndex(pred)\n\tassert.Equal(-1, idx)\n\n\tl = l.Add(1).Add(\"foo\")\n\n\tidx = l.FindIndex(pred)\n\tassert.Equal(1, idx)\n}\n\nfunc TestLength(t *testing.T) {\n\tassert := assert.New(t)\n\tassert.Equal(uint(0), Empty.Length())\n\n\tl := Empty.Add(\"foo\")\n\tassert.Equal(uint(1), l.Length())\n\tl = l.Add(\"bar\").Add(\"baz\")\n\tassert.Equal(uint(3), l.Length())\n}\n\nfunc TestMap(t *testing.T) {\n\tassert := assert.New(t)\n\tf := func(x interface{}) interface{} {\n\t\treturn x.(int) * x.(int)\n\t}\n\tassert.Nil(Empty.Map(f))\n\n\tl := Empty.Add(1).Add(2).Add(3).Add(4)\n\tassert.Equal([]interface{}{1, 4, 9, 16}, l.Map(f))\n}\n" }, { "path": "mock/batcher.go", "content": "/*\nCopyright 2015 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage mock\n\nimport (\n\t\"github.com/stretchr/testify/mock\"\n\n\t\"github.com/Workiva/go-datastructures/batcher\"\n)\n\nvar _ batcher.Batcher = new(Batcher)\n\ntype Batcher struct {\n\tmock.Mock\n\tPutChan chan bool\n}\n\nfunc (m *Batcher) Put(items interface{}) error {\n\targs := m.Called(items)\n\tif m.PutChan != nil {\n\t\tm.PutChan <- true\n\t}\n\treturn args.Error(0)\n}\n\nfunc (m *Batcher) Get() ([]interface{}, error) {\n\targs := m.Called()\n\treturn args.Get(0).([]interface{}), args.Error(1)\n}\n\nfunc (m *Batcher) Flush() error {\n\targs := m.Called()\n\treturn args.Error(0)\n}\n\nfunc (m *Batcher) Dispose() {\n\tm.Called()\n}\n\nfunc (m *Batcher) IsDisposed() bool {\n\targs := m.Called()\n\treturn args.Bool(0)\n}\n" }, { "path": "mock/rangetree.go", "content": "package mock\n\nimport (\n\t\"github.com/stretchr/testify/mock\"\n\n\t\"github.com/Workiva/go-datastructures/rangetree\"\n)\n\ntype RangeTree struct {\n\tmock.Mock\n}\n\nvar _ rangetree.RangeTree = new(RangeTree)\n\nfunc (m *RangeTree) Add(entries ...rangetree.Entry) rangetree.Entries {\n\targs := m.Called(entries)\n\tifc := args.Get(0)\n\tif ifc == nil {\n\t\treturn nil\n\t}\n\n\treturn ifc.(rangetree.Entries)\n}\n\nfunc (m *RangeTree) Len() uint64 {\n\treturn m.Called().Get(0).(uint64)\n}\n\nfunc (m *RangeTree) Delete(entries ...rangetree.Entry) rangetree.Entries {\n\treturn m.Called(entries).Get(0).(rangetree.Entries)\n}\n\nfunc (m *RangeTree) Query(interval rangetree.Interval) rangetree.Entries {\n\targs := m.Called(interval)\n\tifc := args.Get(0)\n\tif ifc == nil {\n\t\treturn nil\n\t}\n\n\treturn ifc.(rangetree.Entries)\n}\n\nfunc (m *RangeTree) InsertAtDimension(dimension uint64, index,\n\tnumber int64) (rangetree.Entries, rangetree.Entries) {\n\n\targs := m.Called(dimension, index, number)\n\treturn args.Get(0).(rangetree.Entries), args.Get(1).(rangetree.Entries)\n}\n\nfunc (m *RangeTree) Apply(interval rangetree.Interval, fn func(rangetree.Entry) bool) {\n\tm.Called(interval, fn)\n}\n\nfunc (m *RangeTree) Get(entries ...rangetree.Entry) rangetree.Entries {\n\tifc := m.Called(entries).Get(0)\n\tif ifc == nil {\n\t\treturn nil\n\t}\n\n\treturn ifc.(rangetree.Entries)\n}\n" }, { "path": "numerics/hilbert/hilbert.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage Hilbert is designed to allow consumers to find the Hilbert\ndistance on the Hilbert curve if given a 2 dimensional coordinate.\nThis could be useful for hashing or constructing a Hilbert R-Tree.\nAlgorithm taken from here:\n\nhttp://en.wikipedia.org/wiki/Hilbert_curve\n\nThis expects coordinates in the range [0, 0] to [MaxInt32, MaxInt32].\nUsing negative values for x and y will have undefinied behavior.\n\nBenchmarks:\nBenchmarkEncode-8\t10000000\t 181 ns/op\nBenchmarkDecode-8\t10000000\t 191 ns/op\n*/\npackage hilbert\n\n// n defines the maximum power of 2 that can define a bound,\n// this is the value for 2-d space if you want to support\n// all hilbert ids with a single integer variable\nconst n = 1 << 31\n\nfunc boolToInt(value bool) int32 {\n\tif value {\n\t\treturn int32(1)\n\t}\n\n\treturn int32(0)\n}\n\nfunc rotate(n, rx, ry int32, x, y *int32) {\n\tif ry == 0 {\n\t\tif rx == 1 {\n\t\t\t*x = n - 1 - *x\n\t\t\t*y = n - 1 - *y\n\t\t}\n\n\t\tt := *x\n\t\t*x = *y\n\t\t*y = t\n\t}\n}\n\n// Encode will encode the provided x and y coordinates into a Hilbert\n// distance.\nfunc Encode(x, y int32) int64 {\n\tvar rx, ry int32\n\tvar d int64\n\tfor s := int32(n / 2); s > 0; s /= 2 {\n\t\trx = boolToInt(x&s > 0)\n\t\try = boolToInt(y&s > 0)\n\t\td += int64(int64(s) * int64(s) * int64(((3 * rx) ^ ry)))\n\t\trotate(s, rx, ry, &x, &y)\n\t}\n\n\treturn d\n}\n\n// Decode will decode the provided Hilbert distance into a corresponding\n// x and y value, respectively.\nfunc Decode(h int64) (int32, int32) {\n\tvar ry, rx int64\n\tvar x, y int32\n\tt := h\n\n\tfor s := int64(1); s < int64(n); s *= 2 {\n\t\trx = 1 & (t / 2)\n\t\try = 1 & (t ^ rx)\n\t\trotate(int32(s), int32(rx), int32(ry), &x, &y)\n\t\tx += int32(s * rx)\n\t\ty += int32(s * ry)\n\t\tt /= 4\n\t}\n\n\treturn x, y\n}\n" }, { "path": "numerics/hilbert/hilbert_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\npackage hilbert\n\nimport (\n\t\"math\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestHilbert(t *testing.T) {\n\th := Encode(0, 0)\n\tx, y := Decode(h)\n\tassert.Equal(t, int64(0), h)\n\tassert.Equal(t, int32(0), x)\n\tassert.Equal(t, int32(0), y)\n\n\th = Encode(1, 0)\n\tx, y = Decode(h)\n\tassert.Equal(t, int64(3), h)\n\tassert.Equal(t, int32(1), x)\n\tassert.Equal(t, int32(0), y)\n\n\th = Encode(1, 1)\n\tx, y = Decode(h)\n\tassert.Equal(t, int64(2), h)\n\tassert.Equal(t, int32(1), x)\n\tassert.Equal(t, int32(1), y)\n\n\th = Encode(0, 1)\n\tx, y = Decode(h)\n\tassert.Equal(t, int64(1), h)\n\tassert.Equal(t, int32(0), x)\n\tassert.Equal(t, int32(1), y)\n}\n\nfunc TestHilbertAtMaxRange(t *testing.T) {\n\tx, y := int32(math.MaxInt32), int32(math.MaxInt32)\n\th := Encode(x, y)\n\tresultx, resulty := Decode(h)\n\tassert.Equal(t, x, resultx)\n\tassert.Equal(t, y, resulty)\n}\n\nfunc BenchmarkEncode(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tEncode(int32(i), int32(i))\n\t}\n}\n\nfunc BenchmarkDecode(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tDecode(int64(i))\n\t}\n}\n" }, { "path": "numerics/optimization/global.go", "content": "package optimization\n\nimport (\n\t\"math\"\n\t\"sort\"\n)\n\ntype pbs []*vertexProbabilityBundle\n\ntype vertexProbabilityBundle struct {\n\tprobability float64\n\tvertex *nmVertex\n}\n\n// calculateVVP will calculate the variable variance probability\n// of the provided vertex based on the previous best guess\n// and the provided sigma. The sigma changes with each run\n// of the optimization algorithm and accounts for a changing\n// number of guesses.\n//\n// VVP is defined as:\n// 1/((2*pi)^(1/2)*sigma)*(1-e^(-dmin^2/2*sigma^2))\n// where dmin = euclidean distance between this vertex and the best guess\n// and sigma = (3*(m^(1/n)))^-1\n//\nfunc calculateVVP(guess, vertex *nmVertex, sigma float64) float64 {\n\tdistance := -guess.euclideanDistance(vertex)\n\tlhs := 1 / (math.Sqrt(2*math.Pi) * sigma)\n\trhs := 1 - math.Exp(math.Pow(distance, 2)/(2*math.Pow(sigma, 2)))\n\treturn rhs * lhs\n}\n\n// calculateSigma will calculate sigma based on the provided information.\n// Typically, sigma will decrease as the number of sampled points\n// increases.\n//\n// sigma = (3*(m^(1/n)))^-1\n//\nfunc calculateSigma(dimensions, guesses int) float64 {\n\treturn math.Pow(3*math.Pow(float64(guesses), 1/float64(dimensions)), -1)\n}\n\nfunc (pbs pbs) calculateProbabilities(bestGuess *nmVertex, sigma float64) {\n\tfor _, v := range pbs {\n\t\tv.probability = calculateVVP(bestGuess, v.vertex, sigma)\n\t}\n}\n\nfunc (pbs pbs) sort() {\n\tsort.Sort(pbs)\n}\n\nfunc (pbs pbs) Less(i, j int) bool {\n\treturn pbs[i].probability < pbs[j].probability\n}\n\nfunc (pbs pbs) Swap(i, j int) {\n\tpbs[i], pbs[j] = pbs[j], pbs[i]\n}\n\nfunc (pbs pbs) Len() int {\n\treturn len(pbs)\n}\n\n// results stores the results of previous iterations of the\n// nelder-mead algorithm\ntype results struct {\n\t// vertices are the results generated by the algorithm\n\tvertices vertices\n\t// config is useful for examining target\n\tconfig NelderMeadConfiguration\n\t// pbs contains the randomly generated guess vertices\n\tpbs pbs\n}\n\n// search will search this list of results based on order, order\n// being defined in the NelderMeadConfiguration, that is a defined\n// target will be treated\nfunc (results *results) search(result *nmVertex) int {\n\treturn sort.Search(len(results.vertices), func(i int) bool {\n\t\treturn !results.vertices[i].less(results.config, result)\n\t})\n}\n\nfunc (results *results) exists(result *nmVertex, hint int) bool {\n\tif hint < 0 {\n\t\thint = results.search(result)\n\t}\n\n\t// maximum hint here should be len(results.vertices)\n\tif hint > 0 && results.vertices[hint-1].approximatelyEqualToVertex(result) {\n\t\treturn true\n\t}\n\n\t// -1 here because if hint == len(vertices) we would've already\n\t// checked the last value in the previous conditional\n\tif hint < len(results.vertices)-1 && results.vertices[hint].approximatelyEqualToVertex(result) {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\nfunc (results *results) insert(vertex *nmVertex) {\n\ti := results.search(vertex)\n\tif results.exists(vertex, i) {\n\t\treturn\n\t}\n\n\tif i == len(results.vertices) {\n\t\tresults.vertices = append(results.vertices, vertex)\n\t\treturn\n\t}\n\n\tresults.vertices = append(results.vertices, nil)\n\tcopy(results.vertices[i+1:], results.vertices[i:])\n\tresults.vertices[i] = vertex\n}\n\nfunc (results *results) grab(num int) vertices {\n\tvs := make(vertices, 0, num)\n\t// first, copy what you want to the list to return\n\t// not returning a sub-slice as we're about to mutate\n\t// the original slice\n\tfor i := 0; i < num; i++ {\n\t\tvs = append(vs, results.pbs[i].vertex)\n\t}\n\t// now we overwrite the vertices that we are taking\n\t// from the beginning\n\tcopy(results.pbs, results.pbs[num:])\n\tlength := len(results.pbs) - num\n\t// this next part is required for the GC\n\tfor i := length; i < len(results.pbs); i++ {\n\t\tresults.pbs[i] = nil\n\t}\n\n\t// and finally set the new slice as a subslice\n\tresults.pbs = results.pbs[:length]\n\treturn vs\n}\n\n// reSort will re-sort the list of possible guess vertices\n// based upon the latest calculated result. It was also\n// add this result to the list of results.\nfunc (results *results) reSort(vertex *nmVertex) {\n\tresults.insert(vertex)\n\n\tbestGuess := results.vertices[0]\n\tsigma := calculateSigma(len(results.config.Vars), len(results.vertices))\n\tresults.pbs.calculateProbabilities(bestGuess, sigma)\n\tresults.pbs.sort()\n}\n\nfunc newResults(guess *nmVertex, config NelderMeadConfiguration, num int) *results {\n\tvertices := make(vertices, 0, num+1)\n\tvertices = append(vertices, guess)\n\tvertices = append(vertices, generateRandomVerticesFromGuess(guess, num)...)\n\n\tbundles := make(pbs, 0, len(vertices))\n\tfor _, v := range vertices {\n\t\tbundles = append(bundles, &vertexProbabilityBundle{vertex: v})\n\t}\n\n\treturn &results{\n\t\tpbs: bundles,\n\t\tconfig: config,\n\t}\n}\n" }, { "path": "numerics/optimization/nelder_mead.go", "content": "package optimization\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"sort\"\n\t\"time\"\n)\n\nconst (\n\talpha = 1 // reflection, must be > 0\n\tbeta = 2 // expansion, must be > 1\n\tgamma = .5 // contraction, 0 < gamma < 1\n\tsigma = .5 // shrink, 0 < sigma < 1\n\tdelta = .0001 // going to use this to determine convergence\n\tmaxRuns = 130\n\tmaxIterations = 5 // maxIterations defines the number of restarts that should\n\t// occur when attempting to find a global critical point\n)\n\nvar (\n\tmin = math.Inf(-1)\n\tmax = math.Inf(1)\n)\n\n// generateRandomVerticesFromGuess will generate num number of vertices\n// with random\nfunc generateRandomVerticesFromGuess(guess *nmVertex, num int) vertices {\n\t// summed allows us to prevent duplicate guesses, checking\n\t// all previous guesses for every guess created would be too\n\t// time consuming so we take an indexed shortcut here. summed\n\t// is a map of a sum of the vars to the vertices that have an\n\t// identical sum. In this way, we can sum the vars of a new guess\n\t// and check only a small subset of previous guesses to determine\n\t// if this is an identical guess.\n\tsummed := make(map[float64]vertices, num)\n\tdimensions := len(guess.vars)\n\tvs := make(vertices, 0, num)\n\ti := 0\n\tr := rand.New(rand.NewSource(time.Now().UnixNano()))\n\nGuess:\n\tfor i < num {\n\t\tsum := float64(0)\n\t\tvars := make([]float64, 0, dimensions)\n\t\tfor j := 0; j < dimensions; j++ {\n\t\t\tv := r.Float64() * 1000\n\t\t\t// we do a separate random check here to determine\n\t\t\t// sign so we don't end up with all high v's one sign\n\t\t\t// and low v's another\n\t\t\tif r.Float64() > .5 {\n\t\t\t\tv = -v\n\t\t\t}\n\t\t\tsum += v\n\t\t\tvars = append(vars, v)\n\t\t}\n\n\t\tguess := &nmVertex{\n\t\t\tvars: vars,\n\t\t}\n\n\t\tif vs, ok := summed[sum]; !ok {\n\t\t\tvs = make(vertices, 0, dimensions) // dimensions is really just a guess, no real way of knowing what this is\n\t\t\tvs = append(vs, guess)\n\t\t\tsummed[sum] = vs\n\t\t} else {\n\t\t\tfor _, vertex := range vs {\n\t\t\t\t// if we've already guessed this, try the loop again\n\t\t\t\tif guess.equalToVertex(vertex) {\n\t\t\t\t\tcontinue Guess\n\t\t\t\t}\n\t\t\t}\n\t\t\tvs = append(vs, guess)\n\t\t}\n\n\t\tvs = append(vs, guess)\n\t\ti++\n\t}\n\n\treturn vs\n}\n\nfunc isInf(num float64) bool {\n\treturn math.IsInf(num, -1) || math.IsInf(num, 1)\n}\n\nfunc findMin(vertices ...*nmVertex) *nmVertex {\n\tmin := vertices[0]\n\tfor _, v := range vertices[1:] {\n\t\tif v.distance < min.distance {\n\t\t\tmin = v\n\t\t}\n\t}\n\n\treturn min\n}\n\n// findMidpoint will find the midpoint of the provided vertices\n// and return a new vertex.\nfunc findMidpoint(vertices ...*nmVertex) *nmVertex {\n\tnum := len(vertices) // this is what we divide by\n\tvars := make([]float64, 0, num)\n\n\tfor i := 0; i < num; i++ {\n\t\tsum := float64(0)\n\t\tfor _, v := range vertices {\n\t\t\tsum += v.vars[i]\n\t\t}\n\t\tvars = append(vars, sum/float64(num))\n\t}\n\n\treturn &nmVertex{\n\t\tvars: vars,\n\t}\n}\n\n// determineDistance will determine the distance between the value\n// and the target. If the target is positive or negative infinity,\n// (ie find max or min), this is clamped to max or min float64.\nfunc determineDistance(value, target float64) float64 {\n\tif math.IsInf(target, 1) { // positive infinity\n\t\ttarget = math.MaxFloat64\n\t} else if math.IsInf(target, -1) { // negative infinity\n\t\ttarget = -math.MaxFloat64\n\t}\n\n\treturn math.Abs(target - value)\n}\n\ntype vertices []*nmVertex\n\n// evaluate will call evaluate on all the verticies in this list\n// and order them by distance to target.\nfunc (vertices vertices) evaluate(config NelderMeadConfiguration) {\n\tfor _, v := range vertices {\n\t\tv.evaluate(config)\n\t}\n\n\tvertices.sort(config)\n}\n\nfunc (vertices vertices) sort(config NelderMeadConfiguration) {\n\tsorter := sorter{\n\t\tconfig: config,\n\t\tvertices: vertices,\n\t}\n\tsorter.sort()\n}\n\ntype sorter struct {\n\tconfig NelderMeadConfiguration\n\tvertices vertices\n}\n\nfunc (sorter sorter) sort() {\n\tsort.Sort(sorter)\n}\n\n// the following methods are required for sort.Interface. We\n// use the standard libraries sort here as it uses an adaptive\n// sort and we really don't expect there to be a ton of dimensions\n// here so mulithreaded sort in this repo really isn't\n// necessary.\n\nfunc (sorter sorter) Less(i, j int) bool {\n\treturn sorter.vertices[i].less(sorter.config, sorter.vertices[j])\n}\n\nfunc (sorter sorter) Len() int {\n\treturn len(sorter.vertices)\n}\n\nfunc (sorter sorter) Swap(i, j int) {\n\tsorter.vertices[i], sorter.vertices[j] = sorter.vertices[j], sorter.vertices[i]\n}\n\n// String prints out a string representation of every vertex in this list.\n// Useful for debugging :).\nfunc (vertices vertices) String() string {\n\tresult := ``\n\tfor i, v := range vertices {\n\t\tresult += fmt.Sprintf(`VERTEX INDEX: %+v, VERTEX: %+v`, i, v)\n\t\tresult += fmt.Sprintln(``)\n\t}\n\n\treturn result\n}\n\n// NelderMeadConfiguration is the struct that must be\n// passed into the NelderMead function. This defines\n// the target value, the function to be run, and a guess\n// of the variables.\ntype NelderMeadConfiguration struct {\n\t// Target is the target we are trying to converge\n\t// to. Set this to positive or negative infinity\n\t// to find the min/max.\n\tTarget float64\n\t// Fn defines the function that Nelder Mead is going\n\t// to call to determine if it is moving closer\n\t// to convergence. In all likelihood, the execution\n\t// of this function is going to be the bottleneck.\n\t// The second value returns a bool indicating if the\n\t// calculated values are \"good\", that is, that no\n\t// constraint has been hit.\n\tFn func([]float64) (float64, bool)\n\t// Vars is a guess and will determine what other\n\t// vertices will be used. By convention, since\n\t// this guess will contain as many numbers as the\n\t// target function requires, the len of Vars determines\n\t// the dimension of this problem.\n\tVars []float64\n}\n\ntype nmVertex struct {\n\t// vars indicates the values used to calculate this vertex.\n\tvars []float64\n\t// distance is the distance between this vertex and the desired\n\t// value. This metric has little meaning if the desired value\n\t// is +- inf.\n\t// result is the calculated result of this vertex. This can\n\t// be used to measure distance or as a metrix to compare two\n\t// vertices if the desired result is a min/max.\n\tdistance, result float64\n\t// good indicates if the calculated values here\n\t// are within all constraints, this should always\n\t// be true if this vertex is in a list of vertices.\n\tgood bool\n}\n\nfunc (nm *nmVertex) evaluate(config NelderMeadConfiguration) {\n\tnm.result, nm.good = config.Fn(nm.vars)\n\tnm.distance = determineDistance(nm.result, config.Target)\n}\n\nfunc (nm *nmVertex) add(other *nmVertex) *nmVertex {\n\tvars := make([]float64, 0, len(nm.vars))\n\tfor i := 0; i < len(nm.vars); i++ {\n\t\tvars = append(vars, nm.vars[i]+other.vars[i])\n\t}\n\n\treturn &nmVertex{\n\t\tvars: vars,\n\t}\n}\n\nfunc (nm *nmVertex) multiply(scalar float64) *nmVertex {\n\tvars := make([]float64, 0, len(nm.vars))\n\tfor i := 0; i < len(nm.vars); i++ {\n\t\tvars = append(vars, nm.vars[i]*scalar)\n\t}\n\n\treturn &nmVertex{\n\t\tvars: vars,\n\t}\n}\n\nfunc (nm *nmVertex) subtract(other *nmVertex) *nmVertex {\n\tvars := make([]float64, 0, len(nm.vars))\n\tfor i := 0; i < len(nm.vars); i++ {\n\t\tvars = append(vars, nm.vars[i]-other.vars[i])\n\t}\n\n\treturn &nmVertex{\n\t\tvars: vars,\n\t}\n}\n\n// less defines a relationship between two points. It is best not to\n// think of less as returning a value indicating absolute relationship between\n// two points, but instead think of less returning a bool indicating\n// if this vertex is *closer* to the desired convergence, or a delta\n// less than the other vertex. For -inf, this returns a value indicating\n// if this vertex has a less absolute value than the other vertex, if +inf\n// less returns a bool indicating if this vertex has a *greater* absolute\n// value than the other vertex. Otherwise, this method returns a bool\n// indicating if this vertex is closer to *converging* upon the desired\n// value.\nfunc (nm *nmVertex) less(config NelderMeadConfiguration, other *nmVertex) bool {\n\tif config.Target == min { // looking for a min\n\t\treturn nm.result < other.result\n\t}\n\tif config.Target == max { // looking for a max\n\t\treturn nm.result > other.result\n\t}\n\n\treturn nm.distance < other.distance\n}\n\nfunc (nm *nmVertex) equal(config NelderMeadConfiguration, other *nmVertex) bool {\n\tif isInf(config.Target) {\n\t\t// if we are looking for a min or max, we compare result\n\t\treturn nm.result == other.result\n\t}\n\n\t// otherwise, we compare distances\n\treturn nm.distance == other.distance\n}\n\n// euclideanDistance determines the euclidean distance between two points.\nfunc (nm *nmVertex) euclideanDistance(other *nmVertex) float64 {\n\tsum := float64(0)\n\t// first we want to sum all the distances between the points\n\tfor i, otherPoint := range other.vars {\n\t\t// distance between points is defined by (qi-ri)^2\n\t\tsum += math.Pow(otherPoint-nm.vars[i], 2)\n\t}\n\n\treturn math.Sqrt(sum)\n}\n\n// equalToVertex will compare this vertex to the provided vertex\n// to determine if the two vertices are actually identical (that is,\n// they fall on the same point).\nfunc (nm *nmVertex) equalToVertex(other *nmVertex) bool {\n\tfor i, n := range nm.vars {\n\t\tif n != other.vars[i] {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// approximatelyEqualToVertex returns a bool indicating if the\n// *result* of this vertex is approximately equal to the vertex\n// provided. Approximately is 2 * delta as the algorithm may\n// cease within a delta distance of the true value, so we may\n// end up with a result that's 2*delta away if we came from\n// the other direction.\nfunc (nm *nmVertex) approximatelyEqualToVertex(other *nmVertex) bool {\n\treturn math.Abs(nm.result-other.result) < 2*delta\n}\n\ntype nelderMead struct {\n\tconfig NelderMeadConfiguration\n\tresults *results\n}\n\n// evaluateWithConstraints will safely evaluate the vertex while\n// conforming to any imposed restraints. If a constraint is found,\n// this method will backtrack the vertex as described here:\n// http://www.iccm-central.org/Proceedings/ICCM16proceedings/contents/pdf/MonK/MoKA1-04ge_ghiasimh224461p.pdf\n// This should work with even non-linear constraints, but it is up to\n// the consumer to check these constraints.\nfunc (nm *nelderMead) evaluateWithConstraints(vertices vertices, vertex *nmVertex) *nmVertex {\n\tvertex.evaluate(nm.config)\n\treturn vertex\n\tif vertex.good {\n\t\treturn vertex\n\t}\n\tbest := vertices[0]\n\tfor i := 0; i < 5; i++ {\n\t\tvertex = best.add((vertex.subtract(best).multiply(alpha)))\n\t\tif vertex.good {\n\t\t\treturn vertex\n\t\t}\n\t}\n\n\treturn best\n}\n\n// reflect will find the reflection point between the two best guesses\n// with the provided midpoint.\nfunc (nm *nelderMead) reflect(vertices vertices, midpoint *nmVertex) *nmVertex {\n\ttoScalar := midpoint.subtract(nm.lastVertex(vertices))\n\ttoScalar = toScalar.multiply(alpha)\n\ttoScalar = midpoint.add(toScalar)\n\treturn nm.evaluateWithConstraints(vertices, toScalar)\n}\n\nfunc (nm *nelderMead) expand(vertices vertices, midpoint, reflection *nmVertex) *nmVertex {\n\ttoScalar := reflection.subtract(midpoint)\n\ttoScalar = toScalar.multiply(beta)\n\ttoScalar = midpoint.add(toScalar)\n\treturn nm.evaluateWithConstraints(vertices, toScalar)\n}\n\n// lastDimensionVertex returns the vertex that is represented by the\n// last dimension, effectively, second to last in the list of\n// vertices.\nfunc (nm *nelderMead) lastDimensionVertex(vertices vertices) *nmVertex {\n\treturn vertices[len(vertices)-2]\n}\n\n// lastVertex returns the last vertex in the list of vertices.\n// It's important to remember that this vertex represents the\n// number of dimensions + 1.\nfunc (nm *nelderMead) lastVertex(vertices vertices) *nmVertex {\n\treturn vertices[len(vertices)-1]\n}\n\nfunc (nm *nelderMead) outsideContract(vertices vertices, midpoint, reflection *nmVertex) *nmVertex {\n\ttoScalar := reflection.subtract(midpoint)\n\ttoScalar = toScalar.multiply(gamma)\n\ttoScalar = midpoint.add(toScalar)\n\treturn nm.evaluateWithConstraints(vertices, toScalar)\n}\n\nfunc (nm *nelderMead) insideContract(vertices vertices, midpoint, reflection *nmVertex) *nmVertex {\n\ttoScalar := reflection.subtract(midpoint)\n\ttoScalar = toScalar.multiply(gamma)\n\ttoScalar = midpoint.subtract(toScalar)\n\treturn nm.evaluateWithConstraints(vertices, toScalar)\n}\n\nfunc (nm *nelderMead) shrink(vertices vertices) {\n\tone := vertices[0]\n\tfor i := 1; i < len(vertices); i++ {\n\t\ttoScalar := vertices[i].subtract(one)\n\t\ttoScalar = toScalar.multiply(sigma)\n\t\tvertices[i] = one.add(toScalar)\n\t}\n}\n\n// checkIteration checks some key values to determine if\n// iteration should be complete. Returns false if iteration\n// should be terminated and true if iteration should continue.\nfunc (nm *nelderMead) checkIteration(vertices vertices) bool {\n\t// this will never be true for += inf\n\tif math.Abs(vertices[0].result-nm.config.Target) < delta {\n\t\treturn false\n\t}\n\n\tbest := vertices[0]\n\t// here we are checking distance convergence. If all vertices\n\t// are near convergence, that is they are all within some delta\n\t// from the expected value, we can go ahead and quit early. This\n\t// can only be performed on convergence checks, not for finding\n\t// min/max.\n\tif !isInf(nm.config.Target) {\n\t\tfor _, v := range vertices[1:] {\n\t\t\tif math.Abs(best.distance-v.distance) >= delta {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\n\t// next we want to check to see if the changes in our polytopes\n\t// dip below some threshold. That is, we want to look at the\n\t// euclidean distances between the best guess and all the other\n\t// guesses to see if they are converged upon some point. If\n\t// all of the vertices have converged close enough, it may be\n\t// worth it to cease iteration.\n\tfor _, v := range vertices[1:] {\n\t\tif best.euclideanDistance(v) >= delta {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc (nm *nelderMead) evaluate() {\n\tvertices := nm.results.grab(len(nm.config.Vars) + 1)\n\t// if the initial guess provided is not good, then\n\t// we are going to die early, leave it up to the user\n\t// to create a good first guess.\n\tvertices[0].evaluate(nm.config)\n\tif !vertices[0].good {\n\t\tnm.results.insert(vertices[0])\n\t\treturn\n\t}\n\n\t// the outer loop controls how hard we try to find\n\t// a global critical point\n\tfor i := 0; i < maxIterations; i++ {\n\t\t// the inner loop controls the degenerate case where\n\t\t// we can't converge to a critical point\n\t\tfor j := 0; j < maxRuns; j++ {\n\t\t\t// TODO: optimize this to prevent duplicate evaluations.\n\t\t\tvertices.evaluate(nm.config)\n\t\t\tbest := vertices[0]\n\t\t\tif !nm.checkIteration(vertices) {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tmidpoint := findMidpoint(vertices[:len(vertices)-1]...)\n\t\t\t// we are guaranteed to have two points here\n\t\t\treflection := nm.reflect(vertices, midpoint)\n\t\t\t// we could not find a reflection that met constraints, the\n\t\t\t// best guess is the best guess.\n\t\t\tif reflection == best {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// in this case, quality-wise, we are between the best\n\t\t\t// and second to best points\n\t\t\tif reflection.less(nm.config, nm.lastDimensionVertex(vertices)) &&\n\t\t\t\t!vertices[0].less(nm.config, reflection) {\n\n\t\t\t\tvertices[len(vertices)-1] = reflection\n\t\t\t}\n\n\t\t\t// midpoint is closer than our previous best guess\n\t\t\tif reflection.less(nm.config, vertices[0]) {\n\t\t\t\texpanded := nm.expand(vertices, midpoint, reflection)\n\t\t\t\t// we could not expand a valid guess, best is the best guess\n\t\t\t\tif expanded == best {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\t// we only need to expand here\n\t\t\t\tif expanded.less(nm.config, reflection) {\n\t\t\t\t\tvertices[len(vertices)-1] = expanded\n\t\t\t\t} else {\n\t\t\t\t\tvertices[len(vertices)-1] = reflection\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// reflection is a bad guess, let's try to contract both\n\t\t\t// inside and outside and see if we can find a better value\n\t\t\tif reflection.less(nm.config, nm.lastVertex(vertices)) {\n\t\t\t\toc := nm.outsideContract(vertices, midpoint, reflection)\n\t\t\t\tif oc == best {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif oc.less(nm.config, reflection) || oc.equal(nm.config, reflection) {\n\t\t\t\t\tvertices[len(vertices)-1] = oc\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t} else if !reflection.less(nm.config, nm.lastVertex(vertices)) {\n\t\t\t\tic := nm.insideContract(vertices, midpoint, reflection)\n\t\t\t\tif ic == best {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif ic.less(nm.config, nm.lastVertex(vertices)) {\n\t\t\t\t\tvertices[len(vertices)-1] = ic\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// we could not guess a better value than nm.vertices[0], so\n\t\t\t// let's converge the other to guesses to our best guess.\n\t\t\tnm.shrink(vertices)\n\t\t}\n\t\tnm.results.reSort(vertices[0])\n\t\tvertices = nm.results.grab(len(nm.config.Vars) + 1)\n\t}\n}\n\nfunc newNelderMead(config NelderMeadConfiguration) *nelderMead {\n\tv := &nmVertex{vars: config.Vars} // construct initial vertex with first guess\n\tresults := newResults(v, config, 1000) // 1000 represents 1000 initial vertex guesses\n\n\treturn &nelderMead{\n\t\tconfig: config,\n\t\tresults: results,\n\t}\n}\n\n// NelderMead takes a configuration and returns a list\n// of floats that can be plugged into the provided function\n// to converge at the target value.\nfunc NelderMead(config NelderMeadConfiguration) []float64 {\n\tnm := newNelderMead(config)\n\tnm.evaluate()\n\treturn nm.results.vertices[0].vars\n}\n" }, { "path": "numerics/optimization/nelder_mead_test.go", "content": "package optimization\n\nimport (\n\t\"math\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestNelderMead(t *testing.T) {\n\tfn := func(vars []float64) (float64, bool) {\n\t\treturn vars[0] * vars[1], true\n\t}\n\tconfig := NelderMeadConfiguration{\n\t\tTarget: float64(9),\n\t\tFn: fn,\n\t\tVars: []float64{2, 4},\n\t}\n\n\tresult, _ := fn(NelderMead(config))\n\tassert.True(t, math.Abs(result-config.Target) <= .01)\n}\n\nfunc TestNelderMeadPolynomial(t *testing.T) {\n\tfn := func(vars []float64) (float64, bool) {\n\t\t// x^2-4x+y^2-y-xy, solution is (3, 2)\n\t\treturn math.Pow(vars[0], 2) - 4*vars[0] + math.Pow(vars[1], 2) - vars[1] - vars[0]*vars[1], true\n\t}\n\tconfig := NelderMeadConfiguration{\n\t\tTarget: float64(-100),\n\t\tFn: fn,\n\t\tVars: []float64{-10, 10},\n\t}\n\n\tresult := NelderMead(config)\n\tcalced, _ := fn(result)\n\tassert.True(t, math.Abs(7-math.Abs(calced)) <= .01)\n\tassert.True(t, math.Abs(3-result[0]) <= .1)\n\tassert.True(t, math.Abs(2-result[1]) <= .1)\n}\n\nfunc TestNelderMeadPolynomialMin(t *testing.T) {\n\tfn := func(vars []float64) (float64, bool) {\n\t\t// x^2-4x+y^2-y-xy, solution is (3, 2)\n\t\treturn math.Pow(vars[0], 2) - 4*vars[0] + math.Pow(vars[1], 2) - vars[1] - vars[0]*vars[1], true\n\t}\n\tconfig := NelderMeadConfiguration{\n\t\tTarget: math.Inf(-1),\n\t\tFn: fn,\n\t\tVars: []float64{-10, 10},\n\t}\n\n\tresult := NelderMead(config)\n\tcalced, _ := fn(result)\n\tassert.True(t, math.Abs(7-math.Abs(calced)) <= .01)\n\tassert.True(t, math.Abs(3-result[0]) <= .01)\n\tassert.True(t, math.Abs(2-result[1]) <= .01)\n}\n\nfunc TestNelderMeadPolynomialMax(t *testing.T) {\n\tfn := func(vars []float64) (float64, bool) {\n\t\t// 3+sin(x)+2cos(y)^2, the min on this equation is 2 and the max is 6\n\t\treturn 3 + math.Sin(vars[0]) + 2*math.Pow(math.Cos(vars[1]), 2), true\n\t}\n\n\tconfig := NelderMeadConfiguration{\n\t\tTarget: math.Inf(1),\n\t\tFn: fn,\n\t\tVars: []float64{-5, 5},\n\t}\n\n\tresult := NelderMead(config)\n\tcalced, _ := fn(result)\n\tassert.True(t, math.Abs(6-math.Abs(calced)) <= .01)\n}\n\nfunc TestNelderMeadConstrained(t *testing.T) {\n\tfn := func(vars []float64) (float64, bool) {\n\t\tif vars[0] < 1 || vars[1] < 1 {\n\t\t\treturn 0, false\n\t\t}\n\t\treturn math.Pow(vars[0], 2) - 4*vars[0] + math.Pow(vars[1], 2) - vars[1] - vars[0]*vars[1], true\n\t}\n\t// by default, converging on this point with the initial\n\t// guess of (6, 3) will converge to (~.46, ~4.75). The\n\t// fn has the added constraint that no guesses may be below\n\t// 1. This should now converge to a point (~8.28, ~4.93).\n\tconfig := NelderMeadConfiguration{\n\t\tTarget: float64(14),\n\t\tFn: fn,\n\t\tVars: []float64{6, 3},\n\t}\n\n\tresult := NelderMead(config)\n\tcalced, _ := fn(result)\n\tassert.True(t, math.Abs(14-math.Abs(calced)) <= .01)\n\tassert.True(t, result[0] >= 1)\n\tassert.True(t, result[1] >= 1)\n\n\tfn = func(vars []float64) (float64, bool) {\n\t\tif vars[0] < 6 || vars[0] > 8 {\n\t\t\treturn 0, false\n\t\t}\n\n\t\tif vars[1] < 0 || vars[1] > 2 {\n\t\t\treturn 0, false\n\t\t}\n\t\treturn math.Pow(vars[0], 2) - 4*vars[0] + math.Pow(vars[1], 2) - vars[1] - vars[0]*vars[1], true\n\t}\n\n\tconfig = NelderMeadConfiguration{\n\t\tTarget: float64(14),\n\t\tFn: fn,\n\t\tVars: []float64{6, .5},\n\t}\n\n\tresult = NelderMead(config)\n\tcalced, _ = fn(result)\n\t// there are two local min here\n\tassert.True(t, math.Abs(14-math.Abs(calced)) <= .01 || math.Abs(8.75-math.Abs(calced)) <= .01)\n\tassert.True(t, result[0] >= 6 && result[0] <= 8)\n\tassert.True(t, result[1] >= 0 && result[1] <= 2)\n}\n\nfunc TestNelderMeadConstrainedBadGuess(t *testing.T) {\n\tfn := func(vars []float64) (float64, bool) {\n\t\tif vars[0] < 1 || vars[1] < 1 {\n\t\t\treturn 0, false\n\t\t}\n\t\treturn math.Pow(vars[0], 2) - 4*vars[0] + math.Pow(vars[1], 2) - vars[1] - vars[0]*vars[1], true\n\t}\n\t// this is a bad guess, as in the initial guess doesn't\n\t// match the constraints. In that case, we return the guessed\n\t// values.\n\tconfig := NelderMeadConfiguration{\n\t\tTarget: float64(14),\n\t\tFn: fn,\n\t\tVars: []float64{0, 3},\n\t}\n\n\tresult := NelderMead(config)\n\tassert.Equal(t, float64(0), result[0])\n\tassert.Equal(t, float64(3), result[1])\n}\n\n// Commenting this function out for now as it's entirely\n// probabilistic. Realistically, we can only say that we'll\n// find the local vs global min/max some percentage of the time\n// and that percentage depends entirely on the function.\n// This is here for debugging purposes.\n/*\nfunc TestNelderMeadFindGlobal(t *testing.T) {\n\tfn := func(vars []float64) (float64, bool) {\n\t\tif vars[0] < -4 || vars[0] > 2 {\n\t\t\treturn 0, false\n\t\t}\n\t\t// x3 + 3x2 − 2x + 1 over [-4, 2] has a global maximum at x = 2\n\t\treturn math.Pow(vars[0], 3) + 3*math.Pow(vars[0], 2) - 2*vars[0] + 1, true\n\t}\n\n\tconfig := NelderMeadConfiguration{\n\t\tTarget: math.Inf(1),\n\t\tFn: fn,\n\t\tVars: []float64{1.5},\n\t}\n\n\tresult := NelderMead(config)\n\tcalced, _ := fn(result)\n\twc, _ := fn([]float64{2})\n\tt.Logf(`RESULT: %+v, CALCED: %+v, WC: %+v`, result, calced, wc)\n\tt.Fail()\n}*/\n" }, { "path": "queue/error.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage queue\n\nimport \"errors\"\n\nvar (\n\t// ErrDisposed is returned when an operation is performed on a disposed\n\t// queue.\n\tErrDisposed = errors.New(`queue: disposed`)\n\n\t// ErrTimeout is returned when an applicable queue operation times out.\n\tErrTimeout = errors.New(`queue: poll timed out`)\n\n\t// ErrEmptyQueue is returned when an non-applicable queue operation was called\n\t// due to the queue's empty item state\n\tErrEmptyQueue = errors.New(`queue: empty queue`)\n)\n" }, { "path": "queue/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage queue\n\ntype mockItem int\n\nfunc (mi mockItem) Compare(other Item) int {\n\tomi := other.(mockItem)\n\tif mi > omi {\n\t\treturn 1\n\t} else if mi == omi {\n\t\treturn 0\n\t}\n\treturn -1\n}\n" }, { "path": "queue/priority_queue.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nThe priority queue is almost a spitting image of the logic\nused for a regular queue. In order to keep the logic fast,\nthis code is repeated instead of using casts to cast to interface{}\nback and forth. If Go had inheritance and generics, this problem\nwould be easier to solve.\n*/\n\npackage queue\n\nimport \"sync\"\n\n// Item is an item that can be added to the priority queue.\ntype Item interface {\n\t// Compare returns a bool that can be used to determine\n\t// ordering in the priority queue. Assuming the queue\n\t// is in ascending order, this should return > logic.\n\t// Return 1 to indicate this object is greater than the\n\t// the other logic, 0 to indicate equality, and -1 to indicate\n\t// less than other.\n\tCompare(other Item) int\n}\n\ntype priorityItems []Item\n\nfunc (items *priorityItems) swap(i, j int) {\n\t(*items)[i], (*items)[j] = (*items)[j], (*items)[i]\n}\n\nfunc (items *priorityItems) pop() Item {\n\tsize := len(*items)\n\n\t// Move last leaf to root, and 'pop' the last item.\n\titems.swap(size-1, 0)\n\titem := (*items)[size-1] // Item to return.\n\t(*items)[size-1], *items = nil, (*items)[:size-1]\n\n\t// 'Bubble down' to restore heap property.\n\tindex := 0\n\tchildL, childR := 2*index+1, 2*index+2\n\tfor len(*items) > childL {\n\t\tchild := childL\n\t\tif len(*items) > childR && (*items)[childR].Compare((*items)[childL]) < 0 {\n\t\t\tchild = childR\n\t\t}\n\n\t\tif (*items)[child].Compare((*items)[index]) < 0 {\n\t\t\titems.swap(index, child)\n\n\t\t\tindex = child\n\t\t\tchildL, childR = 2*index+1, 2*index+2\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn item\n}\n\nfunc (items *priorityItems) get(number int) []Item {\n\treturnItems := make([]Item, 0, number)\n\tfor i := 0; i < number; i++ {\n\t\tif len(*items) == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\treturnItems = append(returnItems, items.pop())\n\t}\n\n\treturn returnItems\n}\n\nfunc (items *priorityItems) push(item Item) {\n\t// Stick the item as the end of the last level.\n\t*items = append(*items, item)\n\n\t// 'Bubble up' to restore heap property.\n\tindex := len(*items) - 1\n\tparent := int((index - 1) / 2)\n\tfor parent >= 0 && (*items)[parent].Compare(item) > 0 {\n\t\titems.swap(index, parent)\n\n\t\tindex = parent\n\t\tparent = int((index - 1) / 2)\n\t}\n}\n\n// PriorityQueue is similar to queue except that it takes\n// items that implement the Item interface and adds them\n// to the queue in priority order.\ntype PriorityQueue struct {\n\twaiters waiters\n\titems priorityItems\n\titemMap map[Item]struct{}\n\tlock sync.Mutex\n\tdisposeLock sync.Mutex\n\tdisposed bool\n\tallowDuplicates bool\n}\n\n// Put adds items to the queue.\nfunc (pq *PriorityQueue) Put(items ...Item) error {\n\tif len(items) == 0 {\n\t\treturn nil\n\t}\n\n\tpq.lock.Lock()\n\tdefer pq.lock.Unlock()\n\n\tif pq.disposed {\n\t\treturn ErrDisposed\n\t}\n\n\tfor _, item := range items {\n\t\tif pq.allowDuplicates {\n\t\t\tpq.items.push(item)\n\t\t} else if _, ok := pq.itemMap[item]; !ok {\n\t\t\tpq.itemMap[item] = struct{}{}\n\t\t\tpq.items.push(item)\n\t\t}\n\t}\n\n\tfor {\n\t\tsema := pq.waiters.get()\n\t\tif sema == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tsema.response.Add(1)\n\t\tsema.ready <- true\n\t\tsema.response.Wait()\n\t\tif len(pq.items) == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Get retrieves items from the queue. If the queue is empty,\n// this call blocks until the next item is added to the queue. This\n// will attempt to retrieve number of items.\nfunc (pq *PriorityQueue) Get(number int) ([]Item, error) {\n\tif number < 1 {\n\t\treturn nil, nil\n\t}\n\n\tpq.lock.Lock()\n\n\tif pq.disposed {\n\t\tpq.lock.Unlock()\n\t\treturn nil, ErrDisposed\n\t}\n\n\tvar items []Item\n\n\t// Remove references to popped items.\n\tdeleteItems := func(items []Item) {\n\t\tfor _, item := range items {\n\t\t\tdelete(pq.itemMap, item)\n\t\t}\n\t}\n\n\tif len(pq.items) == 0 {\n\t\tsema := newSema()\n\t\tpq.waiters.put(sema)\n\t\tpq.lock.Unlock()\n\n\t\t<-sema.ready\n\n\t\tif pq.Disposed() {\n\t\t\treturn nil, ErrDisposed\n\t\t}\n\n\t\titems = pq.items.get(number)\n\t\tif !pq.allowDuplicates {\n\t\t\tdeleteItems(items)\n\t\t}\n\t\tsema.response.Done()\n\t\treturn items, nil\n\t}\n\n\titems = pq.items.get(number)\n\tdeleteItems(items)\n\tpq.lock.Unlock()\n\treturn items, nil\n}\n\n// Peek will look at the next item without removing it from the queue.\nfunc (pq *PriorityQueue) Peek() Item {\n\tpq.lock.Lock()\n\tdefer pq.lock.Unlock()\n\tif len(pq.items) > 0 {\n\t\treturn pq.items[0]\n\t}\n\treturn nil\n}\n\n// Empty returns a bool indicating if there are any items left\n// in the queue.\nfunc (pq *PriorityQueue) Empty() bool {\n\tpq.lock.Lock()\n\tdefer pq.lock.Unlock()\n\n\treturn len(pq.items) == 0\n}\n\n// Len returns a number indicating how many items are in the queue.\nfunc (pq *PriorityQueue) Len() int {\n\tpq.lock.Lock()\n\tdefer pq.lock.Unlock()\n\n\treturn len(pq.items)\n}\n\n// Disposed returns a bool indicating if this queue has been disposed.\nfunc (pq *PriorityQueue) Disposed() bool {\n\tpq.disposeLock.Lock()\n\tdefer pq.disposeLock.Unlock()\n\n\treturn pq.disposed\n}\n\n// Dispose will prevent any further reads/writes to this queue\n// and frees available resources.\nfunc (pq *PriorityQueue) Dispose() {\n\tpq.lock.Lock()\n\tdefer pq.lock.Unlock()\n\n\tpq.disposeLock.Lock()\n\tdefer pq.disposeLock.Unlock()\n\n\tpq.disposed = true\n\tfor _, waiter := range pq.waiters {\n\t\twaiter.response.Add(1)\n\t\twaiter.ready <- true\n\t}\n\n\tpq.items = nil\n\tpq.waiters = nil\n}\n\n// NewPriorityQueue is the constructor for a priority queue.\nfunc NewPriorityQueue(hint int, allowDuplicates bool) *PriorityQueue {\n\treturn &PriorityQueue{\n\t\titems: make(priorityItems, 0, hint),\n\t\titemMap: make(map[Item]struct{}, hint),\n\t\tallowDuplicates: allowDuplicates,\n\t}\n}\n" }, { "path": "queue/priority_queue_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage queue\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestPriorityPut(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\n\tq.Put(mockItem(2))\n\n\tassert.Len(t, q.items, 1)\n\tassert.Equal(t, mockItem(2), q.items[0])\n\n\tq.Put(mockItem(1))\n\n\tif !assert.Len(t, q.items, 2) {\n\t\treturn\n\t}\n\tassert.Equal(t, mockItem(1), q.items[0])\n\tassert.Equal(t, mockItem(2), q.items[1])\n}\n\nfunc TestPriorityGet(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\n\tq.Put(mockItem(2))\n\tresult, err := q.Get(2)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tif !assert.Len(t, result, 1) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, mockItem(2), result[0])\n\tassert.Len(t, q.items, 0)\n\n\tq.Put(mockItem(2))\n\tq.Put(mockItem(1))\n\n\tresult, err = q.Get(1)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tif !assert.Len(t, result, 1) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, mockItem(1), result[0])\n\tassert.Len(t, q.items, 1)\n\n\tresult, err = q.Get(2)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tif !assert.Len(t, result, 1) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, mockItem(2), result[0])\n}\n\nfunc TestAddEmptyPriorityPut(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\n\tq.Put()\n\n\tassert.Len(t, q.items, 0)\n}\n\nfunc TestPriorityGetNonPositiveNumber(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\n\tq.Put(mockItem(1))\n\n\tresult, err := q.Get(0)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Len(t, result, 0)\n\n\tresult, err = q.Get(-1)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Len(t, result, 0)\n}\n\nfunc TestPriorityEmpty(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\tassert.True(t, q.Empty())\n\n\tq.Put(mockItem(1))\n\n\tassert.False(t, q.Empty())\n}\n\nfunc TestPriorityGetEmpty(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\n\tgo func() {\n\t\tq.Put(mockItem(1))\n\t}()\n\n\tresult, err := q.Get(1)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tif !assert.Len(t, result, 1) {\n\t\treturn\n\t}\n\tassert.Equal(t, mockItem(1), result[0])\n}\n\nfunc TestMultiplePriorityGetEmpty(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tresults := make([][]Item, 2)\n\n\tgo func() {\n\t\twg.Done()\n\t\tlocal, _ := q.Get(1)\n\t\tresults[0] = local\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\twg.Done()\n\t\tlocal, _ := q.Get(1)\n\t\tresults[1] = local\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\twg.Add(2)\n\n\tq.Put(mockItem(1), mockItem(3), mockItem(2))\n\twg.Wait()\n\n\tif !assert.Len(t, results[0], 1) || !assert.Len(t, results[1], 1) {\n\t\treturn\n\t}\n\n\tassert.True(\n\t\tt, (results[0][0] == mockItem(1) && results[1][0] == mockItem(2)) ||\n\t\t\tresults[0][0] == mockItem(2) && results[1][0] == mockItem(1),\n\t)\n}\n\nfunc TestEmptyPriorityGetWithDispose(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\tvar err error\n\tgo func() {\n\t\twg.Done()\n\t\t_, err = q.Get(1)\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\twg.Add(1)\n\n\tq.Dispose()\n\n\twg.Wait()\n\n\tassert.IsType(t, ErrDisposed, err)\n}\n\nfunc TestPriorityGetPutDisposed(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\tq.Dispose()\n\n\t_, err := q.Get(1)\n\tassert.IsType(t, ErrDisposed, err)\n\n\terr = q.Put(mockItem(1))\n\tassert.IsType(t, ErrDisposed, err)\n}\n\nfunc BenchmarkPriorityQueue(b *testing.B) {\n\tq := NewPriorityQueue(b.N, false)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\ti := 0\n\n\tgo func() {\n\t\tfor {\n\t\t\tq.Get(1)\n\t\t\ti++\n\t\t\tif i == b.N {\n\t\t\t\twg.Done()\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq.Put(mockItem(i))\n\t}\n\n\twg.Wait()\n}\n\nfunc TestPriorityPeek(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\tq.Put(mockItem(1))\n\n\tassert.Equal(t, mockItem(1), q.Peek())\n}\n\nfunc TestInsertDuplicate(t *testing.T) {\n\tq := NewPriorityQueue(1, false)\n\tq.Put(mockItem(1))\n\tq.Put(mockItem(1))\n\n\tassert.Equal(t, 1, q.Len())\n}\n\nfunc TestAllowDuplicates(t *testing.T) {\n\tq := NewPriorityQueue(2, true)\n\tq.Put(mockItem(1))\n\tq.Put(mockItem(1))\n\n\tassert.Equal(t, 2, q.Len())\n}\n" }, { "path": "queue/queue.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage queue includes a regular queue and a priority queue.\nThese queues rely on waitgroups to pause listening threads\non empty queues until a message is received. If any thread\ncalls Dispose on the queue, any listeners are immediately returned\nwith an error. Any subsequent put to the queue will return an error\nas opposed to panicking as with channels. Queues will grow with unbounded\nbehavior as opposed to channels which can be buffered but will pause\nwhile a thread attempts to put to a full channel.\n\nRecently added is a lockless ring buffer using the same basic C design as\nfound here:\n\nhttp://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue\n\nModified for use with Go with the addition of some dispose semantics providing\nthe capability to release blocked threads. This works for both puts\nand gets, either will return an error if they are blocked and the buffer\nis disposed. This could serve as a signal to kill a goroutine. All threadsafety\nis achieved using CAS operations, making this buffer pretty quick.\n\nBenchmarks:\nBenchmarkPriorityQueue-8\t \t\t2000000\t 782 ns/op\nBenchmarkQueue-8\t \t\t \t\t2000000\t 671 ns/op\nBenchmarkChannel-8\t \t\t \t\t1000000\t 2083 ns/op\nBenchmarkQueuePut-8\t \t\t \t\t20000\t 84299 ns/op\nBenchmarkQueueGet-8\t \t\t \t\t20000\t 80753 ns/op\nBenchmarkExecuteInParallel-8\t 20000\t 68891 ns/op\nBenchmarkRBLifeCycle-8\t\t\t\t10000000\t 177 ns/op\nBenchmarkRBPut-8\t\t\t\t\t30000000\t 58.1 ns/op\nBenchmarkRBGet-8\t\t\t\t\t50000000\t 26.8 ns/op\n\nTODO: We really need a Fibonacci heap for the priority queue.\nTODO: Unify the types of queue to the same interface.\n*/\npackage queue\n\nimport (\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\ntype waiters []*sema\n\nfunc (w *waiters) get() *sema {\n\tif len(*w) == 0 {\n\t\treturn nil\n\t}\n\n\tsema := (*w)[0]\n\tcopy((*w)[0:], (*w)[1:])\n\t(*w)[len(*w)-1] = nil // or the zero value of T\n\t*w = (*w)[:len(*w)-1]\n\treturn sema\n}\n\nfunc (w *waiters) put(sema *sema) {\n\t*w = append(*w, sema)\n}\n\nfunc (w *waiters) remove(sema *sema) {\n\tif len(*w) == 0 {\n\t\treturn\n\t}\n\t// build new slice, copy all except sema\n\tws := *w\n\tnewWs := make(waiters, 0, len(*w))\n\tfor i := range ws {\n\t\tif ws[i] != sema {\n\t\t\tnewWs = append(newWs, ws[i])\n\t\t}\n\t}\n\t*w = newWs\n}\n\ntype items []interface{}\n\nfunc (items *items) get(number int64) []interface{} {\n\treturnItems := make([]interface{}, 0, number)\n\tindex := int64(0)\n\tfor i := int64(0); i < number; i++ {\n\t\tif i >= int64(len(*items)) {\n\t\t\tbreak\n\t\t}\n\n\t\treturnItems = append(returnItems, (*items)[i])\n\t\t(*items)[i] = nil\n\t\tindex++\n\t}\n\n\t*items = (*items)[index:]\n\treturn returnItems\n}\n\nfunc (items *items) peek() (interface{}, bool) {\n\tlength := len(*items)\n\n\tif length == 0 {\n\t\treturn nil, false\n\t}\n\n\treturn (*items)[0], true\n}\n\nfunc (items *items) getUntil(checker func(item interface{}) bool) []interface{} {\n\tlength := len(*items)\n\n\tif len(*items) == 0 {\n\t\t// returning nil here actually wraps that nil in a list\n\t\t// of interfaces... thanks go\n\t\treturn []interface{}{}\n\t}\n\n\treturnItems := make([]interface{}, 0, length)\n\tindex := -1\n\tfor i, item := range *items {\n\t\tif !checker(item) {\n\t\t\tbreak\n\t\t}\n\n\t\treturnItems = append(returnItems, item)\n\t\tindex = i\n\t\t(*items)[i] = nil // prevent memory leak\n\t}\n\n\t*items = (*items)[index+1:]\n\treturn returnItems\n}\n\ntype sema struct {\n\tready chan bool\n\tresponse *sync.WaitGroup\n}\n\nfunc newSema() *sema {\n\treturn &sema{\n\t\tready: make(chan bool, 1),\n\t\tresponse: &sync.WaitGroup{},\n\t}\n}\n\n// Queue is the struct responsible for tracking the state\n// of the queue.\ntype Queue struct {\n\twaiters waiters\n\titems items\n\tlock sync.Mutex\n\tdisposed bool\n}\n\n// Put will add the specified items to the queue.\nfunc (q *Queue) Put(items ...interface{}) error {\n\tif len(items) == 0 {\n\t\treturn nil\n\t}\n\n\tq.lock.Lock()\n\n\tif q.disposed {\n\t\tq.lock.Unlock()\n\t\treturn ErrDisposed\n\t}\n\n\tq.items = append(q.items, items...)\n\tfor {\n\t\tsema := q.waiters.get()\n\t\tif sema == nil {\n\t\t\tbreak\n\t\t}\n\t\tsema.response.Add(1)\n\t\tselect {\n\t\tcase sema.ready <- true:\n\t\t\tsema.response.Wait()\n\t\tdefault:\n\t\t\t// This semaphore timed out.\n\t\t}\n\t\tif len(q.items) == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tq.lock.Unlock()\n\treturn nil\n}\n\n// Get retrieves items from the queue. If there are some items in the\n// queue, get will return a number UP TO the number passed in as a\n// parameter. If no items are in the queue, this method will pause\n// until items are added to the queue.\nfunc (q *Queue) Get(number int64) ([]interface{}, error) {\n\treturn q.Poll(number, 0)\n}\n\n// Poll retrieves items from the queue. If there are some items in the queue,\n// Poll will return a number UP TO the number passed in as a parameter. If no\n// items are in the queue, this method will pause until items are added to the\n// queue or the provided timeout is reached. A non-positive timeout will block\n// until items are added. If a timeout occurs, ErrTimeout is returned.\nfunc (q *Queue) Poll(number int64, timeout time.Duration) ([]interface{}, error) {\n\tif number < 1 {\n\t\t// thanks again go\n\t\treturn []interface{}{}, nil\n\t}\n\n\tq.lock.Lock()\n\n\tif q.disposed {\n\t\tq.lock.Unlock()\n\t\treturn nil, ErrDisposed\n\t}\n\n\tvar items []interface{}\n\n\tif len(q.items) == 0 {\n\t\tsema := newSema()\n\t\tq.waiters.put(sema)\n\t\tq.lock.Unlock()\n\n\t\tvar timeoutC <-chan time.Time\n\t\tif timeout > 0 {\n\t\t\ttimeoutC = time.After(timeout)\n\t\t}\n\t\tselect {\n\t\tcase <-sema.ready:\n\t\t\t// we are now inside the put's lock\n\t\t\tif q.disposed {\n\t\t\t\treturn nil, ErrDisposed\n\t\t\t}\n\t\t\titems = q.items.get(number)\n\t\t\tsema.response.Done()\n\t\t\treturn items, nil\n\t\tcase <-timeoutC:\n\t\t\t// cleanup the sema that was added to waiters\n\t\t\tselect {\n\t\t\tcase sema.ready <- true:\n\t\t\t\t// we called this before Put() could\n\t\t\t\t// Remove sema from waiters.\n\t\t\t\tq.lock.Lock()\n\t\t\t\tq.waiters.remove(sema)\n\t\t\t\tq.lock.Unlock()\n\t\t\tdefault:\n\t\t\t\t// Put() got it already, we need to call Done() so Put() can move on\n\t\t\t\tsema.response.Done()\n\t\t\t}\n\t\t\treturn nil, ErrTimeout\n\t\t}\n\t}\n\n\titems = q.items.get(number)\n\tq.lock.Unlock()\n\treturn items, nil\n}\n\n// Peek returns a the first item in the queue by value\n// without modifying the queue.\nfunc (q *Queue) Peek() (interface{}, error) {\n\tq.lock.Lock()\n\tdefer q.lock.Unlock()\n\n\tif q.disposed {\n\t\treturn nil, ErrDisposed\n\t}\n\n\tpeekItem, ok := q.items.peek()\n\tif !ok {\n\t\treturn nil, ErrEmptyQueue\n\t}\n\n\treturn peekItem, nil\n}\n\n// TakeUntil takes a function and returns a list of items that\n// match the checker until the checker returns false. This does not\n// wait if there are no items in the queue.\nfunc (q *Queue) TakeUntil(checker func(item interface{}) bool) ([]interface{}, error) {\n\tif checker == nil {\n\t\treturn nil, nil\n\t}\n\n\tq.lock.Lock()\n\n\tif q.disposed {\n\t\tq.lock.Unlock()\n\t\treturn nil, ErrDisposed\n\t}\n\n\tresult := q.items.getUntil(checker)\n\tq.lock.Unlock()\n\treturn result, nil\n}\n\n// Empty returns a bool indicating if this bool is empty.\nfunc (q *Queue) Empty() bool {\n\tq.lock.Lock()\n\tdefer q.lock.Unlock()\n\n\treturn len(q.items) == 0\n}\n\n// Len returns the number of items in this queue.\nfunc (q *Queue) Len() int64 {\n\tq.lock.Lock()\n\tdefer q.lock.Unlock()\n\n\treturn int64(len(q.items))\n}\n\n// Disposed returns a bool indicating if this queue\n// has had disposed called on it.\nfunc (q *Queue) Disposed() bool {\n\tq.lock.Lock()\n\tdefer q.lock.Unlock()\n\n\treturn q.disposed\n}\n\n// Dispose will dispose of this queue and returns\n// the items disposed. Any subsequent calls to Get\n// or Put will return an error.\nfunc (q *Queue) Dispose() []interface{} {\n\tq.lock.Lock()\n\tdefer q.lock.Unlock()\n\n\tq.disposed = true\n\tfor _, waiter := range q.waiters {\n\t\twaiter.response.Add(1)\n\t\tselect {\n\t\tcase waiter.ready <- true:\n\t\t\t// release Poll immediately\n\t\tdefault:\n\t\t\t// ignore if it's a timeout or in the get\n\t\t}\n\t}\n\n\tdisposedItems := q.items\n\n\tq.items = nil\n\tq.waiters = nil\n\n\treturn disposedItems\n}\n\n// New is a constructor for a new threadsafe queue.\nfunc New(hint int64) *Queue {\n\treturn &Queue{\n\t\titems: make([]interface{}, 0, hint),\n\t}\n}\n\n// ExecuteInParallel will (in parallel) call the provided function\n// with each item in the queue until the queue is exhausted. When the queue\n// is exhausted execution is complete and all goroutines will be killed.\n// This means that the queue will be disposed so cannot be used again.\nfunc ExecuteInParallel(q *Queue, fn func(interface{})) {\n\tif q == nil {\n\t\treturn\n\t}\n\n\tq.lock.Lock() // so no one touches anything in the middle\n\t// of this process\n\ttodo, done := uint64(len(q.items)), int64(-1)\n\t// this is important or we might face an infinite loop\n\tif todo == 0 {\n\t\treturn\n\t}\n\n\tnumCPU := 1\n\tif runtime.NumCPU() > 1 {\n\t\tnumCPU = runtime.NumCPU() - 1\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(numCPU)\n\titems := q.items\n\n\tfor i := 0; i < numCPU; i++ {\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tindex := atomic.AddInt64(&done, 1)\n\t\t\t\tif index >= int64(todo) {\n\t\t\t\t\twg.Done()\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\tfn(items[index])\n\t\t\t\titems[index] = 0\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n\tq.lock.Unlock()\n\tq.Dispose()\n}\n" }, { "path": "queue/queue_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage queue\n\nimport (\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestPut(t *testing.T) {\n\tq := New(10)\n\n\tq.Put(`test`)\n\tassert.Equal(t, int64(1), q.Len())\n\n\tresults, err := q.Get(1)\n\tassert.Nil(t, err)\n\n\tresult := results[0]\n\tassert.Equal(t, `test`, result)\n\tassert.True(t, q.Empty())\n\n\tq.Put(`test2`)\n\tassert.Equal(t, int64(1), q.Len())\n\n\tresults, err = q.Get(1)\n\tassert.Nil(t, err)\n\n\tresult = results[0]\n\tassert.Equal(t, `test2`, result)\n\tassert.True(t, q.Empty())\n}\n\nfunc TestGet(t *testing.T) {\n\tq := New(10)\n\n\tq.Put(`test`)\n\tresult, err := q.Get(2)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Len(t, result, 1)\n\tassert.Equal(t, `test`, result[0])\n\tassert.Equal(t, int64(0), q.Len())\n\n\tq.Put(`1`)\n\tq.Put(`2`)\n\n\tresult, err = q.Get(1)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Len(t, result, 1)\n\tassert.Equal(t, `1`, result[0])\n\tassert.Equal(t, int64(1), q.Len())\n\n\tresult, err = q.Get(2)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, `2`, result[0])\n}\n\nfunc TestPoll(t *testing.T) {\n\tq := New(10)\n\n\t// should be able to Poll() before anything is present, without breaking future Puts\n\tq.Poll(1, time.Millisecond)\n\n\tq.Put(`test`)\n\tresult, err := q.Poll(2, 0)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Len(t, result, 1)\n\tassert.Equal(t, `test`, result[0])\n\tassert.Equal(t, int64(0), q.Len())\n\n\tq.Put(`1`)\n\tq.Put(`2`)\n\n\tresult, err = q.Poll(1, time.Millisecond)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Len(t, result, 1)\n\tassert.Equal(t, `1`, result[0])\n\tassert.Equal(t, int64(1), q.Len())\n\n\tresult, err = q.Poll(2, time.Millisecond)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, `2`, result[0])\n\n\tbefore := time.Now()\n\t_, err = q.Poll(1, 5*time.Millisecond)\n\t// This delta is normally 1-3 ms but running tests in CI with -race causes\n\t// this to run much slower. For now, just bump up the threshold.\n\tassert.InDelta(t, 5, time.Since(before).Seconds()*1000, 10)\n\tassert.Equal(t, ErrTimeout, err)\n}\n\nfunc TestPollNoMemoryLeak(t *testing.T) {\n\tq := New(0)\n\n\tassert.Len(t, q.waiters, 0)\n\n\tfor i := 0; i < 10; i++ {\n\t\t// Poll() should cleanup waiters after timeout\n\t\tq.Poll(1, time.Nanosecond)\n\t\tassert.Len(t, q.waiters, 0)\n\t}\n}\n\nfunc TestAddEmptyPut(t *testing.T) {\n\tq := New(10)\n\n\tq.Put()\n\n\tif q.Len() != 0 {\n\t\tt.Errorf(`Expected len: %d, received: %d`, 0, q.Len())\n\t}\n}\n\nfunc TestGetNonPositiveNumber(t *testing.T) {\n\tq := New(10)\n\n\tq.Put(`test`)\n\tresult, err := q.Get(0)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tif len(result) != 0 {\n\t\tt.Errorf(`Expected len: %d, received: %d`, 0, len(result))\n\t}\n}\n\nfunc TestEmpty(t *testing.T) {\n\tq := New(10)\n\n\tif !q.Empty() {\n\t\tt.Errorf(`Expected empty queue.`)\n\t}\n\n\tq.Put(`test`)\n\tif q.Empty() {\n\t\tt.Errorf(`Expected non-empty queue.`)\n\t}\n}\n\nfunc TestGetEmpty(t *testing.T) {\n\tq := New(10)\n\n\tgo func() {\n\t\tq.Put(`a`)\n\t}()\n\n\tresult, err := q.Get(2)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Len(t, result, 1)\n\tassert.Equal(t, `a`, result[0])\n}\n\nfunc TestMultipleGetEmpty(t *testing.T) {\n\tq := New(10)\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tresults := make([][]interface{}, 2)\n\n\tgo func() {\n\t\twg.Done()\n\t\tlocal, err := q.Get(1)\n\t\tassert.Nil(t, err)\n\t\tresults[0] = local\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\twg.Done()\n\t\tlocal, err := q.Get(1)\n\t\tassert.Nil(t, err)\n\t\tresults[1] = local\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\twg.Add(2)\n\n\tq.Put(`a`, `b`, `c`)\n\twg.Wait()\n\n\tif assert.Len(t, results[0], 1) && assert.Len(t, results[1], 1) {\n\t\tassert.True(t, (results[0][0] == `a` && results[1][0] == `b`) ||\n\t\t\t(results[0][0] == `b` && results[1][0] == `a`),\n\t\t\t`The array should be a, b or b, a`)\n\t}\n}\n\nfunc TestDispose(t *testing.T) {\n\t// when the queue is empty\n\tq := New(10)\n\titemsDisposed := q.Dispose()\n\n\tassert.Empty(t, itemsDisposed)\n\n\t// when the queue is not empty\n\tq = New(10)\n\tq.Put(`1`)\n\titemsDisposed = q.Dispose()\n\n\texpected := []interface{}{`1`}\n\tassert.Equal(t, expected, itemsDisposed)\n\n\t// when the queue has been disposed\n\titemsDisposed = q.Dispose()\n\tassert.Nil(t, itemsDisposed)\n}\n\nfunc TestEmptyGetWithDispose(t *testing.T) {\n\tq := New(10)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\tvar err error\n\n\tgo func() {\n\t\twg.Done()\n\t\t_, err = q.Get(1)\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\twg.Add(1)\n\n\tq.Dispose()\n\n\twg.Wait()\n\n\tassert.IsType(t, ErrDisposed, err)\n}\n\nfunc TestDisposeAfterEmptyPoll(t *testing.T) {\n\tq := New(10)\n\n\t_, err := q.Poll(1, time.Millisecond)\n\tassert.IsType(t, ErrTimeout, err)\n\n\t// it should not hang\n\tq.Dispose()\n\n\t_, err = q.Poll(1, time.Millisecond)\n\tassert.IsType(t, ErrDisposed, err)\n}\n\nfunc TestGetPutDisposed(t *testing.T) {\n\tq := New(10)\n\n\tq.Dispose()\n\n\t_, err := q.Get(1)\n\tassert.IsType(t, ErrDisposed, err)\n\n\terr = q.Put(`a`)\n\tassert.IsType(t, ErrDisposed, err)\n}\n\nfunc BenchmarkQueue(b *testing.B) {\n\tq := New(int64(b.N))\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\ti := 0\n\n\tgo func() {\n\t\tfor {\n\t\t\tq.Get(1)\n\t\t\ti++\n\t\t\tif i == b.N {\n\t\t\t\twg.Done()\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq.Put(`a`)\n\t}\n\n\twg.Wait()\n}\n\nfunc BenchmarkChannel(b *testing.B) {\n\tch := make(chan interface{}, 1)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\ti := 0\n\n\tgo func() {\n\t\tfor {\n\t\t\t<-ch\n\t\t\ti++\n\t\t\tif i == b.N {\n\t\t\t\twg.Done()\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tch <- `a`\n\t}\n\n\twg.Wait()\n}\n\nfunc TestPeek(t *testing.T) {\n\tq := New(10)\n\tq.Put(`a`)\n\tq.Put(`b`)\n\tq.Put(`c`)\n\tpeekResult, err := q.Peek()\n\tpeekExpected := `a`\n\tassert.Nil(t, err)\n\tassert.Equal(t, q.Len(), int64(3))\n\tassert.Equal(t, peekExpected, peekResult)\n\n\tpopResult, err := q.Get(1)\n\tassert.Nil(t, err)\n\tassert.Equal(t, peekResult, popResult[0])\n\tassert.Equal(t, q.Len(), int64(2))\n}\n\nfunc TestPeekOnDisposedQueue(t *testing.T) {\n\tq := New(10)\n\tq.Dispose()\n\tresult, err := q.Peek()\n\n\tassert.Nil(t, result)\n\tassert.IsType(t, ErrDisposed, err)\n}\n\nfunc TestTakeUntil(t *testing.T) {\n\tq := New(10)\n\tq.Put(`a`, `b`, `c`)\n\tresult, err := q.TakeUntil(func(item interface{}) bool {\n\t\treturn item != `c`\n\t})\n\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\texpected := []interface{}{`a`, `b`}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestTakeUntilEmptyQueue(t *testing.T) {\n\tq := New(10)\n\tresult, err := q.TakeUntil(func(item interface{}) bool {\n\t\treturn item != `c`\n\t})\n\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\texpected := []interface{}{}\n\tassert.Equal(t, expected, result)\n}\n\nfunc TestTakeUntilThenGet(t *testing.T) {\n\tq := New(10)\n\tq.Put(`a`, `b`, `c`)\n\ttakeItems, _ := q.TakeUntil(func(item interface{}) bool {\n\t\treturn item != `c`\n\t})\n\n\trestItems, _ := q.Get(3)\n\tassert.Equal(t, []interface{}{`a`, `b`}, takeItems)\n\tassert.Equal(t, []interface{}{`c`}, restItems)\n}\n\nfunc TestTakeUntilNoMatches(t *testing.T) {\n\tq := New(10)\n\tq.Put(`a`, `b`, `c`)\n\ttakeItems, _ := q.TakeUntil(func(item interface{}) bool {\n\t\treturn item != `a`\n\t})\n\n\trestItems, _ := q.Get(3)\n\tassert.Equal(t, []interface{}{}, takeItems)\n\tassert.Equal(t, []interface{}{`a`, `b`, `c`}, restItems)\n}\n\nfunc TestTakeUntilOnDisposedQueue(t *testing.T) {\n\tq := New(10)\n\tq.Dispose()\n\tresult, err := q.TakeUntil(func(item interface{}) bool {\n\t\treturn true\n\t})\n\n\tassert.Nil(t, result)\n\tassert.IsType(t, ErrDisposed, err)\n}\n\nfunc TestWaiters(t *testing.T) {\n\ts1, s2, s3, s4 := newSema(), newSema(), newSema(), newSema()\n\n\tw := waiters{}\n\tassert.Len(t, w, 0)\n\n\t//\n\t// test put()\n\tw.put(s1)\n\tassert.Equal(t, waiters{s1}, w)\n\n\tw.put(s2)\n\tw.put(s3)\n\tw.put(s4)\n\tassert.Equal(t, waiters{s1, s2, s3, s4}, w)\n\n\t//\n\t// test remove()\n\t//\n\t// remove from middle\n\tw.remove(s2)\n\tassert.Equal(t, waiters{s1, s3, s4}, w)\n\n\t// remove non-existing element\n\tw.remove(s2)\n\tassert.Equal(t, waiters{s1, s3, s4}, w)\n\n\t// remove from beginning\n\tw.remove(s1)\n\tassert.Equal(t, waiters{s3, s4}, w)\n\n\t// remove from end\n\tw.remove(s4)\n\tassert.Equal(t, waiters{s3}, w)\n\n\t// remove last element\n\tw.remove(s3)\n\tassert.Empty(t, w)\n\n\t// remove non-existing element\n\tw.remove(s3)\n\tassert.Empty(t, w)\n\n\t//\n\t// test get()\n\t//\n\t// start with 3 elements in list\n\tw.put(s1)\n\tw.put(s2)\n\tw.put(s3)\n\tassert.Equal(t, waiters{s1, s2, s3}, w)\n\n\t// get() returns each item in insertion order\n\tassert.Equal(t, s1, w.get())\n\tassert.Equal(t, s2, w.get())\n\tw.put(s4) // interleave a put(), item should go to the end\n\tassert.Equal(t, s3, w.get())\n\tassert.Equal(t, s4, w.get())\n\tassert.Empty(t, w)\n\tassert.Nil(t, w.get())\n}\n\nfunc TestExecuteInParallel(t *testing.T) {\n\tq := New(10)\n\tfor i := 0; i < 10; i++ {\n\t\tq.Put(i)\n\t}\n\n\tnumCalls := uint64(0)\n\n\tExecuteInParallel(q, func(item interface{}) {\n\t\tt.Logf(\"ExecuteInParallel called us with %+v\", item)\n\t\tatomic.AddUint64(&numCalls, 1)\n\t})\n\n\tassert.Equal(t, uint64(10), numCalls)\n\tassert.True(t, q.Disposed())\n}\n\nfunc TestExecuteInParallelEmptyQueue(t *testing.T) {\n\tq := New(1)\n\n\t// basically just ensuring we don't deadlock here\n\tExecuteInParallel(q, func(interface{}) {\n\t\tt.Fail()\n\t})\n}\n\nfunc BenchmarkQueuePut(b *testing.B) {\n\tnumItems := int64(1000)\n\n\tqs := make([]*Queue, 0, b.N)\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := New(10)\n\t\tqs = append(qs, q)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tq := qs[i]\n\t\tfor j := int64(0); j < numItems; j++ {\n\t\t\tq.Put(j)\n\t\t}\n\t}\n}\n\nfunc BenchmarkQueueGet(b *testing.B) {\n\tnumItems := int64(1000)\n\n\tqs := make([]*Queue, 0, b.N)\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := New(numItems)\n\t\tfor j := int64(0); j < numItems; j++ {\n\t\t\tq.Put(j)\n\t\t}\n\t\tqs = append(qs, q)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := qs[i]\n\t\tfor j := int64(0); j < numItems; j++ {\n\t\t\tq.Get(1)\n\t\t}\n\t}\n}\n\nfunc BenchmarkQueuePoll(b *testing.B) {\n\tnumItems := int64(1000)\n\n\tqs := make([]*Queue, 0, b.N)\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := New(numItems)\n\t\tfor j := int64(0); j < numItems; j++ {\n\t\t\tq.Put(j)\n\t\t}\n\t\tqs = append(qs, q)\n\t}\n\n\tb.ResetTimer()\n\n\tfor _, q := range qs {\n\t\tfor j := int64(0); j < numItems; j++ {\n\t\t\tq.Poll(1, time.Millisecond)\n\t\t}\n\t}\n}\n\nfunc BenchmarkExecuteInParallel(b *testing.B) {\n\tnumItems := int64(1000)\n\n\tqs := make([]*Queue, 0, b.N)\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := New(numItems)\n\t\tfor j := int64(0); j < numItems; j++ {\n\t\t\tq.Put(j)\n\t\t}\n\t\tqs = append(qs, q)\n\t}\n\n\tvar counter int64\n\tfn := func(ifc interface{}) {\n\t\tc := ifc.(int64)\n\t\tatomic.AddInt64(&counter, c)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tq := qs[i]\n\t\tExecuteInParallel(q, fn)\n\t}\n}\n" }, { "path": "queue/ring.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage queue\n\nimport (\n\t\"runtime\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\n// roundUp takes a uint64 greater than 0 and rounds it up to the next\n// power of 2.\nfunc roundUp(v uint64) uint64 {\n\tv--\n\tv |= v >> 1\n\tv |= v >> 2\n\tv |= v >> 4\n\tv |= v >> 8\n\tv |= v >> 16\n\tv |= v >> 32\n\tv++\n\treturn v\n}\n\ntype node struct {\n\tposition uint64\n\tdata interface{}\n}\n\ntype nodes []node\n\n// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations\n// only. A put on full or get on empty call will block until an item\n// is put or retrieved. Calling Dispose on the RingBuffer will unblock\n// any blocked threads with an error. This buffer is similar to the buffer\n// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue\n// with some minor additions.\ntype RingBuffer struct {\n\t_padding0 [8]uint64\n\tqueue uint64\n\t_padding1 [8]uint64\n\tdequeue uint64\n\t_padding2 [8]uint64\n\tmask, disposed uint64\n\t_padding3 [8]uint64\n\tnodes nodes\n}\n\nfunc (rb *RingBuffer) init(size uint64) {\n\tsize = roundUp(size)\n\trb.nodes = make(nodes, size)\n\tfor i := uint64(0); i < size; i++ {\n\t\trb.nodes[i] = node{position: i}\n\t}\n\trb.mask = size - 1 // so we don't have to do this with every put/get operation\n}\n\n// Put adds the provided item to the queue. If the queue is full, this\n// call will block until an item is added to the queue or Dispose is called\n// on the queue. An error will be returned if the queue is disposed.\nfunc (rb *RingBuffer) Put(item interface{}) error {\n\t_, err := rb.put(item, false)\n\treturn err\n}\n\n// Offer adds the provided item to the queue if there is space. If the queue\n// is full, this call will return false. An error will be returned if the\n// queue is disposed.\nfunc (rb *RingBuffer) Offer(item interface{}) (bool, error) {\n\treturn rb.put(item, true)\n}\n\nfunc (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {\n\tvar n *node\n\tpos := atomic.LoadUint64(&rb.queue)\nL:\n\tfor {\n\t\tif atomic.LoadUint64(&rb.disposed) == 1 {\n\t\t\treturn false, ErrDisposed\n\t\t}\n\n\t\tn = &rb.nodes[pos&rb.mask]\n\t\tseq := atomic.LoadUint64(&n.position)\n\t\tswitch dif := seq - pos; {\n\t\tcase dif == 0:\n\t\t\tif atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {\n\t\t\t\tbreak L\n\t\t\t}\n\t\tcase dif < 0:\n\t\t\tpanic(`Ring buffer in a compromised state during a put operation.`)\n\t\tdefault:\n\t\t\tpos = atomic.LoadUint64(&rb.queue)\n\t\t}\n\n\t\tif offer {\n\t\t\treturn false, nil\n\t\t}\n\n\t\truntime.Gosched() // free up the cpu before the next iteration\n\t}\n\n\tn.data = item\n\tatomic.StoreUint64(&n.position, pos+1)\n\treturn true, nil\n}\n\n// Get will return the next item in the queue. This call will block\n// if the queue is empty. This call will unblock when an item is added\n// to the queue or Dispose is called on the queue. An error will be returned\n// if the queue is disposed.\nfunc (rb *RingBuffer) Get() (interface{}, error) {\n\treturn rb.Poll(0)\n}\n\n// Poll will return the next item in the queue. This call will block\n// if the queue is empty. This call will unblock when an item is added\n// to the queue, Dispose is called on the queue, or the timeout is reached. An\n// error will be returned if the queue is disposed or a timeout occurs. A\n// non-positive timeout will block indefinitely.\nfunc (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {\n\tvar (\n\t\tn *node\n\t\tpos = atomic.LoadUint64(&rb.dequeue)\n\t\tstart time.Time\n\t)\n\tif timeout > 0 {\n\t\tstart = time.Now()\n\t}\nL:\n\tfor {\n\t\tif atomic.LoadUint64(&rb.disposed) == 1 {\n\t\t\treturn nil, ErrDisposed\n\t\t}\n\n\t\tn = &rb.nodes[pos&rb.mask]\n\t\tseq := atomic.LoadUint64(&n.position)\n\t\tswitch dif := seq - (pos + 1); {\n\t\tcase dif == 0:\n\t\t\tif atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {\n\t\t\t\tbreak L\n\t\t\t}\n\t\tcase dif < 0:\n\t\t\tpanic(`Ring buffer in compromised state during a get operation.`)\n\t\tdefault:\n\t\t\tpos = atomic.LoadUint64(&rb.dequeue)\n\t\t}\n\n\t\tif timeout > 0 && time.Since(start) >= timeout {\n\t\t\treturn nil, ErrTimeout\n\t\t}\n\n\t\truntime.Gosched() // free up the cpu before the next iteration\n\t}\n\tdata := n.data\n\tn.data = nil\n\tatomic.StoreUint64(&n.position, pos+rb.mask+1)\n\treturn data, nil\n}\n\n// Len returns the number of items in the queue.\nfunc (rb *RingBuffer) Len() uint64 {\n\treturn atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)\n}\n\n// Cap returns the capacity of this ring buffer.\nfunc (rb *RingBuffer) Cap() uint64 {\n\treturn uint64(len(rb.nodes))\n}\n\n// Dispose will dispose of this queue and free any blocked threads\n// in the Put and/or Get methods. Calling those methods on a disposed\n// queue will return an error.\nfunc (rb *RingBuffer) Dispose() {\n\tatomic.CompareAndSwapUint64(&rb.disposed, 0, 1)\n}\n\n// IsDisposed will return a bool indicating if this queue has been\n// disposed.\nfunc (rb *RingBuffer) IsDisposed() bool {\n\treturn atomic.LoadUint64(&rb.disposed) == 1\n}\n\n// NewRingBuffer will allocate, initialize, and return a ring buffer\n// with the specified size.\nfunc NewRingBuffer(size uint64) *RingBuffer {\n\trb := &RingBuffer{}\n\trb.init(size)\n\treturn rb\n}\n" }, { "path": "queue/ring_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage queue\n\nimport (\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestRingInsert(t *testing.T) {\n\trb := NewRingBuffer(5)\n\tassert.Equal(t, uint64(8), rb.Cap())\n\n\terr := rb.Put(5)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tresult, err := rb.Get()\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, 5, result)\n}\n\nfunc TestRingMultipleInserts(t *testing.T) {\n\trb := NewRingBuffer(5)\n\n\terr := rb.Put(1)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\terr = rb.Put(2)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tresult, err := rb.Get()\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, 1, result)\n\n\tresult, err = rb.Get()\n\tif assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, 2, result)\n}\n\nfunc TestIntertwinedGetAndPut(t *testing.T) {\n\trb := NewRingBuffer(5)\n\terr := rb.Put(1)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tresult, err := rb.Get()\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, 1, result)\n\n\terr = rb.Put(2)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tresult, err = rb.Get()\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, 2, result)\n}\n\nfunc TestPutToFull(t *testing.T) {\n\trb := NewRingBuffer(3)\n\n\tfor i := 0; i < 4; i++ {\n\t\terr := rb.Put(i)\n\t\tif !assert.Nil(t, err) {\n\t\t\treturn\n\t\t}\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\n\tgo func() {\n\t\terr := rb.Put(4)\n\t\tassert.Nil(t, err)\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tresult, err := rb.Get()\n\t\tif !assert.Nil(t, err) {\n\t\t\treturn\n\t\t}\n\n\t\tassert.Equal(t, 0, result)\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestOffer(t *testing.T) {\n\trb := NewRingBuffer(2)\n\n\tok, err := rb.Offer(\"foo\")\n\tassert.True(t, ok)\n\tassert.Nil(t, err)\n\tok, err = rb.Offer(\"bar\")\n\tassert.True(t, ok)\n\tassert.Nil(t, err)\n\tok, err = rb.Offer(\"baz\")\n\tassert.False(t, ok)\n\tassert.Nil(t, err)\n\n\titem, err := rb.Get()\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"foo\", item)\n\titem, err = rb.Get()\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"bar\", item)\n}\n\nfunc TestRingGetEmpty(t *testing.T) {\n\trb := NewRingBuffer(3)\n\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\t// want to kick off this consumer to ensure it blocks\n\tgo func() {\n\t\twg.Done()\n\t\tresult, err := rb.Get()\n\t\tassert.Nil(t, err)\n\t\tassert.Equal(t, 0, result)\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\twg.Add(2)\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\terr := rb.Put(0)\n\t\tassert.Nil(t, err)\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestRingPollEmpty(t *testing.T) {\n\trb := NewRingBuffer(3)\n\n\t_, err := rb.Poll(1)\n\tassert.Equal(t, ErrTimeout, err)\n}\n\nfunc TestRingPoll(t *testing.T) {\n\trb := NewRingBuffer(10)\n\n\t// should be able to Poll() before anything is present, without breaking future Puts\n\trb.Poll(time.Millisecond)\n\n\trb.Put(`test`)\n\tresult, err := rb.Poll(0)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, `test`, result)\n\tassert.Equal(t, uint64(0), rb.Len())\n\n\trb.Put(`1`)\n\trb.Put(`2`)\n\n\tresult, err = rb.Poll(time.Millisecond)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, `1`, result)\n\tassert.Equal(t, uint64(1), rb.Len())\n\n\tresult, err = rb.Poll(time.Millisecond)\n\tif !assert.Nil(t, err) {\n\t\treturn\n\t}\n\n\tassert.Equal(t, `2`, result)\n\n\tbefore := time.Now()\n\t_, err = rb.Poll(5 * time.Millisecond)\n\t// This delta is normally 1-3 ms but running tests in CI with -race causes\n\t// this to run much slower. For now, just bump up the threshold.\n\tassert.InDelta(t, 5, time.Since(before).Seconds()*1000, 10)\n\tassert.Equal(t, ErrTimeout, err)\n}\n\nfunc TestRingLen(t *testing.T) {\n\trb := NewRingBuffer(4)\n\tassert.Equal(t, uint64(0), rb.Len())\n\n\trb.Put(1)\n\tassert.Equal(t, uint64(1), rb.Len())\n\n\trb.Get()\n\tassert.Equal(t, uint64(0), rb.Len())\n\n\tfor i := 0; i < 4; i++ {\n\t\trb.Put(1)\n\t}\n\tassert.Equal(t, uint64(4), rb.Len())\n\n\trb.Get()\n\tassert.Equal(t, uint64(3), rb.Len())\n}\n\nfunc TestDisposeOnGet(t *testing.T) {\n\tnumThreads := 8\n\tvar wg sync.WaitGroup\n\twg.Add(numThreads)\n\trb := NewRingBuffer(4)\n\tvar spunUp sync.WaitGroup\n\tspunUp.Add(numThreads)\n\n\tfor i := 0; i < numThreads; i++ {\n\t\tgo func() {\n\t\t\tspunUp.Done()\n\t\t\tdefer wg.Done()\n\t\t\t_, err := rb.Get()\n\t\t\tassert.NotNil(t, err)\n\t\t}()\n\t}\n\n\tspunUp.Wait()\n\trb.Dispose()\n\n\twg.Wait()\n\tassert.True(t, rb.IsDisposed())\n}\n\nfunc TestDisposeOnPut(t *testing.T) {\n\tnumThreads := 8\n\tvar wg sync.WaitGroup\n\twg.Add(numThreads)\n\trb := NewRingBuffer(4)\n\tvar spunUp sync.WaitGroup\n\tspunUp.Add(numThreads)\n\n\t// fill up the queue\n\tfor i := 0; i < 4; i++ {\n\t\trb.Put(i)\n\t}\n\n\t// it's now full\n\tfor i := 0; i < numThreads; i++ {\n\t\tgo func(i int) {\n\t\t\tspunUp.Done()\n\t\t\tdefer wg.Done()\n\t\t\terr := rb.Put(i)\n\t\t\tassert.NotNil(t, err)\n\t\t}(i)\n\t}\n\n\tspunUp.Wait()\n\n\trb.Dispose()\n\n\twg.Wait()\n\n\tassert.True(t, rb.IsDisposed())\n}\n\nfunc BenchmarkRBLifeCycle(b *testing.B) {\n\trb := NewRingBuffer(64)\n\n\tcounter := uint64(0)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor {\n\t\t\t_, err := rb.Get()\n\t\t\tassert.Nil(b, err)\n\n\t\t\tif atomic.AddUint64(&counter, 1) == uint64(b.N) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trb.Put(i)\n\t}\n\n\twg.Wait()\n}\n\nfunc BenchmarkRBLifeCycleContention(b *testing.B) {\n\trb := NewRingBuffer(64)\n\n\tvar wwg sync.WaitGroup\n\tvar rwg sync.WaitGroup\n\twwg.Add(10)\n\trwg.Add(10)\n\n\tfor i := 0; i < 10; i++ {\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\t_, err := rb.Get()\n\t\t\t\tif err == ErrDisposed {\n\t\t\t\t\trwg.Done()\n\t\t\t\t\treturn\n\t\t\t\t} else {\n\t\t\t\t\tassert.Nil(b, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < 10; i++ {\n\t\tgo func() {\n\t\t\tfor j := 0; j < b.N; j++ {\n\t\t\t\trb.Put(j)\n\t\t\t}\n\t\t\twwg.Done()\n\t\t}()\n\t}\n\n\twwg.Wait()\n\trb.Dispose()\n\trwg.Wait()\n}\n\nfunc BenchmarkRBPut(b *testing.B) {\n\trb := NewRingBuffer(uint64(b.N))\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tok, err := rb.Offer(i)\n\t\tif !ok {\n\t\t\tb.Fail()\n\t\t}\n\t\tif err != nil {\n\t\t\tb.Log(err)\n\t\t\tb.Fail()\n\t\t}\n\t}\n}\n\nfunc BenchmarkRBGet(b *testing.B) {\n\trb := NewRingBuffer(uint64(b.N))\n\n\tfor i := 0; i < b.N; i++ {\n\t\trb.Offer(i)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trb.Get()\n\t}\n}\n\nfunc BenchmarkRBAllocation(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tNewRingBuffer(1024)\n\t}\n}\n" }, { "path": "rangetree/entries.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\nimport \"sync\"\n\nvar entriesPool = sync.Pool{\n\tNew: func() interface{} {\n\t\treturn make(Entries, 0, 10)\n\t},\n}\n\n// Entries is a typed list of Entry that can be reused if Dispose\n// is called.\ntype Entries []Entry\n\n// Dispose will free the resources consumed by this list and\n// allow the list to be reused.\nfunc (entries *Entries) Dispose() {\n\tfor i := 0; i < len(*entries); i++ {\n\t\t(*entries)[i] = nil\n\t}\n\n\t*entries = (*entries)[:0]\n\tentriesPool.Put(*entries)\n}\n\n// NewEntries will return a reused list of entries.\nfunc NewEntries() Entries {\n\treturn entriesPool.Get().(Entries)\n}\n" }, { "path": "rangetree/entries_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestDisposeEntries(t *testing.T) {\n\tentries := NewEntries()\n\tentries = append(entries, constructMockEntry(0, 0))\n\n\tentries.Dispose()\n\n\tassert.Len(t, entries, 0)\n}\n" }, { "path": "rangetree/error.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\nimport \"fmt\"\n\n// NoEntriesError is returned from an operation that requires\n// existing entries when none are found.\ntype NoEntriesError struct{}\n\nfunc (nee NoEntriesError) Error() string {\n\treturn `No entries in this tree.`\n}\n\n// OutOfDimensionError is returned when a requested operation\n// doesn't meet dimensional requirements.\ntype OutOfDimensionError struct {\n\tprovided, max uint64\n}\n\nfunc (oode OutOfDimensionError) Error() string {\n\treturn fmt.Sprintf(`Provided dimension: %d is \n\t\tgreater than max dimension: %d`,\n\t\toode.provided, oode.max,\n\t)\n}\n" }, { "path": "rangetree/immutable.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\nimport \"github.com/Workiva/go-datastructures/slice\"\n\ntype immutableRangeTree struct {\n\tnumber uint64\n\ttop orderedNodes\n\tdimensions uint64\n}\n\nfunc newCache(dimensions uint64) []slice.Int64Slice {\n\tcache := make([]slice.Int64Slice, 0, dimensions-1)\n\tfor i := uint64(0); i < dimensions; i++ {\n\t\tcache = append(cache, slice.Int64Slice{})\n\t}\n\treturn cache\n}\n\nfunc (irt *immutableRangeTree) needNextDimension() bool {\n\treturn irt.dimensions > 1\n}\n\nfunc (irt *immutableRangeTree) add(nodes *orderedNodes, cache []slice.Int64Slice, entry Entry, added *uint64) {\n\tvar node *node\n\tlist := nodes\n\n\tfor i := uint64(1); i <= irt.dimensions; i++ {\n\t\tif isLastDimension(irt.dimensions, i) {\n\t\t\tif i != 1 && !cache[i-1].Exists(node.value) {\n\t\t\t\tnodes := make(orderedNodes, len(*list))\n\t\t\t\tcopy(nodes, *list)\n\t\t\t\tlist = &nodes\n\t\t\t\tcache[i-1].Insert(node.value)\n\t\t\t}\n\n\t\t\tnewNode := newNode(entry.ValueAtDimension(i), entry, false)\n\t\t\toverwritten := list.add(newNode)\n\t\t\tif overwritten == nil {\n\t\t\t\t*added++\n\t\t\t}\n\t\t\tif node != nil {\n\t\t\t\tnode.orderedNodes = *list\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\tif i != 1 && !cache[i-1].Exists(node.value) {\n\t\t\tnodes := make(orderedNodes, len(*list))\n\t\t\tcopy(nodes, *list)\n\t\t\tlist = &nodes\n\t\t\tcache[i-1].Insert(node.value)\n\t\t\tnode.orderedNodes = *list\n\t\t}\n\n\t\tnode, _ = list.getOrAdd(entry, i, irt.dimensions)\n\t\tlist = &node.orderedNodes\n\t}\n}\n\n// Add will add the provided entries into the tree and return\n// a new tree with those entries added.\nfunc (irt *immutableRangeTree) Add(entries ...Entry) *immutableRangeTree {\n\tif len(entries) == 0 {\n\t\treturn irt\n\t}\n\n\tcache := newCache(irt.dimensions)\n\ttop := make(orderedNodes, len(irt.top))\n\tcopy(top, irt.top)\n\tadded := uint64(0)\n\tfor _, entry := range entries {\n\t\tirt.add(&top, cache, entry, &added)\n\t}\n\n\ttree := newImmutableRangeTree(irt.dimensions)\n\ttree.top = top\n\ttree.number = irt.number + added\n\treturn tree\n}\n\n// InsertAtDimension will increment items at and above the given index\n// by the number provided. Provide a negative number to to decrement.\n// Returned are two lists and the modified tree. The first list is a\n// list of entries that were moved. The second is a list entries that\n// were deleted. These lists are exclusive.\nfunc (irt *immutableRangeTree) InsertAtDimension(dimension uint64,\n\tindex, number int64) (*immutableRangeTree, Entries, Entries) {\n\n\tif dimension > irt.dimensions || number == 0 {\n\t\treturn irt, nil, nil\n\t}\n\n\tmodified, deleted := make(Entries, 0, 100), make(Entries, 0, 100)\n\n\ttree := newImmutableRangeTree(irt.dimensions)\n\ttree.top = irt.top.immutableInsert(\n\t\tdimension, 1, irt.dimensions,\n\t\tindex, number,\n\t\t&modified, &deleted,\n\t)\n\ttree.number = irt.number - uint64(len(deleted))\n\n\treturn tree, modified, deleted\n}\n\ntype immutableNodeBundle struct {\n\tlist *orderedNodes\n\tindex int\n\tpreviousNode *node\n\tnewNode *node\n}\n\nfunc (irt *immutableRangeTree) Delete(entries ...Entry) *immutableRangeTree {\n\tcache := newCache(irt.dimensions)\n\ttop := make(orderedNodes, len(irt.top))\n\tcopy(top, irt.top)\n\tdeleted := uint64(0)\n\tfor _, entry := range entries {\n\t\tirt.delete(&top, cache, entry, &deleted)\n\t}\n\n\ttree := newImmutableRangeTree(irt.dimensions)\n\ttree.top = top\n\ttree.number = irt.number - deleted\n\treturn tree\n}\n\nfunc (irt *immutableRangeTree) delete(top *orderedNodes,\n\tcache []slice.Int64Slice, entry Entry, deleted *uint64) {\n\n\tpath := make([]*immutableNodeBundle, 0, 5)\n\tvar index int\n\tvar n *node\n\tvar local *node\n\tlist := top\n\n\tfor i := uint64(1); i <= irt.dimensions; i++ {\n\t\tvalue := entry.ValueAtDimension(i)\n\t\tlocal, index = list.get(value)\n\t\tif local == nil { // there's nothing to delete\n\t\t\treturn\n\t\t}\n\n\t\tnb := &immutableNodeBundle{\n\t\t\tlist: list,\n\t\t\tindex: index,\n\t\t\tpreviousNode: n,\n\t\t}\n\t\tpath = append(path, nb)\n\t\tn = local\n\t\tlist = &n.orderedNodes\n\t}\n\n\t*deleted++\n\n\tfor i := len(path) - 1; i >= 0; i-- {\n\t\tnb := path[i]\n\t\tif nb.previousNode != nil {\n\t\t\tnodes := make(orderedNodes, len(*nb.list))\n\t\t\tcopy(nodes, *nb.list)\n\t\t\tnb.list = &nodes\n\t\t\tif len(*nb.list) == 1 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tnn := newNode(\n\t\t\t\tnb.previousNode.value,\n\t\t\t\tnb.previousNode.entry,\n\t\t\t\t!isLastDimension(irt.dimensions, uint64(i)+1),\n\t\t\t)\n\t\t\tnn.orderedNodes = nodes\n\t\t\tpath[i-1].newNode = nn\n\t\t}\n\t}\n\n\tfor _, nb := range path {\n\t\tif nb.newNode == nil {\n\t\t\tnb.list.deleteAt(nb.index)\n\t\t} else {\n\t\t\t(*nb.list)[nb.index] = nb.newNode\n\t\t}\n\t}\n}\n\nfunc (irt *immutableRangeTree) apply(list orderedNodes, interval Interval,\n\tdimension uint64, fn func(*node) bool) bool {\n\n\tlow, high := interval.LowAtDimension(dimension), interval.HighAtDimension(dimension)\n\n\tif isLastDimension(irt.dimensions, dimension) {\n\t\tif !list.apply(low, high, fn) {\n\t\t\treturn false\n\t\t}\n\t} else {\n\t\tif !list.apply(low, high, func(n *node) bool {\n\t\t\tif !irt.apply(n.orderedNodes, interval, dimension+1, fn) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn true\n\t\t}) {\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t}\n\n\treturn true\n}\n\n// Query will return an ordered list of results in the given\n// interval.\nfunc (irt *immutableRangeTree) Query(interval Interval) Entries {\n\tentries := NewEntries()\n\n\tirt.apply(irt.top, interval, 1, func(n *node) bool {\n\t\tentries = append(entries, n.entry)\n\t\treturn true\n\t})\n\n\treturn entries\n}\n\nfunc (irt *immutableRangeTree) get(entry Entry) Entry {\n\ton := irt.top\n\tfor i := uint64(1); i <= irt.dimensions; i++ {\n\t\tn, _ := on.get(entry.ValueAtDimension(i))\n\t\tif n == nil {\n\t\t\treturn nil\n\t\t}\n\t\tif i == irt.dimensions {\n\t\t\treturn n.entry\n\t\t}\n\t\ton = n.orderedNodes\n\t}\n\n\treturn nil\n}\n\n// Get returns any entries that exist at the addresses provided by the\n// given entries. Entries are returned in the order in which they are\n// received. If an entry cannot be found, a nil is returned in its\n// place.\nfunc (irt *immutableRangeTree) Get(entries ...Entry) Entries {\n\tresult := make(Entries, 0, len(entries))\n\tfor _, entry := range entries {\n\t\tresult = append(result, irt.get(entry))\n\t}\n\n\treturn result\n}\n\n// Len returns the number of items in this tree.\nfunc (irt *immutableRangeTree) Len() uint64 {\n\treturn irt.number\n}\n\nfunc newImmutableRangeTree(dimensions uint64) *immutableRangeTree {\n\treturn &immutableRangeTree{\n\t\tdimensions: dimensions,\n\t}\n}\n" }, { "path": "rangetree/immutable_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestImmutableSingleDimensionAdd(t *testing.T) {\n\ttree := newImmutableRangeTree(1)\n\tentry := constructMockEntry(0, int64(0), int64(0))\n\ttree2 := tree.Add(entry)\n\n\tresult := tree.Query(\n\t\tconstructMockInterval(dimension{0, 10}, dimension{0, 10}),\n\t)\n\tassert.Len(t, result, 0)\n\n\tresult = tree2.Query(\n\t\tconstructMockInterval(dimension{0, 10}, dimension{0, 10}),\n\t)\n\tassert.Equal(t, Entries{entry}, result)\n}\n\nfunc TestImmutableSingleDimensionMultipleAdds(t *testing.T) {\n\ttree := newImmutableRangeTree(1)\n\te1 := constructMockEntry(0, int64(0), int64(0))\n\te2 := constructMockEntry(0, int64(1), int64(1))\n\te3 := constructMockEntry(0, int64(2), int64(2))\n\n\ttree1 := tree.Add(e1)\n\ttree2 := tree1.Add(e2)\n\ttree3 := tree2.Add(e3)\n\n\tiv := constructMockInterval(dimension{0, 10}, dimension{0, 10})\n\n\tresult := tree1.Query(iv)\n\tassert.Equal(t, Entries{e1}, result)\n\tassert.Equal(t, uint64(1), tree1.Len())\n\n\tresult = tree2.Query(iv)\n\tassert.Equal(t, Entries{e1, e2}, result)\n\tassert.Equal(t, uint64(2), tree2.Len())\n\n\tresult = tree3.Query(iv)\n\tassert.Equal(t, Entries{e1, e2, e3}, result)\n\tassert.Equal(t, uint64(3), tree3.Len())\n}\n\nfunc TestImmutableSingleDimensionBulkAdd(t *testing.T) {\n\ttree := newImmutableRangeTree(1)\n\te1 := constructMockEntry(0, int64(0), int64(0))\n\te2 := constructMockEntry(0, int64(1), int64(1))\n\te3 := constructMockEntry(0, int64(2), int64(2))\n\n\tentries := Entries{e1, e2, e3}\n\n\ttree1 := tree.Add(entries...)\n\n\tresult := tree1.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\tassert.Equal(t, entries, result)\n\tassert.Equal(t, uint64(3), tree1.Len())\n}\n\nfunc TestImmutableMultiDimensionAdd(t *testing.T) {\n\ttree := newImmutableRangeTree(2)\n\tentry := constructMockEntry(0, int64(0), int64(0))\n\ttree2 := tree.Add(entry)\n\n\tresult := tree.Query(\n\t\tconstructMockInterval(dimension{0, 10}, dimension{0, 10}),\n\t)\n\tassert.Len(t, result, 0)\n\n\tresult = tree2.Query(\n\t\tconstructMockInterval(dimension{0, 10}, dimension{0, 10}),\n\t)\n\tassert.Equal(t, Entries{entry}, result)\n}\n\nfunc TestImmutableMultiDimensionMultipleAdds(t *testing.T) {\n\ttree := newImmutableRangeTree(2)\n\te1 := constructMockEntry(0, int64(0), int64(0))\n\te2 := constructMockEntry(0, int64(1), int64(1))\n\te3 := constructMockEntry(0, int64(2), int64(2))\n\n\ttree1 := tree.Add(e1)\n\ttree2 := tree1.Add(e2)\n\ttree3 := tree2.Add(e3)\n\n\tiv := constructMockInterval(dimension{0, 10}, dimension{0, 10})\n\n\tresult := tree1.Query(iv)\n\tassert.Equal(t, Entries{e1}, result)\n\tassert.Equal(t, uint64(1), tree1.Len())\n\n\tresult = tree2.Query(iv)\n\tassert.Equal(t, Entries{e1, e2}, result)\n\tassert.Equal(t, uint64(2), tree2.Len())\n\n\tresult = tree3.Query(iv)\n\tassert.Equal(t, Entries{e1, e2, e3}, result)\n\tassert.Equal(t, uint64(3), tree3.Len())\n}\n\nfunc TestImmutableMultiDimensionBulkAdd(t *testing.T) {\n\ttree := newImmutableRangeTree(2)\n\te1 := constructMockEntry(0, int64(0), int64(0))\n\te2 := constructMockEntry(0, int64(1), int64(1))\n\te3 := constructMockEntry(0, int64(2), int64(2))\n\n\tentries := Entries{e1, e2, e3}\n\n\ttree1 := tree.Add(entries...)\n\n\tresult := tree1.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\tassert.Equal(t, entries, result)\n\tassert.Equal(t, uint64(3), tree1.Len())\n}\n\nfunc BenchmarkImmutableMultiDimensionInserts(b *testing.B) {\n\tnumItems := int64(1000)\n\n\tentries := make(Entries, 0, numItems)\n\tfor i := int64(0); i < numItems; i++ {\n\t\te := constructMockEntry(uint64(i), i, i)\n\t\tentries = append(entries, e)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree := newImmutableRangeTree(2)\n\t\tfor _, e := range entries {\n\t\t\ttree = tree.Add(e)\n\t\t}\n\t}\n}\n\nfunc BenchmarkImmutableMultiDimensionBulkInsert(b *testing.B) {\n\tnumItems := int64(100000)\n\n\tentries := make(Entries, 0, numItems)\n\tfor i := int64(0); i < numItems; i++ {\n\t\te := constructMockEntry(uint64(i), i, i)\n\t\tentries = append(entries, e)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree := newImmutableRangeTree(2)\n\t\ttree.Add(entries...)\n\t}\n}\n\nfunc BenchmarkMultiDimensionBulkInsert(b *testing.B) {\n\tnumItems := int64(100000)\n\n\tentries := make(Entries, 0, numItems)\n\tfor i := int64(0); i < numItems; i++ {\n\t\te := constructMockEntry(uint64(i), i, i)\n\t\tentries = append(entries, e)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree := newOrderedTree(2)\n\t\ttree.Add(entries...)\n\t}\n}\n\nfunc TestImmutableSingleDimensionDelete(t *testing.T) {\n\ttree := newImmutableRangeTree(1)\n\tentry := constructMockEntry(0, int64(0), int64(0))\n\ttree2 := tree.Add(entry)\n\ttree3 := tree2.Delete(entry)\n\n\tiv := constructMockInterval(dimension{0, 10}, dimension{0, 10})\n\n\tresult := tree3.Query(iv)\n\tassert.Len(t, result, 0)\n}\n\nfunc TestImmutableSingleDimensionMultipleDeletes(t *testing.T) {\n\ttree := newImmutableRangeTree(1)\n\te1 := constructMockEntry(0, int64(0), int64(0))\n\te2 := constructMockEntry(0, int64(1), int64(1))\n\te3 := constructMockEntry(0, int64(2), int64(2))\n\n\ttree1 := tree.Add(e1)\n\ttree2 := tree1.Add(e2)\n\ttree3 := tree2.Add(e3)\n\n\tiv := constructMockInterval(dimension{0, 10}, dimension{0, 10})\n\n\ttree4 := tree3.Delete(e3)\n\tresult := tree4.Query(iv)\n\tassert.Equal(t, Entries{e1, e2}, result)\n\tassert.Equal(t, uint64(2), tree4.Len())\n\n\ttree5 := tree4.Delete(e2)\n\tresult = tree5.Query(iv)\n\tassert.Equal(t, Entries{e1}, result)\n\tassert.Equal(t, uint64(1), tree5.Len())\n\n\ttree6 := tree5.Delete(e1)\n\tresult = tree6.Query(iv)\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(0), tree6.Len())\n\n\tresult = tree3.Query(iv)\n\tassert.Equal(t, Entries{e1, e2, e3}, result)\n\tassert.Equal(t, uint64(3), tree3.Len())\n\n\ttree7 := tree3.Delete(constructMockEntry(0, int64(3), int64(3)))\n\tassert.Equal(t, tree3, tree7)\n}\n\nfunc TestImmutableSingleDimensionBulkDeletes(t *testing.T) {\n\ttree := newImmutableRangeTree(1)\n\te1 := constructMockEntry(0, int64(0), int64(0))\n\te2 := constructMockEntry(0, int64(1), int64(1))\n\te3 := constructMockEntry(0, int64(2), int64(2))\n\n\ttree1 := tree.Add(e1, e2, e3)\n\ttree2 := tree1.Delete(e2, e3)\n\n\tiv := constructMockInterval(dimension{0, 10}, dimension{0, 10})\n\n\tresult := tree2.Query(iv)\n\tassert.Equal(t, Entries{e1}, result)\n\tassert.Equal(t, uint64(1), tree2.Len())\n\n\ttree3 := tree2.Delete(e1)\n\n\tresult = tree3.Query(iv)\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(0), tree3.Len())\n}\n\nfunc TestImmutableMultiDimensionDelete(t *testing.T) {\n\ttree := newImmutableRangeTree(2)\n\tentry := constructMockEntry(0, int64(0), int64(0))\n\ttree2 := tree.Add(entry)\n\ttree3 := tree2.Delete(entry)\n\n\tiv := constructMockInterval(dimension{0, 10}, dimension{0, 10})\n\n\tresult := tree3.Query(iv)\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(0), tree3.Len())\n}\n\nfunc TestImmutableMultiDimensionMultipleDeletes(t *testing.T) {\n\ttree := newImmutableRangeTree(2)\n\te1 := constructMockEntry(0, int64(0), int64(0))\n\te2 := constructMockEntry(0, int64(1), int64(1))\n\te3 := constructMockEntry(0, int64(2), int64(2))\n\n\ttree1 := tree.Add(e1)\n\ttree2 := tree1.Add(e2)\n\ttree3 := tree2.Add(e3)\n\n\tiv := constructMockInterval(dimension{0, 10}, dimension{0, 10})\n\ttree4 := tree3.Delete(e3)\n\n\tresult := tree4.Query(iv)\n\tassert.Equal(t, Entries{e1, e2}, result)\n\tassert.Equal(t, uint64(2), tree4.Len())\n\n\ttree5 := tree4.Delete(e2)\n\tresult = tree5.Query(iv)\n\tassert.Equal(t, Entries{e1}, result)\n\tassert.Equal(t, uint64(1), tree5.Len())\n\n\ttree6 := tree5.Delete(e1)\n\tresult = tree6.Query(iv)\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(0), tree6.Len())\n\n\tresult = tree3.Query(iv)\n\tassert.Equal(t, Entries{e1, e2, e3}, result)\n\tassert.Equal(t, uint64(3), tree3.Len())\n\n\ttree7 := tree3.Delete(constructMockEntry(0, int64(3), int64(3)))\n\tassert.Equal(t, tree3, tree7)\n}\n\nfunc TestImmutableMultiDimensionBulkDeletes(t *testing.T) {\n\ttree := newImmutableRangeTree(2)\n\te1 := constructMockEntry(0, int64(0), int64(0))\n\te2 := constructMockEntry(0, int64(1), int64(1))\n\te3 := constructMockEntry(0, int64(2), int64(2))\n\n\ttree1 := tree.Add(e1, e2, e3)\n\ttree2 := tree1.Delete(e2, e3)\n\n\tiv := constructMockInterval(dimension{0, 10}, dimension{0, 10})\n\n\tresult := tree2.Query(iv)\n\tassert.Equal(t, Entries{e1}, result)\n\tassert.Equal(t, uint64(1), tree2.Len())\n\n\ttree3 := tree2.Delete(e1)\n\n\tresult = tree3.Query(iv)\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(0), tree3.Len())\n}\n\nfunc constructMultiDimensionalImmutableTree(number int64) (*immutableRangeTree, Entries) {\n\ttree := newImmutableRangeTree(2)\n\tentries := make(Entries, 0, number)\n\tfor i := int64(0); i < number; i++ {\n\t\tentries = append(entries, constructMockEntry(uint64(i), i, i))\n\t}\n\n\treturn tree.Add(entries...), entries\n}\n\nfunc TestImmutableInsertPositiveIndexFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(2)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(1, 1, 1)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, entries[1:], modified)\n\n\tresult := tree1.Query(constructMockInterval(dimension{2, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[1:], result)\n\n\tresult = tree.Query(constructMockInterval(dimension{2, 10}, dimension{0, 10}))\n\tassert.Len(t, result, 0)\n}\n\nfunc TestImmutableInsertPositiveIndexSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(2, 1, 1)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, entries[1:], modified)\n\n\tresult := tree1.Query(constructMockInterval(dimension{1, 10}, dimension{2, 10}))\n\tassert.Equal(t, entries[1:], result)\n\n\tresult = tree.Query(constructMockInterval(dimension{1, 10}, dimension{2, 10}))\n\tassert.Equal(t, entries[2:], result)\n}\n\nfunc TestImmutableInsertPositiveIndexOutOfBoundsFirstDimension(t *testing.T) {\n\ttree, _ := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(1, 4, 1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, tree, tree1)\n}\n\nfunc TestImmutableInsertPositiveIndexOutOfBoundsSecondDimension(t *testing.T) {\n\ttree, _ := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(2, 4, 1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, tree, tree1)\n}\n\nfunc TestImmutableInsertMultiplePositiveIndexFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(1, 1, 2)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, entries[1:], modified)\n\n\tresult := tree1.Query(constructMockInterval(dimension{3, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[1:], result)\n\n\tresult = tree.Query(constructMockInterval(dimension{3, 10}, dimension{1, 10}))\n\tassert.Len(t, result, 0)\n}\n\nfunc TestImmutableInsertMultiplePositiveIndexSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(2, 1, 2)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, entries[1:], modified)\n\n\tresult := tree1.Query(constructMockInterval(dimension{1, 10}, dimension{3, 10}))\n\tassert.Equal(t, entries[1:], result)\n\n\tresult = tree.Query(constructMockInterval(dimension{1, 10}, dimension{3, 10}))\n\tassert.Len(t, result, 0)\n}\n\nfunc TestImmutableInsertNegativeIndexFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(1, 1, -1)\n\tassert.Equal(t, entries[1:2], deleted)\n\tassert.Equal(t, entries[2:], modified)\n\n\tresult := tree1.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[2:], result)\n\n\tresult = tree1.Query(constructMockInterval(dimension{2, 10}, dimension{1, 10}))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(2), tree1.Len())\n\n\tresult = tree.Query(constructMockInterval(dimension{2, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[2:], result)\n\tassert.Equal(t, uint64(3), tree.Len())\n}\n\nfunc TestImmutableInsertNegativeIndexSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(2, 1, -1)\n\tassert.Equal(t, entries[1:2], deleted)\n\tassert.Equal(t, entries[2:], modified)\n\n\tresult := tree1.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[2:], result)\n\n\tresult = tree1.Query(constructMockInterval(dimension{1, 10}, dimension{2, 10}))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(2), tree1.Len())\n\n\tresult = tree.Query(constructMockInterval(dimension{1, 10}, dimension{2, 10}))\n\tassert.Equal(t, entries[2:], result)\n\tassert.Equal(t, uint64(3), tree.Len())\n}\n\nfunc TestImmutableInsertNegativeIndexOutOfBoundsFirstDimension(t *testing.T) {\n\ttree, _ := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(1, 4, -1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, tree, tree1)\n}\n\nfunc TestImmutableInsertNegativeIndexOutOfBoundsSecondDimension(t *testing.T) {\n\ttree, _ := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(2, 4, -1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, tree, tree1)\n}\n\nfunc TestImmutableInsertMultipleNegativeIndexFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(1, 1, -2)\n\tassert.Equal(t, entries[1:], deleted)\n\tassert.Len(t, modified, 0)\n\n\tresult := tree1.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(1), tree1.Len())\n\n\tresult = tree.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[1:], result)\n}\n\nfunc TestImmutableInsertMultipleNegativeIndexSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(2, 1, -2)\n\tassert.Equal(t, entries[1:], deleted)\n\tassert.Len(t, modified, 0)\n\n\tresult := tree1.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(1), tree1.Len())\n\n\tresult = tree.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[1:], result)\n}\n\nfunc TestImmutableInsertInvalidDimension(t *testing.T) {\n\ttree, _ := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(3, 1, -1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, tree, tree1)\n}\n\nfunc TestImmutableInsertInvalidNumber(t *testing.T) {\n\ttree, _ := constructMultiDimensionalImmutableTree(3)\n\n\ttree1, modified, deleted := tree.InsertAtDimension(1, 1, 0)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, tree, tree1)\n}\n\nfunc TestImmutableGet(t *testing.T) {\n\ttree, entries := constructMultiDimensionalImmutableTree(2)\n\n\tresult := tree.Get(entries...)\n\tassert.Equal(t, entries, result)\n\n\tresult = tree.Get(constructMockEntry(10000, 5000, 5000))\n\tassert.Equal(t, Entries{nil}, result)\n}\n\nfunc BenchmarkImmutableInsertFirstDimension(b *testing.B) {\n\tnumItems := int64(100000)\n\n\ttree, _ := constructMultiDimensionalImmutableTree(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.InsertAtDimension(1, 0, 1)\n\t}\n}\n\nfunc BenchmarkImmutableInsertSecondDimension(b *testing.B) {\n\tnumItems := int64(100000)\n\n\ttree, _ := constructMultiDimensionalImmutableTree(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.InsertAtDimension(2, 0, 1)\n\t}\n}\n" }, { "path": "rangetree/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage rangetree is designed to store n-dimensional data in an easy-to-query\nway. Given this package's primary use as representing cartesian data, this\ninformation is represented by int64s at n-dimensions. This implementation\nis not actually a tree but a sparse n-dimensional list. This package also\nincludes two implementations of this sparse list, one mutable (and not threadsafe)\nand another that is immutable copy-on-write which is threadsafe. The mutable\nversion is obviously faster but will likely have write contention for any\nconsumer that needs a threadsafe rangetree.\n\nTODO: unify both implementations with the same interface.\n*/\npackage rangetree\n\n// Entry defines items that can be added to the rangetree.\ntype Entry interface {\n\t// ValueAtDimension returns the value of this entry\n\t// at the specified dimension.\n\tValueAtDimension(dimension uint64) int64\n}\n\n// Interval describes the methods required to query the rangetree. Note that\n// all ranges are inclusive.\ntype Interval interface {\n\t// LowAtDimension returns an integer representing the lower bound\n\t// at the requested dimension.\n\tLowAtDimension(dimension uint64) int64\n\t// HighAtDimension returns an integer representing the higher bound\n\t// at the request dimension.\n\tHighAtDimension(dimension uint64) int64\n}\n\n// RangeTree describes the methods available to the rangetree.\ntype RangeTree interface {\n\t// Add will add the provided entries to the tree. Any entries that\n\t// were overwritten will be returned in the order in which they\n\t// were overwritten. If an entry's addition does not overwrite, a nil\n\t// is returned for that entry's index in the provided cells.\n\tAdd(entries ...Entry) Entries\n\t// Len returns the number of entries in the tree.\n\tLen() uint64\n\t// Delete will remove the provided entries from the tree.\n\t// Any entries that were deleted will be returned in the order in\n\t// which they were deleted. If an entry does not exist to be deleted,\n\t// a nil is returned for that entry's index in the provided cells.\n\tDelete(entries ...Entry) Entries\n\t// Query will return a list of entries that fall within\n\t// the provided interval. The values at dimensions are inclusive.\n\tQuery(interval Interval) Entries\n\t// Apply will call the provided function with each entry that exists\n\t// within the provided range, in order. Return false at any time to\n\t// cancel iteration. Altering the entry in such a way that its location\n\t// changes will result in undefined behavior.\n\tApply(interval Interval, fn func(Entry) bool)\n\t// Get returns any entries that exist at the addresses provided by the\n\t// given entries. Entries are returned in the order in which they are\n\t// received. If an entry cannot be found, a nil is returned in its\n\t// place.\n\tGet(entries ...Entry) Entries\n\t// InsertAtDimension will increment items at and above the given index\n\t// by the number provided. Provide a negative number to to decrement.\n\t// Returned are two lists. The first list is a list of entries that\n\t// were moved. The second is a list entries that were deleted. These\n\t// lists are exclusive.\n\tInsertAtDimension(dimension uint64, index, number int64) (Entries, Entries)\n}\n" }, { "path": "rangetree/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\ntype mockEntry struct {\n\tid uint64\n\tdimensions []int64\n}\n\nfunc (me *mockEntry) ID() uint64 {\n\treturn me.id\n}\n\nfunc (me *mockEntry) ValueAtDimension(dimension uint64) int64 {\n\treturn me.dimensions[dimension-1]\n}\n\nfunc constructMockEntry(id uint64, values ...int64) *mockEntry {\n\treturn &mockEntry{\n\t\tid: id,\n\t\tdimensions: values,\n\t}\n}\n\ntype dimension struct {\n\tlow, high int64\n}\n\ntype mockInterval struct {\n\tdimensions []dimension\n}\n\nfunc (mi *mockInterval) LowAtDimension(dimension uint64) int64 {\n\treturn mi.dimensions[dimension-1].low\n}\n\nfunc (mi *mockInterval) HighAtDimension(dimension uint64) int64 {\n\treturn mi.dimensions[dimension-1].high\n}\n\nfunc constructMockInterval(dimensions ...dimension) *mockInterval {\n\treturn &mockInterval{\n\t\tdimensions: dimensions,\n\t}\n}\n" }, { "path": "rangetree/node.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\ntype nodes []*node\n\ntype node struct {\n\tvalue int64\n\tentry Entry\n\torderedNodes orderedNodes\n}\n\nfunc newNode(value int64, entry Entry, needNextDimension bool) *node {\n\tn := &node{}\n\tn.value = value\n\tif needNextDimension {\n\t\tn.orderedNodes = make(orderedNodes, 0, 10)\n\t} else {\n\t\tn.entry = entry\n\t}\n\n\treturn n\n}\n" }, { "path": "rangetree/ordered.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\nimport \"sort\"\n\n// orderedNodes represents an ordered list of points living\n// at the last dimension. No duplicates can be inserted here.\ntype orderedNodes nodes\n\nfunc (nodes orderedNodes) search(value int64) int {\n\treturn sort.Search(\n\t\tlen(nodes),\n\t\tfunc(i int) bool { return nodes[i].value >= value },\n\t)\n}\n\n// addAt will add the provided node at the provided index. Returns\n// a node if one was overwritten.\nfunc (nodes *orderedNodes) addAt(i int, node *node) *node {\n\tif i == len(*nodes) {\n\t\t*nodes = append(*nodes, node)\n\t\treturn nil\n\t}\n\n\tif (*nodes)[i].value == node.value {\n\t\toverwritten := (*nodes)[i]\n\t\t// this is a duplicate, there can't be a duplicate\n\t\t// point in the last dimension\n\t\t(*nodes)[i] = node\n\t\treturn overwritten\n\t}\n\n\t*nodes = append(*nodes, nil)\n\tcopy((*nodes)[i+1:], (*nodes)[i:])\n\t(*nodes)[i] = node\n\treturn nil\n}\n\nfunc (nodes *orderedNodes) add(node *node) *node {\n\ti := nodes.search(node.value)\n\treturn nodes.addAt(i, node)\n}\n\nfunc (nodes *orderedNodes) deleteAt(i int) *node {\n\tif i >= len(*nodes) { // no matching found\n\t\treturn nil\n\t}\n\n\tdeleted := (*nodes)[i]\n\tcopy((*nodes)[i:], (*nodes)[i+1:])\n\t(*nodes)[len(*nodes)-1] = nil\n\t*nodes = (*nodes)[:len(*nodes)-1]\n\treturn deleted\n}\n\nfunc (nodes *orderedNodes) delete(value int64) *node {\n\ti := nodes.search(value)\n\n\tif (*nodes)[i].value != value || i == len(*nodes) {\n\t\treturn nil\n\t}\n\n\treturn nodes.deleteAt(i)\n}\n\nfunc (nodes orderedNodes) apply(low, high int64, fn func(*node) bool) bool {\n\tindex := nodes.search(low)\n\tif index == len(nodes) {\n\t\treturn true\n\t}\n\n\tfor ; index < len(nodes); index++ {\n\t\tif nodes[index].value > high {\n\t\t\tbreak\n\t\t}\n\n\t\tif !fn(nodes[index]) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc (nodes orderedNodes) get(value int64) (*node, int) {\n\ti := nodes.search(value)\n\tif i == len(nodes) {\n\t\treturn nil, i\n\t}\n\n\tif nodes[i].value == value {\n\t\treturn nodes[i], i\n\t}\n\n\treturn nil, i\n}\n\nfunc (nodes *orderedNodes) getOrAdd(entry Entry,\n\tdimension, lastDimension uint64) (*node, bool) {\n\n\tisLastDimension := isLastDimension(lastDimension, dimension)\n\tvalue := entry.ValueAtDimension(dimension)\n\n\ti := nodes.search(value)\n\tif i == len(*nodes) {\n\t\tnode := newNode(value, entry, !isLastDimension)\n\t\t*nodes = append(*nodes, node)\n\t\treturn node, true\n\t}\n\n\tif (*nodes)[i].value == value {\n\t\treturn (*nodes)[i], false\n\t}\n\n\tnode := newNode(value, entry, !isLastDimension)\n\t*nodes = append(*nodes, nil)\n\tcopy((*nodes)[i+1:], (*nodes)[i:])\n\t(*nodes)[i] = node\n\treturn node, true\n}\n\nfunc (nodes orderedNodes) flatten(entries *Entries) {\n\tfor _, node := range nodes {\n\t\tif node.orderedNodes != nil {\n\t\t\tnode.orderedNodes.flatten(entries)\n\t\t} else {\n\t\t\t*entries = append(*entries, node.entry)\n\t\t}\n\t}\n}\n\nfunc (nodes *orderedNodes) insert(insertDimension, dimension, maxDimension uint64,\n\tindex, number int64, modified, deleted *Entries) {\n\n\tlastDimension := isLastDimension(maxDimension, dimension)\n\n\tif insertDimension == dimension {\n\t\ti := nodes.search(index)\n\t\tvar toDelete []int\n\n\t\tfor j := i; j < len(*nodes); j++ {\n\t\t\t(*nodes)[j].value += number\n\t\t\tif (*nodes)[j].value < index {\n\t\t\t\ttoDelete = append(toDelete, j)\n\t\t\t\tif lastDimension {\n\t\t\t\t\t*deleted = append(*deleted, (*nodes)[j].entry)\n\t\t\t\t} else {\n\t\t\t\t\t(*nodes)[j].orderedNodes.flatten(deleted)\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif lastDimension {\n\t\t\t\t*modified = append(*modified, (*nodes)[j].entry)\n\t\t\t} else {\n\t\t\t\t(*nodes)[j].orderedNodes.flatten(modified)\n\t\t\t}\n\t\t}\n\n\t\tfor i, index := range toDelete {\n\t\t\tnodes.deleteAt(index - i)\n\t\t}\n\n\t\treturn\n\t}\n\n\tfor _, node := range *nodes {\n\t\tnode.orderedNodes.insert(\n\t\t\tinsertDimension, dimension+1, maxDimension,\n\t\t\tindex, number, modified, deleted,\n\t\t)\n\t}\n}\n\nfunc (nodes orderedNodes) immutableInsert(insertDimension, dimension, maxDimension uint64,\n\tindex, number int64, modified, deleted *Entries) orderedNodes {\n\n\tlastDimension := isLastDimension(maxDimension, dimension)\n\n\tcp := make(orderedNodes, len(nodes))\n\tcopy(cp, nodes)\n\n\tif insertDimension == dimension {\n\t\ti := cp.search(index)\n\t\tvar toDelete []int\n\n\t\tfor j := i; j < len(cp); j++ {\n\t\t\tnn := newNode(cp[j].value+number, cp[j].entry, !lastDimension)\n\t\t\tnn.orderedNodes = cp[j].orderedNodes\n\t\t\tcp[j] = nn\n\t\t\tif cp[j].value < index {\n\t\t\t\ttoDelete = append(toDelete, j)\n\t\t\t\tif lastDimension {\n\t\t\t\t\t*deleted = append(*deleted, cp[j].entry)\n\t\t\t\t} else {\n\t\t\t\t\tcp[j].orderedNodes.flatten(deleted)\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif lastDimension {\n\t\t\t\t*modified = append(*modified, cp[j].entry)\n\t\t\t} else {\n\t\t\t\tcp[j].orderedNodes.flatten(modified)\n\t\t\t}\n\t\t}\n\n\t\tfor _, index := range toDelete {\n\t\t\tcp.deleteAt(index)\n\t\t}\n\n\t\treturn cp\n\t}\n\n\tfor i := 0; i < len(cp); i++ {\n\t\toldNode := nodes[i]\n\t\tnn := newNode(oldNode.value, oldNode.entry, !lastDimension)\n\t\tnn.orderedNodes = oldNode.orderedNodes.immutableInsert(\n\t\t\tinsertDimension, dimension+1,\n\t\t\tmaxDimension,\n\t\t\tindex, number,\n\t\t\tmodified, deleted,\n\t\t)\n\t\tcp[i] = nn\n\t}\n\n\treturn cp\n}\n" }, { "path": "rangetree/ordered_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestOrderedAdd(t *testing.T) {\n\tnodes := make(orderedNodes, 0)\n\n\tn1 := newNode(4, constructMockEntry(1, 4), false)\n\tn2 := newNode(1, constructMockEntry(2, 1), false)\n\n\toverwritten := nodes.add(n1)\n\tassert.Nil(t, overwritten)\n\n\toverwritten = nodes.add(n2)\n\tassert.Nil(t, overwritten)\n\n\tassert.Equal(t, orderedNodes{n2, n1}, nodes)\n\n\tn3 := newNode(4, constructMockEntry(1, 4), false)\n\n\toverwritten = nodes.add(n3)\n\n\tassert.True(t, n1 == overwritten)\n\tassert.Equal(t, orderedNodes{n2, n3}, nodes)\n}\n\nfunc TestOrderedDelete(t *testing.T) {\n\tnodes := make(orderedNodes, 0)\n\n\tn1 := newNode(4, constructMockEntry(1, 4), false)\n\tn2 := newNode(1, constructMockEntry(2, 1), false)\n\n\tnodes.add(n1)\n\tnodes.add(n2)\n\n\tdeleted := nodes.delete(n2.value)\n\n\tassert.Equal(t, orderedNodes{n1}, nodes)\n\tassert.Equal(t, n2, deleted)\n\n\tmissingValue := int64(3)\n\tdeleted = nodes.delete(missingValue)\n\n\tassert.Equal(t, orderedNodes{n1}, nodes)\n\tassert.Nil(t, deleted)\n\n\tdeleted = nodes.delete(n1.value)\n\n\tassert.Empty(t, nodes)\n\tassert.Equal(t, n1, deleted)\n}\n\nfunc TestApply(t *testing.T) {\n\tns := make(orderedNodes, 0)\n\n\tn1 := newNode(4, constructMockEntry(1, 4), false)\n\tn2 := newNode(1, constructMockEntry(2, 1), false)\n\n\tns.add(n1)\n\tns.add(n2)\n\n\tresults := make(nodes, 0, 2)\n\n\tns.apply(1, 1, func(n *node) bool {\n\t\tresults = append(results, n)\n\t\treturn true\n\t})\n\n\tassert.Equal(t, nodes{n2}, results)\n\n\tresults = results[:0]\n\n\tns.apply(0, 0, func(n *node) bool {\n\t\tresults = append(results, n)\n\t\treturn true\n\t})\n\n\tassert.Len(t, results, 0)\n\tresults = results[:0]\n\n\tns.apply(2, 3, func(n *node) bool {\n\t\tresults = append(results, n)\n\t\treturn true\n\t})\n\n\tassert.Len(t, results, 0)\n\tresults = results[:0]\n\n\tns.apply(4, 5, func(n *node) bool {\n\t\tresults = append(results, n)\n\t\treturn true\n\t})\n\n\tassert.Equal(t, nodes{n1}, results)\n\tresults = results[:0]\n\n\tns.apply(0, 5, func(n *node) bool {\n\t\tresults = append(results, n)\n\t\treturn true\n\t})\n\n\tassert.Equal(t, nodes{n2, n1}, results)\n\tresults = results[:0]\n\n\tns.apply(5, 10, func(n *node) bool {\n\t\tresults = append(results, n)\n\t\treturn true\n\t})\n\n\tassert.Len(t, results, 0)\n\tresults = results[:0]\n\n\tns.apply(0, 100, func(n *node) bool {\n\t\tresults = append(results, n)\n\t\treturn false\n\t})\n\n\tassert.Equal(t, nodes{n2}, results)\n}\n\nfunc TestInsertDelete(t *testing.T) {\n\tns := make(orderedNodes, 0)\n\n\tn1 := newNode(4, constructMockEntry(1, 4), false)\n\tn2 := newNode(1, constructMockEntry(2, 1), false)\n\tn3 := newNode(2, constructMockEntry(3, 2), false)\n\n\tns.add(n1)\n\tns.add(n2)\n\tns.add(n3)\n\n\tmodified := make(Entries, 0, 1)\n\tdeleted := make(Entries, 0, 1)\n\n\tns.insert(2, 2, 2, 0, -5, &modified, &deleted)\n\n\tassert.Len(t, ns, 0)\n\tassert.Equal(t, Entries{n2.entry, n3.entry, n1.entry}, deleted)\n}\n\nfunc BenchmarkPrepend(b *testing.B) {\n\tnumItems := 100000\n\tns := make(orderedNodes, 0, numItems)\n\n\tfor i := b.N; i < b.N+numItems; i++ {\n\t\tns.add(newNode(int64(i), constructMockEntry(uint64(i), int64(i)), false))\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tns.add(newNode(int64(i), constructMockEntry(uint64(i), int64(i)), false))\n\t}\n}\n" }, { "path": "rangetree/orderedtree.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\npackage rangetree\n\nfunc isLastDimension(value, test uint64) bool {\n\treturn test >= value\n}\n\ntype nodeBundle struct {\n\tlist *orderedNodes\n\tindex int\n}\n\ntype orderedTree struct {\n\ttop orderedNodes\n\tnumber uint64\n\tdimensions uint64\n\tpath []*nodeBundle\n}\n\nfunc (ot *orderedTree) resetPath() {\n\tot.path = ot.path[:0]\n}\n\nfunc (ot *orderedTree) needNextDimension() bool {\n\treturn ot.dimensions > 1\n}\n\n// add will add the provided entry to the rangetree and return an\n// entry if one was overwritten.\nfunc (ot *orderedTree) add(entry Entry) *node {\n\tvar node *node\n\tlist := &ot.top\n\n\tfor i := uint64(1); i <= ot.dimensions; i++ {\n\t\tif isLastDimension(ot.dimensions, i) {\n\t\t\toverwritten := list.add(\n\t\t\t\tnewNode(entry.ValueAtDimension(i), entry, false),\n\t\t\t)\n\t\t\tif overwritten == nil {\n\t\t\t\tot.number++\n\t\t\t}\n\t\t\treturn overwritten\n\t\t}\n\t\tnode, _ = list.getOrAdd(entry, i, ot.dimensions)\n\t\tlist = &node.orderedNodes\n\t}\n\n\treturn nil\n}\n\n// Add will add the provided entries to the tree. This method\n// returns a list of entries that were overwritten in the order\n// in which entries were received. If an entry doesn't overwrite\n// anything, a nil will be returned for that entry in the returned\n// slice.\nfunc (ot *orderedTree) Add(entries ...Entry) Entries {\n\tif len(entries) == 0 {\n\t\treturn nil\n\t}\n\n\toverwrittens := make(Entries, len(entries))\n\tfor i, entry := range entries {\n\t\tif entry == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\toverwritten := ot.add(entry)\n\t\tif overwritten != nil {\n\t\t\toverwrittens[i] = overwritten.entry\n\t\t}\n\t}\n\n\treturn overwrittens\n}\n\nfunc (ot *orderedTree) delete(entry Entry) *node {\n\tot.resetPath()\n\tvar index int\n\tvar node *node\n\tlist := &ot.top\n\n\tfor i := uint64(1); i <= ot.dimensions; i++ {\n\t\tvalue := entry.ValueAtDimension(i)\n\t\tnode, index = list.get(value)\n\t\tif node == nil { // there's nothing to delete\n\t\t\treturn nil\n\t\t}\n\n\t\tnb := &nodeBundle{list: list, index: index}\n\t\tot.path = append(ot.path, nb)\n\n\t\tlist = &node.orderedNodes\n\t}\n\n\tot.number--\n\n\tfor i := len(ot.path) - 1; i >= 0; i-- {\n\t\tnb := ot.path[i]\n\t\tnb.list.deleteAt(nb.index)\n\t\tif len(*nb.list) > 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn node\n}\n\nfunc (ot *orderedTree) get(entry Entry) Entry {\n\ton := ot.top\n\tfor i := uint64(1); i <= ot.dimensions; i++ {\n\t\tn, _ := on.get(entry.ValueAtDimension(i))\n\t\tif n == nil {\n\t\t\treturn nil\n\t\t}\n\t\tif i == ot.dimensions {\n\t\t\treturn n.entry\n\t\t}\n\t\ton = n.orderedNodes\n\t}\n\n\treturn nil\n}\n\n// Get returns any entries that exist at the addresses provided by the\n// given entries. Entries are returned in the order in which they are\n// received. If an entry cannot be found, a nil is returned in its\n// place.\nfunc (ot *orderedTree) Get(entries ...Entry) Entries {\n\tresult := make(Entries, 0, len(entries))\n\tfor _, entry := range entries {\n\t\tresult = append(result, ot.get(entry))\n\t}\n\n\treturn result\n}\n\n// Delete will remove the provided entries from the tree.\n// Any entries that were deleted will be returned in the order in\n// which they were deleted. If an entry does not exist to be deleted,\n// a nil is returned for that entry's index in the provided cells.\nfunc (ot *orderedTree) Delete(entries ...Entry) Entries {\n\tif len(entries) == 0 {\n\t\treturn nil\n\t}\n\n\tdeletedEntries := make(Entries, len(entries))\n\tfor i, entry := range entries {\n\t\tif entry == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tdeleted := ot.delete(entry)\n\t\tif deleted != nil {\n\t\t\tdeletedEntries[i] = deleted.entry\n\t\t}\n\t}\n\n\treturn deletedEntries\n}\n\n// Len returns the number of items in the tree.\nfunc (ot *orderedTree) Len() uint64 {\n\treturn ot.number\n}\n\nfunc (ot *orderedTree) apply(list orderedNodes, interval Interval,\n\tdimension uint64, fn func(*node) bool) bool {\n\n\tlow, high := interval.LowAtDimension(dimension), interval.HighAtDimension(dimension)\n\n\tif isLastDimension(ot.dimensions, dimension) {\n\t\tif !list.apply(low, high, fn) {\n\t\t\treturn false\n\t\t}\n\t} else {\n\t\tif !list.apply(low, high, func(n *node) bool {\n\t\t\tif !ot.apply(n.orderedNodes, interval, dimension+1, fn) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn true\n\t\t}) {\n\t\t\treturn false\n\t\t}\n\t\treturn true\n\t}\n\n\treturn true\n}\n\n// Apply will call (in order) the provided function to every\n// entry that falls within the provided interval. Any alteration\n// the the entry that would result in different answers to the\n// interface methods results in undefined behavior.\nfunc (ot *orderedTree) Apply(interval Interval, fn func(Entry) bool) {\n\tot.apply(ot.top, interval, 1, func(n *node) bool {\n\t\treturn fn(n.entry)\n\t})\n}\n\n// Query will return an ordered list of results in the given\n// interval.\nfunc (ot *orderedTree) Query(interval Interval) Entries {\n\tentries := NewEntries()\n\n\tot.apply(ot.top, interval, 1, func(n *node) bool {\n\t\tentries = append(entries, n.entry)\n\t\treturn true\n\t})\n\n\treturn entries\n}\n\n// InsertAtDimension will increment items at and above the given index\n// by the number provided. Provide a negative number to to decrement.\n// Returned are two lists. The first list is a list of entries that\n// were moved. The second is a list entries that were deleted. These\n// lists are exclusive.\nfunc (ot *orderedTree) InsertAtDimension(dimension uint64,\n\tindex, number int64) (Entries, Entries) {\n\n\t// TODO: perhaps return an error here?\n\tif dimension > ot.dimensions || number == 0 {\n\t\treturn nil, nil\n\t}\n\n\tmodified := make(Entries, 0, 100)\n\tdeleted := make(Entries, 0, 100)\n\n\tot.top.insert(dimension, 1, ot.dimensions,\n\t\tindex, number, &modified, &deleted,\n\t)\n\n\tot.number -= uint64(len(deleted))\n\n\treturn modified, deleted\n}\n\nfunc newOrderedTree(dimensions uint64) *orderedTree {\n\treturn &orderedTree{\n\t\tdimensions: dimensions,\n\t\tpath: make([]*nodeBundle, 0, dimensions),\n\t}\n}\n\n// New is the constructor to create a new rangetree with\n// the provided number of dimensions.\nfunc New(dimensions uint64) RangeTree {\n\treturn newOrderedTree(dimensions)\n}\n" }, { "path": "rangetree/orderedtree_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rangetree\n\nimport (\n\t\"math/rand\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc constructMultiDimensionalOrderedTree(number uint64) (\n\t*orderedTree, Entries) {\n\n\ttree := newOrderedTree(2)\n\n\tentries := make(Entries, 0, number)\n\tfor i := uint64(0); i < number; i++ {\n\t\tentries = append(entries, constructMockEntry(i, int64(i), int64(i)))\n\t}\n\n\ttree.Add(entries...)\n\n\treturn tree, entries\n}\n\nfunc TestOTRootAddMultipleDimensions(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(1)\n\n\tassert.Equal(t, uint64(1), tree.Len())\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 0}, dimension{0, 0}))\n\tassert.Equal(t, Entries{entries[0]}, result)\n}\n\nfunc TestOTMultipleAddMultipleDimensions(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(4)\n\n\tassert.Equal(t, uint64(4), tree.Len())\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 0}, dimension{0, 0}))\n\tassert.Equal(t, Entries{entries[0]}, result)\n\n\tresult = tree.Query(constructMockInterval(dimension{3, 4}, dimension{3, 4}))\n\tassert.Equal(t, Entries{entries[3]}, result)\n\n\tresult = tree.Query(constructMockInterval(dimension{0, 4}, dimension{0, 4}))\n\tassert.Equal(t, entries, result)\n\n\tresult = tree.Query(constructMockInterval(dimension{1, 2}, dimension{1, 2}))\n\tassert.Equal(t, Entries{entries[1], entries[2]}, result)\n\n\tresult = tree.Query(constructMockInterval(dimension{0, 2}, dimension{10, 20}))\n\tassert.Len(t, result, 0)\n\n\tresult = tree.Query(constructMockInterval(dimension{10, 20}, dimension{0, 2}))\n\tassert.Len(t, result, 0)\n\n\tresult = tree.Query(constructMockInterval(dimension{0, 1}, dimension{0, 0}))\n\tassert.Equal(t, Entries{entries[0]}, result)\n\n\tresult = tree.Query(constructMockInterval(dimension{0, 0}, dimension{0, 1}))\n\tassert.Equal(t, Entries{entries[0]}, result)\n}\n\nfunc TestOTAddInOrderMultiDimensions(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(10)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\tassert.Equal(t, uint64(10), tree.Len())\n\tassert.Len(t, result, 10)\n\tassert.Equal(t, entries, result)\n}\n\nfunc TestOTAddReverseOrderMultiDimensions(t *testing.T) {\n\ttree := newOrderedTree(2)\n\n\tfor i := uint64(10); i > 0; i-- {\n\t\ttree.Add(constructMockEntry(i, int64(i), int64(i)))\n\t}\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 11}, dimension{0, 11}))\n\tassert.Len(t, result, 10)\n\tassert.Equal(t, uint64(10), tree.Len())\n}\n\nfunc TestOTAddRandomOrderMultiDimensions(t *testing.T) {\n\ttree := newOrderedTree(2)\n\n\tstarts := []uint64{0, 4, 2, 1, 3}\n\n\tfor _, start := range starts {\n\t\ttree.Add(constructMockEntry(start, int64(start), int64(start)))\n\t}\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 5}, dimension{0, 5}))\n\tassert.Len(t, result, 5)\n\tassert.Equal(t, uint64(5), tree.Len())\n}\n\nfunc TestOTAddLargeNumbersMultiDimension(t *testing.T) {\n\tnumItems := uint64(1000)\n\ttree := newOrderedTree(2)\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\ttree.Add(constructMockEntry(i, int64(i), int64(i)))\n\t}\n\n\tresult := tree.Query(\n\t\tconstructMockInterval(\n\t\t\tdimension{0, int64(numItems)},\n\t\t\tdimension{0, int64(numItems)},\n\t\t),\n\t)\n\tassert.Equal(t, numItems, tree.Len())\n\tassert.Len(t, result, int(numItems))\n}\n\nfunc TestOTAddReturnsOverwritten(t *testing.T) {\n\ttree := newOrderedTree(2)\n\n\tstarts := []uint64{0, 4, 2, 1, 3}\n\n\tentries := make(Entries, 0, len(starts))\n\tfor _, start := range starts {\n\t\tentries = append(entries, constructMockEntry(start, int64(start), int64(start)))\n\t}\n\n\toverwritten := tree.Add(entries...)\n\n\tassert.Equal(t, Entries{nil, nil, nil, nil, nil}, overwritten)\n\n\toldEntry := entries[2]\n\tnewEntry := constructMockEntry(10, oldEntry.ValueAtDimension(1),\n\t\toldEntry.ValueAtDimension(2))\n\toverwritten = tree.Add(newEntry)\n\n\tassert.Equal(t, Entries{oldEntry}, overwritten)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 5}, dimension{0, 5}))\n\tassert.Len(t, result, 5)\n\tassert.Equal(t, uint64(5), tree.Len())\n}\n\nfunc BenchmarkOTAddItemsMultiDimensions(b *testing.B) {\n\tnumItems := b.N\n\tentries := make(Entries, 0, numItems)\n\n\tfor i := uint64(0); i < uint64(numItems); i++ {\n\t\tvalue := rand.Int63()\n\t\tentries = append(entries, constructMockEntry(i, value, value))\n\t}\n\n\trt := newOrderedTree(2)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trt.Add(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkOTQueryItemsMultiDimensions(b *testing.B) {\n\tnumItems := uint64(1000)\n\tentries := make(Entries, 0, numItems)\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tentries = append(entries, constructMockEntry(i, int64(i), int64(i)))\n\t}\n\n\ttree := newOrderedTree(2)\n\ttree.Add(entries...)\n\tiv := constructMockInterval(\n\t\tdimension{0, int64(numItems)},\n\t\tdimension{0, int64(numItems)},\n\t)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Query(iv)\n\t}\n}\n\nfunc TestOTRootDeleteMultiDimensions(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(1)\n\ttree.Delete(entries...)\n\n\tassert.Equal(t, uint64(0), tree.Len())\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 100}, dimension{0, 100}))\n\tassert.Len(t, result, 0)\n}\n\nfunc TestOTDeleteMultiDimensions(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(4)\n\n\ttree.Delete(entries[2])\n\n\tassert.Equal(t, uint64(3), tree.Len())\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 4}, dimension{0, 4}))\n\tassert.Equal(t, Entries{entries[0], entries[1], entries[3]}, result)\n\n\tresult = tree.Query(constructMockInterval(dimension{3, 4}, dimension{3, 4}))\n\tassert.Equal(t, Entries{entries[3]}, result)\n\n\tresult = tree.Query(constructMockInterval(dimension{0, 2}, dimension{0, 2}))\n\tassert.Equal(t, Entries{entries[0], entries[1]}, result)\n}\n\nfunc TestOTDeleteInOrderMultiDimensions(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(10)\n\n\ttree.Delete(entries[5])\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\tassert.Len(t, result, 9)\n\tassert.Equal(t, uint64(9), tree.Len())\n\n\tassert.NotContains(t, result, entries[5])\n}\n\nfunc TestOTDeleteReverseOrderMultiDimensions(t *testing.T) {\n\ttree := newOrderedTree(2)\n\n\tentries := NewEntries()\n\tfor i := uint64(10); i > 0; i-- {\n\t\tentries = append(entries, constructMockEntry(i, int64(i), int64(i)))\n\t}\n\n\ttree.Add(entries...)\n\n\ttree.Delete(entries[5])\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 11}, dimension{0, 11}))\n\tassert.Len(t, result, 9)\n\tassert.Equal(t, uint64(9), tree.Len())\n\n\tassert.NotContains(t, result, entries[5])\n}\n\nfunc TestOTDeleteRandomOrderMultiDimensions(t *testing.T) {\n\ttree := newOrderedTree(2)\n\n\tentries := NewEntries()\n\tstarts := []uint64{0, 4, 2, 1, 3}\n\tfor _, start := range starts {\n\t\tentries = append(entries, constructMockEntry(start, int64(start), int64(start)))\n\t}\n\n\ttree.Add(entries...)\n\n\ttree.Delete(entries[2])\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 11}, dimension{0, 11}))\n\n\tassert.Len(t, result, 4)\n\tassert.Equal(t, uint64(4), tree.Len())\n\n\tassert.NotContains(t, result, entries[2])\n}\n\nfunc TestOTDeleteEmptyTreeMultiDimensions(t *testing.T) {\n\ttree := newOrderedTree(2)\n\n\ttree.Delete(constructMockEntry(0, 0, 0))\n\n\tassert.Equal(t, uint64(0), tree.Len())\n}\n\nfunc TestOTDeleteReturnsDeleted(t *testing.T) {\n\ttree := newOrderedTree(2)\n\n\tentries := NewEntries()\n\tstarts := []uint64{0, 4, 2, 1, 3}\n\tfor _, start := range starts {\n\t\tentries = append(entries, constructMockEntry(start, int64(start), int64(start)))\n\t}\n\n\ttree.Add(entries...)\n\n\tdeleted := tree.Delete(entries[2], constructMockEntry(10, 10, 10))\n\n\tassert.Equal(t, Entries{entries[2], nil}, deleted)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 11}, dimension{0, 11}))\n\n\tassert.Len(t, result, 4)\n\tassert.Equal(t, uint64(4), tree.Len())\n\n\tassert.NotContains(t, result, entries[2])\n}\n\nfunc BenchmarkOTDeleteItemsMultiDimensions(b *testing.B) {\n\tnumItems := uint64(1000)\n\tentries := make(Entries, 0, numItems)\n\n\tfor i := uint64(0); i < numItems; i++ {\n\t\tentries = append(entries, constructMockEntry(i, int64(i), int64(i)))\n\t}\n\n\ttrees := make([]*orderedTree, 0, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\ttree := newOrderedTree(2)\n\t\ttree.Add(entries...)\n\t\ttrees = append(trees, tree)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttrees[i].Delete(entries...)\n\t}\n}\n\nfunc TestOverwrites(t *testing.T) {\n\ttree, _ := constructMultiDimensionalOrderedTree(1)\n\n\tentry := constructMockEntry(10, 10, 10)\n\n\toverwritten := tree.Add(entry)\n\tassert.Equal(t, Entries{nil}, overwritten)\n\n\tresults := tree.Query(constructMockInterval(dimension{10, 11}, dimension{10, 11}))\n\n\tassert.Equal(t, Entries{entry}, results)\n\tassert.Equal(t, uint64(2), tree.Len())\n\n\tnewEntry := constructMockEntry(10, 10, 10)\n\n\toverwritten = tree.Add(newEntry)\n\tassert.Equal(t, Entries{entry}, overwritten)\n}\n\nfunc TestGet(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(2)\n\n\tresult := tree.Get(entries...)\n\tassert.Equal(t, entries, result)\n\n\tresult = tree.Get(constructMockEntry(10000, 5000, 5000))\n\tassert.Equal(t, Entries{nil}, result)\n}\n\nfunc TestTreeApply(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(2)\n\n\tresult := make(Entries, 0, len(entries))\n\n\ttree.Apply(constructMockInterval(dimension{0, 100}, dimension{0, 100}),\n\t\tfunc(e Entry) bool {\n\t\t\tresult = append(result, e)\n\t\t\treturn true\n\t\t},\n\t)\n\n\tassert.Equal(t, entries, result)\n}\n\nfunc TestApplyWithBail(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(2)\n\n\tresult := make(Entries, 0, 1)\n\n\ttree.Apply(constructMockInterval(dimension{0, 100}, dimension{0, 100}),\n\t\tfunc(e Entry) bool {\n\t\t\tresult = append(result, e)\n\t\t\treturn false\n\t\t},\n\t)\n\n\tassert.Equal(t, entries[:1], result)\n}\n\nfunc BenchmarkApply(b *testing.B) {\n\tnumItems := 1000\n\n\ttree, _ := constructMultiDimensionalOrderedTree(uint64(numItems))\n\n\tiv := constructMockInterval(\n\t\tdimension{0, int64(numItems)}, dimension{0, int64(numItems)},\n\t)\n\tfn := func(Entry) bool { return true }\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Apply(iv, fn)\n\t}\n}\n\nfunc TestInsertPositiveIndexFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(2)\n\n\tmodified, deleted := tree.InsertAtDimension(1, 1, 1)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, entries[1:], modified)\n\n\tresult := tree.Query(constructMockInterval(dimension{2, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[1:], result)\n}\n\nfunc TestInsertPositiveIndexSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(2, 1, 1)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, entries[1:], modified)\n\n\tresult := tree.Query(constructMockInterval(dimension{1, 10}, dimension{2, 10}))\n\tassert.Equal(t, entries[1:], result)\n}\n\nfunc TestInsertPositiveIndexOutOfBoundsFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(1, 4, 1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\n\tassert.Equal(t, entries, result)\n}\n\nfunc TestInsertPositiveIndexOutOfBoundsSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(2, 4, 1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\n\tassert.Equal(t, entries, result)\n}\n\nfunc TestInsertMultiplePositiveIndexFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(1, 1, 2)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, entries[1:], modified)\n\n\tresult := tree.Query(constructMockInterval(dimension{3, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[1:], result)\n}\n\nfunc TestInsertMultiplePositiveIndexSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(2, 1, 2)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, entries[1:], modified)\n\n\tresult := tree.Query(constructMockInterval(dimension{1, 10}, dimension{3, 10}))\n\tassert.Equal(t, entries[1:], result)\n}\n\nfunc TestInsertNegativeIndexFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(1, 1, -1)\n\tassert.Equal(t, entries[1:2], deleted)\n\tassert.Equal(t, entries[2:], modified)\n\n\tresult := tree.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[2:], result)\n\n\tresult = tree.Query(constructMockInterval(dimension{2, 10}, dimension{1, 10}))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(2), tree.Len())\n}\n\nfunc TestInsertNegativeIndexSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(2, 1, -1)\n\tassert.Equal(t, entries[1:2], deleted)\n\tassert.Equal(t, entries[2:], modified)\n\n\tresult := tree.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Equal(t, entries[2:], result)\n\n\tresult = tree.Query(constructMockInterval(dimension{1, 10}, dimension{2, 10}))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(2), tree.Len())\n}\n\nfunc TestInsertNegativeIndexOutOfBoundsFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(1, 4, -1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\n\tassert.Equal(t, entries, result)\n\tassert.Equal(t, uint64(3), tree.Len())\n}\n\nfunc TestInsertNegativeIndexOutOfBoundsSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(2, 4, -1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\n\tassert.Equal(t, entries, result)\n\tassert.Equal(t, uint64(3), tree.Len())\n}\n\nfunc TestInsertMultipleNegativeIndexFirstDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(1, 1, -2)\n\tassert.Equal(t, entries[1:], deleted)\n\tassert.Len(t, modified, 0)\n\n\tresult := tree.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(1), tree.Len())\n}\n\nfunc TestInsertMultipleNegativeIndexSecondDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(2, 1, -2)\n\tassert.Equal(t, entries[1:], deleted)\n\tassert.Len(t, modified, 0)\n\n\tresult := tree.Query(constructMockInterval(dimension{1, 10}, dimension{1, 10}))\n\tassert.Len(t, result, 0)\n\tassert.Equal(t, uint64(1), tree.Len())\n}\n\nfunc TestInsertInvalidDimension(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(3, 1, -1)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\tassert.Equal(t, entries, result)\n}\n\nfunc TestInsertInvalidNumber(t *testing.T) {\n\ttree, entries := constructMultiDimensionalOrderedTree(3)\n\n\tmodified, deleted := tree.InsertAtDimension(1, 1, 0)\n\tassert.Len(t, modified, 0)\n\tassert.Len(t, deleted, 0)\n\n\tresult := tree.Query(constructMockInterval(dimension{0, 10}, dimension{0, 10}))\n\tassert.Equal(t, entries, result)\n}\n\nfunc BenchmarkInsertFirstDimension(b *testing.B) {\n\tnumItems := uint64(100000)\n\n\ttree, _ := constructMultiDimensionalOrderedTree(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.InsertAtDimension(1, 0, 1)\n\t}\n}\n\nfunc BenchmarkInsertSecondDimension(b *testing.B) {\n\tnumItems := uint64(100000)\n\n\ttree, _ := constructMultiDimensionalOrderedTree(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.InsertAtDimension(2, 0, 1)\n\t}\n}\n\nfunc BenchmarkDeleteFirstDimension(b *testing.B) {\n\tnumItems := uint64(100000)\n\n\ttree, _ := constructMultiDimensionalOrderedTree(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.InsertAtDimension(1, 0, -1)\n\t}\n}\n\nfunc BenchmarkDeleteSecondDimension(b *testing.B) {\n\tnumItems := uint64(100000)\n\n\ttree, _ := constructMultiDimensionalOrderedTree(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.InsertAtDimension(2, 0, -1)\n\t}\n}\n\nfunc BenchmarkGetMultiDimensions(b *testing.B) {\n\tnumItemsX := 10000\n\tnumItemsY := 100\n\n\ttree := newOrderedTree(2)\n\tentries := make(Entries, 0, numItemsY*numItemsX)\n\n\tfor i := 0; i < numItemsX; i++ {\n\t\tfor j := 0; j < numItemsY; j++ {\n\t\t\te := constructMockEntry(uint64(j*numItemsY+i), int64(i), int64(j))\n\t\t\tentries = append(entries, e)\n\t\t}\n\t}\n\n\ttree.Add(entries...)\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Get(entries[i%len(entries)])\n\t}\n}\n" }, { "path": "rangetree/skiplist/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage skiplist\n\ntype mockEntry struct {\n\tvalues []int64\n}\n\nfunc (me *mockEntry) ValueAtDimension(dimension uint64) int64 {\n\treturn me.values[dimension]\n}\n\nfunc newMockEntry(values ...int64) *mockEntry {\n\treturn &mockEntry{values: values}\n}\n\ntype mockInterval struct {\n\tlows, highs []int64\n}\n\nfunc (mi *mockInterval) LowAtDimension(dimension uint64) int64 {\n\treturn mi.lows[dimension]\n}\n\nfunc (mi *mockInterval) HighAtDimension(dimension uint64) int64 {\n\treturn mi.highs[dimension]\n}\n\nfunc newMockInterval(lows, highs []int64) *mockInterval {\n\treturn &mockInterval{\n\t\tlows: lows,\n\t\thighs: highs,\n\t}\n}\n" }, { "path": "rangetree/skiplist/skiplist.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage skiplist implements an n-dimensional rangetree based on a skip\nlist. This should be faster than a straight slice implementation as\nmemcopy is avoided.\n\nTime complexities revolve around the ability to quickly find items\nin the n-dimensional skiplist. That time can be defined by the\nnumber of items in any dimension. Let N1, N2,... Nn define the\nnumber of dimensions.\n\nPerformance characteristics:\nSpace: O(n)\nSearch: O(log N1 + log N2 + ...log Nn) = O(log N1*N2*...Nn)\nInsert: O(log N1 + log N2 + ...log Nn) = O(log N1*N2*...Nn)\nDelete: O(log N1 + log N2 + ...log Nn) = O(log N1*N2*...Nn)\n*/\npackage skiplist\n\nimport (\n\t\"github.com/Workiva/go-datastructures/common\"\n\t\"github.com/Workiva/go-datastructures/rangetree\"\n\t\"github.com/Workiva/go-datastructures/slice/skip\"\n)\n\n// keyed is required as in the rangetree code we often want to compare\n// two different types of bundles and this allows us to do so without\n// checking for each one.\ntype keyed interface {\n\tkey() uint64\n}\n\ntype skipEntry uint64\n\n// Compare is required by the Comparator interface.\nfunc (se skipEntry) Compare(other common.Comparator) int {\n\totherSe := other.(skipEntry)\n\tif se == otherSe {\n\t\treturn 0\n\t}\n\n\tif se > otherSe {\n\t\treturn 1\n\t}\n\n\treturn -1\n}\n\nfunc (se skipEntry) key() uint64 {\n\treturn uint64(se)\n}\n\n// isLastDimension simply returns dimension == lastDimension-1.\n// This panics if dimension >= lastDimension.\nfunc isLastDimension(dimension, lastDimension uint64) bool {\n\tif dimension >= lastDimension { // useful in testing and denotes a serious problem\n\t\tpanic(`Dimension is greater than possible dimensions.`)\n\t}\n\n\treturn dimension == lastDimension-1\n}\n\n// needsDeletion returns a bool indicating if the provided value\n// needs to be deleted based on the provided index and number.\nfunc needsDeletion(value, index, number int64) bool {\n\tif number > 0 {\n\t\treturn false\n\t}\n\n\tnumber = -number // get the magnitude\n\toffset := value - index\n\n\treturn offset >= 0 && offset < number\n}\n\n// dimensionalBundle is an intermediate holder up to the last\n// dimension and represents a wrapper around a skiplist.\ntype dimensionalBundle struct {\n\tid uint64\n\tsl *skip.SkipList\n}\n\n// Compare returns a value indicating the relative relationship and the\n// provided bundle.\nfunc (db *dimensionalBundle) Compare(e common.Comparator) int {\n\tkeyed := e.(keyed)\n\tif db.id == keyed.key() {\n\t\treturn 0\n\t}\n\n\tif db.id > keyed.key() {\n\t\treturn 1\n\t}\n\n\treturn -1\n}\n\n// key returns the key for this bundle.\nfunc (db *dimensionalBundle) key() uint64 {\n\treturn db.id\n}\n\n// lastBundle represents a bundle living at the last dimension\n// of the tree.\ntype lastBundle struct {\n\tid uint64\n\tentry rangetree.Entry\n}\n\n// Compare returns a value indicating the relative relationship and the\n// provided bundle.\nfunc (lb *lastBundle) Compare(e common.Comparator) int {\n\tkeyed := e.(keyed)\n\tif lb.id == keyed.key() {\n\t\treturn 0\n\t}\n\n\tif lb.id > keyed.key() {\n\t\treturn 1\n\t}\n\n\treturn -1\n}\n\n// Key returns the key for this bundle.\nfunc (lb *lastBundle) key() uint64 {\n\treturn lb.id\n}\n\ntype skipListRT struct {\n\ttop *skip.SkipList\n\tdimensions, number uint64\n}\n\nfunc (rt *skipListRT) init(dimensions uint64) {\n\trt.dimensions = dimensions\n\trt.top = skip.New(uint64(0))\n}\n\nfunc (rt *skipListRT) add(entry rangetree.Entry) rangetree.Entry {\n\tvar (\n\t\tvalue int64\n\t\te common.Comparator\n\t\tsl = rt.top\n\t\tdb *dimensionalBundle\n\t\tlb *lastBundle\n\t)\n\n\tfor i := uint64(0); i < rt.dimensions; i++ {\n\t\tvalue = entry.ValueAtDimension(i)\n\t\te = sl.Get(skipEntry(value))[0]\n\t\tif isLastDimension(i, rt.dimensions) {\n\t\t\tif e != nil { // this is an overwrite\n\t\t\t\tlb = e.(*lastBundle)\n\t\t\t\toldEntry := lb.entry\n\t\t\t\tlb.entry = entry\n\t\t\t\treturn oldEntry\n\t\t\t}\n\n\t\t\t// need to add new sl entry\n\t\t\tlb = &lastBundle{id: uint64(value), entry: entry}\n\t\t\trt.number++\n\t\t\tsl.Insert(lb)\n\t\t\treturn nil\n\t\t}\n\n\t\tif e == nil { // we need the intermediate dimension\n\t\t\tdb = &dimensionalBundle{id: uint64(value), sl: skip.New(uint64(0))}\n\t\t\tsl.Insert(db)\n\t\t} else {\n\t\t\tdb = e.(*dimensionalBundle)\n\t\t}\n\n\t\tsl = db.sl\n\t}\n\n\tpanic(`Ran out of dimensions before for loop completed.`)\n}\n\n// Add will add the provided entries to the tree. This method\n// returns a list of entries that were overwritten in the order\n// in which entries were received. If an entry doesn't overwrite\n// anything, a nil will be returned for that entry in the returned\n// slice.\nfunc (rt *skipListRT) Add(entries ...rangetree.Entry) rangetree.Entries {\n\toverwritten := make(rangetree.Entries, len(entries))\n\tfor i, e := range entries {\n\t\toverwritten[i] = rt.add(e)\n\t}\n\n\treturn overwritten\n}\n\nfunc (rt *skipListRT) get(entry rangetree.Entry) rangetree.Entry {\n\tvar (\n\t\tsl = rt.top\n\t\te common.Comparator\n\t\tvalue uint64\n\t)\n\tfor i := uint64(0); i < rt.dimensions; i++ {\n\t\tvalue = uint64(entry.ValueAtDimension(i))\n\t\te = sl.Get(skipEntry(value))[0]\n\t\tif e == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\tif isLastDimension(i, rt.dimensions) {\n\t\t\treturn e.(*lastBundle).entry\n\t\t}\n\n\t\tsl = e.(*dimensionalBundle).sl\n\t}\n\n\tpanic(`Reached past for loop without finding last dimension.`)\n}\n\n// Get will return any rangetree.Entries matching the provided entries.\n// Similar in functionality to a key lookup, this returns nil for any\n// entry that could not be found.\nfunc (rt *skipListRT) Get(entries ...rangetree.Entry) rangetree.Entries {\n\tresults := make(rangetree.Entries, 0, len(entries))\n\tfor _, e := range entries {\n\t\tresults = append(results, rt.get(e))\n\t}\n\n\treturn results\n}\n\n// Len returns the number of entries in the tree.\nfunc (rt *skipListRT) Len() uint64 {\n\treturn rt.number\n}\n\n// deleteRecursive is used by the delete logic. The recursion depth\n// only goes as far as the number of dimensions, so this shouldn't be an\n// issue.\nfunc (rt *skipListRT) deleteRecursive(sl *skip.SkipList, dimension uint64,\n\tentry rangetree.Entry) rangetree.Entry {\n\n\tvalue := entry.ValueAtDimension(dimension)\n\tif isLastDimension(dimension, rt.dimensions) {\n\t\tentries := sl.Delete(skipEntry(value))\n\t\tif entries[0] == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\trt.number--\n\t\treturn entries[0].(*lastBundle).entry\n\t}\n\n\tdb, ok := sl.Get(skipEntry(value))[0].(*dimensionalBundle)\n\tif !ok { // value was not found\n\t\treturn nil\n\t}\n\n\tresult := rt.deleteRecursive(db.sl, dimension+1, entry)\n\tif result == nil {\n\t\treturn nil\n\t}\n\n\tif db.sl.Len() == 0 {\n\t\tsl.Delete(db)\n\t}\n\n\treturn result\n}\n\nfunc (rt *skipListRT) delete(entry rangetree.Entry) rangetree.Entry {\n\treturn rt.deleteRecursive(rt.top, 0, entry)\n}\n\n// Delete will remove the provided entries from the tree.\n// Any entries that were deleted will be returned in the order in\n// which they were deleted. If an entry does not exist to be deleted,\n// a nil is returned for that entry's index in the provided cells.\nfunc (rt *skipListRT) Delete(entries ...rangetree.Entry) rangetree.Entries {\n\tdeletedEntries := make(rangetree.Entries, len(entries))\n\tfor i, e := range entries {\n\t\tdeletedEntries[i] = rt.delete(e)\n\t}\n\n\treturn deletedEntries\n}\n\nfunc (rt *skipListRT) apply(sl *skip.SkipList, dimension uint64,\n\tinterval rangetree.Interval, fn func(rangetree.Entry) bool) bool {\n\n\tlowValue, highValue := interval.LowAtDimension(dimension), interval.HighAtDimension(dimension)\n\n\tvar e common.Comparator\n\n\tfor iter := sl.Iter(skipEntry(lowValue)); iter.Next(); {\n\t\te = iter.Value()\n\t\tif int64(e.(keyed).key()) >= highValue {\n\t\t\tbreak\n\t\t}\n\n\t\tif isLastDimension(dimension, rt.dimensions) {\n\t\t\tif !fn(e.(*lastBundle).entry) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\n\t\t\tif !rt.apply(e.(*dimensionalBundle).sl, dimension+1, interval, fn) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\n\treturn true\n}\n\n// Apply will call the provided function with each entry that exists\n// within the provided range, in order. Return false at any time to\n// cancel iteration. Altering the entry in such a way that its location\n// changes will result in undefined behavior.\nfunc (rt *skipListRT) Apply(interval rangetree.Interval, fn func(rangetree.Entry) bool) {\n\trt.apply(rt.top, 0, interval, fn)\n}\n\n// Query will return a list of entries that fall within\n// the provided interval.\nfunc (rt *skipListRT) Query(interval rangetree.Interval) rangetree.Entries {\n\tentries := make(rangetree.Entries, 0, 100)\n\trt.apply(rt.top, 0, interval, func(e rangetree.Entry) bool {\n\t\tentries = append(entries, e)\n\t\treturn true\n\t})\n\n\treturn entries\n}\n\nfunc (rt *skipListRT) flatten(sl *skip.SkipList, dimension uint64, entries *rangetree.Entries) {\n\tlastDimension := isLastDimension(dimension, rt.dimensions)\n\tfor iter := sl.Iter(skipEntry(0)); iter.Next(); {\n\t\tif lastDimension {\n\t\t\t*entries = append(*entries, iter.Value().(*lastBundle).entry)\n\t\t} else {\n\t\t\trt.flatten(iter.Value().(*dimensionalBundle).sl, dimension+1, entries)\n\t\t}\n\t}\n}\n\nfunc (rt *skipListRT) insert(sl *skip.SkipList, dimension, insertDimension uint64,\n\tindex, number int64, deleted, affected *rangetree.Entries) {\n\n\tvar e common.Comparator\n\tlastDimension := isLastDimension(dimension, rt.dimensions)\n\taffectedDimension := dimension == insertDimension\n\tvar iter skip.Iterator\n\tif dimension == insertDimension {\n\t\titer = sl.Iter(skipEntry(index))\n\t} else {\n\t\titer = sl.Iter(skipEntry(0))\n\t}\n\n\tvar toDelete common.Comparators\n\tif number < 0 {\n\t\ttoDelete = make(common.Comparators, 0, 100)\n\t}\n\n\tfor iter.Next() {\n\t\te = iter.Value()\n\t\tif !affectedDimension {\n\t\t\trt.insert(e.(*dimensionalBundle).sl, dimension+1,\n\t\t\t\tinsertDimension, index, number, deleted, affected,\n\t\t\t)\n\t\t\tcontinue\n\t\t}\n\t\tif needsDeletion(int64(e.(keyed).key()), index, number) {\n\t\t\ttoDelete = append(toDelete, e)\n\t\t\tcontinue\n\t\t}\n\n\t\tif lastDimension {\n\t\t\te.(*lastBundle).id += uint64(number)\n\t\t\t*affected = append(*affected, e.(*lastBundle).entry)\n\t\t} else {\n\t\t\te.(*dimensionalBundle).id += uint64(number)\n\t\t\trt.flatten(e.(*dimensionalBundle).sl, dimension+1, affected)\n\t\t}\n\t}\n\n\tif len(toDelete) > 0 {\n\t\tfor _, e := range toDelete {\n\t\t\tif lastDimension {\n\t\t\t\t*deleted = append(*deleted, e.(*lastBundle).entry)\n\t\t\t} else {\n\t\t\t\trt.flatten(e.(*dimensionalBundle).sl, dimension+1, deleted)\n\t\t\t}\n\t\t}\n\n\t\tsl.Delete(toDelete...)\n\t}\n}\n\n// InsertAtDimension will increment items at and above the given index\n// by the number provided. Provide a negative number to to decrement.\n// Returned are two lists. The first list is a list of entries that\n// were moved. The second is a list entries that were deleted. These\n// lists are exclusive.\nfunc (rt *skipListRT) InsertAtDimension(dimension uint64,\n\tindex, number int64) (rangetree.Entries, rangetree.Entries) {\n\n\tif dimension >= rt.dimensions || number == 0 {\n\t\treturn rangetree.Entries{}, rangetree.Entries{}\n\t}\n\n\taffected := make(rangetree.Entries, 0, 100)\n\tvar deleted rangetree.Entries\n\tif number < 0 {\n\t\tdeleted = make(rangetree.Entries, 0, 100)\n\t}\n\n\trt.insert(rt.top, 0, dimension, index, number, &deleted, &affected)\n\trt.number -= uint64(len(deleted))\n\treturn affected, deleted\n}\n\nfunc new(dimensions uint64) *skipListRT {\n\tsl := &skipListRT{}\n\tsl.init(dimensions)\n\treturn sl\n}\n\n// New will allocate, initialize, and return a new rangetree.RangeTree\n// with the provided number of dimensions.\nfunc New(dimensions uint64) rangetree.RangeTree {\n\treturn new(dimensions)\n}\n" }, { "path": "rangetree/skiplist/skiplist_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage skiplist\n\nimport (\n\t\"math\"\n\t\"math/rand\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/Workiva/go-datastructures/rangetree\"\n)\n\nfunc generateMultiDimensionalEntries(num int) rangetree.Entries {\n\tentries := make(rangetree.Entries, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tentries = append(entries, newMockEntry(int64(i), int64(i)))\n\t}\n\n\treturn entries\n}\n\nfunc generateRandomMultiDimensionalEntries(num int) rangetree.Entries {\n\tentries := make(rangetree.Entries, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tvalue := rand.Int63()\n\t\tentries = append(entries, newMockEntry(value, value))\n\t}\n\n\treturn entries\n}\n\nfunc TestRTSingleDimensionAdd(t *testing.T) {\n\trt := new(1)\n\tm1 := newMockEntry(3)\n\tm2 := newMockEntry(5)\n\n\toverwritten := rt.Add(m1, m2)\n\tassert.Equal(t, rangetree.Entries{nil, nil}, overwritten)\n\tassert.Equal(t, uint64(2), rt.Len())\n\tassert.Equal(t, rangetree.Entries{m1, m2}, rt.Get(m1, m2))\n}\n\nfunc TestRTMultiDimensionAdd(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 5)\n\tm2 := newMockEntry(4, 6)\n\n\toverwritten := rt.Add(m1, m2)\n\tassert.Equal(t, rangetree.Entries{nil, nil}, overwritten)\n\tassert.Equal(t, uint64(2), rt.Len())\n\tassert.Equal(t, rangetree.Entries{m1, m2}, rt.Get(m1, m2))\n}\n\nfunc TestRTSingleDimensionOverwrite(t *testing.T) {\n\trt := new(1)\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(5)\n\n\toverwritten := rt.Add(m1)\n\tassert.Equal(t, rangetree.Entries{nil}, overwritten)\n\tassert.Equal(t, uint64(1), rt.Len())\n\n\toverwritten = rt.Add(m2)\n\tassert.Equal(t, rangetree.Entries{m1}, overwritten)\n\tassert.Equal(t, uint64(1), rt.Len())\n\tassert.Equal(t, rangetree.Entries{m2}, rt.Get(m2))\n}\n\nfunc TestRTMultiDimensionOverwrite(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(5, 6)\n\tm2 := newMockEntry(5, 6)\n\n\toverwritten := rt.Add(m1)\n\tassert.Equal(t, rangetree.Entries{nil}, overwritten)\n\tassert.Equal(t, uint64(1), rt.Len())\n\n\toverwritten = rt.Add(m2)\n\tassert.Equal(t, rangetree.Entries{m1}, overwritten)\n\tassert.Equal(t, uint64(1), rt.Len())\n\tassert.Equal(t, rangetree.Entries{m2}, rt.Get(m2))\n}\n\nfunc TestRTSingleDimensionDelete(t *testing.T) {\n\trt := new(1)\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(2)\n\trt.Add(m1, m2)\n\n\tdeleted := rt.Delete(m1, newMockEntry(10), m2)\n\tassert.Equal(t, rangetree.Entries{m1, nil, m2}, deleted)\n\tassert.Equal(t, uint64(0), rt.Len())\n\tassert.Equal(t, rangetree.Entries{nil, nil}, rt.Get(m1, m2))\n}\n\nfunc TestRTMultiDimensionDelete(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 5)\n\tm2 := newMockEntry(4, 6)\n\trt.Add(m1, m2)\n\n\tdeleted := rt.Delete(m1, newMockEntry(10, 10), m2)\n\tassert.Equal(t, rangetree.Entries{m1, nil, m2}, deleted)\n\tassert.Equal(t, uint64(0), rt.Len())\n\tassert.Equal(t, rangetree.Entries{nil, nil}, rt.Get(m1, m2))\n}\n\nfunc TestRTSingleDimensionQuery(t *testing.T) {\n\trt := new(1)\n\tm1 := newMockEntry(3)\n\tm2 := newMockEntry(6)\n\tm3 := newMockEntry(9)\n\trt.Add(m1, m2, m3)\n\n\tresult := rt.Query(newMockInterval([]int64{1}, []int64{7}))\n\tassert.Equal(t, rangetree.Entries{m1, m2}, result)\n\n\tresult = rt.Query(newMockInterval([]int64{6}, []int64{10}))\n\tassert.Equal(t, rangetree.Entries{m2, m3}, result)\n\n\tresult = rt.Query(newMockInterval([]int64{9}, []int64{11}))\n\tassert.Equal(t, rangetree.Entries{m3}, result)\n\n\tresult = rt.Query(newMockInterval([]int64{0}, []int64{3}))\n\tassert.Len(t, result, 0)\n\n\tresult = rt.Query(newMockInterval([]int64{10}, []int64{13}))\n\tassert.Len(t, result, 0)\n}\n\nfunc TestRTMultiDimensionQuery(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 3)\n\tm2 := newMockEntry(6, 6)\n\tm3 := newMockEntry(9, 9)\n\trt.Add(m1, m2, m3)\n\n\tresult := rt.Query(newMockInterval([]int64{1, 1}, []int64{7, 7}))\n\tassert.Equal(t, rangetree.Entries{m1, m2}, result)\n\n\tresult = rt.Query(newMockInterval([]int64{6, 6}, []int64{10, 10}))\n\tassert.Equal(t, rangetree.Entries{m2, m3}, result)\n\n\tresult = rt.Query(newMockInterval([]int64{9, 9}, []int64{11, 11}))\n\tassert.Equal(t, rangetree.Entries{m3}, result)\n\n\tresult = rt.Query(newMockInterval([]int64{0, 0}, []int64{3, 3}))\n\tassert.Len(t, result, 0)\n\n\tresult = rt.Query(newMockInterval([]int64{10, 10}, []int64{13, 13}))\n\tassert.Len(t, result, 0)\n\n\tresult = rt.Query(newMockInterval([]int64{0, 0}, []int64{3, 3}))\n\tassert.Len(t, result, 0)\n\n\tresult = rt.Query(newMockInterval([]int64{6, 1}, []int64{7, 6}))\n\tassert.Len(t, result, 0)\n\n\tresult = rt.Query(newMockInterval([]int64{0, 0}, []int64{7, 4}))\n\tassert.Equal(t, rangetree.Entries{m1}, result)\n}\n\nfunc TestRTSingleDimensionInsert(t *testing.T) {\n\trt := new(1)\n\tm1 := newMockEntry(3)\n\tm2 := newMockEntry(6)\n\tm3 := newMockEntry(9)\n\trt.Add(m1, m2, m3)\n\n\taffected, deleted := rt.InsertAtDimension(0, 0, 1)\n\tassert.Equal(t, rangetree.Entries{m1, m2, m3}, affected)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, uint64(3), rt.Len())\n\tassert.Equal(t, rangetree.Entries{nil, nil, nil}, rt.Get(m1, m2, m3))\n\te1 := newMockEntry(4)\n\te2 := newMockEntry(7)\n\te3 := newMockEntry(10)\n\tassert.Equal(t, rangetree.Entries{m1, m2, m3}, rt.Get(e1, e2, e3))\n}\n\nfunc TestRTSingleDimensionInsertNegative(t *testing.T) {\n\trt := new(1)\n\tm1 := newMockEntry(3)\n\tm2 := newMockEntry(6)\n\tm3 := newMockEntry(9)\n\trt.Add(m1, m2, m3)\n\n\taffected, deleted := rt.InsertAtDimension(0, 6, -2)\n\tassert.Equal(t, rangetree.Entries{m3}, affected)\n\tassert.Equal(t, rangetree.Entries{m2}, deleted)\n\tassert.Equal(t, uint64(2), rt.Len())\n\tassert.Equal(t, rangetree.Entries{m1, nil}, rt.Get(m1, m2))\n\n\te2 := newMockEntry(4)\n\te3 := newMockEntry(7)\n\tassert.Equal(t, rangetree.Entries{nil, m3}, rt.Get(e2, e3))\n}\n\nfunc TestRTMultiDimensionInsert(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 3)\n\tm2 := newMockEntry(6, 6)\n\tm3 := newMockEntry(9, 9)\n\trt.Add(m1, m2, m3)\n\n\taffected, deleted := rt.InsertAtDimension(1, 4, 2)\n\tassert.Equal(t, rangetree.Entries{m2, m3}, affected)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, uint64(3), rt.Len())\n\n\te2 := newMockEntry(6, 8)\n\te3 := newMockEntry(9, 11)\n\tassert.Equal(t, rangetree.Entries{m1, nil, nil}, rt.Get(m1, m2, m3))\n\tassert.Equal(t, rangetree.Entries{m2, m3}, rt.Get(e2, e3))\n}\n\nfunc TestRTMultiDimensionInsertNegative(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 3)\n\tm2 := newMockEntry(6, 6)\n\tm3 := newMockEntry(9, 9)\n\trt.Add(m1, m2, m3)\n\n\taffected, deleted := rt.InsertAtDimension(1, 6, -2)\n\tassert.Equal(t, rangetree.Entries{m3}, affected)\n\tassert.Equal(t, rangetree.Entries{m2}, deleted)\n\tassert.Equal(t, uint64(2), rt.Len())\n\tassert.Equal(t, rangetree.Entries{m1, nil, nil}, rt.Get(m1, m2, m3))\n\n\te2 := newMockEntry(6, 4)\n\te3 := newMockEntry(9, 7)\n\tassert.Equal(t, rangetree.Entries{nil, m3}, rt.Get(e2, e3))\n}\n\nfunc TestRTInsertInZeroDimensionMultiDimensionList(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 3)\n\tm2 := newMockEntry(6, 6)\n\tm3 := newMockEntry(9, 9)\n\trt.Add(m1, m2, m3)\n\n\taffected, deleted := rt.InsertAtDimension(0, 4, 2)\n\tassert.Equal(t, rangetree.Entries{m2, m3}, affected)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, uint64(3), rt.Len())\n\tassert.Equal(t, rangetree.Entries{m1, nil, nil}, rt.Get(m1, m2, m3))\n\n\te2 := newMockEntry(8, 6)\n\te3 := newMockEntry(11, 9)\n\tassert.Equal(t, rangetree.Entries{m2, m3}, rt.Get(e2, e3))\n}\n\nfunc TestRTInsertNegativeInZeroDimensionMultiDimensionList(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 3)\n\tm2 := newMockEntry(6, 6)\n\tm3 := newMockEntry(9, 9)\n\trt.Add(m1, m2, m3)\n\n\taffected, deleted := rt.InsertAtDimension(0, 6, -2)\n\tassert.Equal(t, rangetree.Entries{m3}, affected)\n\tassert.Equal(t, rangetree.Entries{m2}, deleted)\n\tassert.Equal(t, uint64(2), rt.Len())\n\tassert.Equal(t, rangetree.Entries{m1, nil, nil}, rt.Get(m1, m2, m3))\n\n\te2 := newMockEntry(4, 6)\n\te3 := newMockEntry(7, 9)\n\tassert.Equal(t, rangetree.Entries{nil, m3}, rt.Get(e2, e3))\n}\n\nfunc TestRTInsertBeyondDimension(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 3)\n\trt.Add(m1)\n\n\taffected, deleted := rt.InsertAtDimension(4, 0, 1)\n\tassert.Len(t, affected, 0)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, rangetree.Entries{m1}, rt.Get(m1))\n}\n\nfunc TestRTInsertZero(t *testing.T) {\n\trt := new(2)\n\tm1 := newMockEntry(3, 3)\n\trt.Add(m1)\n\n\taffected, deleted := rt.InsertAtDimension(1, 0, 0)\n\tassert.Len(t, affected, 0)\n\tassert.Len(t, deleted, 0)\n\tassert.Equal(t, rangetree.Entries{m1}, rt.Get(m1))\n}\n\nfunc BenchmarkMultiDimensionInsert(b *testing.B) {\n\tnumItems := b.N\n\trt := new(2)\n\tentries := generateMultiDimensionalEntries(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trt.Add(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkMultiDimensionInsertReverse(b *testing.B) {\n\tnumItems := b.N\n\trt := new(2)\n\tentries := generateMultiDimensionalEntries(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tindex := numItems - (i % numItems) - 1\n\t\trt.Add(entries[index])\n\t}\n}\n\nfunc BenchmarkMultiDimensionRandomInsert(b *testing.B) {\n\tnumItems := b.N\n\trt := new(2)\n\tentries := generateRandomMultiDimensionalEntries(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trt.Add(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkMultiDimensionalGet(b *testing.B) {\n\tnumItems := b.N\n\trt := new(2)\n\tentries := generateRandomMultiDimensionalEntries(numItems)\n\trt.Add(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trt.Get(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkMultiDimensionDelete(b *testing.B) {\n\tnumItems := b.N\n\trt := new(2)\n\tentries := generateRandomMultiDimensionalEntries(numItems)\n\trt.Add(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trt.Delete(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkMultiDimensionQuery(b *testing.B) {\n\tnumItems := b.N\n\trt := new(2)\n\tentries := generateRandomMultiDimensionalEntries(numItems)\n\trt.Add(entries...)\n\tiv := newMockInterval([]int64{0, 0}, []int64{math.MaxInt64, math.MaxInt64})\n\tvar result rangetree.Entries\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tresult = rt.Query(iv)\n\t}\n\n\tassert.Len(b, result, numItems)\n}\n\nfunc BenchmarkMultiDimensionInsertAtZeroDimension(b *testing.B) {\n\tnumItems := b.N\n\trt := new(2)\n\tentries := generateRandomMultiDimensionalEntries(numItems)\n\trt.Add(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trt.InsertAtDimension(0, 0, 1)\n\t}\n}\n\nfunc BenchmarkMultiDimensionInsertNegativeAtZeroDimension(b *testing.B) {\n\tnumItems := b.N\n\trt := new(2)\n\tentries := generateRandomMultiDimensionalEntries(numItems)\n\trt.Add(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\trt.InsertAtDimension(0, 0, -1)\n\t}\n}\n" }, { "path": "rtree/hilbert/action.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage hilbert\n\nimport (\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/Workiva/go-datastructures/rtree\"\n)\n\ntype actions []action\n\ntype action interface {\n\toperation() operation\n\tkeys() hilberts\n\trects() []*hilbertBundle\n\tcomplete()\n\taddNode(int64, *node)\n\tnodes() []*node\n}\n\ntype getAction struct {\n\tresult rtree.Rectangles\n\tcompleter *sync.WaitGroup\n\tlookup *rectangle\n}\n\nfunc (ga *getAction) complete() {\n\tga.completer.Done()\n}\n\nfunc (ga *getAction) operation() operation {\n\treturn get\n}\n\nfunc (ga *getAction) keys() hilberts {\n\treturn nil\n}\n\nfunc (ga *getAction) addNode(i int64, n *node) {\n\treturn // not necessary for gets\n}\n\nfunc (ga *getAction) nodes() []*node {\n\treturn nil\n}\n\nfunc (ga *getAction) rects() []*hilbertBundle {\n\treturn []*hilbertBundle{&hilbertBundle{}}\n}\n\nfunc newGetAction(rect rtree.Rectangle) *getAction {\n\tr := newRectangeFromRect(rect)\n\tga := &getAction{\n\t\tcompleter: new(sync.WaitGroup),\n\t\tlookup: r,\n\t}\n\tga.completer.Add(1)\n\treturn ga\n}\n\ntype insertAction struct {\n\trs []*hilbertBundle\n\tcompleter *sync.WaitGroup\n\tns []*node\n}\n\nfunc (ia *insertAction) complete() {\n\tia.completer.Done()\n}\n\nfunc (ia *insertAction) operation() operation {\n\treturn add\n}\n\nfunc (ia *insertAction) keys() hilberts {\n\treturn nil\n}\n\nfunc (ia *insertAction) addNode(i int64, n *node) {\n\tia.ns[i] = n\n}\n\nfunc (ia *insertAction) nodes() []*node {\n\treturn ia.ns\n}\n\nfunc (ia *insertAction) rects() []*hilbertBundle {\n\treturn ia.rs\n}\n\nfunc newInsertAction(rects rtree.Rectangles) *insertAction {\n\tia := &insertAction{\n\t\trs: bundlesFromRects(rects...),\n\t\tcompleter: new(sync.WaitGroup),\n\t\tns: make([]*node, len(rects)),\n\t}\n\tia.completer.Add(1)\n\treturn ia\n}\n\ntype removeAction struct {\n\t*insertAction\n}\n\nfunc (ra *removeAction) operation() operation {\n\treturn remove\n}\n\nfunc newRemoveAction(rects rtree.Rectangles) *removeAction {\n\treturn &removeAction{\n\t\tnewInsertAction(rects),\n\t}\n}\n\nfunc minUint64(choices ...uint64) uint64 {\n\tmin := choices[0]\n\tfor i := 1; i < len(choices); i++ {\n\t\tif choices[i] < min {\n\t\t\tmin = choices[i]\n\t\t}\n\t}\n\n\treturn min\n}\n\ntype interfaces []interface{}\n\nfunc executeInterfacesInParallel(ifs interfaces, fn func(interface{})) {\n\tif len(ifs) == 0 {\n\t\treturn\n\t}\n\n\tdone := int64(-1)\n\tnumCPU := uint64(runtime.NumCPU())\n\tif numCPU > 1 {\n\t\tnumCPU--\n\t}\n\n\tnumCPU = minUint64(numCPU, uint64(len(ifs)))\n\n\tvar wg sync.WaitGroup\n\twg.Add(int(numCPU))\n\n\tfor i := uint64(0); i < numCPU; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\n\t\t\tfor {\n\t\t\t\ti := atomic.AddInt64(&done, 1)\n\t\t\t\tif i >= int64(len(ifs)) {\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tfn(ifs[i])\n\t\t\t}\n\t\t}()\n\t}\n\n\twg.Wait()\n}\n\nfunc executeInterfacesInSerial(ifs interfaces, fn func(interface{})) {\n\tif len(ifs) == 0 {\n\t\treturn\n\t}\n\n\tfor _, ifc := range ifs {\n\t\tfn(ifc)\n\t}\n}\n" }, { "path": "rtree/hilbert/hilbert.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage hilbert\n\nimport (\n\t\"runtime\"\n\t\"sync\"\n\n\th \"github.com/Workiva/go-datastructures/numerics/hilbert\"\n\t\"github.com/Workiva/go-datastructures/rtree\"\n)\n\nfunc getCenter(rect rtree.Rectangle) (int32, int32) {\n\txlow, ylow := rect.LowerLeft()\n\txhigh, yhigh := rect.UpperRight()\n\n\treturn (xhigh + xlow) / 2, (yhigh + ylow) / 2\n}\n\ntype hilbertBundle struct {\n\thilbert hilbert\n\trect rtree.Rectangle\n}\n\nfunc bundlesFromRects(rects ...rtree.Rectangle) []*hilbertBundle {\n\tchunks := chunkRectangles(rects, int64(runtime.NumCPU()))\n\tbundleChunks := make([][]*hilbertBundle, len(chunks))\n\tvar wg sync.WaitGroup\n\twg.Add(len(chunks))\n\n\tfor i := 0; i < runtime.NumCPU(); i++ {\n\t\tif len(chunks[i]) == 0 {\n\t\t\tbundleChunks[i] = []*hilbertBundle{}\n\t\t\twg.Done()\n\t\t\tcontinue\n\t\t}\n\t\tgo func(i int) {\n\t\t\tbundles := make([]*hilbertBundle, 0, len(chunks[i]))\n\t\t\tfor _, r := range chunks[i] {\n\t\t\t\th := h.Encode(getCenter(r))\n\t\t\t\tbundles = append(bundles, &hilbertBundle{hilbert(h), r})\n\t\t\t}\n\t\t\tbundleChunks[i] = bundles\n\t\t\twg.Done()\n\t\t}(i)\n\t}\n\n\twg.Wait()\n\n\tbundles := make([]*hilbertBundle, 0, len(rects))\n\tfor _, bc := range bundleChunks {\n\t\tbundles = append(bundles, bc...)\n\t}\n\n\treturn bundles\n}\n\n// chunkRectangles takes a slice of rtree.Rectangle values and chunks it into `numParts` subslices.\nfunc chunkRectangles(slice rtree.Rectangles, numParts int64) []rtree.Rectangles {\n\tparts := make([]rtree.Rectangles, numParts)\n\tfor i := int64(0); i < numParts; i++ {\n\t\tparts[i] = slice[i*int64(len(slice))/numParts : (i+1)*int64(len(slice))/numParts]\n\t}\n\treturn parts\n}\n" }, { "path": "rtree/hilbert/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage hilbert\n\ntype mockRectangle struct {\n\txlow, ylow, xhigh, yhigh int32\n}\n\nfunc (mr *mockRectangle) LowerLeft() (int32, int32) {\n\treturn mr.xlow, mr.ylow\n}\n\nfunc (mr *mockRectangle) UpperRight() (int32, int32) {\n\treturn mr.xhigh, mr.yhigh\n}\n\nfunc newMockRectangle(xlow, ylow, xhigh, yhigh int32) *mockRectangle {\n\treturn &mockRectangle{\n\t\txlow: xlow,\n\t\tylow: ylow,\n\t\txhigh: xhigh,\n\t\tyhigh: yhigh,\n\t}\n}\n" }, { "path": "rtree/hilbert/node.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage hilbert\n\nimport (\n\t\"sort\"\n\n\t\"github.com/Workiva/go-datastructures/rtree\"\n)\n\ntype hilbert int64\n\ntype hilberts []hilbert\n\nfunc getParent(parent *node, key hilbert, r1 rtree.Rectangle) *node {\n\tvar n *node\n\tfor parent != nil && !parent.isLeaf {\n\t\tn = parent.searchNode(key)\n\t\tparent = n\n\t}\n\n\tif parent != nil && r1 != nil { // must be leaf and we need exact match\n\t\t// we are safe to travel to the right\n\t\ti := parent.search(key)\n\t\tfor parent.keys.byPosition(i) == key {\n\t\t\tif equal(parent.nodes.list[i], r1) {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\ti++\n\t\t\tif i == parent.keys.len() {\n\t\t\t\tif parent.right == nil { // we are far to the right\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\tif parent.right.keys.byPosition(0) != key {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\tparent = parent.right\n\t\t\t\ti = 0\n\t\t\t}\n\t\t}\n\t}\n\n\treturn parent\n}\n\ntype nodes struct {\n\tlist rtree.Rectangles\n}\n\nfunc (ns *nodes) push(n rtree.Rectangle) {\n\tns.list = append(ns.list, n)\n}\n\nfunc (ns *nodes) splitAt(i, capacity uint64) (*nodes, *nodes) {\n\ti++\n\tright := make(rtree.Rectangles, uint64(len(ns.list))-i, capacity)\n\tcopy(right, ns.list[i:])\n\tfor j := i; j < uint64(len(ns.list)); j++ {\n\t\tns.list[j] = nil\n\t}\n\tns.list = ns.list[:i]\n\treturn ns, &nodes{list: right}\n}\n\nfunc (ns *nodes) byPosition(pos uint64) *node {\n\tif pos >= uint64(len(ns.list)) {\n\t\treturn nil\n\t}\n\n\treturn ns.list[pos].(*node)\n}\n\nfunc (ns *nodes) insertAt(i uint64, n rtree.Rectangle) {\n\tns.list = append(ns.list, nil)\n\tcopy(ns.list[i+1:], ns.list[i:])\n\tns.list[i] = n\n}\n\nfunc (ns *nodes) replaceAt(i uint64, n rtree.Rectangle) {\n\tns.list[i] = n\n}\n\nfunc (ns *nodes) len() uint64 {\n\treturn uint64(len(ns.list))\n}\n\nfunc (ns *nodes) deleteAt(i uint64) {\n\tcopy(ns.list[i:], ns.list[i+1:])\n\tns.list = ns.list[:len(ns.list)-1]\n}\n\nfunc newNodes(size uint64) *nodes {\n\treturn &nodes{\n\t\tlist: make(rtree.Rectangles, 0, size),\n\t}\n}\n\ntype keys struct {\n\tlist hilberts\n}\n\nfunc (ks *keys) splitAt(i, capacity uint64) (*keys, *keys) {\n\ti++\n\tright := make(hilberts, uint64(len(ks.list))-i, capacity)\n\tcopy(right, ks.list[i:])\n\tks.list = ks.list[:i]\n\treturn ks, &keys{list: right}\n}\n\nfunc (ks *keys) len() uint64 {\n\treturn uint64(len(ks.list))\n}\n\nfunc (ks *keys) byPosition(i uint64) hilbert {\n\tif i >= uint64(len(ks.list)) {\n\t\treturn -1\n\t}\n\treturn ks.list[i]\n}\n\nfunc (ks *keys) deleteAt(i uint64) {\n\tcopy(ks.list[i:], ks.list[i+1:])\n\tks.list = ks.list[:len(ks.list)-1]\n}\n\nfunc (ks *keys) delete(k hilbert) hilbert {\n\ti := ks.search(k)\n\tif i >= uint64(len(ks.list)) {\n\t\treturn -1\n\t}\n\n\tif ks.list[i] != k {\n\t\treturn -1\n\t}\n\told := ks.list[i]\n\tks.deleteAt(i)\n\treturn old\n}\n\nfunc (ks *keys) search(key hilbert) uint64 {\n\ti := sort.Search(len(ks.list), func(i int) bool {\n\t\treturn ks.list[i] >= key\n\t})\n\n\treturn uint64(i)\n}\n\nfunc (ks *keys) insert(key hilbert) (hilbert, uint64) {\n\ti := ks.search(key)\n\tif i == uint64(len(ks.list)) {\n\t\tks.list = append(ks.list, key)\n\t\treturn -1, i\n\t}\n\n\tvar old hilbert\n\tif ks.list[i] == key {\n\t\told = ks.list[i]\n\t\tks.list[i] = key\n\t} else {\n\t\tks.insertAt(i, key)\n\t}\n\n\treturn old, i\n}\n\nfunc (ks *keys) last() hilbert {\n\treturn ks.list[len(ks.list)-1]\n}\n\nfunc (ks *keys) insertAt(i uint64, k hilbert) {\n\tks.list = append(ks.list, -1)\n\tcopy(ks.list[i+1:], ks.list[i:])\n\tks.list[i] = k\n}\n\nfunc (ks *keys) withPosition(k hilbert) (hilbert, uint64) {\n\ti := ks.search(k)\n\tif i == uint64(len(ks.list)) {\n\t\treturn -1, i\n\t}\n\tif ks.list[i] == k {\n\t\treturn ks.list[i], i\n\t}\n\n\treturn -1, i\n}\n\nfunc newKeys(size uint64) *keys {\n\treturn &keys{\n\t\tlist: make(hilberts, 0, size),\n\t}\n}\n\ntype node struct {\n\tkeys *keys\n\tnodes *nodes\n\tisLeaf bool\n\tparent, right *node\n\tmbr *rectangle\n\tmaxHilbert hilbert\n}\n\nfunc (n *node) insert(kb *keyBundle) rtree.Rectangle {\n\ti := n.keys.search(kb.key)\n\tif n.isLeaf { // we can have multiple keys with the same hilbert number\n\t\tfor i < n.keys.len() && n.keys.list[i] == kb.key {\n\t\t\tif equal(n.nodes.list[i], kb.left) {\n\t\t\t\told := n.nodes.list[i]\n\t\t\t\tn.nodes.list[i] = kb.left\n\t\t\t\treturn old\n\t\t\t}\n\t\t\ti++\n\t\t}\n\t}\n\n\tif i == n.keys.len() {\n\t\tn.maxHilbert = kb.key\n\t}\n\n\tn.keys.insertAt(i, kb.key)\n\tif n.isLeaf {\n\t\tn.nodes.insertAt(i, kb.left)\n\t} else {\n\t\tif n.nodes.len() == 0 {\n\t\t\tn.nodes.push(kb.left)\n\t\t\tn.nodes.push(kb.right)\n\t\t} else {\n\t\t\tn.nodes.replaceAt(i, kb.left)\n\t\t\tn.nodes.insertAt(i+1, kb.right)\n\t\t}\n\t\tn.mbr.adjust(kb.left)\n\t\tn.mbr.adjust(kb.right)\n\t\tif kb.right.(*node).maxHilbert > n.maxHilbert {\n\t\t\tn.maxHilbert = kb.right.(*node).maxHilbert\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (n *node) delete(kb *keyBundle) rtree.Rectangle {\n\ti := n.keys.search(kb.key)\n\tif n.keys.byPosition(i) != kb.key { // hilbert value not found\n\t\treturn nil\n\t}\n\n\tif !equal(n.nodes.list[i], kb.left) {\n\t\treturn nil\n\t}\n\n\told := n.nodes.list[i]\n\tn.keys.deleteAt(i)\n\tn.nodes.deleteAt(i)\n\treturn old\n}\n\nfunc (n *node) LowerLeft() (int32, int32) {\n\treturn n.mbr.xlow, n.mbr.ylow\n}\n\nfunc (n *node) UpperRight() (int32, int32) {\n\treturn n.mbr.xhigh, n.mbr.yhigh\n}\n\nfunc (n *node) needsSplit(ary uint64) bool {\n\treturn n.keys.len() >= ary\n}\n\nfunc (n *node) splitLeaf(i, capacity uint64) (hilbert, *node, *node) {\n\tkey := n.keys.byPosition(i)\n\t_, rightKeys := n.keys.splitAt(i, capacity)\n\t_, rightNodes := n.nodes.splitAt(i, capacity)\n\tnn := &node{\n\t\tkeys: rightKeys,\n\t\tnodes: rightNodes,\n\t\tisLeaf: true,\n\t\tright: n.right,\n\t\tparent: n.parent,\n\t}\n\tn.right = nn\n\tnn.mbr = newRectangleFromRects(rightNodes.list)\n\tn.mbr = newRectangleFromRects(n.nodes.list)\n\tnn.maxHilbert = rightKeys.last()\n\tn.maxHilbert = n.keys.last()\n\treturn key, n, nn\n}\n\nfunc (n *node) splitInternal(i, capacity uint64) (hilbert, *node, *node) {\n\tkey := n.keys.byPosition(i)\n\tn.keys.delete(key)\n\n\t_, rightKeys := n.keys.splitAt(i-1, capacity)\n\t_, rightNodes := n.nodes.splitAt(i, capacity)\n\n\tnn := newNode(false, rightKeys, rightNodes)\n\tfor _, n := range rightNodes.list {\n\t\tn.(*node).parent = nn\n\t}\n\tnn.mbr = newRectangleFromRects(rightNodes.list)\n\tn.mbr = newRectangleFromRects(n.nodes.list)\n\tnn.maxHilbert = nn.keys.last()\n\tn.maxHilbert = n.keys.last()\n\n\treturn key, n, nn\n}\n\nfunc (n *node) split(i, capacity uint64) (hilbert, *node, *node) {\n\tif n.isLeaf {\n\t\treturn n.splitLeaf(i, capacity)\n\t}\n\n\treturn n.splitInternal(i, capacity)\n}\n\nfunc (n *node) search(key hilbert) uint64 {\n\treturn n.keys.search(key)\n}\n\nfunc (n *node) searchNode(key hilbert) *node {\n\ti := n.search(key)\n\n\treturn n.nodes.byPosition(uint64(i))\n}\n\nfunc (n *node) searchRects(r *rectangle) rtree.Rectangles {\n\trects := make(rtree.Rectangles, 0, n.nodes.len())\n\tfor _, child := range n.nodes.list {\n\t\tif intersect(r, child) {\n\t\t\trects = append(rects, child)\n\t\t}\n\t}\n\n\treturn rects\n}\n\nfunc (n *node) key() hilbert {\n\treturn n.keys.last()\n}\n\nfunc newNode(isLeaf bool, keys *keys, ns *nodes) *node {\n\treturn &node{\n\t\tisLeaf: isLeaf,\n\t\tkeys: keys,\n\t\tnodes: ns,\n\t}\n}\n" }, { "path": "rtree/hilbert/rectangle.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage hilbert\n\nimport \"github.com/Workiva/go-datastructures/rtree\"\n\ntype rectangle struct {\n\txlow, xhigh, ylow, yhigh int32\n}\n\nfunc (r *rectangle) adjust(rect rtree.Rectangle) {\n\tx, y := rect.LowerLeft()\n\tif x < r.xlow {\n\t\tr.xlow = x\n\t}\n\tif y < r.ylow {\n\t\tr.ylow = y\n\t}\n\n\tx, y = rect.UpperRight()\n\tif x > r.xhigh {\n\t\tr.xhigh = x\n\t}\n\n\tif y > r.yhigh {\n\t\tr.yhigh = y\n\t}\n}\n\nfunc equal(r1, r2 rtree.Rectangle) bool {\n\txlow1, ylow1 := r1.LowerLeft()\n\txhigh2, yhigh2 := r2.UpperRight()\n\n\txhigh1, yhigh1 := r1.UpperRight()\n\txlow2, ylow2 := r2.LowerLeft()\n\n\treturn xlow1 == xlow2 && xhigh1 == xhigh2 && ylow1 == ylow2 && yhigh1 == yhigh2\n}\n\nfunc intersect(rect1 *rectangle, rect2 rtree.Rectangle) bool {\n\txhigh2, yhigh2 := rect2.UpperRight()\n\txlow2, ylow2 := rect2.LowerLeft()\n\n\treturn xhigh2 >= rect1.xlow && xlow2 <= rect1.xhigh && yhigh2 >= rect1.ylow && ylow2 <= rect1.yhigh\n}\n\nfunc newRectangeFromRect(rect rtree.Rectangle) *rectangle {\n\tr := &rectangle{}\n\tx, y := rect.LowerLeft()\n\tr.xlow = x\n\tr.ylow = y\n\n\tx, y = rect.UpperRight()\n\tr.xhigh = x\n\tr.yhigh = y\n\n\treturn r\n}\n\nfunc newRectangleFromRects(rects rtree.Rectangles) *rectangle {\n\tif len(rects) == 0 {\n\t\tpanic(`Cannot construct rectangle with no dimensions.`)\n\t}\n\n\txlow, ylow := rects[0].LowerLeft()\n\txhigh, yhigh := rects[0].UpperRight()\n\tr := &rectangle{\n\t\txlow: xlow,\n\t\txhigh: xhigh,\n\t\tylow: ylow,\n\t\tyhigh: yhigh,\n\t}\n\n\tfor i := 1; i < len(rects); i++ {\n\t\tr.adjust(rects[i])\n\t}\n\n\treturn r\n}\n" }, { "path": "rtree/hilbert/tree.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage hilbert implements a Hilbert R-tree based on PALM principles\nto improve multithreaded performance. This package is not quite complete\nand some optimization and delete codes remain to be completed.\n\nThis serves as a potential replacement for the interval tree and\nrangetree.\n\nBenchmarks:\nBenchmarkBulkAddPoints-8\t 500\t 2589270 ns/op\nBenchmarkBulkUpdatePoints-8\t 2000\t 1212641 ns/op\nBenchmarkPointInsertion-8\t 200000\t 9135 ns/op\nBenchmarkQueryPoints-8\t \t 500000\t 3122 ns/op\n\n*/\npackage hilbert\n\nimport (\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/Workiva/go-datastructures/queue\"\n\t\"github.com/Workiva/go-datastructures/rtree\"\n)\n\ntype operation int\n\nconst (\n\tget operation = iota\n\tadd\n\tremove\n)\n\nconst multiThreadAt = 1000 // number of keys before we multithread lookups\n\ntype keyBundle struct {\n\tkey hilbert\n\tleft, right rtree.Rectangle\n}\n\ntype tree struct {\n\troot *node\n\t_ [8]uint64\n\tnumber uint64\n\t_ [8]uint64\n\tary, bufferSize uint64\n\tactions *queue.RingBuffer\n\tcache []interface{}\n\t_ [8]uint64\n\tdisposed uint64\n\t_ [8]uint64\n\trunning uint64\n}\n\nfunc (tree *tree) checkAndRun(action action) {\n\tif tree.actions.Len() > 0 {\n\t\tif action != nil {\n\t\t\ttree.actions.Put(action)\n\t\t}\n\t\tif atomic.CompareAndSwapUint64(&tree.running, 0, 1) {\n\t\t\tvar a interface{}\n\t\t\tvar err error\n\t\t\tfor tree.actions.Len() > 0 {\n\t\t\t\ta, err = tree.actions.Get()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\ttree.cache = append(tree.cache, a)\n\t\t\t\tif uint64(len(tree.cache)) >= tree.bufferSize {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tgo tree.operationRunner(tree.cache, true)\n\t\t}\n\t} else if action != nil {\n\t\tif atomic.CompareAndSwapUint64(&tree.running, 0, 1) {\n\t\t\tswitch action.operation() {\n\t\t\tcase get:\n\t\t\t\tga := action.(*getAction)\n\t\t\t\tresult := tree.search(ga.lookup)\n\t\t\t\tga.result = result\n\t\t\t\taction.complete()\n\t\t\t\ttree.reset()\n\t\t\tcase add, remove:\n\t\t\t\tif len(action.keys()) > multiThreadAt {\n\t\t\t\t\ttree.operationRunner(interfaces{action}, true)\n\t\t\t\t} else {\n\t\t\t\t\ttree.operationRunner(interfaces{action}, false)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\ttree.actions.Put(action)\n\t\t\ttree.checkAndRun(nil)\n\t\t}\n\t}\n}\n\nfunc (tree *tree) init(bufferSize, ary uint64) {\n\ttree.bufferSize = bufferSize\n\ttree.ary = ary\n\ttree.cache = make([]interface{}, 0, bufferSize)\n\ttree.root = newNode(true, newKeys(ary), newNodes(ary))\n\ttree.root.mbr = &rectangle{}\n\ttree.actions = queue.NewRingBuffer(tree.bufferSize)\n}\n\nfunc (tree *tree) operationRunner(xns interfaces, threaded bool) {\n\twriteOperations, deleteOperations, toComplete := tree.fetchKeys(xns, threaded)\n\ttree.recursiveMutate(writeOperations, deleteOperations, false, threaded)\n\tfor _, a := range toComplete {\n\t\ta.complete()\n\t}\n\n\ttree.reset()\n}\n\nfunc (tree *tree) fetchKeys(xns interfaces, inParallel bool) (map[*node][]*keyBundle, map[*node][]*keyBundle, actions) {\n\tif inParallel {\n\t\ttree.fetchKeysInParallel(xns)\n\t} else {\n\t\ttree.fetchKeysInSerial(xns)\n\t}\n\n\twriteOperations := make(map[*node][]*keyBundle)\n\tdeleteOperations := make(map[*node][]*keyBundle)\n\ttoComplete := make(actions, 0, len(xns)/2)\n\tfor _, ifc := range xns {\n\t\taction := ifc.(action)\n\t\tswitch action.operation() {\n\t\tcase add:\n\t\t\tfor i, n := range action.nodes() {\n\t\t\t\twriteOperations[n] = append(writeOperations[n], &keyBundle{key: action.rects()[i].hilbert, left: action.rects()[i].rect})\n\t\t\t}\n\t\t\ttoComplete = append(toComplete, action)\n\t\tcase remove:\n\t\t\tfor i, n := range action.nodes() {\n\t\t\t\tdeleteOperations[n] = append(deleteOperations[n], &keyBundle{key: action.rects()[i].hilbert, left: action.rects()[i].rect})\n\t\t\t}\n\t\t\ttoComplete = append(toComplete, action)\n\t\tcase get:\n\t\t\taction.complete()\n\t\t}\n\t}\n\n\treturn writeOperations, deleteOperations, toComplete\n}\n\nfunc (tree *tree) fetchKeysInSerial(xns interfaces) {\n\tfor _, ifc := range xns {\n\t\taction := ifc.(action)\n\t\tswitch action.operation() {\n\t\tcase add, remove:\n\t\t\tfor i, key := range action.rects() {\n\t\t\t\tn := getParent(tree.root, key.hilbert, key.rect)\n\t\t\t\taction.addNode(int64(i), n)\n\t\t\t}\n\t\tcase get:\n\t\t\tga := action.(*getAction)\n\t\t\trects := tree.search(ga.lookup)\n\t\t\tga.result = rects\n\t\t}\n\t}\n}\n\nfunc (tree *tree) reset() {\n\tfor i := range tree.cache {\n\t\ttree.cache[i] = nil\n\t}\n\n\ttree.cache = tree.cache[:0]\n\tatomic.StoreUint64(&tree.running, 0)\n\ttree.checkAndRun(nil)\n}\n\nfunc (tree *tree) fetchKeysInParallel(xns []interface{}) {\n\tvar forCache struct {\n\t\ti int64\n\t\tbuffer [8]uint64 // different cache lines\n\t\tjs []int64\n\t}\n\n\tfor j := 0; j < len(xns); j++ {\n\t\tforCache.js = append(forCache.js, -1)\n\t}\n\tnumCPU := runtime.NumCPU()\n\tif numCPU > 1 {\n\t\tnumCPU--\n\t}\n\tvar wg sync.WaitGroup\n\twg.Add(numCPU)\n\n\tfor k := 0; k < numCPU; k++ {\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tindex := atomic.LoadInt64(&forCache.i)\n\t\t\t\tif index >= int64(len(xns)) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\taction := xns[index].(action)\n\n\t\t\t\tj := atomic.AddInt64(&forCache.js[index], 1)\n\t\t\t\tif j > int64(len(action.rects())) { // someone else is updating i\n\t\t\t\t\tcontinue\n\t\t\t\t} else if j == int64(len(action.rects())) {\n\t\t\t\t\tatomic.StoreInt64(&forCache.i, index+1)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tswitch action.operation() {\n\t\t\t\tcase add, remove:\n\t\t\t\t\thb := action.rects()[j]\n\t\t\t\t\tn := getParent(tree.root, hb.hilbert, hb.rect)\n\t\t\t\t\taction.addNode(j, n)\n\t\t\t\tcase get:\n\t\t\t\t\tga := action.(*getAction)\n\t\t\t\t\tresult := tree.search(ga.lookup)\n\t\t\t\t\tga.result = result\n\t\t\t\t}\n\t\t\t}\n\t\t\twg.Done()\n\t\t}()\n\t}\n\twg.Wait()\n}\n\nfunc (tree *tree) splitNode(n, parent *node, nodes *[]*node, keys *hilberts) {\n\tif !n.needsSplit(tree.ary) {\n\t\treturn\n\t}\n\n\tlength := n.keys.len()\n\tsplitAt := tree.ary - 1\n\n\tfor i := splitAt; i < length; i += splitAt {\n\t\toffset := length - i\n\t\tk, left, right := n.split(offset, tree.ary)\n\t\tleft.right = right\n\t\t*keys = append(*keys, k)\n\t\t*nodes = append(*nodes, left, right)\n\t\tleft.parent = parent\n\t\tright.parent = parent\n\t}\n}\n\nfunc (tree *tree) applyNode(n *node, adds, deletes []*keyBundle) {\n\tfor _, kb := range deletes {\n\t\tif n.keys.len() == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tdeleted := n.delete(kb)\n\t\tif deleted != nil {\n\t\t\tatomic.AddUint64(&tree.number, ^uint64(0))\n\t\t}\n\t}\n\n\tfor _, kb := range adds {\n\t\told := n.insert(kb)\n\t\tif n.isLeaf && old == nil {\n\t\t\tatomic.AddUint64(&tree.number, 1)\n\t\t}\n\t}\n}\n\nfunc (tree *tree) recursiveMutate(adds, deletes map[*node][]*keyBundle, setRoot, inParallel bool) {\n\tif len(adds) == 0 && len(deletes) == 0 {\n\t\treturn\n\t}\n\n\tif setRoot && len(adds) > 1 {\n\t\tpanic(`SHOULD ONLY HAVE ONE ROOT`)\n\t}\n\n\tifs := make(interfaces, 0, len(adds))\n\tfor n := range adds {\n\t\tif n.parent == nil {\n\t\t\tsetRoot = true\n\t\t}\n\t\tifs = append(ifs, n)\n\t}\n\n\tfor n := range deletes {\n\t\tif n.parent == nil {\n\t\t\tsetRoot = true\n\t\t}\n\n\t\tif _, ok := adds[n]; !ok {\n\t\t\tifs = append(ifs, n)\n\t\t}\n\t}\n\n\tvar dummyRoot *node\n\tif setRoot {\n\t\tdummyRoot = &node{\n\t\t\tkeys: newKeys(tree.ary),\n\t\t\tnodes: newNodes(tree.ary),\n\t\t\tmbr: &rectangle{},\n\t\t}\n\t}\n\n\tvar write sync.Mutex\n\tnextLayerWrite := make(map[*node][]*keyBundle)\n\tnextLayerDelete := make(map[*node][]*keyBundle)\n\n\tvar mutate func(interfaces, func(interface{}))\n\tif inParallel {\n\t\tmutate = executeInterfacesInParallel\n\t} else {\n\t\tmutate = executeInterfacesInSerial\n\t}\n\n\tmutate(ifs, func(ifc interface{}) {\n\t\tn := ifc.(*node)\n\t\tadds := adds[n]\n\t\tdeletes := deletes[n]\n\n\t\tif len(adds) == 0 && len(deletes) == 0 {\n\t\t\treturn\n\t\t}\n\n\t\tif setRoot {\n\t\t\ttree.root = n\n\t\t}\n\n\t\tparent := n.parent\n\t\tif parent == nil {\n\t\t\tparent = dummyRoot\n\t\t\tsetRoot = true\n\t\t}\n\n\t\ttree.applyNode(n, adds, deletes)\n\n\t\tif n.needsSplit(tree.ary) {\n\t\t\tkeys := make(hilberts, 0, n.keys.len())\n\t\t\tnodes := make([]*node, 0, n.nodes.len())\n\t\t\ttree.splitNode(n, parent, &nodes, &keys)\n\t\t\twrite.Lock()\n\t\t\tfor i, k := range keys {\n\t\t\t\tnextLayerWrite[parent] = append(nextLayerWrite[parent], &keyBundle{key: k, left: nodes[i*2], right: nodes[i*2+1]})\n\t\t\t}\n\t\t\twrite.Unlock()\n\t\t}\n\t})\n\n\ttree.recursiveMutate(nextLayerWrite, nextLayerDelete, setRoot, inParallel)\n}\n\n// Insert will add the provided keys to the tree.\nfunc (tree *tree) Insert(rects ...rtree.Rectangle) {\n\tia := newInsertAction(rects)\n\ttree.checkAndRun(ia)\n\tia.completer.Wait()\n}\n\n// Delete will remove the provided keys from the tree. If no\n// matching key is found, this is a no-op.\nfunc (tree *tree) Delete(rects ...rtree.Rectangle) {\n\tra := newRemoveAction(rects)\n\ttree.checkAndRun(ra)\n\tra.completer.Wait()\n}\n\nfunc (tree *tree) search(r *rectangle) rtree.Rectangles {\n\tif tree.root == nil {\n\t\treturn rtree.Rectangles{}\n\t}\n\n\tresult := make(rtree.Rectangles, 0, 10)\n\twhs := tree.root.searchRects(r)\n\tfor len(whs) > 0 {\n\t\twh := whs[0]\n\t\tif n, ok := wh.(*node); ok {\n\t\t\twhs = append(whs, n.searchRects(r)...)\n\t\t} else {\n\t\t\tresult = append(result, wh)\n\t\t}\n\t\twhs = whs[1:]\n\t}\n\n\treturn result\n}\n\n// Search will return a list of rectangles that intersect the provided\n// rectangle.\nfunc (tree *tree) Search(rect rtree.Rectangle) rtree.Rectangles {\n\tga := newGetAction(rect)\n\ttree.checkAndRun(ga)\n\tga.completer.Wait()\n\treturn ga.result\n}\n\n// Len returns the number of items in the tree.\nfunc (tree *tree) Len() uint64 {\n\treturn atomic.LoadUint64(&tree.number)\n}\n\n// Dispose will clean up any resources used by this tree. This\n// must be called to prevent a memory leak.\nfunc (tree *tree) Dispose() {\n\ttree.actions.Dispose()\n\tatomic.StoreUint64(&tree.disposed, 1)\n}\n\nfunc newTree(bufferSize, ary uint64) *tree {\n\ttree := &tree{}\n\ttree.init(bufferSize, ary)\n\treturn tree\n}\n\n// New will construct a new Hilbert R-Tree and return it.\nfunc New(bufferSize, ary uint64) rtree.RTree {\n\treturn newTree(bufferSize, ary)\n}\n" }, { "path": "rtree/hilbert/tree_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage hilbert\n\nimport (\n\t\"log\"\n\t\"math\"\n\t\"math/rand\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/Workiva/go-datastructures/rtree\"\n)\n\nfunc getConsoleLogger() *log.Logger {\n\treturn log.New(os.Stderr, \"\", log.LstdFlags)\n}\n\nfunc (n *node) print(log *log.Logger) {\n\tlog.Printf(`NODE: %+v, MBR: %+v, %p`, n, n.mbr, n)\n\tif n.isLeaf {\n\t\tfor i, wh := range n.nodes.list {\n\t\t\txlow, ylow := wh.LowerLeft()\n\t\t\txhigh, yhigh := wh.UpperRight()\n\t\t\tlog.Printf(`KEY: %+v, XLOW: %+v, YLOW: %+v, XHIGH: %+v, YHIGH: %+v`, n.keys.list[i], xlow, ylow, xhigh, yhigh)\n\t\t}\n\t} else {\n\t\tfor _, wh := range n.nodes.list {\n\t\t\twh.(*node).print(log)\n\t\t}\n\t}\n}\n\nfunc (t *tree) print(log *log.Logger) {\n\tlog.Println(`PRINTING TREE`)\n\tif t.root == nil {\n\t\tlog.Println(`EMPTY TREE.`)\n\t\treturn\n\t}\n\n\tt.root.print(log)\n}\n\nfunc constructMockPoints(num int) rtree.Rectangles {\n\trects := make(rtree.Rectangles, 0, num)\n\tfor i := int32(0); i < int32(num); i++ {\n\t\trects = append(rects, newMockRectangle(i, i, i, i))\n\t}\n\treturn rects\n}\n\nfunc constructRandomMockPoints(num int) rtree.Rectangles {\n\trects := make(rtree.Rectangles, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tr := rand.Int31()\n\t\trects = append(rects, newMockRectangle(r, r, r, r))\n\t}\n\n\treturn rects\n}\n\nfunc constructInfiniteRect() rtree.Rectangle {\n\treturn newMockRectangle(0, 0, math.MaxInt32, math.MaxInt32)\n}\n\nfunc TestSimpleInsert(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 10, 10)\n\ttree := newTree(3, 3)\n\n\ttree.Insert(r1)\n\tassert.Equal(t, uint64(1), tree.Len())\n\n\tq := newMockRectangle(5, 5, 15, 15)\n\tresult := tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r1}, result)\n}\n\nfunc TestSimpleDelete(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 10, 10)\n\ttree := newTree(3, 3)\n\ttree.Insert(r1)\n\n\ttree.Delete(r1)\n\tassert.Equal(t, uint64(0), tree.Len())\n\tq := newMockRectangle(5, 5, 15, 15)\n\tresult := tree.Search(q)\n\tassert.Len(t, result, 0)\n}\n\nfunc TestDeleteIdenticalHilbergNumber(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 20, 20)\n\tr2 := newMockRectangle(5, 5, 15, 15)\n\ttree := newTree(3, 3)\n\ttree.Insert(r1)\n\n\ttree.Delete(r2)\n\tassert.Equal(t, uint64(1), tree.Len())\n\tresult := tree.Search(r2)\n\tassert.Equal(t, rtree.Rectangles{r1}, result)\n\n\ttree.Delete(r1)\n\tassert.Equal(t, uint64(0), tree.Len())\n\tresult = tree.Search(r1)\n\tassert.Len(t, result, 0)\n}\n\nfunc TestDeleteAll(t *testing.T) {\n\tpoints := constructRandomMockPoints(3)\n\ttree := newTree(3, 3)\n\ttree.Insert(points...)\n\tassert.Equal(t, uint64(len(points)), tree.Len())\n\n\ttree.Delete(points...)\n\tassert.Equal(t, uint64(0), tree.Len())\n\tresult := tree.Search(constructInfiniteRect())\n\tassert.Len(t, result, 0)\n}\n\nfunc TestTwoInsert(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 10, 10)\n\tr2 := newMockRectangle(5, 5, 15, 15)\n\ttree := newTree(3, 3)\n\n\ttree.Insert(r1, r2)\n\tassert.Equal(t, uint64(2), tree.Len())\n\n\tq := newMockRectangle(0, 0, 20, 20)\n\tresult := tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r1, r2}, result)\n\n\tq = newMockRectangle(0, 0, 4, 4)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r1}, result)\n\n\tq = newMockRectangle(11, 11, 20, 20)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r2}, result)\n}\n\nfunc TestInsertCausesRootSplitOddAry(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 10, 10)\n\tr2 := newMockRectangle(5, 5, 15, 15)\n\tr3 := newMockRectangle(10, 10, 20, 20)\n\ttree := newTree(3, 3)\n\n\ttree.Insert(r1, r2, r3)\n\tassert.Equal(t, uint64(3), tree.Len())\n\n\tq := newMockRectangle(0, 0, 20, 20)\n\tresult := tree.Search(q)\n\tassert.Contains(t, result, r1)\n\tassert.Contains(t, result, r2)\n\tassert.Contains(t, result, r3)\n}\n\nfunc TestInsertCausesRootSplitEvenAry(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 10, 10)\n\tr2 := newMockRectangle(5, 5, 15, 15)\n\tr3 := newMockRectangle(10, 10, 20, 20)\n\tr4 := newMockRectangle(15, 15, 25, 25)\n\ttree := newTree(4, 4)\n\n\ttree.Insert(r1, r2, r3, r4)\n\tassert.Equal(t, uint64(4), tree.Len())\n\n\tq := newMockRectangle(0, 0, 25, 25)\n\tresult := tree.Search(q)\n\tassert.Contains(t, result, r1)\n\tassert.Contains(t, result, r2)\n\tassert.Contains(t, result, r3)\n\tassert.Contains(t, result, r4)\n}\n\nfunc TestQueryWithLine(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 10, 10)\n\tr2 := newMockRectangle(5, 5, 15, 15)\n\ttree := newTree(3, 3)\n\ttree.Insert(r1, r2)\n\n\t// vertical line at x=5\n\tq := newMockRectangle(5, 0, 5, 10)\n\tresult := tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r1, r2}, result)\n\n\t// horizontal line at y=5\n\tq = newMockRectangle(0, 5, 10, 5)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r1, r2}, result)\n\n\t// vertical line at x=15\n\tq = newMockRectangle(15, 0, 15, 20)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r2}, result)\n\n\t// horizontal line at y=15\n\tq = newMockRectangle(0, 15, 20, 15)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r2}, result)\n\n\t// vertical line on the y-axis\n\tq = newMockRectangle(0, 0, 0, 10)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r1}, result)\n\n\t// horizontal line on the x-axis\n\tq = newMockRectangle(0, 0, 10, 0)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r1}, result)\n\n\t// vertical line at x=20\n\tq = newMockRectangle(20, 0, 20, 20)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{}, result)\n\n\t// horizontal line at y=20\n\tq = newMockRectangle(0, 20, 20, 20)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{}, result)\n}\n\nfunc TestQueryForPoint(t *testing.T) {\n\tr1 := newMockRectangle(5, 5, 5, 5) // (5, 5)\n\tr2 := newMockRectangle(10, 10, 10, 10) // (10, 10)\n\ttree := newTree(3, 3)\n\ttree.Insert(r1, r2)\n\n\tq := newMockRectangle(0, 0, 5, 5)\n\tresult := tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r1}, result)\n\n\tq = newMockRectangle(0, 0, 20, 20)\n\tresult = tree.Search(q)\n\tassert.Contains(t, result, r1)\n\tassert.Contains(t, result, r2)\n\n\tq = newMockRectangle(6, 6, 20, 20)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{r2}, result)\n\n\tq = newMockRectangle(20, 20, 30, 30)\n\tresult = tree.Search(q)\n\tassert.Equal(t, rtree.Rectangles{}, result)\n}\n\nfunc TestMultipleInsertsCauseInternalSplitOddAry(t *testing.T) {\n\tpoints := constructMockPoints(100)\n\ttree := newTree(3, 3)\n\n\ttree.Insert(points...)\n\n\tassert.Equal(t, uint64(len(points)), tree.Len())\n\n\tq := newMockRectangle(0, 0, int32(len(points)), int32(len(points)))\n\tresult := tree.Search(q)\n\tsucceeded := true\n\tfor _, p := range points {\n\t\tif !assert.Contains(t, result, p) {\n\t\t\tsucceeded = false\n\t\t}\n\t}\n\n\tif !succeeded {\n\t\ttree.print(getConsoleLogger())\n\t}\n}\n\nfunc TestMultipleInsertsCauseInternalSplitOddAryRandomPoints(t *testing.T) {\n\tpoints := constructRandomMockPoints(100)\n\ttree := newTree(3, 3)\n\n\ttree.Insert(points...)\n\n\tassert.Equal(t, uint64(len(points)), tree.Len())\n\n\tq := newMockRectangle(0, 0, math.MaxInt32, math.MaxInt32)\n\tresult := tree.Search(q)\n\tsucceeded := true\n\tfor _, p := range points {\n\t\tif !assert.Contains(t, result, p) {\n\t\t\tsucceeded = false\n\t\t}\n\t}\n\n\tif !succeeded {\n\t\ttree.print(getConsoleLogger())\n\t}\n}\n\nfunc TestMultipleInsertsCauseInternalSplitEvenAry(t *testing.T) {\n\tpoints := constructMockPoints(100)\n\ttree := newTree(4, 4)\n\n\ttree.Insert(points...)\n\n\tassert.Equal(t, uint64(len(points)), tree.Len())\n\n\tq := newMockRectangle(0, 0, math.MaxInt32, math.MaxInt32)\n\tresult := tree.Search(q)\n\tsucceeded := true\n\tfor _, p := range points {\n\t\tif !assert.Contains(t, result, p) {\n\t\t\tsucceeded = false\n\t\t}\n\t}\n\n\tif !succeeded {\n\t\ttree.print(getConsoleLogger())\n\t}\n}\n\nfunc TestMultipleInsertsCauseInternalSplitEvenAryRandomOrder(t *testing.T) {\n\tpoints := constructRandomMockPoints(100)\n\ttree := newTree(4, 4)\n\n\ttree.Insert(points...)\n\n\tassert.Equal(t, uint64(len(points)), tree.Len())\n\n\tq := newMockRectangle(0, 0, math.MaxInt32, math.MaxInt32)\n\tresult := tree.Search(q)\n\tsucceeded := true\n\tfor _, p := range points {\n\t\tif !assert.Contains(t, result, p) {\n\t\t\tsucceeded = false\n\t\t}\n\t}\n\n\tif !succeeded {\n\t\ttree.print(getConsoleLogger())\n\t}\n}\n\nfunc TestInsertDuplicateHilbert(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 20, 20)\n\tr2 := newMockRectangle(1, 1, 19, 19)\n\tr3 := newMockRectangle(2, 2, 18, 18)\n\tr4 := newMockRectangle(3, 3, 17, 17)\n\ttree := newTree(3, 3)\n\ttree.Insert(r1)\n\ttree.Insert(r2)\n\ttree.Insert(r3)\n\ttree.Insert(r4)\n\n\tassert.Equal(t, uint64(4), tree.Len())\n\tq := newMockRectangle(0, 0, 30, 30)\n\tresult := tree.Search(q)\n\tassert.Len(t, result, 4)\n\tassert.Contains(t, result, r1)\n\tassert.Contains(t, result, r2)\n\tassert.Contains(t, result, r3)\n\tassert.Contains(t, result, r4)\n}\n\nfunc TestDeleteAllDuplicateHilbert(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 20, 20)\n\tr2 := newMockRectangle(1, 1, 19, 19)\n\tr3 := newMockRectangle(2, 2, 18, 18)\n\tr4 := newMockRectangle(3, 3, 17, 17)\n\ttree := newTree(3, 3)\n\ttree.Insert(r1)\n\ttree.Insert(r2)\n\ttree.Insert(r3)\n\ttree.Insert(r4)\n\n\ttree.Delete(r1, r2, r3, r4)\n\tassert.Equal(t, uint64(0), tree.Len())\n\tresult := tree.Search(constructInfiniteRect())\n\tassert.Len(t, result, 0)\n}\n\nfunc TestInsertDuplicateRect(t *testing.T) {\n\tr1 := newMockRectangle(0, 0, 20, 20)\n\tr2 := newMockRectangle(0, 0, 20, 20)\n\ttree := newTree(3, 3)\n\ttree.Insert(r1)\n\ttree.Insert(r2)\n\n\tassert.Equal(t, uint64(1), tree.Len())\n\tresult := tree.Search(constructInfiniteRect())\n\tassert.Equal(t, rtree.Rectangles{r2}, result)\n}\n\nfunc BenchmarkBulkAddPoints(b *testing.B) {\n\tnumItems := 1000\n\tpoints := constructMockPoints(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree := newTree(8, 8)\n\t\ttree.Insert(points...)\n\t}\n}\n\nfunc BenchmarkBulkUpdatePoints(b *testing.B) {\n\tnumItems := 1000\n\tpoints := constructMockPoints(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(points...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(points...)\n\t}\n}\n\nfunc BenchmarkPointInsertion(b *testing.B) {\n\tnumItems := b.N\n\tpoints := constructMockPoints(numItems)\n\ttree := newTree(8, 8)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Insert(points[i%numItems])\n\t}\n}\n\nfunc BenchmarkQueryPoints(b *testing.B) {\n\tnumItems := b.N\n\tpoints := constructMockPoints(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(points...)\n\n\tb.ResetTimer()\n\n\tfor i := int32(0); i < int32(b.N); i++ {\n\t\ttree.Search(newMockRectangle(i, i, i+10, i+10))\n\t}\n}\n\nfunc BenchmarkQueryBulkPoints(b *testing.B) {\n\tnumItems := b.N\n\tpoints := constructMockPoints(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(points...)\n\n\tb.ResetTimer()\n\n\tfor i := int32(0); i < int32(b.N); i++ {\n\t\ttree.Search(newMockRectangle(i, i, int32(numItems), int32(numItems)))\n\t}\n}\n\nfunc BenchmarkDelete(b *testing.B) {\n\tnumItems := b.N\n\tpoints := constructMockPoints(numItems)\n\ttree := newTree(8, 8)\n\ttree.Insert(points...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ttree.Delete(points[i%numItems])\n\t}\n}\n" }, { "path": "rtree/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage rtree\n\n// Rectangles is a typed list of Rectangle.\ntype Rectangles []Rectangle\n\n// Rectangle describes a two-dimensional bound.\ntype Rectangle interface {\n\t// LowerLeft describes the lower left coordinate of this rectangle.\n\tLowerLeft() (int32, int32)\n\t// UpperRight describes the upper right coordinate of this rectangle.\n\tUpperRight() (int32, int32)\n}\n\n// RTree defines an object that can be returned from any subpackage\n// of this package.\ntype RTree interface {\n\t// Search will perform an intersection search of the given\n\t// rectangle and return any rectangles that intersect.\n\tSearch(Rectangle) Rectangles\n\t// Len returns in the number of items in the RTree.\n\tLen() uint64\n\t// Dispose will clean up any objects used by the RTree.\n\tDispose()\n\t// Delete will remove the provided rectangles from the RTree.\n\tDelete(...Rectangle)\n\t// Insert will add the provided rectangles to the RTree.\n\tInsert(...Rectangle)\n}\n" }, { "path": "set/dict.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage set is a simple unordered set implemented with a map. This set\nis threadsafe which decreases performance.\n\nTODO: Actually write custom hashmap using the hash/fnv hasher.\n\nTODO: Our Set implementation Could be further optimized by getting the uintptr\nof the generic interface{} used and using that as the key; Golang maps handle\nuintptr much better than the generic interface{} key.\n*/\n\npackage set\n\nimport \"sync\"\n\nvar pool = sync.Pool{}\n\n// Set is an implementation of ISet using the builtin map type. Set is threadsafe.\ntype Set struct {\n\titems map[interface{}]struct{}\n\tlock sync.RWMutex\n\tflattened []interface{}\n}\n\n// Add will add the provided items to the set.\nfunc (set *Set) Add(items ...interface{}) {\n\tset.lock.Lock()\n\tdefer set.lock.Unlock()\n\n\tset.flattened = nil\n\tfor _, item := range items {\n\t\tset.items[item] = struct{}{}\n\t}\n}\n\n// Remove will remove the given items from the set.\nfunc (set *Set) Remove(items ...interface{}) {\n\tset.lock.Lock()\n\tdefer set.lock.Unlock()\n\n\tset.flattened = nil\n\tfor _, item := range items {\n\t\tdelete(set.items, item)\n\t}\n}\n\n// Exists returns a bool indicating if the given item exists in the set.\nfunc (set *Set) Exists(item interface{}) bool {\n\tset.lock.RLock()\n\n\t_, ok := set.items[item]\n\n\tset.lock.RUnlock()\n\n\treturn ok\n}\n\n// Flatten will return a list of the items in the set.\nfunc (set *Set) Flatten() []interface{} {\n\tset.lock.Lock()\n\tdefer set.lock.Unlock()\n\n\tif set.flattened != nil {\n\t\treturn set.flattened\n\t}\n\n\tset.flattened = make([]interface{}, 0, len(set.items))\n\tfor item := range set.items {\n\t\tset.flattened = append(set.flattened, item)\n\t}\n\treturn set.flattened\n}\n\n// Len returns the number of items in the set.\nfunc (set *Set) Len() int64 {\n\tset.lock.RLock()\n\n\tsize := int64(len(set.items))\n\n\tset.lock.RUnlock()\n\n\treturn size\n}\n\n// Clear will remove all items from the set.\nfunc (set *Set) Clear() {\n\tset.lock.Lock()\n\n\tset.items = map[interface{}]struct{}{}\n\n\tset.lock.Unlock()\n}\n\n// All returns a bool indicating if all of the supplied items exist in the set.\nfunc (set *Set) All(items ...interface{}) bool {\n\tset.lock.RLock()\n\tdefer set.lock.RUnlock()\n\n\tfor _, item := range items {\n\t\tif _, ok := set.items[item]; !ok {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// Dispose will add this set back into the pool.\nfunc (set *Set) Dispose() {\n\tset.lock.Lock()\n\tdefer set.lock.Unlock()\n\n\tfor k := range set.items {\n\t\tdelete(set.items, k)\n\t}\n\n\t//this is so we don't hang onto any references\n\tfor i := 0; i < len(set.flattened); i++ {\n\t\tset.flattened[i] = nil\n\t}\n\n\tset.flattened = set.flattened[:0]\n\tpool.Put(set)\n}\n\n// New is the constructor for sets. It will pull from a reuseable memory pool if it can.\n// Takes a list of items to initialize the set with.\nfunc New(items ...interface{}) *Set {\n\tset := pool.Get().(*Set)\n\tfor _, item := range items {\n\t\tset.items[item] = struct{}{}\n\t}\n\n\tif len(items) > 0 {\n\t\tset.flattened = nil\n\t}\n\n\treturn set\n}\n\nfunc init() {\n\tpool.New = func() interface{} {\n\t\treturn &Set{\n\t\t\titems: make(map[interface{}]struct{}, 10),\n\t\t}\n\t}\n}\n" }, { "path": "set/dict_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage set\n\nimport (\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestAddDuplicateItem(t *testing.T) {\n\tset := New()\n\tset.Add(`test`)\n\tset.Add(`test`)\n\n\tif !reflect.DeepEqual([]interface{}{`test`}, set.Flatten()) {\n\t\tt.Errorf(`Incorrect result returned: %+v`, set.Flatten())\n\t}\n}\n\nfunc TestAddItems(t *testing.T) {\n\tset := New()\n\tset.Add(`test`)\n\tset.Add(`test1`)\n\n\tfirstSeen := false\n\tsecondSeen := false\n\t// order is not guaranteed\n\tfor _, item := range set.Flatten() {\n\t\tif item.(string) == `test` {\n\t\t\tfirstSeen = true\n\t\t} else if item.(string) == `test1` {\n\t\t\tsecondSeen = true\n\t\t}\n\t}\n\n\tif !firstSeen || !secondSeen {\n\t\tt.Errorf(`Not all items seen in set.`)\n\t}\n}\n\nfunc TestRemove(t *testing.T) {\n\tset := New()\n\tset.Add(`test`)\n\tset.Remove(`test`)\n\n\tif !reflect.DeepEqual([]interface{}{}, set.Flatten()) {\n\t\tt.Errorf(`Incorrect result returned: %+v`, set.Flatten())\n\t}\n}\n\nfunc TestExists(t *testing.T) {\n\tset := New()\n\tset.Add(`test`)\n\n\tif !set.Exists(`test`) {\n\t\tt.Errorf(`Correct existence not determined`)\n\t}\n\n\tif set.Exists(`test1`) {\n\t\tt.Errorf(`Correct nonexistence not determined.`)\n\t}\n}\n\nfunc TestExists_WithNewItems(t *testing.T) {\n\tset := New(`test`, `test1`)\n\n\tif !set.Exists(`test`) {\n\t\tt.Errorf(`Correct existence not determined`)\n\t}\n\n\tif !set.Exists(`test1`) {\n\t\tt.Errorf(`Correct existence not determined`)\n\t}\n\n\tif set.Exists(`test2`) {\n\t\tt.Errorf(`Correct nonexistence not determined.`)\n\t}\n}\n\nfunc TestLen(t *testing.T) {\n\tset := New()\n\tset.Add(`test`)\n\n\tif set.Len() != 1 {\n\t\tt.Errorf(`Expected len: %d, received: %d`, 1, set.Len())\n\t}\n\n\tset.Add(`test1`)\n\tif set.Len() != 2 {\n\t\tt.Errorf(`Expected len: %d, received: %d`, 2, set.Len())\n\t}\n}\n\nfunc TestFlattenCaches(t *testing.T) {\n\tset := New()\n\titem := `test`\n\tset.Add(item)\n\n\tset.Flatten()\n\n\tif len(set.flattened) != 1 {\n\t\tt.Errorf(`Expected len: %d, received: %d`, 1, len(set.flattened))\n\t}\n}\n\nfunc TestFlattenCaches_CacheReturn(t *testing.T) {\n\tset := New()\n\titem := `test`\n\tset.Add(item)\n\n\tflatten1 := set.Flatten()\n\tflatten2 := set.Flatten()\n\n\tif !reflect.DeepEqual(flatten1, flatten2) {\n\t\tt.Errorf(`Flatten cache is not the same as original result. Got %+v, expected %+v`, flatten2, flatten1)\n\t}\n}\n\nfunc TestAddClearsCache(t *testing.T) {\n\tset := New()\n\titem := `test`\n\tset.Add(item)\n\tset.Flatten()\n\n\tset.Add(item)\n\n\tif len(set.flattened) != 0 {\n\t\tt.Errorf(`Expected len: %d, received: %d`, 0, len(set.flattened))\n\t}\n\n\titem = `test2`\n\tset.Add(item)\n\n\tif set.flattened != nil {\n\t\tt.Errorf(`Cache not cleared.`)\n\t}\n}\n\nfunc TestDeleteClearsCache(t *testing.T) {\n\tset := New()\n\titem := `test`\n\tset.Add(item)\n\tset.Flatten()\n\n\tset.Remove(item)\n\n\tif set.flattened != nil {\n\t\tt.Errorf(`Cache not cleared.`)\n\t}\n}\n\nfunc TestAll(t *testing.T) {\n\tset := New()\n\titem := `test`\n\tset.Add(item)\n\n\tresult := set.All(item)\n\tif !result {\n\t\tt.Errorf(`Expected true.`)\n\t}\n\n\titemTwo := `test1`\n\n\tresult = set.All(item, itemTwo)\n\tif result {\n\t\tt.Errorf(`Expected false.`)\n\t}\n}\n\nfunc TestClear(t *testing.T) {\n\tset := New()\n\tset.Add(`test`)\n\n\tset.Clear()\n\n\tif set.Len() != 0 {\n\t\tt.Errorf(`Expected len: %d, received: %d`, 0, set.Len())\n\t}\n}\n\nfunc BenchmarkFlatten(b *testing.B) {\n\tset := New()\n\tfor i := 0; i < 50; i++ {\n\t\titem := strconv.Itoa(i)\n\t\tset.Add(item)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tset.Flatten()\n\t}\n}\n\nfunc BenchmarkLen(b *testing.B) {\n\tset := New()\n\tfor i := 0; i < 50; i++ {\n\t\titem := strconv.Itoa(i)\n\t\tset.Add(item)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tset.Len()\n\t}\n}\n\nfunc BenchmarkExists(b *testing.B) {\n\tset := New()\n\tset.Add(1)\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tset.Exists(1)\n\t}\n}\n\nfunc BenchmarkClear(b *testing.B) {\n\tset := New()\n\tfor i := 0; i < b.N; i++ {\n\t\tset.Clear()\n\t}\n}\n" }, { "path": "slice/int64.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage Int64 simply adds an Int64-typed version of the standard library's\nsort/IntSlice implementation.\n\nAlso added is an Insert method.\n*/\npackage slice\n\nimport \"sort\"\n\n// Int64Slice is a slice that fulfills the sort.Interface interface.\ntype Int64Slice []int64\n\n// Len returns the len of this slice. Required by sort.Interface.\nfunc (s Int64Slice) Len() int {\n\treturn len(s)\n}\n\n// Less returns a bool indicating if the value at position i\n// is less than at position j. Required by sort.Interface.\nfunc (s Int64Slice) Less(i, j int) bool {\n\treturn s[i] < s[j]\n}\n\n// Search will search this slice and return an index that corresponds\n// to the lowest position of that value. You'll need to check\n// separately if the value at that position is equal to x. The\n// behavior of this method is undefinited if the slice is not sorted.\nfunc (s Int64Slice) Search(x int64) int {\n\treturn sort.Search(len(s), func(i int) bool {\n\t\treturn s[i] >= x\n\t})\n}\n\n// Sort will in-place sort this list of int64s.\nfunc (s Int64Slice) Sort() {\n\tsort.Sort(s)\n}\n\n// Swap will swap the elements at positions i and j. This is required\n// by sort.Interface.\nfunc (s Int64Slice) Swap(i, j int) {\n\ts[i], s[j] = s[j], s[i]\n}\n\n// Exists returns a bool indicating if the provided value exists\n// in this list. This has undefined behavior if the list is not\n// sorted.\nfunc (s Int64Slice) Exists(x int64) bool {\n\ti := s.Search(x)\n\tif i == len(s) {\n\t\treturn false\n\t}\n\n\treturn s[i] == x\n}\n\n// Insert will insert x into the sorted position in this list\n// and return a list with the value added. If this slice has not\n// been sorted Insert's behavior is undefined.\nfunc (s Int64Slice) Insert(x int64) Int64Slice {\n\ti := s.Search(x)\n\tif i == len(s) {\n\t\treturn append(s, x)\n\t}\n\n\tif s[i] == x {\n\t\treturn s\n\t}\n\n\ts = append(s, 0)\n\tcopy(s[i+1:], s[i:])\n\ts[i] = x\n\treturn s\n}\n" }, { "path": "slice/int64_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slice\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestSort(t *testing.T) {\n\ts := Int64Slice{3, 6, 1, 0, -1}\n\ts.Sort()\n\n\tassert.Equal(t, Int64Slice{-1, 0, 1, 3, 6}, s)\n}\n\nfunc TestSearch(t *testing.T) {\n\ts := Int64Slice{1, 3, 6}\n\n\tassert.Equal(t, 1, s.Search(3))\n\tassert.Equal(t, 1, s.Search(2))\n\tassert.Equal(t, 3, s.Search(7))\n}\n\nfunc TestExists(t *testing.T) {\n\ts := Int64Slice{1, 3, 6}\n\n\tassert.True(t, s.Exists(3))\n\tassert.False(t, s.Exists(4))\n}\n\nfunc TestInsert(t *testing.T) {\n\ts := Int64Slice{1, 3, 6}\n\ts = s.Insert(2)\n\tassert.Equal(t, Int64Slice{1, 2, 3, 6}, s)\n\n\ts = s.Insert(7)\n\tassert.Equal(t, Int64Slice{1, 2, 3, 6, 7}, s)\n}\n" }, { "path": "slice/skip/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage skip\n\nimport \"github.com/Workiva/go-datastructures/common\"\n\n// Iterator defines an interface that allows a consumer to iterate\n// all results of a query. All values will be visited in-order.\ntype Iterator interface {\n\t// Next returns a bool indicating if there is future value\n\t// in the iterator and moves the iterator to that value.\n\tNext() bool\n\t// Value returns a Comparator representing the iterator's current\n\t// position. If there is no value, this returns nil.\n\tValue() common.Comparator\n\t// exhaust is a helper method that will iterate this iterator\n\t// to completion and return a list of resulting Entries\n\t// in order.\n\texhaust() common.Comparators\n}\n" }, { "path": "slice/skip/iterator.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage skip\n\nimport \"github.com/Workiva/go-datastructures/common\"\n\nconst iteratorExhausted = -2\n\n// iterator represents an object that can be iterated. It will\n// return false on Next and nil on Value if there are no further\n// values to be iterated.\ntype iterator struct {\n\tfirst bool\n\tn *node\n}\n\n// Next returns a bool indicating if there are any further values\n// in this iterator.\nfunc (iter *iterator) Next() bool {\n\tif iter.first {\n\t\titer.first = false\n\t\treturn iter.n != nil\n\t}\n\n\tif iter.n == nil {\n\t\treturn false\n\t}\n\n\titer.n = iter.n.forward[0]\n\treturn iter.n != nil\n}\n\n// Value returns a Comparator representing the iterator's present\n// position in the query. Returns nil if no values remain to iterate.\nfunc (iter *iterator) Value() common.Comparator {\n\tif iter.n == nil {\n\t\treturn nil\n\t}\n\n\treturn iter.n.entry\n}\n\n// exhaust is a helper method to exhaust this iterator and return\n// all remaining entries.\nfunc (iter *iterator) exhaust() common.Comparators {\n\tentries := make(common.Comparators, 0, 10)\n\tfor i := iter; i.Next(); {\n\t\tentries = append(entries, i.Value())\n\t}\n\n\treturn entries\n}\n\n// nilIterator returns an iterator that will always return false\n// for Next and nil for Value.\nfunc nilIterator() *iterator {\n\treturn &iterator{}\n}\n" }, { "path": "slice/skip/iterator_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage skip\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestIterate(t *testing.T) {\n\te1 := newMockEntry(1)\n\tn1 := newNode(e1, 8)\n\titer := &iterator{\n\t\tn: n1,\n\t\tfirst: true,\n\t}\n\n\tassert.True(t, iter.Next())\n\tassert.Equal(t, e1, iter.Value())\n\tassert.False(t, iter.Next())\n\tassert.Nil(t, iter.Value())\n\n\te2 := newMockEntry(2)\n\tn2 := newNode(e2, 8)\n\tn1.forward[0] = n2\n\n\titer = &iterator{\n\t\tn: n1,\n\t\tfirst: true,\n\t}\n\n\tassert.True(t, iter.Next())\n\tassert.Equal(t, e1, iter.Value())\n\tassert.True(t, iter.Next())\n\tassert.Equal(t, e2, iter.Value())\n\tassert.False(t, iter.Next())\n\tassert.Nil(t, iter.Value())\n\n\titer = nilIterator()\n\tassert.False(t, iter.Next())\n\tassert.Nil(t, iter.Value())\n}\n" }, { "path": "slice/skip/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage skip\n\nimport (\n\t\"github.com/stretchr/testify/mock\"\n\n\t\"github.com/Workiva/go-datastructures/common\"\n)\n\ntype mockEntry uint64\n\nfunc (me mockEntry) Compare(other common.Comparator) int {\n\totherU := other.(mockEntry)\n\tif me == otherU {\n\t\treturn 0\n\t}\n\n\tif me > otherU {\n\t\treturn 1\n\t}\n\n\treturn -1\n}\n\nfunc newMockEntry(key uint64) mockEntry {\n\treturn mockEntry(key)\n}\n\ntype mockIterator struct {\n\tmock.Mock\n}\n\nfunc (mi *mockIterator) Next() bool {\n\targs := mi.Called()\n\treturn args.Bool(0)\n}\n\nfunc (mi *mockIterator) Value() common.Comparator {\n\targs := mi.Called()\n\tresult, ok := args.Get(0).(common.Comparator)\n\tif !ok {\n\t\treturn nil\n\t}\n\n\treturn result\n}\n\nfunc (mi *mockIterator) exhaust() common.Comparators {\n\treturn nil\n}\n" }, { "path": "slice/skip/node.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage skip\n\nimport \"github.com/Workiva/go-datastructures/common\"\n\ntype widths []uint64\n\ntype nodes []*node\n\ntype node struct {\n\t// forward denotes the forward pointing pointers in this\n\t// node.\n\tforward nodes\n\t// widths keeps track of the distance between this pointer\n\t// and the forward pointers so we can access skip list\n\t// values by position in logarithmic time.\n\twidths widths\n\t// entry is the associated value with this node.\n\tentry common.Comparator\n}\n\nfunc (n *node) Compare(e common.Comparator) int {\n\treturn n.entry.Compare(e)\n}\n\n// newNode will allocate and return a new node with the entry\n// provided. maxLevels will determine the length of the forward\n// pointer list associated with this node.\nfunc newNode(cmp common.Comparator, maxLevels uint8) *node {\n\treturn &node{\n\t\tentry: cmp,\n\t\tforward: make(nodes, maxLevels),\n\t\twidths: make(widths, maxLevels),\n\t}\n}\n" }, { "path": "slice/skip/skip.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage skip defines a skiplist datastructure. That is, a data structure\nthat probabilistically determines relationships between keys. By doing\nso, it becomes easier to program than a binary search tree but maintains\nsimilar speeds.\n\nPerformance characteristics:\nInsert: O(log n)\nSearch: O(log n)\nDelete: O(log n)\nSpace: O(n)\n\nRecently added is the capability to address, insert, and replace an\nentry by position. This capability is achieved by saving the width\nof the \"gap\" between two nodes. Searching for an item by position is\nvery similar to searching by value in that the same basic algorithm is\nused but we are searching for width instead of value. Because this avoids\nthe overhead associated with Golang interfaces, operations by position\nare about twice as fast as operations by value. Time complexities listed\nbelow.\n\nSearchByPosition: O(log n)\nInsertByPosition: O(log n)\n\nMore information here: http://cglab.ca/~morin/teaching/5408/refs/p90b.pdf\n\nBenchmarks:\nBenchmarkInsert-8\t \t\t 2000000\t 930 ns/op\nBenchmarkGet-8\t \t\t\t 2000000\t 989 ns/op\nBenchmarkDelete-8\t \t\t 3000000\t 600 ns/op\nBenchmarkPrepend-8\t \t\t 1000000\t 1468 ns/op\nBenchmarkByPosition-8\t\t10000000\t 202 ns/op\nBenchmarkInsertAtPosition-8\t 3000000\t 485 ns/op\n\nCPU profiling has shown that the most expensive thing we do here\nis call Compare. A potential optimization for gets only is to\ndo a binary search in the forward/width lists instead of visiting\nevery value. We could also use generics if Golang had them and\nlet the consumer specify primitive types, which would speed up\nthese operation dramatically.\n*/\npackage skip\n\nimport (\n\t\"math/rand\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/Workiva/go-datastructures/common\"\n)\n\nconst p = .5 // the p level defines the probability that a node\n// with a value at level i also has a value at i+1. This number\n// is also important in determining max level. Max level will\n// be defined as L(N) where L = log base (1/p) of n where n\n// is the number of items in the list and N is the number of possible\n// items in the universe. If p = .5 then maxlevel = 32 is appropriate\n// for uint32.\n\n// lockedSource is an implementation of rand.Source that is safe for\n// concurrent use by multiple goroutines. The code is modeled after\n// https://golang.org/src/math/rand/rand.go.\ntype lockedSource struct {\n\tmu sync.Mutex\n\tsrc rand.Source\n}\n\n// Int63 implements the rand.Source interface.\nfunc (ls *lockedSource) Int63() (n int64) {\n\tls.mu.Lock()\n\tn = ls.src.Int63()\n\tls.mu.Unlock()\n\treturn\n}\n\n// Seed implements the rand.Source interface.\nfunc (ls *lockedSource) Seed(seed int64) {\n\tls.mu.Lock()\n\tls.src.Seed(seed)\n\tls.mu.Unlock()\n}\n\n// generator will be the common generator to create random numbers. It\n// is seeded with unix nanosecond when this line is executed at runtime,\n// and only executed once ensuring all random numbers come from the same\n// randomly seeded generator.\nvar generator = rand.New(&lockedSource{src: rand.NewSource(time.Now().UnixNano())})\n\nfunc generateLevel(maxLevel uint8) uint8 {\n\tvar level uint8\n\tfor level = uint8(1); level < maxLevel-1; level++ {\n\t\tif generator.Float64() >= p {\n\n\t\t\treturn level\n\t\t}\n\t}\n\n\treturn level\n}\n\nfunc insertNode(sl *SkipList, n *node, cmp common.Comparator, pos uint64, cache nodes, posCache widths, allowDuplicate bool) common.Comparator {\n\tif !allowDuplicate && n != nil && n.Compare(cmp) == 0 { // a simple update in this case\n\t\toldEntry := n.entry\n\t\tn.entry = cmp\n\t\treturn oldEntry\n\t}\n\tatomic.AddUint64(&sl.num, 1)\n\n\tnodeLevel := generateLevel(sl.maxLevel)\n\tif nodeLevel > sl.level {\n\t\tfor i := sl.level; i < nodeLevel; i++ {\n\t\t\tcache[i] = sl.head\n\t\t}\n\t\tsl.level = nodeLevel\n\t}\n\n\tnn := newNode(cmp, nodeLevel)\n\tfor i := uint8(0); i < nodeLevel; i++ {\n\t\tnn.forward[i] = cache[i].forward[i]\n\t\tcache[i].forward[i] = nn\n\t\tformerWidth := cache[i].widths[i]\n\t\tif formerWidth == 0 {\n\t\t\tnn.widths[i] = 0\n\t\t} else {\n\t\t\tnn.widths[i] = posCache[i] + formerWidth + 1 - pos\n\t\t}\n\n\t\tif cache[i].forward[i] != nil {\n\t\t\tcache[i].widths[i] = pos - posCache[i]\n\t\t}\n\n\t}\n\n\tfor i := nodeLevel; i < sl.level; i++ {\n\t\tif cache[i].forward[i] == nil {\n\t\t\tcontinue\n\t\t}\n\t\tcache[i].widths[i]++\n\t}\n\treturn nil\n}\n\nfunc splitAt(sl *SkipList, index uint64) (*SkipList, *SkipList) {\n\tright := &SkipList{}\n\tright.maxLevel = sl.maxLevel\n\tright.level = sl.level\n\tright.cache = make(nodes, sl.maxLevel)\n\tright.posCache = make(widths, sl.maxLevel)\n\tright.head = newNode(nil, sl.maxLevel)\n\tsl.searchByPosition(index, sl.cache, sl.posCache) // populate the cache that needs updating\n\n\tfor i := uint8(0); i <= sl.level; i++ {\n\t\tright.head.forward[i] = sl.cache[i].forward[i]\n\t\tif sl.cache[i].forward[i] != nil {\n\t\t\tright.head.widths[i] = sl.cache[i].widths[i] - (index - sl.posCache[i])\n\t\t}\n\t\tsl.cache[i].widths[i] = 0\n\t\tsl.cache[i].forward[i] = nil\n\t}\n\n\tright.num = sl.Len() - index // right is not in user's hands yet\n\tatomic.AddUint64(&sl.num, -right.num)\n\n\tsl.resetMaxLevel()\n\tright.resetMaxLevel()\n\n\treturn sl, right\n}\n\n// Skip list is a datastructure that probabalistically determines\n// relationships between nodes. This results in a structure\n// that performs similarly to a BST but is much easier to build\n// from a programmatic perspective (no rotations).\ntype SkipList struct {\n\tmaxLevel, level uint8\n\thead *node\n\tnum uint64\n\t// a list of nodes that can be reused, should reduce\n\t// the number of allocations in the insert/delete case.\n\tcache nodes\n\tposCache widths\n}\n\n// init will initialize this skiplist. The parameter is expected\n// to be of some uint type which will set this skiplist's maximum\n// level.\nfunc (sl *SkipList) init(ifc interface{}) {\n\tswitch ifc.(type) {\n\tcase uint8:\n\t\tsl.maxLevel = 8\n\tcase uint16:\n\t\tsl.maxLevel = 16\n\tcase uint32:\n\t\tsl.maxLevel = 32\n\tcase uint64, uint:\n\t\tsl.maxLevel = 64\n\t}\n\tsl.cache = make(nodes, sl.maxLevel)\n\tsl.posCache = make(widths, sl.maxLevel)\n\tsl.head = newNode(nil, sl.maxLevel)\n}\n\nfunc (sl *SkipList) search(cmp common.Comparator, update nodes, widths widths) (*node, uint64) {\n\tif sl.Len() == 0 { // nothing in the list\n\t\treturn nil, 1\n\t}\n\n\tvar pos uint64 = 0\n\tvar offset uint8\n\tvar alreadyChecked *node\n\tn := sl.head\n\tfor i := uint8(0); i <= sl.level; i++ {\n\t\toffset = sl.level - i\n\t\tfor n.forward[offset] != nil && n.forward[offset] != alreadyChecked && n.forward[offset].Compare(cmp) < 0 {\n\t\t\tpos += n.widths[offset]\n\t\t\tn = n.forward[offset]\n\t\t}\n\n\t\talreadyChecked = n\n\t\tif update != nil {\n\t\t\tupdate[offset] = n\n\t\t\twidths[offset] = pos\n\t\t}\n\t}\n\n\treturn n.forward[0], pos + 1\n}\n\nfunc (sl *SkipList) resetMaxLevel() {\n\tif sl.level < 1 {\n\t\tsl.level = 1\n\t\treturn\n\t}\n\tfor sl.head.forward[sl.level-1] == nil && sl.level > 1 {\n\t\tsl.level--\n\t}\n}\n\nfunc (sl *SkipList) searchByPosition(position uint64, update nodes, widths widths) (*node, uint64) {\n\tif sl.Len() == 0 { // nothing in the list\n\t\treturn nil, 1\n\t}\n\n\tif position > sl.Len() {\n\t\treturn nil, 1\n\t}\n\n\tvar pos uint64 = 0\n\tvar offset uint8\n\tn := sl.head\n\tfor i := uint8(0); i <= sl.level; i++ {\n\t\toffset = sl.level - i\n\t\tfor n.forward[offset] != nil && pos+n.widths[offset] <= position {\n\t\t\tpos += n.widths[offset]\n\t\t\tn = n.forward[offset]\n\t\t}\n\n\t\tif update != nil {\n\t\t\tupdate[offset] = n\n\t\t\twidths[offset] = pos\n\t\t}\n\t}\n\n\treturn n, pos + 1\n}\n\n// Get will retrieve values associated with the keys provided. If an\n// associated value could not be found, a nil is returned in its place.\n// This is an O(log n) operation.\nfunc (sl *SkipList) Get(comparators ...common.Comparator) common.Comparators {\n\tresult := make(common.Comparators, 0, len(comparators))\n\n\tvar n *node\n\tfor _, cmp := range comparators {\n\t\tn, _ = sl.search(cmp, nil, nil)\n\t\tif n != nil && n.Compare(cmp) == 0 {\n\t\t\tresult = append(result, n.entry)\n\t\t} else {\n\t\t\tresult = append(result, nil)\n\t\t}\n\t}\n\n\treturn result\n}\n\n// GetWithPosition will retrieve the value with the provided key and\n// return the position of that value within the list. Returns nil, 0\n// if an associated value could not be found.\nfunc (sl *SkipList) GetWithPosition(cmp common.Comparator) (common.Comparator, uint64) {\n\tn, pos := sl.search(cmp, nil, nil)\n\tif n == nil {\n\t\treturn nil, 0\n\t}\n\n\treturn n.entry, pos - 1\n}\n\n// ByPosition returns the Comparator at the given position.\nfunc (sl *SkipList) ByPosition(position uint64) common.Comparator {\n\tn, _ := sl.searchByPosition(position+1, nil, nil)\n\tif n == nil {\n\t\treturn nil\n\t}\n\n\treturn n.entry\n}\n\nfunc (sl *SkipList) insert(cmp common.Comparator) common.Comparator {\n\tn, pos := sl.search(cmp, sl.cache, sl.posCache)\n\treturn insertNode(sl, n, cmp, pos, sl.cache, sl.posCache, false)\n}\n\n// Insert will insert the provided comparators into the list. Returned\n// is a list of comparators that were overwritten. This is expected to\n// be an O(log n) operation.\nfunc (sl *SkipList) Insert(comparators ...common.Comparator) common.Comparators {\n\toverwritten := make(common.Comparators, 0, len(comparators))\n\tfor _, cmp := range comparators {\n\t\toverwritten = append(overwritten, sl.insert(cmp))\n\t}\n\n\treturn overwritten\n}\n\nfunc (sl *SkipList) insertAtPosition(position uint64, cmp common.Comparator) {\n\tif position > sl.Len() {\n\t\tposition = sl.Len()\n\t}\n\tn, pos := sl.searchByPosition(position, sl.cache, sl.posCache)\n\tinsertNode(sl, n, cmp, pos, sl.cache, sl.posCache, true)\n}\n\n// InsertAtPosition will insert the provided Comparator at the provided position.\n// If position is greater than the length of the skiplist, the Comparator\n// is appended. This method bypasses order checks and checks for\n// duplicates so use with caution.\nfunc (sl *SkipList) InsertAtPosition(position uint64, cmp common.Comparator) {\n\tsl.insertAtPosition(position, cmp)\n}\n\nfunc (sl *SkipList) replaceAtPosition(position uint64, cmp common.Comparator) {\n\tn, _ := sl.searchByPosition(position+1, nil, nil)\n\tif n == nil {\n\t\treturn\n\t}\n\n\tn.entry = cmp\n}\n\n// Replace at position will replace the Comparator at the provided position\n// with the provided Comparator. If the provided position does not exist,\n// this operation is a no-op.\nfunc (sl *SkipList) ReplaceAtPosition(position uint64, cmp common.Comparator) {\n\tsl.replaceAtPosition(position, cmp)\n}\n\nfunc (sl *SkipList) delete(cmp common.Comparator) common.Comparator {\n\tn, _ := sl.search(cmp, sl.cache, sl.posCache)\n\n\tif n == nil || n.Compare(cmp) != 0 {\n\t\treturn nil\n\t}\n\n\tatomic.AddUint64(&sl.num, ^uint64(0)) // decrement\n\n\tfor i := uint8(0); i <= sl.level; i++ {\n\t\tif sl.cache[i].forward[i] != n {\n\t\t\tif sl.cache[i].forward[i] != nil {\n\t\t\t\tsl.cache[i].widths[i]--\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tsl.cache[i].widths[i] += n.widths[i] - 1\n\t\tsl.cache[i].forward[i] = n.forward[i]\n\t}\n\n\tfor sl.level > 1 && sl.head.forward[sl.level-1] == nil {\n\t\tsl.head.widths[sl.level] = 0\n\t\tsl.level--\n\t}\n\n\treturn n.entry\n}\n\n// Delete will remove the provided keys from the skiplist and return\n// a list of in-order Comparators that were deleted. This is a no-op if\n// an associated key could not be found. This is an O(log n) operation.\nfunc (sl *SkipList) Delete(comparators ...common.Comparator) common.Comparators {\n\tdeleted := make(common.Comparators, 0, len(comparators))\n\n\tfor _, cmp := range comparators {\n\t\tdeleted = append(deleted, sl.delete(cmp))\n\t}\n\n\treturn deleted\n}\n\n// Len returns the number of items in this skiplist.\nfunc (sl *SkipList) Len() uint64 {\n\treturn atomic.LoadUint64(&sl.num)\n}\n\nfunc (sl *SkipList) iterAtPosition(pos uint64) *iterator {\n\tn, _ := sl.searchByPosition(pos, nil, nil)\n\tif n == nil || n.entry == nil {\n\t\treturn nilIterator()\n\t}\n\n\treturn &iterator{\n\t\tfirst: true,\n\t\tn: n,\n\t}\n}\n\n// IterAtPosition is the sister method to Iter only the user defines\n// a position in the skiplist to begin iteration instead of a value.\nfunc (sl *SkipList) IterAtPosition(pos uint64) Iterator {\n\treturn sl.iterAtPosition(pos + 1)\n}\n\nfunc (sl *SkipList) iter(cmp common.Comparator) *iterator {\n\tn, _ := sl.search(cmp, nil, nil)\n\tif n == nil {\n\t\treturn nilIterator()\n\t}\n\n\treturn &iterator{\n\t\tfirst: true,\n\t\tn: n,\n\t}\n}\n\n// Iter will return an iterator that can be used to iterate\n// over all the values with a key equal to or greater than\n// the key provided.\nfunc (sl *SkipList) Iter(cmp common.Comparator) Iterator {\n\treturn sl.iter(cmp)\n}\n\n// SplitAt will split the current skiplist into two lists. The first\n// skiplist returned is the \"left\" list and the second is the \"right.\"\n// The index defines the last item in the left list. If index is greater\n// then the length of this list, only the left skiplist is returned\n// and the right will be nil. This is a mutable operation and modifies\n// the content of this list.\nfunc (sl *SkipList) SplitAt(index uint64) (*SkipList, *SkipList) {\n\tindex++ // 0-index offset\n\tif index >= sl.Len() {\n\t\treturn sl, nil\n\t}\n\treturn splitAt(sl, index)\n}\n\n// New will allocate, initialize, and return a new skiplist.\n// The provided parameter should be of type uint and will determine\n// the maximum possible level that will be created to ensure\n// a random and quick distribution of levels. Parameter must\n// be a uint type.\nfunc New(ifc interface{}) *SkipList {\n\tsl := &SkipList{}\n\tsl.init(ifc)\n\treturn sl\n}\n" }, { "path": "slice/skip/skip_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage skip\n\nimport (\n\t\"math/rand\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/Workiva/go-datastructures/common\"\n)\n\nfunc generateMockEntries(num int) common.Comparators {\n\tentries := make(common.Comparators, 0, num)\n\tfor i := uint64(0); i < uint64(num); i++ {\n\t\tentries = append(entries, newMockEntry(i))\n\t}\n\n\treturn entries\n}\n\nfunc generateRandomMockEntries(num int) common.Comparators {\n\tentries := make(common.Comparators, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tentries = append(entries, newMockEntry(uint64(rand.Int())))\n\t}\n\n\treturn entries\n}\n\nfunc TestInsertByPosition(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(6)\n\tm3 := newMockEntry(2)\n\tsl := New(uint8(0))\n\tsl.InsertAtPosition(2, m1)\n\tsl.InsertAtPosition(0, m2)\n\tsl.InsertAtPosition(0, m3)\n\n\tassert.Equal(t, m3, sl.ByPosition(0))\n\tassert.Equal(t, m2, sl.ByPosition(1))\n\tassert.Equal(t, m1, sl.ByPosition(2))\n\tassert.Nil(t, sl.ByPosition(3))\n}\n\nfunc TestGetByPosition(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(6)\n\tsl := New(uint8(0))\n\tsl.Insert(m1, m2)\n\n\tassert.Equal(t, m1, sl.ByPosition(0))\n\tassert.Equal(t, m2, sl.ByPosition(1))\n\tassert.Nil(t, sl.ByPosition(2))\n}\n\nfunc TestSplitAt(t *testing.T) {\n\tm1 := newMockEntry(3)\n\tm2 := newMockEntry(5)\n\tm3 := newMockEntry(7)\n\tsl := New(uint8(0))\n\tsl.Insert(m1, m2, m3)\n\n\tleft, right := sl.SplitAt(1)\n\tassert.Equal(t, uint64(2), left.Len())\n\tassert.Equal(t, uint64(1), right.Len())\n\tassert.Equal(t, common.Comparators{m1, m2, nil}, left.Get(m1, m2, m3))\n\tassert.Equal(t, common.Comparators{nil, nil, m3}, right.Get(m1, m2, m3))\n\tassert.Equal(t, m1, left.ByPosition(0))\n\tassert.Equal(t, m2, left.ByPosition(1))\n\tassert.Equal(t, m3, right.ByPosition(0))\n\tassert.Equal(t, nil, left.ByPosition(2))\n\tassert.Equal(t, nil, right.ByPosition(1))\n}\n\nfunc TestSplitLargeSkipList(t *testing.T) {\n\tentries := generateMockEntries(100)\n\tleftEntries := entries[:50]\n\trightEntries := entries[50:]\n\tsl := New(uint64(0))\n\tsl.Insert(entries...)\n\n\tleft, right := sl.SplitAt(49)\n\tassert.Equal(t, uint64(50), left.Len())\n\tfor _, le := range leftEntries {\n\t\tresult, index := left.GetWithPosition(le)\n\t\tassert.Equal(t, le, result)\n\t\tassert.Equal(t, le, left.ByPosition(index))\n\t}\n\n\tassert.Equal(t, uint64(50), right.Len())\n\tfor _, re := range rightEntries {\n\t\tresult, index := right.GetWithPosition(re)\n\t\tassert.Equal(t, re, result)\n\t\tassert.Equal(t, re, right.ByPosition(index))\n\t}\n}\n\nfunc TestSplitLargeSkipListOddNumber(t *testing.T) {\n\tentries := generateMockEntries(99)\n\tleftEntries := entries[:50]\n\trightEntries := entries[50:]\n\tsl := New(uint64(0))\n\tsl.Insert(entries...)\n\n\tleft, right := sl.SplitAt(49)\n\tassert.Equal(t, uint64(50), left.Len())\n\tfor _, le := range leftEntries {\n\t\tresult, index := left.GetWithPosition(le)\n\t\tassert.Equal(t, le, result)\n\t\tassert.Equal(t, le, left.ByPosition(index))\n\t}\n\n\tassert.Equal(t, uint64(49), right.Len())\n\tfor _, re := range rightEntries {\n\t\tresult, index := right.GetWithPosition(re)\n\t\tassert.Equal(t, re, result)\n\t\tassert.Equal(t, re, right.ByPosition(index))\n\t}\n}\n\nfunc TestSplitAtSkipListLength(t *testing.T) {\n\tentries := generateMockEntries(5)\n\tsl := New(uint64(0))\n\tsl.Insert(entries...)\n\n\tleft, right := sl.SplitAt(4)\n\tassert.Equal(t, sl, left)\n\tassert.Nil(t, right)\n}\n\nfunc TestGetWithPosition(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(6)\n\tsl := New(uint8(0))\n\tsl.Insert(m1, m2)\n\n\te, pos := sl.GetWithPosition(m1)\n\tassert.Equal(t, m1, e)\n\tassert.Equal(t, uint64(0), pos)\n\n\te, pos = sl.GetWithPosition(m2)\n\tassert.Equal(t, m2, e)\n\tassert.Equal(t, uint64(1), pos)\n}\n\nfunc TestReplaceAtPosition(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(6)\n\tsl := New(uint8(0))\n\n\tsl.Insert(m1, m2)\n\tm3 := newMockEntry(9)\n\tsl.ReplaceAtPosition(0, m3)\n\tassert.Equal(t, m3, sl.ByPosition(0))\n\tassert.Equal(t, m2, sl.ByPosition(1))\n}\n\nfunc TestInsertRandomGetByPosition(t *testing.T) {\n\tentries := generateRandomMockEntries(100)\n\tsl := New(uint64(0))\n\tsl.Insert(entries...)\n\n\tfor _, e := range entries {\n\t\t_, pos := sl.GetWithPosition(e)\n\t\tassert.Equal(t, e, sl.ByPosition(pos))\n\t}\n}\n\nfunc TestGetManyByPosition(t *testing.T) {\n\tentries := generateMockEntries(10)\n\tsl := New(uint64(0))\n\tsl.Insert(entries...)\n\n\tfor i, e := range entries {\n\t\tassert.Equal(t, e, sl.ByPosition(uint64(i)))\n\t}\n}\n\nfunc TestGetPositionAfterDelete(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(6)\n\tsl := New(uint8(0))\n\tsl.Insert(m1, m2)\n\n\tsl.Delete(m1)\n\tassert.Equal(t, m2, sl.ByPosition(0))\n\tassert.Nil(t, sl.ByPosition(1))\n\n\tsl.Delete(m2)\n\tassert.Nil(t, sl.ByPosition(0))\n\tassert.Nil(t, sl.ByPosition(1))\n}\n\nfunc TestGetPositionBulkDelete(t *testing.T) {\n\tes := generateMockEntries(20)\n\te1 := es[:10]\n\te2 := es[10:]\n\tsl := New(uint64(0))\n\tsl.Insert(e1...)\n\tsl.Insert(e2...)\n\n\tfor _, e := range e1 {\n\t\tsl.Delete(e)\n\t}\n\tfor i, e := range e2 {\n\t\tassert.Equal(t, e, sl.ByPosition(uint64(i)))\n\t}\n}\n\nfunc TestSimpleInsert(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(6)\n\n\tsl := New(uint8(0))\n\n\toverwritten := sl.Insert(m1)\n\tassert.Equal(t, common.Comparators{m1}, sl.Get(m1))\n\tassert.Equal(t, uint64(1), sl.Len())\n\tassert.Equal(t, common.Comparators{nil}, overwritten)\n\tassert.Equal(t, common.Comparators{nil}, sl.Get(mockEntry(1)))\n\n\toverwritten = sl.Insert(m2)\n\tassert.Equal(t, common.Comparators{m2}, sl.Get(m2))\n\tassert.Equal(t, common.Comparators{nil}, sl.Get(mockEntry(7)))\n\tassert.Equal(t, uint64(2), sl.Len())\n\tassert.Equal(t, common.Comparators{nil}, overwritten)\n}\n\nfunc TestSimpleOverwrite(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(5)\n\n\tsl := New(uint8(0))\n\n\toverwritten := sl.Insert(m1)\n\tassert.Equal(t, common.Comparators{nil}, overwritten)\n\tassert.Equal(t, uint64(1), sl.Len())\n\n\toverwritten = sl.Insert(m2)\n\tassert.Equal(t, common.Comparators{m1}, overwritten)\n\tassert.Equal(t, uint64(1), sl.Len())\n}\n\nfunc TestInsertOutOfOrder(t *testing.T) {\n\tm1 := newMockEntry(6)\n\tm2 := newMockEntry(5)\n\n\tsl := New(uint8(0))\n\n\toverwritten := sl.Insert(m1, m2)\n\tassert.Equal(t, common.Comparators{nil, nil}, overwritten)\n\n\tassert.Equal(t, common.Comparators{m1, m2}, sl.Get(m1, m2))\n}\n\nfunc TestSimpleDelete(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tsl := New(uint8(0))\n\tsl.Insert(m1)\n\n\tdeleted := sl.Delete(m1)\n\tassert.Equal(t, common.Comparators{m1}, deleted)\n\tassert.Equal(t, uint64(0), sl.Len())\n\tassert.Equal(t, common.Comparators{nil}, sl.Get(m1))\n\n\tdeleted = sl.Delete(m1)\n\tassert.Equal(t, common.Comparators{nil}, deleted)\n}\n\nfunc TestDeleteAll(t *testing.T) {\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(6)\n\tsl := New(uint8(0))\n\tsl.Insert(m1, m2)\n\n\tdeleted := sl.Delete(m1, m2)\n\tassert.Equal(t, common.Comparators{m1, m2}, deleted)\n\tassert.Equal(t, uint64(0), sl.Len())\n\tassert.Equal(t, common.Comparators{nil, nil}, sl.Get(m1, m2))\n}\n\nfunc TestIter(t *testing.T) {\n\tsl := New(uint8(0))\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(10)\n\n\tsl.Insert(m1, m2)\n\n\titer := sl.Iter(mockEntry(0))\n\tassert.Equal(t, common.Comparators{m1, m2}, iter.exhaust())\n\n\titer = sl.Iter(mockEntry(5))\n\tassert.Equal(t, common.Comparators{m1, m2}, iter.exhaust())\n\n\titer = sl.Iter(mockEntry(6))\n\tassert.Equal(t, common.Comparators{m2}, iter.exhaust())\n\n\titer = sl.Iter(mockEntry(10))\n\tassert.Equal(t, common.Comparators{m2}, iter.exhaust())\n\n\titer = sl.Iter(mockEntry(11))\n\tassert.Equal(t, common.Comparators{}, iter.exhaust())\n}\n\nfunc TestIterAtPosition(t *testing.T) {\n\tsl := New(uint8(0))\n\tm1 := newMockEntry(5)\n\tm2 := newMockEntry(10)\n\n\tsl.Insert(m1, m2)\n\n\titer := sl.IterAtPosition(0)\n\tassert.Equal(t, common.Comparators{m1, m2}, iter.exhaust())\n\n\titer = sl.IterAtPosition(1)\n\tassert.Equal(t, common.Comparators{m2}, iter.exhaust())\n\n\titer = sl.IterAtPosition(2)\n\tassert.Equal(t, common.Comparators{}, iter.exhaust())\n}\n\nfunc BenchmarkInsert(b *testing.B) {\n\tnumItems := b.N\n\tsl := New(uint64(0))\n\n\tentries := generateMockEntries(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.Insert(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkGet(b *testing.B) {\n\tnumItems := b.N\n\tsl := New(uint64(0))\n\n\tentries := generateMockEntries(numItems)\n\tsl.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.Get(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkDelete(b *testing.B) {\n\tnumItems := b.N\n\tsl := New(uint64(0))\n\n\tentries := generateMockEntries(numItems)\n\tsl.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.Delete(entries[i])\n\t}\n}\n\nfunc BenchmarkPrepend(b *testing.B) {\n\tnumItems := b.N\n\tsl := New(uint64(0))\n\n\tentries := make(common.Comparators, 0, numItems)\n\tfor i := b.N; i < b.N+numItems; i++ {\n\t\tentries = append(entries, newMockEntry(uint64(i)))\n\t}\n\n\tsl.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.Insert(newMockEntry(uint64(i)))\n\t}\n}\n\nfunc BenchmarkByPosition(b *testing.B) {\n\tnumItems := b.N\n\tsl := New(uint64(0))\n\tentries := generateMockEntries(numItems)\n\tsl.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.ByPosition(uint64(i % numItems))\n\t}\n}\n\nfunc BenchmarkInsertAtPosition(b *testing.B) {\n\tnumItems := b.N\n\tsl := New(uint64(0))\n\tentries := generateRandomMockEntries(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.InsertAtPosition(0, entries[i%numItems])\n\t}\n}\n" }, { "path": "sort/interface.go", "content": "package merge\n\n// Comparators defines a typed list of type Comparator.\ntype Comparators []Comparator\n\n// Less returns a bool indicating if the comparator at index i\n// is less than the comparator at index j.\nfunc (c Comparators) Less(i, j int) bool {\n\treturn c[i].Compare(c[j]) < 0\n}\n\n// Len returns an int indicating the length of this list\n// of comparators.\nfunc (c Comparators) Len() int {\n\treturn len(c)\n}\n\n// Swap swaps the values at positions i and j.\nfunc (c Comparators) Swap(i, j int) {\n\tc[j], c[i] = c[i], c[j]\n}\n\n// Comparator defines items that can be sorted. It contains\n// a single method allowing the compare logic to compare one\n// comparator to another.\ntype Comparator interface {\n\t// Compare will return a value indicating how this comparator\n\t// compares with the provided comparator. A negative number\n\t// indicates this comparator is less than the provided comparator,\n\t// a 0 indicates equality, and a positive number indicates this\n\t// comparator is greater than the provided comparator.\n\tCompare(Comparator) int\n}\n" }, { "path": "sort/sort.go", "content": "package merge\n\nimport (\n\t\"runtime\"\n\t\"sort\"\n\t\"sync\"\n)\n\nfunc sortBucket(comparators Comparators) {\n\tsort.Sort(comparators)\n}\n\nfunc copyChunk(chunk []Comparators) []Comparators {\n\tcp := make([]Comparators, len(chunk))\n\tcopy(cp, chunk)\n\treturn cp\n}\n\n// MultithreadedSortComparators will take a list of comparators\n// and sort it using as many threads as are available. The list\n// is split into buckets for a bucket sort and then recursively\n// merged using SymMerge.\nfunc MultithreadedSortComparators(comparators Comparators) Comparators {\n\ttoBeSorted := make(Comparators, len(comparators))\n\tcopy(toBeSorted, comparators)\n\n\tvar wg sync.WaitGroup\n\n\tnumCPU := int64(runtime.NumCPU())\n\tif numCPU == 1 { // single core machine\n\t\tnumCPU = 2\n\t} else {\n\t\t// otherwise this algo only works with a power of two\n\t\tnumCPU = int64(prevPowerOfTwo(uint64(numCPU)))\n\t}\n\n\tchunks := chunk(toBeSorted, numCPU)\n\twg.Add(len(chunks))\n\tfor i := 0; i < len(chunks); i++ {\n\t\tgo func(i int) {\n\t\t\tsortBucket(chunks[i])\n\t\t\twg.Done()\n\t\t}(i)\n\t}\n\n\twg.Wait()\n\ttodo := make([]Comparators, len(chunks)/2)\n\tfor {\n\t\ttodo = todo[:len(chunks)/2]\n\t\twg.Add(len(chunks) / 2)\n\t\tfor i := 0; i < len(chunks); i += 2 {\n\t\t\tgo func(i int) {\n\t\t\t\ttodo[i/2] = SymMerge(chunks[i], chunks[i+1])\n\t\t\t\twg.Done()\n\t\t\t}(i)\n\t\t}\n\n\t\twg.Wait()\n\n\t\tchunks = copyChunk(todo)\n\t\tif len(chunks) == 1 {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn chunks[0]\n}\n\nfunc chunk(comparators Comparators, numParts int64) []Comparators {\n\tparts := make([]Comparators, numParts)\n\tfor i := int64(0); i < numParts; i++ {\n\t\tparts[i] = comparators[i*int64(len(comparators))/numParts : (i+1)*int64(len(comparators))/numParts]\n\t}\n\treturn parts\n}\n\nfunc prevPowerOfTwo(x uint64) uint64 {\n\tx = x | (x >> 1)\n\tx = x | (x >> 2)\n\tx = x | (x >> 4)\n\tx = x | (x >> 8)\n\tx = x | (x >> 16)\n\tx = x | (x >> 32)\n\treturn x - (x >> 1)\n}\n" }, { "path": "sort/sort_test.go", "content": "package merge\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestMultiThreadedSortEvenNumber(t *testing.T) {\n\tcomparators := constructOrderedMockComparators(10)\n\tcomparators = reverseComparators(comparators)\n\n\tresult := MultithreadedSortComparators(comparators)\n\tcomparators = reverseComparators(comparators)\n\n\tassert.Equal(t, comparators, result)\n}\n\nfunc TestMultiThreadedSortOddNumber(t *testing.T) {\n\tcomparators := constructOrderedMockComparators(9)\n\tcomparators = reverseComparators(comparators)\n\n\tresult := MultithreadedSortComparators(comparators)\n\tcomparators = reverseComparators(comparators)\n\n\tassert.Equal(t, comparators, result)\n}\n\nfunc BenchmarkMultiThreadedSort(b *testing.B) {\n\tnumCells := 100000\n\n\tcomparators := constructOrderedMockComparators(numCells)\n\tcomparators = reverseComparators(comparators)\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMultithreadedSortComparators(comparators)\n\t}\n}\n" }, { "path": "sort/symmerge.go", "content": "package merge\n\nimport (\n\t\"math\"\n\t\"sync\"\n)\n\n// symSearch is like symBinarySearch but operates\n// on two sorted lists instead of a sorted list and an index.\n// It's duplication of code but you buy performance.\nfunc symSearch(u, w Comparators) int {\n\tstart, stop, p := 0, len(u), len(w)-1\n\tfor start < stop {\n\t\tmid := (start + stop) / 2\n\t\tif u[mid].Compare(w[p-mid]) <= 0 {\n\t\t\tstart = mid + 1\n\t\t} else {\n\t\t\tstop = mid\n\t\t}\n\t}\n\n\treturn start\n}\n\n// swap will swap positions of the two lists from index\n// to the end of the list. It expects that these lists\n// are the same size or one different.\nfunc swap(u, w Comparators, index int) {\n\tfor i := index; i < len(u); i++ {\n\t\tu[i], w[i-index] = w[i-index], u[i]\n\t}\n}\n\n// decomposeForSymMerge pulls an active site out of the list\n// of length in size. W becomes the active site for future sym\n// merges and v1, v2 are decomposed and split among the other\n// list to be merged and w.\nfunc decomposeForSymMerge(length int,\n\tcomparators Comparators) (v1 Comparators,\n\tw Comparators, v2 Comparators) {\n\n\tif length >= len(comparators) {\n\t\tpanic(`INCORRECT PARAMS FOR SYM MERGE.`)\n\t}\n\n\toverhang := (len(comparators) - length) / 2\n\tv1 = comparators[:overhang]\n\tw = comparators[overhang : overhang+length]\n\tv2 = comparators[overhang+length:]\n\treturn\n}\n\n// symBinarySearch will perform a binary search between the provided\n// indices and find the index at which a rotation should occur.\nfunc symBinarySearch(u Comparators, start, stop, total int) int {\n\tfor start < stop {\n\t\tmid := (start + stop) / 2\n\t\tif u[mid].Compare(u[total-mid]) <= 0 {\n\t\t\tstart = mid + 1\n\t\t} else {\n\t\t\tstop = mid\n\t\t}\n\t}\n\n\treturn start\n}\n\n// symSwap will perform a rotation or swap between the provided\n// indices. Again, there is duplication here with swap, but\n// we are buying performance.\nfunc symSwap(u Comparators, start1, start2, end int) {\n\tfor i := 0; i < end; i++ {\n\t\tu[start1+i], u[start2+i] = u[start2+i], u[start1+i]\n\t}\n}\n\n// symRotate determines the indices to use in a symSwap and\n// performs the swap.\nfunc symRotate(u Comparators, start1, start2, end int) {\n\ti := start2 - start1\n\tif i == 0 {\n\t\treturn\n\t}\n\n\tj := end - start2\n\tif j == 0 {\n\t\treturn\n\t}\n\n\tif i == j {\n\t\tsymSwap(u, start1, start2, i)\n\t\treturn\n\t}\n\n\tp := start1 + i\n\tfor i != j {\n\t\tif i > j {\n\t\t\tsymSwap(u, p-i, p, j)\n\t\t\ti -= j\n\t\t} else {\n\t\t\tsymSwap(u, p-i, p+j-i, i)\n\t\t\tj -= i\n\t\t}\n\t}\n\tsymSwap(u, p-i, p, i)\n}\n\n// symMerge is the recursive and internal form of SymMerge.\nfunc symMerge(u Comparators, start1, start2, last int) {\n\tif start1 < start2 && start2 < last {\n\t\tmid := (start1 + last) / 2\n\t\tn := mid + start2\n\t\tvar start int\n\t\tif start2 > mid {\n\t\t\tstart = symBinarySearch(u, n-last, mid, n-1)\n\t\t} else {\n\t\t\tstart = symBinarySearch(u, start1, start2, n-1)\n\t\t}\n\t\tend := n - start\n\n\t\tsymRotate(u, start, start2, end)\n\t\tsymMerge(u, start1, start, mid)\n\t\tsymMerge(u, mid, end, last)\n\t}\n}\n\n// SymMerge will perform a symmetrical merge of the two provided\n// lists. It is expected that these lists are pre-sorted. Failure\n// to do so will result in undefined behavior. This function does\n// make use of goroutines, so multithreading can aid merge time.\n// This makes M*log(N/M+1) comparisons where M is the length\n// of the shorter list and N is the length of the longer list.\nfunc SymMerge(u, w Comparators) Comparators {\n\tlenU, lenW := len(u), len(w)\n\tif lenU == 0 {\n\t\treturn w\n\t}\n\n\tif lenW == 0 {\n\t\treturn u\n\t}\n\n\tdiff := lenU - lenW\n\tif math.Abs(float64(diff)) > 1 {\n\t\tu1, w1, u2, w2 := prepareForSymMerge(u, w)\n\n\t\tlenU1 := len(u1)\n\t\tlenU2 := len(u2)\n\t\tu = append(u1, w1...)\n\t\tw = append(u2, w2...)\n\t\tvar wg sync.WaitGroup\n\t\twg.Add(2)\n\t\tgo func() {\n\t\t\tsymMerge(u, 0, lenU1, len(u))\n\t\t\twg.Done()\n\t\t}()\n\t\tgo func() {\n\t\t\tsymMerge(w, 0, lenU2, len(w))\n\t\t\twg.Done()\n\t\t}()\n\n\t\twg.Wait()\n\t\tu = append(u, w...)\n\t\treturn u\n\t}\n\n\tu = append(u, w...)\n\tsymMerge(u, 0, lenU, len(u))\n\treturn u\n}\n\n// prepareForSymMerge performs a symmetrical decomposition on two\n// lists of different sizes. It breaks apart the longer list into\n// an active site (equal to the size of the shorter list) and performs\n// a symmetrical rotation with the active site and the shorter list.\n// The two stubs are then split between the active site and shorter list\n// ensuring two equally sized lists where every value in u' is less\n// than w'.\nfunc prepareForSymMerge(u, w Comparators) (u1, w1, u2, w2 Comparators) {\n\tif u.Len() > w.Len() {\n\t\tu, w = w, u\n\t}\n\tv1, w, v2 := decomposeForSymMerge(len(u), w)\n\n\ti := symSearch(u, w)\n\n\tu1 = make(Comparators, i)\n\tcopy(u1, u[:i])\n\tw1 = append(v1, w[:len(w)-i]...)\n\n\tu2 = make(Comparators, len(u)-i)\n\tcopy(u2, u[i:])\n\n\tw2 = append(w[len(w)-i:], v2...)\n\treturn\n}\n" }, { "path": "sort/symmerge_test.go", "content": "package merge\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\ntype mockComparator int\n\nfunc (mc mockComparator) Compare(other Comparator) int {\n\tif mc == other.(mockComparator) {\n\t\treturn 0\n\t}\n\n\tif mc > other.(mockComparator) {\n\t\treturn 1\n\t}\n\n\treturn -1\n}\n\nfunc constructMockComparators(values ...int) Comparators {\n\tcomparators := make(Comparators, 0, len(values))\n\tfor _, v := range values {\n\t\tcomparators = append(comparators, mockComparator(v))\n\t}\n\n\treturn comparators\n}\n\nfunc constructOrderedMockComparators(upTo int) Comparators {\n\tcomparators := make(Comparators, 0, upTo)\n\tfor i := 0; i < upTo; i++ {\n\t\tcomparators = append(comparators, mockComparator(i))\n\t}\n\n\treturn comparators\n}\n\nfunc reverseComparators(comparators Comparators) Comparators {\n\tfor i := 0; i < len(comparators); i++ {\n\t\tli := len(comparators) - i - 1\n\t\tcomparators[i], comparators[li] = comparators[li], comparators[i]\n\t}\n\treturn comparators\n}\n\nfunc TestDecomposeForSymMergeOddNumber(t *testing.T) {\n\tcomparators := constructOrderedMockComparators(7)\n\n\tv1, w, v2 := decomposeForSymMerge(3, comparators)\n\tassert.Equal(t, comparators[:2], v1)\n\tassert.Equal(t, comparators[2:5], w)\n\tassert.Equal(t, comparators[5:], v2)\n}\n\nfunc TestDecomposeForSymMergeEvenNumber(t *testing.T) {\n\tcomparators := constructOrderedMockComparators(8)\n\tv1, w, v2 := decomposeForSymMerge(5, comparators)\n\n\tassert.Equal(t, comparators[:1], v1)\n\tassert.Equal(t, comparators[1:6], w)\n\tassert.Equal(t, comparators[6:], v2)\n}\n\nfunc TestNearCompleteDecomposeForSymMerge(t *testing.T) {\n\tcomparators := constructOrderedMockComparators(8)\n\tv1, w, v2 := decomposeForSymMerge(7, comparators)\n\n\tassert.Len(t, v1, 0)\n\tassert.Equal(t, comparators[:7], w)\n\tassert.Equal(t, comparators[7:], v2)\n}\n\nfunc TestDecomposePanicsWithWrongLength(t *testing.T) {\n\tcomparators := constructOrderedMockComparators(8)\n\tassert.Panics(t, func() {\n\t\tdecomposeForSymMerge(8, comparators)\n\t})\n}\n\nfunc TestSymSearch(t *testing.T) {\n\tu := constructMockComparators(1, 5, 7, 9)\n\tw := constructMockComparators(1, 3, 9, 10)\n\n\tresult := symSearch(u, w)\n\tassert.Equal(t, 2, result)\n\n\tu = constructMockComparators(1, 5, 7)\n\tw = constructMockComparators(1, 3, 9)\n\n\tresult = symSearch(u, w)\n\tassert.Equal(t, 1, result)\n}\n\nfunc TestSwap(t *testing.T) {\n\tu := constructMockComparators(1, 5, 7, 9)\n\tw := constructMockComparators(2, 8, 11, 13)\n\tu1 := constructMockComparators(1, 5, 2, 8)\n\tw1 := constructMockComparators(7, 9, 11, 13)\n\n\tswap(u, w, 2)\n\n\tassert.Equal(t, u1, u)\n\tassert.Equal(t, w1, w)\n}\n\nfunc TestSymMergeSmallLists(t *testing.T) {\n\tu := constructMockComparators(1, 5)\n\tw := constructMockComparators(2, 8)\n\texpected := constructMockComparators(1, 2, 5, 8)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestSymMergeAlreadySorted(t *testing.T) {\n\tu := constructMockComparators(1, 5)\n\tw := constructMockComparators(6, 7)\n\texpected := constructMockComparators(1, 5, 6, 7)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestSymMergeAlreadySortedReverseOrder(t *testing.T) {\n\tu := constructMockComparators(6, 7)\n\tw := constructMockComparators(1, 5)\n\texpected := constructMockComparators(1, 5, 6, 7)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestSymMergeUnevenLists(t *testing.T) {\n\tu := constructMockComparators(1, 3, 7)\n\tw := constructMockComparators(2, 4)\n\texpected := constructMockComparators(1, 2, 3, 4, 7)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestSymMergeUnevenListsWrongOrder(t *testing.T) {\n\tu := constructMockComparators(2, 4)\n\tw := constructMockComparators(1, 3, 7)\n\texpected := constructMockComparators(1, 2, 3, 4, 7)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestMergeVeryUnevenLists(t *testing.T) {\n\tu := constructMockComparators(1, 3, 7, 12, 15)\n\tw := constructMockComparators(2, 4)\n\texpected := constructMockComparators(1, 2, 3, 4, 7, 12, 15)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestMergeVeryUnevenListsWrongOrder(t *testing.T) {\n\tu := constructMockComparators(2, 4)\n\tw := constructMockComparators(1, 3, 7, 12, 15)\n\texpected := constructMockComparators(1, 2, 3, 4, 7, 12, 15)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestMergeVeryUnevenListsAlreadySorted(t *testing.T) {\n\tu := constructMockComparators(2, 4)\n\tw := constructMockComparators(5, 7, 9, 10, 11, 12)\n\texpected := constructMockComparators(2, 4, 5, 7, 9, 10, 11, 12)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestMergeVeryUnevenListsAlreadySortedWrongOrder(t *testing.T) {\n\tw := constructMockComparators(2, 4)\n\tu := constructMockComparators(5, 7, 9, 10, 11, 12)\n\texpected := constructMockComparators(2, 4, 5, 7, 9, 10, 11, 12)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestMergeVeryUnevenListIsSubset(t *testing.T) {\n\tu := constructMockComparators(2, 4)\n\tw := constructMockComparators(1, 3, 5, 7, 9)\n\texpected := constructMockComparators(1, 2, 3, 4, 5, 7, 9)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestMergeVeryUnevenListIsSubsetReverseOrder(t *testing.T) {\n\tw := constructMockComparators(2, 4)\n\tu := constructMockComparators(1, 3, 5, 7, 9)\n\texpected := constructMockComparators(1, 2, 3, 4, 5, 7, 9)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestMergeUnevenOneListIsOne(t *testing.T) {\n\tu := constructMockComparators(1)\n\tw := constructMockComparators(0, 3, 5, 7, 9)\n\texpected := constructMockComparators(0, 1, 3, 5, 7, 9)\n\n\tu = SymMerge(u, w)\n\tassert.Equal(t, expected, u)\n}\n\nfunc TestMergeEmptyList(t *testing.T) {\n\tu := constructMockComparators(1, 3, 5)\n\texpected := constructMockComparators(1, 3, 5)\n\n\tu = SymMerge(u, nil)\n\tassert.Equal(t, expected, u)\n}\n" }, { "path": "threadsafe/err/error.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage err implements a threadsafe error interface. In my places,\nI found myself needing a lock to protect writing to a common error interface\nfrom multiple go routines (channels are great but slow). This just makes\nthat process more convenient.\n*/\npackage err\n\nimport \"sync\"\n\n// Error is a struct that holds an error and allows this error\n// to be set and retrieved in a threadsafe manner.\ntype Error struct {\n\tlock sync.RWMutex\n\terr error\n}\n\n// Set will set the error of this structure to the provided\n// value.\nfunc (e *Error) Set(err error) {\n\te.lock.Lock()\n\tdefer e.lock.Unlock()\n\n\te.err = err\n}\n\n// Get will return any error associated with this structure.\nfunc (e *Error) Get() error {\n\te.lock.RLock()\n\tdefer e.lock.RUnlock()\n\n\treturn e.err\n}\n\n// New is a constructor to generate a new error object\n// that can be set and retrieved in a threadsafe manner.\nfunc New() *Error {\n\treturn &Error{}\n}\n" }, { "path": "threadsafe/err/error_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage err\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestGetSetError(t *testing.T) {\n\te := New()\n\tassert.Nil(t, e.Get())\n\n\terr := fmt.Errorf(`test`)\n\te.Set(err)\n\n\tassert.Equal(t, err, e.Get())\n}\n" }, { "path": "tree/avl/avl.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage avl includes an immutable AVL tree.\n\nAVL trees can be used as the foundation for many functional data types.\nCombined with a B+ tree, you can make an immutable index which serves as the\nbackbone for many different kinds of key/value stores.\n\nTime complexities:\nSpace: O(n)\nInsert: O(log n)\nDelete: O(log n)\nGet: O(log n)\n\nThe immutable version of the AVL tree is obviously going to be slower than\nthe mutable version but should offer higher read availability.\n*/\npackage avl\n\nimport \"math\"\n\n// Immutable represents an immutable AVL tree. This is achieved\n// by branch copying.\ntype Immutable struct {\n\troot *node\n\tnumber uint64\n\tdummy node // helper for inserts.\n}\n\n// copy returns a copy of this immutable tree with a copy\n// of the root and a new dummy helper for the insertion operation.\nfunc (immutable *Immutable) copy() *Immutable {\n\tvar root *node\n\tif immutable.root != nil {\n\t\troot = immutable.root.copy()\n\t}\n\tcp := &Immutable{\n\t\troot: root,\n\t\tnumber: immutable.number,\n\t\tdummy: *newNode(nil),\n\t}\n\treturn cp\n}\n\nfunc (immutable *Immutable) resetDummy() {\n\timmutable.dummy.children[0], immutable.dummy.children[1] = nil, nil\n\timmutable.dummy.balance = 0\n}\n\nfunc (immutable *Immutable) init() {\n\timmutable.dummy = node{\n\t\tchildren: [2]*node{},\n\t}\n}\n\nfunc (immutable *Immutable) get(entry Entry) Entry {\n\tn := immutable.root\n\tvar result int\n\tfor n != nil {\n\t\tswitch result = n.entry.Compare(entry); {\n\t\tcase result == 0:\n\t\t\treturn n.entry\n\t\tcase result > 0:\n\t\t\tn = n.children[0]\n\t\tcase result < 0:\n\t\t\tn = n.children[1]\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Get will get the provided Entries from the tree. If no matching\n// Entry is found, a nil is returned in its place.\nfunc (immutable *Immutable) Get(entries ...Entry) Entries {\n\tresult := make(Entries, 0, len(entries))\n\tfor _, e := range entries {\n\t\tresult = append(result, immutable.get(e))\n\t}\n\n\treturn result\n}\n\n// Len returns the number of items in this immutable.\nfunc (immutable *Immutable) Len() uint64 {\n\treturn immutable.number\n}\n\nfunc (immutable *Immutable) insert(entry Entry) Entry {\n\t// TODO: check cache to see if a node has already been copied.\n\tif immutable.root == nil {\n\t\timmutable.root = newNode(entry)\n\t\timmutable.number++\n\t\treturn nil\n\t}\n\n\timmutable.resetDummy()\n\tvar (\n\t\tdummy = immutable.dummy\n\t\tp, s, q *node\n\t\tdir, normalized int\n\t\thelper = &dummy\n\t)\n\n\t// set this AFTER clearing dummy\n\thelper.children[1] = immutable.root\n\t// we'll go ahead and copy on the way down as we'll need to branch\n\t// copy no matter what.\n\tfor s, p = helper.children[1], helper.children[1]; ; {\n\t\tdir = p.entry.Compare(entry)\n\n\t\tnormalized = normalizeComparison(dir)\n\t\tif dir > 0 { // go left\n\t\t\tif p.children[0] != nil {\n\t\t\t\tq = p.children[0].copy()\n\t\t\t\tp.children[0] = q\n\t\t\t} else {\n\t\t\t\tq = nil\n\t\t\t}\n\t\t} else if dir < 0 { // go right\n\t\t\tif p.children[1] != nil {\n\t\t\t\tq = p.children[1].copy()\n\t\t\t\tp.children[1] = q\n\t\t\t} else {\n\t\t\t\tq = nil\n\t\t\t}\n\t\t} else { // equality\n\t\t\toldEntry := p.entry\n\t\t\tp.entry = entry\n\t\t\treturn oldEntry\n\t\t}\n\t\tif q == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tif q.balance != 0 {\n\t\t\thelper = p\n\t\t\ts = q\n\t\t}\n\t\tp = q\n\t}\n\n\timmutable.number++\n\tq = newNode(entry)\n\tp.children[normalized] = q\n\n\timmutable.root = dummy.children[1]\n\tfor p = s; p != q; p = p.children[normalized] {\n\t\tnormalized = normalizeComparison(p.entry.Compare(entry))\n\t\tif normalized == 0 {\n\t\t\tp.balance--\n\t\t} else {\n\t\t\tp.balance++\n\t\t}\n\t}\n\n\tq = s\n\n\tif math.Abs(float64(s.balance)) > 1 {\n\t\tnormalized = normalizeComparison(s.entry.Compare(entry))\n\t\ts = insertBalance(s, normalized)\n\t}\n\n\tif q == dummy.children[1] {\n\t\timmutable.root = s\n\t} else {\n\t\thelper.children[intFromBool(helper.children[1] == q)] = s\n\t}\n\treturn nil\n}\n\n// Insert will add the provided entries into the tree and return the new\n// state. Also returned is a list of Entries that were overwritten. If\n// nothing was overwritten for an Entry, a nil is returned in its place.\nfunc (immutable *Immutable) Insert(entries ...Entry) (*Immutable, Entries) {\n\tif len(entries) == 0 {\n\t\treturn immutable, Entries{}\n\t}\n\n\toverwritten := make(Entries, 0, len(entries))\n\tcp := immutable.copy()\n\tfor _, e := range entries {\n\t\toverwritten = append(overwritten, cp.insert(e))\n\t}\n\n\treturn cp, overwritten\n}\n\nfunc (immutable *Immutable) delete(entry Entry) Entry {\n\t// TODO: reuse cache and dirs, check cache to see if nodes\n\t// really need to be copied.\n\tif immutable.root == nil { // easy case, nothing to remove\n\t\treturn nil\n\t}\n\n\tvar (\n\t\t// we are going to make a list here representing our stack.\n\t\t// This means we don't have to copy if a value wasn't found.\n\t\tcache = make(nodes, 64)\n\t\tit, p, q *node\n\t\ttop, done, dir, normalized int\n\t\tdirs = make([]int, 64)\n\t\toldEntry Entry\n\t)\n\n\tit = immutable.root\n\n\tfor {\n\t\tif it == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\tdir = it.entry.Compare(entry)\n\t\tif dir == 0 {\n\t\t\tbreak\n\t\t}\n\t\tnormalized = normalizeComparison(dir)\n\t\tdirs[top] = normalized\n\t\tcache[top] = it\n\t\ttop++\n\t\tit = it.children[normalized]\n\t}\n\timmutable.number--\n\toldEntry = it.entry // we need to return this\n\n\t// we need to make a branch copy now\n\tfor i := 0; i < top; i++ { // first item will be root\n\t\tp = cache[i]\n\t\tif p.children[dirs[i]] != nil {\n\t\t\tq = p.children[dirs[i]].copy()\n\t\t\tp.children[dirs[i]] = q\n\t\t\tif i != top-1 {\n\t\t\t\tcache[i+1] = q\n\t\t\t}\n\t\t}\n\t}\n\tit = it.copy() // the node we found needs to be copied\n\n\toldTop := top\n\tif it.children[0] == nil || it.children[1] == nil { // need to set children on parent, splicing out\n\t\tdir = intFromBool(it.children[0] == nil)\n\t\tif top != 0 {\n\t\t\tcache[top-1].children[dirs[top-1]] = it.children[dir]\n\t\t} else {\n\t\t\timmutable.root = it.children[dir]\n\t\t}\n\t} else { // climb up and set heirs\n\t\their := it.children[1]\n\t\tdirs[top] = 1\n\t\tcache[top] = it\n\t\ttop++\n\n\t\tfor heir.children[0] != nil {\n\t\t\tdirs[top] = 0\n\t\t\tcache[top] = heir\n\t\t\ttop++\n\t\t\their = heir.children[0]\n\t\t}\n\n\t\tit.entry = heir.entry\n\t\tif oldTop != 0 {\n\t\t\tcache[oldTop-1].children[dirs[oldTop-1]] = it\n\t\t} else {\n\t\t\timmutable.root = it\n\t\t}\n\t\tcache[top-1].children[intFromBool(cache[top-1] == it)] = heir.children[1]\n\t}\n\n\tfor top-1 >= 0 && done == 0 {\n\t\ttop--\n\t\t// set bounded balance\n\t\tif dirs[top] != 0 {\n\t\t\tcache[top].balance--\n\t\t} else {\n\t\t\tcache[top].balance++\n\t\t}\n\n\t\tif math.Abs(float64(cache[top].balance)) == 1 {\n\t\t\tbreak\n\t\t} else if math.Abs(float64(cache[top].balance)) > 1 {\n\t\t\t// any rotations done here\n\t\t\tcache[top] = removeBalance(cache[top], dirs[top], &done)\n\n\t\t\tif top != 0 {\n\t\t\t\tcache[top-1].children[dirs[top-1]] = cache[top]\n\t\t\t} else {\n\t\t\t\timmutable.root = cache[0]\n\t\t\t}\n\t\t}\n\n\t}\n\n\treturn oldEntry\n}\n\n// Delete will remove the provided entries from this AVL tree and\n// return a new tree and any entries removed. If an entry could not\n// be found, nil is returned in its place.\nfunc (immutable *Immutable) Delete(entries ...Entry) (*Immutable, Entries) {\n\tif len(entries) == 0 {\n\t\treturn immutable, Entries{}\n\t}\n\n\tdeleted := make(Entries, 0, len(entries))\n\tcp := immutable.copy()\n\tfor _, e := range entries {\n\t\tdeleted = append(deleted, cp.delete(e))\n\t}\n\n\treturn cp, deleted\n}\n\nfunc insertBalance(root *node, dir int) *node {\n\tn := root.children[dir]\n\tvar bal int8\n\tif dir == 0 {\n\t\tbal = -1\n\t} else {\n\t\tbal = 1\n\t}\n\n\tif n.balance == bal {\n\t\troot.balance, n.balance = 0, 0\n\t\troot = rotate(root, takeOpposite(dir))\n\t} else {\n\t\tadjustBalance(root, dir, int(bal))\n\t\troot = doubleRotate(root, takeOpposite(dir))\n\t}\n\n\treturn root\n}\n\nfunc removeBalance(root *node, dir int, done *int) *node {\n\tn := root.children[takeOpposite(dir)].copy()\n\troot.children[takeOpposite(dir)] = n\n\tvar bal int8\n\tif dir == 0 {\n\t\tbal = -1\n\t} else {\n\t\tbal = 1\n\t}\n\n\tif n.balance == -bal {\n\t\troot.balance, n.balance = 0, 0\n\t\troot = rotate(root, dir)\n\t} else if n.balance == bal {\n\t\tadjustBalance(root, takeOpposite(dir), int(-bal))\n\t\troot = doubleRotate(root, dir)\n\t} else {\n\t\troot.balance = -bal\n\t\tn.balance = bal\n\t\troot = rotate(root, dir)\n\t\t*done = 1\n\t}\n\n\treturn root\n}\n\nfunc intFromBool(value bool) int {\n\tif value {\n\t\treturn 1\n\t}\n\n\treturn 0\n}\n\nfunc takeOpposite(value int) int {\n\treturn 1 - value\n}\n\nfunc adjustBalance(root *node, dir, bal int) {\n\tn := root.children[dir]\n\tnn := n.children[takeOpposite(dir)]\n\n\tif nn.balance == 0 {\n\t\troot.balance, n.balance = 0, 0\n\t} else if int(nn.balance) == bal {\n\t\troot.balance = int8(-bal)\n\t\tn.balance = 0\n\t} else {\n\t\troot.balance = 0\n\t\tn.balance = int8(bal)\n\t}\n\tnn.balance = 0\n}\n\nfunc rotate(parent *node, dir int) *node {\n\totherDir := takeOpposite(dir)\n\n\tchild := parent.children[otherDir]\n\tparent.children[otherDir] = child.children[dir]\n\tchild.children[dir] = parent\n\n\treturn child\n}\n\nfunc doubleRotate(parent *node, dir int) *node {\n\totherDir := takeOpposite(dir)\n\n\tparent.children[otherDir] = rotate(parent.children[otherDir], otherDir)\n\treturn rotate(parent, dir)\n}\n\n// normalizeComparison converts the value returned from Entry.Compare\n// into a direction, ie, left or right, 0 or 1.\nfunc normalizeComparison(i int) int {\n\tif i < 0 {\n\t\treturn 1\n\t}\n\n\tif i > 0 {\n\t\treturn 0\n\t}\n\n\treturn -1\n}\n\n// NewImmutable allocates, initializes, and returns a new immutable\n// AVL tree.\nfunc NewImmutable() *Immutable {\n\timmutable := &Immutable{}\n\timmutable.init()\n\treturn immutable\n}\n" }, { "path": "tree/avl/avl_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage avl\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc generateMockEntries(num int) Entries {\n\tentries := make(Entries, 0, num)\n\tfor i := 0; i < num; i++ {\n\t\tentries = append(entries, mockEntry(i))\n\t}\n\n\treturn entries\n}\n\nfunc TestAVLSimpleInsert(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(5)\n\tm2 := mockEntry(10)\n\n\ti2, overwritten := i1.Insert(m1, m2)\n\tassert.Equal(t, Entries{nil, nil}, overwritten)\n\tassert.Equal(t, uint64(2), i2.Len())\n\tassert.Equal(t, uint64(0), i1.Len())\n\tassert.Equal(t, Entries{nil, nil}, i1.Get(m1, m2))\n\tassert.Equal(t, Entries{m1, m2}, i2.Get(m1, m2))\n\n\tm3 := mockEntry(1)\n\n\ti3, overwritten := i2.Insert(m3)\n\tassert.Equal(t, Entries{nil}, overwritten)\n\tassert.Equal(t, uint64(3), i3.Len())\n\tassert.Equal(t, uint64(2), i2.Len())\n\tassert.Equal(t, uint64(0), i1.Len())\n\tassert.Equal(t, Entries{m1, m2, m3}, i3.Get(m1, m2, m3))\n}\n\nfunc TestAVLInsertRightLeaning(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(1)\n\tm2 := mockEntry(5)\n\tm3 := mockEntry(10)\n\n\ti2, overwritten := i1.Insert(m1, m2, m3)\n\tassert.Equal(t, Entries{nil, nil, nil}, overwritten)\n\tassert.Equal(t, uint64(0), i1.Len())\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, Entries{m1, m2, m3}, i2.Get(m1, m2, m3))\n\tassert.Equal(t, Entries{nil, nil, nil}, i1.Get(m1, m2, m3))\n\n\tm4 := mockEntry(15)\n\tm5 := mockEntry(20)\n\n\ti3, overwritten := i2.Insert(m4, m5)\n\tassert.Equal(t, Entries{nil, nil}, overwritten)\n\tassert.Equal(t, uint64(5), i3.Len())\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, Entries{nil, nil}, i2.Get(m4, m5))\n\tassert.Equal(t, Entries{m4, m5}, i3.Get(m4, m5))\n}\n\nfunc TestAVLInsertRightLeaningDoubleRotation(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(1)\n\tm2 := mockEntry(10)\n\tm3 := mockEntry(5)\n\n\ti2, overwritten := i1.Insert(m1, m2, m3)\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, Entries{nil, nil, nil}, overwritten)\n\tassert.Equal(t, Entries{nil, nil, nil}, i1.Get(m1, m2, m3))\n\tassert.Equal(t, Entries{m1, m2, m3}, i2.Get(m1, m2, m3))\n\n\tm4 := mockEntry(20)\n\tm5 := mockEntry(15)\n\n\ti3, overwritten := i2.Insert(m4, m5)\n\tassert.Equal(t, Entries{nil, nil}, overwritten)\n\tassert.Equal(t, uint64(5), i3.Len())\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, Entries{nil, nil}, i2.Get(m4, m5))\n\tassert.Equal(t, Entries{m4, m5}, i3.Get(m4, m5))\n}\n\nfunc TestAVLInsertLeftLeaning(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(20)\n\tm2 := mockEntry(15)\n\tm3 := mockEntry(10)\n\n\ti2, overwritten := i1.Insert(m1, m2, m3)\n\tassert.Equal(t, Entries{nil, nil, nil}, overwritten)\n\tassert.Equal(t, uint64(0), i1.Len())\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, Entries{nil, nil, nil}, i1.Get(m1, m2, m3))\n\tassert.Equal(t, Entries{m1, m2, m3}, i2.Get(m1, m2, m3))\n\n\tm4 := mockEntry(5)\n\tm5 := mockEntry(1)\n\n\ti3, overwritten := i2.Insert(m4, m5)\n\tassert.Equal(t, Entries{nil, nil}, overwritten)\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, uint64(5), i3.Len())\n\tassert.Equal(t, Entries{nil, nil}, i2.Get(m4, m5))\n\tassert.Equal(t, Entries{m4, m5}, i3.Get(m4, m5))\n}\n\nfunc TestAVLInsertLeftLeaningDoubleRotation(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(20)\n\tm2 := mockEntry(10)\n\tm3 := mockEntry(15)\n\n\ti2, overwritten := i1.Insert(m1, m2, m3)\n\tassert.Equal(t, Entries{nil, nil, nil}, overwritten)\n\tassert.Equal(t, uint64(0), i1.Len())\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, Entries{nil, nil, nil}, i1.Get(m1, m2, m3))\n\tassert.Equal(t, Entries{m1, m2, m3}, i2.Get(m1, m2, m3))\n\n\tm4 := mockEntry(1)\n\tm5 := mockEntry(5)\n\n\ti3, overwritten := i2.Insert(m4, m5)\n\tassert.Equal(t, Entries{nil, nil}, overwritten)\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, uint64(5), i3.Len())\n\tassert.Equal(t, Entries{nil, nil}, i2.Get(m4, m5))\n\tassert.Equal(t, Entries{m4, m5}, i3.Get(m4, m5))\n\tassert.Equal(t, Entries{m1, m2, m3}, i3.Get(m1, m2, m3))\n}\n\nfunc TestAVLInsertOverwrite(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(20)\n\tm2 := mockEntry(10)\n\tm3 := mockEntry(15)\n\n\ti2, _ := i1.Insert(m1, m2, m3)\n\tm4 := mockEntry(15)\n\n\ti3, overwritten := i2.Insert(m4)\n\tassert.Equal(t, Entries{m3}, overwritten)\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, uint64(3), i3.Len())\n\tassert.Equal(t, Entries{m4}, i3.Get(m4))\n\tassert.Equal(t, Entries{m3}, i2.Get(m3))\n}\n\nfunc TestAVLSimpleDelete(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(10)\n\tm2 := mockEntry(15)\n\tm3 := mockEntry(20)\n\n\ti2, _ := i1.Insert(m1, m2, m3)\n\n\ti3, deleted := i2.Delete(m3)\n\tassert.Equal(t, Entries{m3}, deleted)\n\tassert.Equal(t, uint64(3), i2.Len())\n\tassert.Equal(t, uint64(2), i3.Len())\n\tassert.Equal(t, Entries{m1, m2, m3}, i2.Get(m1, m2, m3))\n\tassert.Equal(t, Entries{m1, m2, nil}, i3.Get(m1, m2, m3))\n\n\ti4, deleted := i3.Delete(m2)\n\tassert.Equal(t, Entries{m2}, deleted)\n\tassert.Equal(t, uint64(2), i3.Len())\n\tassert.Equal(t, uint64(1), i4.Len())\n\tassert.Equal(t, Entries{m1, m2, nil}, i3.Get(m1, m2, m3))\n\tassert.Equal(t, Entries{m1, nil, nil}, i4.Get(m1, m2, m3))\n\n\ti5, deleted := i4.Delete(m1)\n\tassert.Equal(t, Entries{m1}, deleted)\n\tassert.Equal(t, uint64(0), i5.Len())\n\tassert.Equal(t, uint64(1), i4.Len())\n\tassert.Equal(t, Entries{m1, nil, nil}, i4.Get(m1, m2, m3))\n\tassert.Equal(t, Entries{nil, nil, nil}, i5.Get(m1, m2, m3))\n}\n\nfunc TestAVLDeleteWithRotation(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(1)\n\tm2 := mockEntry(5)\n\tm3 := mockEntry(10)\n\tm4 := mockEntry(15)\n\tm5 := mockEntry(20)\n\n\ti2, _ := i1.Insert(m1, m2, m3, m4, m5)\n\tassert.Equal(t, uint64(5), i2.Len())\n\n\ti3, deleted := i2.Delete(m1)\n\tassert.Equal(t, uint64(4), i3.Len())\n\tassert.Equal(t, Entries{m1}, deleted)\n\tassert.Equal(t, Entries{m1, m2, m3, m4, m5}, i2.Get(m1, m2, m3, m4, m5))\n\tassert.Equal(t, Entries{nil, m2, m3, m4, m5}, i3.Get(m1, m2, m3, m4, m5))\n}\n\nfunc TestAVLDeleteWithDoubleRotation(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(1)\n\tm2 := mockEntry(5)\n\tm3 := mockEntry(10)\n\tm4 := mockEntry(15)\n\n\ti2, _ := i1.Insert(m2, m1, m3, m4)\n\tassert.Equal(t, uint64(4), i2.Len())\n\n\ti3, deleted := i2.Delete(m1)\n\tassert.Equal(t, Entries{m1}, deleted)\n\tassert.Equal(t, uint64(3), i3.Len())\n\tassert.Equal(t, Entries{m1, m2, m3, m4}, i2.Get(m1, m2, m3, m4))\n\tassert.Equal(t, Entries{nil, m2, m3, m4}, i3.Get(m1, m2, m3, m4))\n}\n\nfunc TestAVLDeleteAll(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(1)\n\tm2 := mockEntry(5)\n\tm3 := mockEntry(10)\n\tm4 := mockEntry(15)\n\n\ti2, _ := i1.Insert(m2, m1, m3, m4)\n\tassert.Equal(t, uint64(4), i2.Len())\n\n\ti3, deleted := i2.Delete(m1, m2, m3, m4)\n\tassert.Equal(t, Entries{m1, m2, m3, m4}, deleted)\n\tassert.Equal(t, uint64(0), i3.Len())\n\tassert.Equal(t, Entries{nil, nil, nil, nil}, i3.Get(m1, m2, m3, m4))\n\tassert.Equal(t, Entries{m1, m2, m3, m4}, i2.Get(m1, m2, m3, m4))\n}\n\nfunc TestAVLDeleteNotLeaf(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(1)\n\tm2 := mockEntry(5)\n\tm3 := mockEntry(10)\n\tm4 := mockEntry(15)\n\n\ti2, _ := i1.Insert(m2, m1, m3, m4)\n\ti3, deleted := i2.Delete(m3)\n\tassert.Equal(t, Entries{m3}, deleted)\n\tassert.Equal(t, uint64(3), i3.Len())\n}\n\nfunc TestAVLBulkDeleteAll(t *testing.T) {\n\ti1 := NewImmutable()\n\tentries := generateMockEntries(5)\n\ti2, _ := i1.Insert(entries...)\n\n\ti3, deleted := i2.Delete(entries...)\n\tassert.Equal(t, entries, deleted)\n\tassert.Equal(t, uint64(0), i3.Len())\n\n\ti3, deleted = i2.Delete(entries...)\n\tassert.Equal(t, entries, deleted)\n\tassert.Equal(t, uint64(0), i3.Len())\n}\n\nfunc TestAVLDeleteReplay(t *testing.T) {\n\ti1 := NewImmutable()\n\tm1 := mockEntry(1)\n\tm2 := mockEntry(5)\n\tm3 := mockEntry(10)\n\tm4 := mockEntry(15)\n\n\ti2, _ := i1.Insert(m2, m1, m3, m4)\n\n\ti3, deleted := i2.Delete(m3)\n\tassert.Equal(t, uint64(3), i3.Len())\n\tassert.Equal(t, Entries{m3}, deleted)\n\tassert.Equal(t, uint64(4), i2.Len())\n\n\ti3, deleted = i2.Delete(m3)\n\tassert.Equal(t, uint64(3), i3.Len())\n\tassert.Equal(t, Entries{m3}, deleted)\n\tassert.Equal(t, uint64(4), i2.Len())\n}\n\nfunc TestAVLFails(t *testing.T) {\n\tkeys := []mockEntry{\n\t\tmockEntry(0),\n\t\tmockEntry(1),\n\t\tmockEntry(3),\n\t\tmockEntry(4),\n\t\tmockEntry(5),\n\t\tmockEntry(6),\n\t\tmockEntry(7),\n\t\tmockEntry(2),\n\t}\n\ti1 := NewImmutable()\n\tfor _, k := range keys {\n\t\ti1, _ = i1.Insert(k)\n\t}\n\n\tfor _, k := range keys {\n\t\tvar deleted Entries\n\t\ti1, deleted = i1.Delete(k)\n\t\tassert.Equal(t, Entries{k}, deleted)\n\t}\n}\n\nfunc BenchmarkImmutableInsert(b *testing.B) {\n\tnumItems := b.N\n\tsl := NewImmutable()\n\n\tentries := generateMockEntries(numItems)\n\tsl, _ = sl.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl, _ = sl.Insert(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkImmutableGet(b *testing.B) {\n\tnumItems := b.N\n\tsl := NewImmutable()\n\n\tentries := generateMockEntries(numItems)\n\tsl, _ = sl.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.Get(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkImmutableBulkInsert(b *testing.B) {\n\tnumItems := b.N\n\tsl := NewImmutable()\n\n\tentries := generateMockEntries(numItems)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.Insert(entries...)\n\t}\n}\n\nfunc BenchmarkImmutableDelete(b *testing.B) {\n\tnumItems := b.N\n\tsl := NewImmutable()\n\n\tentries := generateMockEntries(numItems)\n\tsl, _ = sl.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl, _ = sl.Delete(entries[i%numItems])\n\t}\n}\n\nfunc BenchmarkImmutableBulkDelete(b *testing.B) {\n\tnumItems := b.N\n\tsl := NewImmutable()\n\n\tentries := generateMockEntries(numItems)\n\tsl, _ = sl.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tsl.Delete(entries...)\n\t}\n}\n" }, { "path": "tree/avl/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage avl\n\n// Entries is a list of type Entry.\ntype Entries []Entry\n\n// Entry represents all items that can be placed into the AVL tree.\n// They must implement a Compare method that can be used to determine\n// the Entry's correct place in the tree. Any object can implement\n// Compare.\ntype Entry interface {\n\t// Compare should return a value indicating the relationship\n\t// of this Entry to the provided Entry. A -1 means this entry\n\t// is less than, 0 means equality, and 1 means greater than.\n\tCompare(Entry) int\n}\n" }, { "path": "tree/avl/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage avl\n\ntype mockEntry int\n\nfunc (me mockEntry) Compare(other Entry) int {\n\totherMe := other.(mockEntry)\n\tif me > otherMe {\n\t\treturn 1\n\t}\n\n\tif me < otherMe {\n\t\treturn -1\n\t}\n\n\treturn 0\n}\n" }, { "path": "tree/avl/node.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage avl\n\ntype nodes []*node\n\nfunc (ns nodes) reset() {\n\tfor i := range ns {\n\t\tns[i] = nil\n\t}\n}\n\ntype node struct {\n\tbalance int8 // bounded, |balance| should be <= 1\n\tchildren [2]*node\n\tentry Entry\n}\n\n// copy returns a copy of this node with pointers to the original\n// children.\nfunc (n *node) copy() *node {\n\treturn &node{\n\t\tbalance: n.balance,\n\t\tchildren: [2]*node{n.children[0], n.children[1]},\n\t\tentry: n.entry,\n\t}\n}\n\n// newNode returns a new node for the provided entry. A nil\n// entry is used to represent the dummy node.\nfunc newNode(entry Entry) *node {\n\treturn &node{\n\t\tentry: entry,\n\t\tchildren: [2]*node{},\n\t}\n}\n" }, { "path": "trie/ctrie/ctrie.go", "content": "/*\nCopyright 2015 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage ctrie provides an implementation of the Ctrie data structure, which is\na concurrent, lock-free hash trie. This data structure was originally presented\nin the paper Concurrent Tries with Efficient Non-Blocking Snapshots:\n\nhttps://axel22.github.io/resources/docs/ctries-snapshot.pdf\n*/\npackage ctrie\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"hash\"\n\t\"hash/fnv\"\n\t\"sync/atomic\"\n\t\"unsafe\"\n\n\t\"github.com/Workiva/go-datastructures/list\"\n)\n\nconst (\n\t// w controls the number of branches at a node (2^w branches).\n\tw = 5\n\n\t// exp2 is 2^w, which is the hashcode space.\n\texp2 = 32\n)\n\n// HashFactory returns a new Hash32 used to hash keys.\ntype HashFactory func() hash.Hash32\n\nfunc defaultHashFactory() hash.Hash32 {\n\treturn fnv.New32a()\n}\n\n// Ctrie is a concurrent, lock-free hash trie. By default, keys are hashed\n// using FNV-1a unless a HashFactory is provided to New.\ntype Ctrie struct {\n\troot *iNode\n\treadOnly bool\n\thashFactory HashFactory\n}\n\n// generation demarcates Ctrie snapshots. We use a heap-allocated reference\n// instead of an integer to avoid integer overflows. Struct must have a field\n// on it since two distinct zero-size variables may have the same address in\n// memory.\ntype generation struct{ _ int }\n\n// iNode is an indirection node. I-nodes remain present in the Ctrie even as\n// nodes above and below change. Thread-safety is achieved in part by\n// performing CAS operations on the I-node instead of the internal node array.\ntype iNode struct {\n\tmain *mainNode\n\tgen *generation\n\n\t// rdcss is set during an RDCSS operation. The I-node is actually a wrapper\n\t// around the descriptor in this case so that a single type is used during\n\t// CAS operations on the root.\n\trdcss *rdcssDescriptor\n}\n\n// copyToGen returns a copy of this I-node copied to the given generation.\nfunc (i *iNode) copyToGen(gen *generation, ctrie *Ctrie) *iNode {\n\tnin := &iNode{gen: gen}\n\tmain := gcasRead(i, ctrie)\n\tatomic.StorePointer(\n\t\t(*unsafe.Pointer)(unsafe.Pointer(&nin.main)), unsafe.Pointer(main))\n\treturn nin\n}\n\n// mainNode is either a cNode, tNode, lNode, or failed node which makes up an\n// I-node.\ntype mainNode struct {\n\tcNode *cNode\n\ttNode *tNode\n\tlNode *lNode\n\tfailed *mainNode\n\n\t// prev is set as a failed main node when we attempt to CAS and the\n\t// I-node's generation does not match the root generation. This signals\n\t// that the GCAS failed and the I-node's main node must be set back to the\n\t// previous value.\n\tprev *mainNode\n}\n\n// cNode is an internal main node containing a bitmap and the array with\n// references to branch nodes. A branch node is either another I-node or a\n// singleton S-node.\ntype cNode struct {\n\tbmp uint32\n\tarray []branch\n\tgen *generation\n}\n\n// newMainNode is a recursive constructor which creates a new mainNode. This\n// mainNode will consist of cNodes as long as the hashcode chunks of the two\n// keys are equal at the given level. If the level exceeds 2^w, an lNode is\n// created.\nfunc newMainNode(x *sNode, xhc uint32, y *sNode, yhc uint32, lev uint, gen *generation) *mainNode {\n\tif lev < exp2 {\n\t\txidx := (xhc >> lev) & 0x1f\n\t\tyidx := (yhc >> lev) & 0x1f\n\t\tbmp := uint32((1 << xidx) | (1 << yidx))\n\n\t\tif xidx == yidx {\n\t\t\t// Recurse when indexes are equal.\n\t\t\tmain := newMainNode(x, xhc, y, yhc, lev+w, gen)\n\t\t\tiNode := &iNode{main: main, gen: gen}\n\t\t\treturn &mainNode{cNode: &cNode{bmp, []branch{iNode}, gen}}\n\t\t}\n\t\tif xidx < yidx {\n\t\t\treturn &mainNode{cNode: &cNode{bmp, []branch{x, y}, gen}}\n\t\t}\n\t\treturn &mainNode{cNode: &cNode{bmp, []branch{y, x}, gen}}\n\t}\n\tl := list.Empty.Add(x).Add(y)\n\treturn &mainNode{lNode: &lNode{l}}\n}\n\n// inserted returns a copy of this cNode with the new entry at the given\n// position.\nfunc (c *cNode) inserted(pos, flag uint32, br branch, gen *generation) *cNode {\n\tlength := uint32(len(c.array))\n\tbmp := c.bmp\n\tarray := make([]branch, length+1)\n\tcopy(array, c.array)\n\tarray[pos] = br\n\tfor i, x := pos, uint32(0); x < length-pos; i++ {\n\t\tarray[i+1] = c.array[i]\n\t\tx++\n\t}\n\tncn := &cNode{bmp: bmp | flag, array: array, gen: gen}\n\treturn ncn\n}\n\n// updated returns a copy of this cNode with the entry at the given index\n// updated.\nfunc (c *cNode) updated(pos uint32, br branch, gen *generation) *cNode {\n\tarray := make([]branch, len(c.array))\n\tcopy(array, c.array)\n\tarray[pos] = br\n\tncn := &cNode{bmp: c.bmp, array: array, gen: gen}\n\treturn ncn\n}\n\n// removed returns a copy of this cNode with the entry at the given index\n// removed.\nfunc (c *cNode) removed(pos, flag uint32, gen *generation) *cNode {\n\tlength := uint32(len(c.array))\n\tbmp := c.bmp\n\tarray := make([]branch, length-1)\n\tfor i := uint32(0); i < pos; i++ {\n\t\tarray[i] = c.array[i]\n\t}\n\tfor i, x := pos, uint32(0); x < length-pos-1; i++ {\n\t\tarray[i] = c.array[i+1]\n\t\tx++\n\t}\n\tncn := &cNode{bmp: bmp ^ flag, array: array, gen: gen}\n\treturn ncn\n}\n\n// renewed returns a copy of this cNode with the I-nodes below it copied to the\n// given generation.\nfunc (c *cNode) renewed(gen *generation, ctrie *Ctrie) *cNode {\n\tarray := make([]branch, len(c.array))\n\tfor i, br := range c.array {\n\t\tswitch t := br.(type) {\n\t\tcase *iNode:\n\t\t\tarray[i] = t.copyToGen(gen, ctrie)\n\t\tdefault:\n\t\t\tarray[i] = br\n\t\t}\n\t}\n\treturn &cNode{bmp: c.bmp, array: array, gen: gen}\n}\n\n// tNode is tomb node which is a special node used to ensure proper ordering\n// during removals.\ntype tNode struct {\n\t*sNode\n}\n\n// untombed returns the S-node contained by the T-node.\nfunc (t *tNode) untombed() *sNode {\n\treturn &sNode{&Entry{Key: t.Key, hash: t.hash, Value: t.Value}}\n}\n\n// lNode is a list node which is a leaf node used to handle hashcode\n// collisions by keeping such keys in a persistent list.\ntype lNode struct {\n\tlist.PersistentList\n}\n\n// entry returns the first S-node contained in the L-node.\nfunc (l *lNode) entry() *sNode {\n\thead, _ := l.Head()\n\treturn head.(*sNode)\n}\n\n// lookup returns the value at the given entry in the L-node or returns false\n// if it's not contained.\nfunc (l *lNode) lookup(e *Entry) (interface{}, bool) {\n\tfound, ok := l.Find(func(sn interface{}) bool {\n\t\treturn bytes.Equal(e.Key, sn.(*sNode).Key)\n\t})\n\tif !ok {\n\t\treturn nil, false\n\t}\n\treturn found.(*sNode).Value, true\n}\n\n// inserted creates a new L-node with the added entry.\nfunc (l *lNode) inserted(entry *Entry) *lNode {\n\treturn &lNode{l.removed(entry).Add(&sNode{entry})}\n}\n\n// removed creates a new L-node with the entry removed.\nfunc (l *lNode) removed(e *Entry) *lNode {\n\tidx := l.FindIndex(func(sn interface{}) bool {\n\t\treturn bytes.Equal(e.Key, sn.(*sNode).Key)\n\t})\n\tif idx < 0 {\n\t\treturn l\n\t}\n\tnl, _ := l.Remove(uint(idx))\n\treturn &lNode{nl}\n}\n\n// length returns the L-node list length.\nfunc (l *lNode) length() uint {\n\treturn l.Length()\n}\n\n// branch is either an iNode or sNode.\ntype branch interface{}\n\n// Entry contains a Ctrie key-value pair.\ntype Entry struct {\n\tKey []byte\n\tValue interface{}\n\thash uint32\n}\n\n// sNode is a singleton node which contains a single key and value.\ntype sNode struct {\n\t*Entry\n}\n\n// New creates an empty Ctrie which uses the provided HashFactory for key\n// hashing. If nil is passed in, it will default to FNV-1a hashing.\nfunc New(hashFactory HashFactory) *Ctrie {\n\tif hashFactory == nil {\n\t\thashFactory = defaultHashFactory\n\t}\n\troot := &iNode{main: &mainNode{cNode: &cNode{}}}\n\treturn newCtrie(root, hashFactory, false)\n}\n\nfunc newCtrie(root *iNode, hashFactory HashFactory, readOnly bool) *Ctrie {\n\treturn &Ctrie{\n\t\troot: root,\n\t\thashFactory: hashFactory,\n\t\treadOnly: readOnly,\n\t}\n}\n\n// Insert adds the key-value pair to the Ctrie, replacing the existing value if\n// the key already exists.\nfunc (c *Ctrie) Insert(key []byte, value interface{}) {\n\tc.assertReadWrite()\n\tc.insert(&Entry{\n\t\tKey: key,\n\t\tValue: value,\n\t\thash: c.hash(key),\n\t})\n}\n\n// Lookup returns the value for the associated key or returns false if the key\n// doesn't exist.\nfunc (c *Ctrie) Lookup(key []byte) (interface{}, bool) {\n\treturn c.lookup(&Entry{Key: key, hash: c.hash(key)})\n}\n\n// Remove deletes the value for the associated key, returning true if it was\n// removed or false if the entry doesn't exist.\nfunc (c *Ctrie) Remove(key []byte) (interface{}, bool) {\n\tc.assertReadWrite()\n\treturn c.remove(&Entry{Key: key, hash: c.hash(key)})\n}\n\n// Snapshot returns a stable, point-in-time snapshot of the Ctrie. If the Ctrie\n// is read-only, the returned Ctrie will also be read-only.\nfunc (c *Ctrie) Snapshot() *Ctrie {\n\treturn c.snapshot(c.readOnly)\n}\n\n// ReadOnlySnapshot returns a stable, point-in-time snapshot of the Ctrie which\n// is read-only. Write operations on a read-only snapshot will panic.\nfunc (c *Ctrie) ReadOnlySnapshot() *Ctrie {\n\treturn c.snapshot(true)\n}\n\n// snapshot wraps up the CAS logic to make a snapshot or a read-only snapshot.\nfunc (c *Ctrie) snapshot(readOnly bool) *Ctrie {\n\tif readOnly && c.readOnly {\n\t\treturn c\n\t}\n\tfor {\n\t\troot := c.readRoot()\n\t\tmain := gcasRead(root, c)\n\t\tif c.rdcssRoot(root, main, root.copyToGen(&generation{}, c)) {\n\t\t\tif readOnly {\n\t\t\t\t// For a read-only snapshot, we can share the old generation\n\t\t\t\t// root.\n\t\t\t\treturn newCtrie(root, c.hashFactory, readOnly)\n\t\t\t}\n\t\t\t// For a read-write snapshot, we need to take a copy of the root\n\t\t\t// in the new generation.\n\t\t\treturn newCtrie(c.readRoot().copyToGen(&generation{}, c), c.hashFactory, readOnly)\n\t\t}\n\t}\n}\n\n// Clear removes all keys from the Ctrie.\nfunc (c *Ctrie) Clear() {\n\tfor {\n\t\troot := c.readRoot()\n\t\tgen := &generation{}\n\t\tnewRoot := &iNode{\n\t\t\tmain: &mainNode{cNode: &cNode{array: make([]branch, 0), gen: gen}},\n\t\t\tgen: gen,\n\t\t}\n\t\tif c.rdcssRoot(root, gcasRead(root, c), newRoot) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Iterator returns a channel which yields the Entries of the Ctrie. If a\n// cancel channel is provided, closing it will terminate and close the iterator\n// channel. Note that if a cancel channel is not used and not every entry is\n// read from the iterator, a goroutine will leak.\nfunc (c *Ctrie) Iterator(cancel <-chan struct{}) <-chan *Entry {\n\tch := make(chan *Entry)\n\tsnapshot := c.ReadOnlySnapshot()\n\tgo func() {\n\t\tsnapshot.traverse(snapshot.readRoot(), ch, cancel)\n\t\tclose(ch)\n\t}()\n\treturn ch\n}\n\n// Size returns the number of keys in the Ctrie.\nfunc (c *Ctrie) Size() uint {\n\t// TODO: The size operation can be optimized further by caching the size\n\t// information in main nodes of a read-only Ctrie – this reduces the\n\t// amortized complexity of the size operation to O(1) because the size\n\t// computation is amortized across the update operations that occurred\n\t// since the last snapshot.\n\tsize := uint(0)\n\tfor _ = range c.Iterator(nil) {\n\t\tsize++\n\t}\n\treturn size\n}\n\nvar errCanceled = errors.New(\"canceled\")\n\nfunc (c *Ctrie) traverse(i *iNode, ch chan<- *Entry, cancel <-chan struct{}) error {\n\tmain := gcasRead(i, c)\n\tswitch {\n\tcase main.cNode != nil:\n\t\tfor _, br := range main.cNode.array {\n\t\t\tswitch b := br.(type) {\n\t\t\tcase *iNode:\n\t\t\t\tif err := c.traverse(b, ch, cancel); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\tcase *sNode:\n\t\t\t\tselect {\n\t\t\t\tcase ch <- b.Entry:\n\t\t\t\tcase <-cancel:\n\t\t\t\t\treturn errCanceled\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\tcase main.lNode != nil:\n\t\tfor _, e := range main.lNode.Map(func(sn interface{}) interface{} {\n\t\t\treturn sn.(*sNode).Entry\n\t\t}) {\n\t\t\tselect {\n\t\t\tcase ch <- e.(*Entry):\n\t\t\tcase <-cancel:\n\t\t\t\treturn errCanceled\n\t\t\t}\n\t\t}\n\tcase main.tNode != nil:\n\t\tselect {\n\t\tcase ch <- main.tNode.Entry:\n\t\tcase <-cancel:\n\t\t\treturn errCanceled\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c *Ctrie) assertReadWrite() {\n\tif c.readOnly {\n\t\tpanic(\"Cannot modify read-only snapshot\")\n\t}\n}\n\nfunc (c *Ctrie) insert(entry *Entry) {\n\troot := c.readRoot()\n\tif !c.iinsert(root, entry, 0, nil, root.gen) {\n\t\tc.insert(entry)\n\t}\n}\n\nfunc (c *Ctrie) lookup(entry *Entry) (interface{}, bool) {\n\troot := c.readRoot()\n\tresult, exists, ok := c.ilookup(root, entry, 0, nil, root.gen)\n\tfor !ok {\n\t\treturn c.lookup(entry)\n\t}\n\treturn result, exists\n}\n\nfunc (c *Ctrie) remove(entry *Entry) (interface{}, bool) {\n\troot := c.readRoot()\n\tresult, exists, ok := c.iremove(root, entry, 0, nil, root.gen)\n\tfor !ok {\n\t\treturn c.remove(entry)\n\t}\n\treturn result, exists\n}\n\nfunc (c *Ctrie) hash(k []byte) uint32 {\n\thasher := c.hashFactory()\n\thasher.Write(k)\n\treturn hasher.Sum32()\n}\n\n// iinsert attempts to insert the entry into the Ctrie. If false is returned,\n// the operation should be retried.\nfunc (c *Ctrie) iinsert(i *iNode, entry *Entry, lev uint, parent *iNode, startGen *generation) bool {\n\t// Linearization point.\n\tmain := gcasRead(i, c)\n\tswitch {\n\tcase main.cNode != nil:\n\t\tcn := main.cNode\n\t\tflag, pos := flagPos(entry.hash, lev, cn.bmp)\n\t\tif cn.bmp&flag == 0 {\n\t\t\t// If the relevant bit is not in the bitmap, then a copy of the\n\t\t\t// cNode with the new entry is created. The linearization point is\n\t\t\t// a successful CAS.\n\t\t\trn := cn\n\t\t\tif cn.gen != i.gen {\n\t\t\t\trn = cn.renewed(i.gen, c)\n\t\t\t}\n\t\t\tncn := &mainNode{cNode: rn.inserted(pos, flag, &sNode{entry}, i.gen)}\n\t\t\treturn gcas(i, main, ncn, c)\n\t\t}\n\t\t// If the relevant bit is present in the bitmap, then its corresponding\n\t\t// branch is read from the array.\n\t\tbranch := cn.array[pos]\n\t\tswitch branch.(type) {\n\t\tcase *iNode:\n\t\t\t// If the branch is an I-node, then iinsert is called recursively.\n\t\t\tin := branch.(*iNode)\n\t\t\tif startGen == in.gen {\n\t\t\t\treturn c.iinsert(in, entry, lev+w, i, startGen)\n\t\t\t}\n\t\t\tif gcas(i, main, &mainNode{cNode: cn.renewed(startGen, c)}, c) {\n\t\t\t\treturn c.iinsert(i, entry, lev, parent, startGen)\n\t\t\t}\n\t\t\treturn false\n\t\tcase *sNode:\n\t\t\tsn := branch.(*sNode)\n\t\t\tif !bytes.Equal(sn.Key, entry.Key) {\n\t\t\t\t// If the branch is an S-node and its key is not equal to the\n\t\t\t\t// key being inserted, then the Ctrie has to be extended with\n\t\t\t\t// an additional level. The C-node is replaced with its updated\n\t\t\t\t// version, created using the updated function that adds a new\n\t\t\t\t// I-node at the respective position. The new Inode has its\n\t\t\t\t// main node pointing to a C-node with both keys. The\n\t\t\t\t// linearization point is a successful CAS.\n\t\t\t\trn := cn\n\t\t\t\tif cn.gen != i.gen {\n\t\t\t\t\trn = cn.renewed(i.gen, c)\n\t\t\t\t}\n\t\t\t\tnsn := &sNode{entry}\n\t\t\t\tnin := &iNode{main: newMainNode(sn, sn.hash, nsn, nsn.hash, lev+w, i.gen), gen: i.gen}\n\t\t\t\tncn := &mainNode{cNode: rn.updated(pos, nin, i.gen)}\n\t\t\t\treturn gcas(i, main, ncn, c)\n\t\t\t}\n\t\t\t// If the key in the S-node is equal to the key being inserted,\n\t\t\t// then the C-node is replaced with its updated version with a new\n\t\t\t// S-node. The linearization point is a successful CAS.\n\t\t\tncn := &mainNode{cNode: cn.updated(pos, &sNode{entry}, i.gen)}\n\t\t\treturn gcas(i, main, ncn, c)\n\t\tdefault:\n\t\t\tpanic(\"Ctrie is in an invalid state\")\n\t\t}\n\tcase main.tNode != nil:\n\t\tclean(parent, lev-w, c)\n\t\treturn false\n\tcase main.lNode != nil:\n\t\tnln := &mainNode{lNode: main.lNode.inserted(entry)}\n\t\treturn gcas(i, main, nln, c)\n\tdefault:\n\t\tpanic(\"Ctrie is in an invalid state\")\n\t}\n}\n\n// ilookup attempts to fetch the entry from the Ctrie. The first two return\n// values are the entry value and whether or not the entry was contained in the\n// Ctrie. The last bool indicates if the operation succeeded. False means it\n// should be retried.\nfunc (c *Ctrie) ilookup(i *iNode, entry *Entry, lev uint, parent *iNode, startGen *generation) (interface{}, bool, bool) {\n\t// Linearization point.\n\tmain := gcasRead(i, c)\n\tswitch {\n\tcase main.cNode != nil:\n\t\tcn := main.cNode\n\t\tflag, pos := flagPos(entry.hash, lev, cn.bmp)\n\t\tif cn.bmp&flag == 0 {\n\t\t\t// If the bitmap does not contain the relevant bit, a key with the\n\t\t\t// required hashcode prefix is not present in the trie.\n\t\t\treturn nil, false, true\n\t\t}\n\t\t// Otherwise, the relevant branch at index pos is read from the array.\n\t\tbranch := cn.array[pos]\n\t\tswitch branch.(type) {\n\t\tcase *iNode:\n\t\t\t// If the branch is an I-node, the ilookup procedure is called\n\t\t\t// recursively at the next level.\n\t\t\tin := branch.(*iNode)\n\t\t\tif c.readOnly || startGen == in.gen {\n\t\t\t\treturn c.ilookup(in, entry, lev+w, i, startGen)\n\t\t\t}\n\t\t\tif gcas(i, main, &mainNode{cNode: cn.renewed(startGen, c)}, c) {\n\t\t\t\treturn c.ilookup(i, entry, lev, parent, startGen)\n\t\t\t}\n\t\t\treturn nil, false, false\n\t\tcase *sNode:\n\t\t\t// If the branch is an S-node, then the key within the S-node is\n\t\t\t// compared with the key being searched – these two keys have the\n\t\t\t// same hashcode prefixes, but they need not be equal. If they are\n\t\t\t// equal, the corresponding value from the S-node is\n\t\t\t// returned and a NOTFOUND value otherwise.\n\t\t\tsn := branch.(*sNode)\n\t\t\tif bytes.Equal(sn.Key, entry.Key) {\n\t\t\t\treturn sn.Value, true, true\n\t\t\t}\n\t\t\treturn nil, false, true\n\t\tdefault:\n\t\t\tpanic(\"Ctrie is in an invalid state\")\n\t\t}\n\tcase main.tNode != nil:\n\t\treturn cleanReadOnly(main.tNode, lev, parent, c, entry)\n\tcase main.lNode != nil:\n\t\t// Hash collisions are handled using L-nodes, which are essentially\n\t\t// persistent linked lists.\n\t\tval, ok := main.lNode.lookup(entry)\n\t\treturn val, ok, true\n\tdefault:\n\t\tpanic(\"Ctrie is in an invalid state\")\n\t}\n}\n\n// iremove attempts to remove the entry from the Ctrie. The first two return\n// values are the entry value and whether or not the entry was contained in the\n// Ctrie. The last bool indicates if the operation succeeded. False means it\n// should be retried.\nfunc (c *Ctrie) iremove(i *iNode, entry *Entry, lev uint, parent *iNode, startGen *generation) (interface{}, bool, bool) {\n\t// Linearization point.\n\tmain := gcasRead(i, c)\n\tswitch {\n\tcase main.cNode != nil:\n\t\tcn := main.cNode\n\t\tflag, pos := flagPos(entry.hash, lev, cn.bmp)\n\t\tif cn.bmp&flag == 0 {\n\t\t\t// If the bitmap does not contain the relevant bit, a key with the\n\t\t\t// required hashcode prefix is not present in the trie.\n\t\t\treturn nil, false, true\n\t\t}\n\t\t// Otherwise, the relevant branch at index pos is read from the array.\n\t\tbranch := cn.array[pos]\n\t\tswitch branch.(type) {\n\t\tcase *iNode:\n\t\t\t// If the branch is an I-node, the iremove procedure is called\n\t\t\t// recursively at the next level.\n\t\t\tin := branch.(*iNode)\n\t\t\tif startGen == in.gen {\n\t\t\t\treturn c.iremove(in, entry, lev+w, i, startGen)\n\t\t\t}\n\t\t\tif gcas(i, main, &mainNode{cNode: cn.renewed(startGen, c)}, c) {\n\t\t\t\treturn c.iremove(i, entry, lev, parent, startGen)\n\t\t\t}\n\t\t\treturn nil, false, false\n\t\tcase *sNode:\n\t\t\t// If the branch is an S-node, its key is compared against the key\n\t\t\t// being removed.\n\t\t\tsn := branch.(*sNode)\n\t\t\tif !bytes.Equal(sn.Key, entry.Key) {\n\t\t\t\t// If the keys are not equal, the NOTFOUND value is returned.\n\t\t\t\treturn nil, false, true\n\t\t\t}\n\t\t\t// If the keys are equal, a copy of the current node without the\n\t\t\t// S-node is created. The contraction of the copy is then created\n\t\t\t// using the toContracted procedure. A successful CAS will\n\t\t\t// substitute the old C-node with the copied C-node, thus removing\n\t\t\t// the S-node with the given key from the trie – this is the\n\t\t\t// linearization point\n\t\t\tncn := cn.removed(pos, flag, i.gen)\n\t\t\tcntr := toContracted(ncn, lev)\n\t\t\tif gcas(i, main, cntr, c) {\n\t\t\t\tif parent != nil {\n\t\t\t\t\tmain = gcasRead(i, c)\n\t\t\t\t\tif main.tNode != nil {\n\t\t\t\t\t\tcleanParent(parent, i, entry.hash, lev-w, c, startGen)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn sn.Value, true, true\n\t\t\t}\n\t\t\treturn nil, false, false\n\t\tdefault:\n\t\t\tpanic(\"Ctrie is in an invalid state\")\n\t\t}\n\tcase main.tNode != nil:\n\t\tclean(parent, lev-w, c)\n\t\treturn nil, false, false\n\tcase main.lNode != nil:\n\t\tnln := &mainNode{lNode: main.lNode.removed(entry)}\n\t\tif nln.lNode.length() == 1 {\n\t\t\tnln = entomb(nln.lNode.entry())\n\t\t}\n\t\tif gcas(i, main, nln, c) {\n\t\t\tval, ok := main.lNode.lookup(entry)\n\t\t\treturn val, ok, true\n\t\t}\n\t\treturn nil, false, true\n\tdefault:\n\t\tpanic(\"Ctrie is in an invalid state\")\n\t}\n}\n\n// toContracted ensures that every I-node except the root points to a C-node\n// with at least one branch. If a given C-Node has only a single S-node below\n// it and is not at the root level, a T-node which wraps the S-node is\n// returned.\nfunc toContracted(cn *cNode, lev uint) *mainNode {\n\tif lev > 0 && len(cn.array) == 1 {\n\t\tbranch := cn.array[0]\n\t\tswitch branch.(type) {\n\t\tcase *sNode:\n\t\t\treturn entomb(branch.(*sNode))\n\t\tdefault:\n\t\t\treturn &mainNode{cNode: cn}\n\t\t}\n\t}\n\treturn &mainNode{cNode: cn}\n}\n\n// toCompressed compacts the C-node as a performance optimization.\nfunc toCompressed(cn *cNode, lev uint) *mainNode {\n\ttmpArray := make([]branch, len(cn.array))\n\tfor i, sub := range cn.array {\n\t\tswitch sub.(type) {\n\t\tcase *iNode:\n\t\t\tinode := sub.(*iNode)\n\t\t\tmainPtr := (*unsafe.Pointer)(unsafe.Pointer(&inode.main))\n\t\t\tmain := (*mainNode)(atomic.LoadPointer(mainPtr))\n\t\t\ttmpArray[i] = resurrect(inode, main)\n\t\tcase *sNode:\n\t\t\ttmpArray[i] = sub\n\t\tdefault:\n\t\t\tpanic(\"Ctrie is in an invalid state\")\n\t\t}\n\t}\n\n\treturn toContracted(&cNode{bmp: cn.bmp, array: tmpArray}, lev)\n}\n\nfunc entomb(m *sNode) *mainNode {\n\treturn &mainNode{tNode: &tNode{m}}\n}\n\nfunc resurrect(iNode *iNode, main *mainNode) branch {\n\tif main.tNode != nil {\n\t\treturn main.tNode.untombed()\n\t}\n\treturn iNode\n}\n\nfunc clean(i *iNode, lev uint, ctrie *Ctrie) bool {\n\tmain := gcasRead(i, ctrie)\n\tif main.cNode != nil {\n\t\treturn gcas(i, main, toCompressed(main.cNode, lev), ctrie)\n\t}\n\treturn true\n}\n\nfunc cleanReadOnly(tn *tNode, lev uint, p *iNode, ctrie *Ctrie, entry *Entry) (val interface{}, exists bool, ok bool) {\n\tif !ctrie.readOnly {\n\t\tclean(p, lev-5, ctrie)\n\t\treturn nil, false, false\n\t}\n\tif tn.hash == entry.hash && bytes.Equal(tn.Key, entry.Key) {\n\t\treturn tn.Value, true, true\n\t}\n\treturn nil, false, true\n}\n\nfunc cleanParent(p, i *iNode, hc uint32, lev uint, ctrie *Ctrie, startGen *generation) {\n\tvar (\n\t\tmainPtr = (*unsafe.Pointer)(unsafe.Pointer(&i.main))\n\t\tmain = (*mainNode)(atomic.LoadPointer(mainPtr))\n\t\tpMainPtr = (*unsafe.Pointer)(unsafe.Pointer(&p.main))\n\t\tpMain = (*mainNode)(atomic.LoadPointer(pMainPtr))\n\t)\n\tif pMain.cNode != nil {\n\t\tflag, pos := flagPos(hc, lev, pMain.cNode.bmp)\n\t\tif pMain.cNode.bmp&flag != 0 {\n\t\t\tsub := pMain.cNode.array[pos]\n\t\t\tif sub == i && main.tNode != nil {\n\t\t\t\tncn := pMain.cNode.updated(pos, resurrect(i, main), i.gen)\n\t\t\t\tif !gcas(p, pMain, toContracted(ncn, lev), ctrie) && ctrie.readRoot().gen == startGen {\n\t\t\t\t\tcleanParent(p, i, hc, lev, ctrie, startGen)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc flagPos(hashcode uint32, lev uint, bmp uint32) (uint32, uint32) {\n\tidx := (hashcode >> lev) & 0x1f\n\tflag := uint32(1) << uint32(idx)\n\tmask := uint32(flag - 1)\n\tpos := bitCount(bmp & mask)\n\treturn flag, pos\n}\n\nfunc bitCount(x uint32) uint32 {\n\tx -= (x >> 1) & 0x55555555\n\tx = ((x >> 2) & 0x33333333) + (x & 0x33333333)\n\tx = ((x >> 4) + x) & 0x0f0f0f0f\n\tx *= 0x01010101\n\treturn x >> 24\n}\n\n// gcas is a generation-compare-and-swap which has semantics similar to RDCSS,\n// but it does not create the intermediate object except in the case of\n// failures that occur due to the snapshot being taken. This ensures that the\n// write occurs only if the Ctrie root generation has remained the same in\n// addition to the I-node having the expected value.\nfunc gcas(in *iNode, old, n *mainNode, ct *Ctrie) bool {\n\tprevPtr := (*unsafe.Pointer)(unsafe.Pointer(&n.prev))\n\tatomic.StorePointer(prevPtr, unsafe.Pointer(old))\n\tif atomic.CompareAndSwapPointer(\n\t\t(*unsafe.Pointer)(unsafe.Pointer(&in.main)),\n\t\tunsafe.Pointer(old), unsafe.Pointer(n)) {\n\t\tgcasComplete(in, n, ct)\n\t\treturn atomic.LoadPointer(prevPtr) == nil\n\t}\n\treturn false\n}\n\n// gcasRead performs a GCAS-linearizable read of the I-node's main node.\nfunc gcasRead(in *iNode, ctrie *Ctrie) *mainNode {\n\tm := (*mainNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&in.main))))\n\tprev := (*mainNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&m.prev))))\n\tif prev == nil {\n\t\treturn m\n\t}\n\treturn gcasComplete(in, m, ctrie)\n}\n\n// gcasComplete commits the GCAS operation.\nfunc gcasComplete(i *iNode, m *mainNode, ctrie *Ctrie) *mainNode {\n\tfor {\n\t\tif m == nil {\n\t\t\treturn nil\n\t\t}\n\t\tprev := (*mainNode)(atomic.LoadPointer(\n\t\t\t(*unsafe.Pointer)(unsafe.Pointer(&m.prev))))\n\t\troot := ctrie.rdcssReadRoot(true)\n\t\tif prev == nil {\n\t\t\treturn m\n\t\t}\n\n\t\tif prev.failed != nil {\n\t\t\t// Signals GCAS failure. Swap old value back into I-node.\n\t\t\tfn := prev.failed\n\t\t\tif atomic.CompareAndSwapPointer((*unsafe.Pointer)(unsafe.Pointer(&i.main)),\n\t\t\t\tunsafe.Pointer(m), unsafe.Pointer(fn)) {\n\t\t\t\treturn fn\n\t\t\t}\n\t\t\tm = (*mainNode)(atomic.LoadPointer(\n\t\t\t\t(*unsafe.Pointer)(unsafe.Pointer(&i.main))))\n\t\t\tcontinue\n\t\t}\n\n\t\tif root.gen == i.gen && !ctrie.readOnly {\n\t\t\t// Commit GCAS.\n\t\t\tif atomic.CompareAndSwapPointer(\n\t\t\t\t(*unsafe.Pointer)(unsafe.Pointer(&m.prev)), unsafe.Pointer(prev), nil) {\n\t\t\t\treturn m\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// Generations did not match. Store failed node on prev to signal\n\t\t// I-node's main node must be set back to the previous value.\n\t\tatomic.CompareAndSwapPointer(\n\t\t\t(*unsafe.Pointer)(unsafe.Pointer(&m.prev)),\n\t\t\tunsafe.Pointer(prev),\n\t\t\tunsafe.Pointer(&mainNode{failed: prev}))\n\t\tm = (*mainNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&i.main))))\n\t\treturn gcasComplete(i, m, ctrie)\n\t}\n}\n\n// rdcssDescriptor is an intermediate struct which communicates the intent to\n// replace the value in an I-node and check that the root's generation has not\n// changed before committing to the new value.\ntype rdcssDescriptor struct {\n\told *iNode\n\texpected *mainNode\n\tnv *iNode\n\tcommitted int32\n}\n\n// readRoot performs a linearizable read of the Ctrie root. This operation is\n// prioritized so that if another thread performs a GCAS on the root, a\n// deadlock does not occur.\nfunc (c *Ctrie) readRoot() *iNode {\n\treturn c.rdcssReadRoot(false)\n}\n\n// rdcssReadRoot performs a RDCSS-linearizable read of the Ctrie root with the\n// given priority.\nfunc (c *Ctrie) rdcssReadRoot(abort bool) *iNode {\n\tr := (*iNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&c.root))))\n\tif r.rdcss != nil {\n\t\treturn c.rdcssComplete(abort)\n\t}\n\treturn r\n}\n\n// rdcssRoot performs a RDCSS on the Ctrie root. This is used to create a\n// snapshot of the Ctrie by copying the root I-node and setting it to a new\n// generation.\nfunc (c *Ctrie) rdcssRoot(old *iNode, expected *mainNode, nv *iNode) bool {\n\tdesc := &iNode{\n\t\trdcss: &rdcssDescriptor{\n\t\t\told: old,\n\t\t\texpected: expected,\n\t\t\tnv: nv,\n\t\t},\n\t}\n\tif c.casRoot(old, desc) {\n\t\tc.rdcssComplete(false)\n\t\treturn atomic.LoadInt32(&desc.rdcss.committed) == 1\n\t}\n\treturn false\n}\n\n// rdcssComplete commits the RDCSS operation.\nfunc (c *Ctrie) rdcssComplete(abort bool) *iNode {\n\tfor {\n\t\tr := (*iNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&c.root))))\n\t\tif r.rdcss == nil {\n\t\t\treturn r\n\t\t}\n\n\t\tvar (\n\t\t\tdesc = r.rdcss\n\t\t\tov = desc.old\n\t\t\texp = desc.expected\n\t\t\tnv = desc.nv\n\t\t)\n\n\t\tif abort {\n\t\t\tif c.casRoot(r, ov) {\n\t\t\t\treturn ov\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\toldeMain := gcasRead(ov, c)\n\t\tif oldeMain == exp {\n\t\t\t// Commit the RDCSS.\n\t\t\tif c.casRoot(r, nv) {\n\t\t\t\tatomic.StoreInt32(&desc.committed, 1)\n\t\t\t\treturn nv\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif c.casRoot(r, ov) {\n\t\t\treturn ov\n\t\t}\n\t\tcontinue\n\t}\n}\n\n// casRoot performs a CAS on the Ctrie root.\nfunc (c *Ctrie) casRoot(ov, nv *iNode) bool {\n\tc.assertReadWrite()\n\treturn atomic.CompareAndSwapPointer(\n\t\t(*unsafe.Pointer)(unsafe.Pointer(&c.root)), unsafe.Pointer(ov), unsafe.Pointer(nv))\n}\n" }, { "path": "trie/ctrie/ctrie_test.go", "content": "/*\nCopyright 2015 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage ctrie\n\nimport (\n\t\"hash\"\n\t\"hash/fnv\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestCtrie(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(nil)\n\n\t_, ok := ctrie.Lookup([]byte(\"foo\"))\n\tassert.False(ok)\n\n\tctrie.Insert([]byte(\"foo\"), \"bar\")\n\tval, ok := ctrie.Lookup([]byte(\"foo\"))\n\tassert.True(ok)\n\tassert.Equal(\"bar\", val)\n\n\tctrie.Insert([]byte(\"fooooo\"), \"baz\")\n\tval, ok = ctrie.Lookup([]byte(\"foo\"))\n\tassert.True(ok)\n\tassert.Equal(\"bar\", val)\n\tval, ok = ctrie.Lookup([]byte(\"fooooo\"))\n\tassert.True(ok)\n\tassert.Equal(\"baz\", val)\n\n\tfor i := 0; i < 100; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), \"blah\")\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\tval, ok = ctrie.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(\"blah\", val)\n\t}\n\n\tval, ok = ctrie.Lookup([]byte(\"foo\"))\n\tassert.True(ok)\n\tassert.Equal(\"bar\", val)\n\tctrie.Insert([]byte(\"foo\"), \"qux\")\n\tval, ok = ctrie.Lookup([]byte(\"foo\"))\n\tassert.True(ok)\n\tassert.Equal(\"qux\", val)\n\n\tval, ok = ctrie.Remove([]byte(\"foo\"))\n\tassert.True(ok)\n\tassert.Equal(\"qux\", val)\n\n\t_, ok = ctrie.Remove([]byte(\"foo\"))\n\tassert.False(ok)\n\n\tval, ok = ctrie.Remove([]byte(\"fooooo\"))\n\tassert.True(ok)\n\tassert.Equal(\"baz\", val)\n\n\tfor i := 0; i < 100; i++ {\n\t\tctrie.Remove([]byte(strconv.Itoa(i)))\n\t}\n}\n\ntype mockHash32 struct {\n\thash.Hash32\n}\n\nfunc (m *mockHash32) Sum32() uint32 {\n\treturn 0\n}\n\nfunc mockHashFactory() hash.Hash32 {\n\treturn &mockHash32{fnv.New32a()}\n}\n\nfunc TestInsertLNode(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(mockHashFactory)\n\n\tfor i := 0; i < 10; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tval, ok := ctrie.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n\t_, ok := ctrie.Lookup([]byte(\"11\"))\n\tassert.False(ok)\n\n\tfor i := 0; i < 10; i++ {\n\t\tval, ok := ctrie.Remove([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n}\n\nfunc TestInsertTNode(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(nil)\n\n\tfor i := 0; i < 10000; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\n\tfor i := 0; i < 5000; i++ {\n\t\tctrie.Remove([]byte(strconv.Itoa(i)))\n\t}\n\n\tfor i := 0; i < 10000; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\n\tfor i := 0; i < 10000; i++ {\n\t\tval, ok := ctrie.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n}\n\nfunc TestConcurrency(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(nil)\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\n\tgo func() {\n\t\tfor i := 0; i < 10000; i++ {\n\t\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\tfor i := 0; i < 10000; i++ {\n\t\t\tval, ok := ctrie.Lookup([]byte(strconv.Itoa(i)))\n\t\t\tif ok {\n\t\t\t\tassert.Equal(i, val)\n\t\t\t}\n\t\t}\n\t\twg.Done()\n\t}()\n\n\tfor i := 0; i < 10000; i++ {\n\t\ttime.Sleep(5)\n\t\tctrie.Remove([]byte(strconv.Itoa(i)))\n\t}\n\n\twg.Wait()\n}\n\nfunc TestConcurrency2(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(nil)\n\tvar wg sync.WaitGroup\n\twg.Add(4)\n\n\tgo func() {\n\t\tfor i := 0; i < 10000; i++ {\n\t\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\tfor i := 0; i < 10000; i++ {\n\t\t\tval, ok := ctrie.Lookup([]byte(strconv.Itoa(i)))\n\t\t\tif ok {\n\t\t\t\tassert.Equal(i, val)\n\t\t\t}\n\t\t}\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\tfor i := 0; i < 10000; i++ {\n\t\t\tctrie.Snapshot()\n\t\t}\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\tfor i := 0; i < 10000; i++ {\n\t\t\tctrie.ReadOnlySnapshot()\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\tassert.Equal(uint(10000), ctrie.Size())\n}\n\nfunc TestSnapshot(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(nil)\n\tfor i := 0; i < 100; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\n\tsnapshot := ctrie.Snapshot()\n\n\t// Ensure snapshot contains expected keys.\n\tfor i := 0; i < 100; i++ {\n\t\tval, ok := snapshot.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n\n\t// Now remove the values from the original.\n\tfor i := 0; i < 100; i++ {\n\t\tctrie.Remove([]byte(strconv.Itoa(i)))\n\t}\n\n\t// Ensure snapshot was unaffected by removals.\n\tfor i := 0; i < 100; i++ {\n\t\tval, ok := snapshot.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n\n\t// New Ctrie and snapshot.\n\tctrie = New(nil)\n\tfor i := 0; i < 100; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\tsnapshot = ctrie.Snapshot()\n\n\t// Ensure snapshot is mutable.\n\tfor i := 0; i < 100; i++ {\n\t\tsnapshot.Remove([]byte(strconv.Itoa(i)))\n\t}\n\tsnapshot.Insert([]byte(\"bat\"), \"man\")\n\n\tfor i := 0; i < 100; i++ {\n\t\t_, ok := snapshot.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.False(ok)\n\t}\n\tval, ok := snapshot.Lookup([]byte(\"bat\"))\n\tassert.True(ok)\n\tassert.Equal(\"man\", val)\n\n\t// Ensure original Ctrie was unaffected.\n\tfor i := 0; i < 100; i++ {\n\t\tval, ok := ctrie.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n\t_, ok = ctrie.Lookup([]byte(\"bat\"))\n\tassert.False(ok)\n\n\t// Ensure snapshots-of-snapshots work as expected.\n\tsnapshot2 := snapshot.Snapshot()\n\tfor i := 0; i < 100; i++ {\n\t\t_, ok := snapshot2.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.False(ok)\n\t}\n\tval, ok = snapshot2.Lookup([]byte(\"bat\"))\n\tassert.True(ok)\n\tassert.Equal(\"man\", val)\n\n\tsnapshot2.Remove([]byte(\"bat\"))\n\t_, ok = snapshot2.Lookup([]byte(\"bat\"))\n\tassert.False(ok)\n\tval, ok = snapshot.Lookup([]byte(\"bat\"))\n\tassert.True(ok)\n\tassert.Equal(\"man\", val)\n}\n\nfunc TestReadOnlySnapshot(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(nil)\n\tfor i := 0; i < 100; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\n\tsnapshot := ctrie.ReadOnlySnapshot()\n\n\t// Ensure snapshot contains expected keys.\n\tfor i := 0; i < 100; i++ {\n\t\tval, ok := snapshot.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n\n\tfor i := 0; i < 50; i++ {\n\t\tctrie.Remove([]byte(strconv.Itoa(i)))\n\t}\n\n\t// Ensure snapshot was unaffected by removals.\n\tfor i := 0; i < 100; i++ {\n\t\tval, ok := snapshot.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n\n\t// Ensure read-only snapshots panic on writes.\n\tfunc() {\n\t\tdefer func() {\n\t\t\tassert.NotNil(recover())\n\t\t}()\n\t\tsnapshot.Remove([]byte(\"blah\"))\n\t}()\n\n\t// Ensure snapshots-of-snapshots work as expected.\n\tsnapshot2 := snapshot.Snapshot()\n\tfor i := 50; i < 100; i++ {\n\t\tctrie.Remove([]byte(strconv.Itoa(i)))\n\t}\n\tfor i := 0; i < 100; i++ {\n\t\tval, ok := snapshot2.Lookup([]byte(strconv.Itoa(i)))\n\t\tassert.True(ok)\n\t\tassert.Equal(i, val)\n\t}\n\n\t// Ensure snapshots of read-only snapshots panic on writes.\n\tfunc() {\n\t\tdefer func() {\n\t\t\tassert.NotNil(recover())\n\t\t}()\n\t\tsnapshot2.Remove([]byte(\"blah\"))\n\t}()\n}\n\nfunc TestIterator(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(nil)\n\tfor i := 0; i < 10; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\texpected := map[string]int{\n\t\t\"0\": 0,\n\t\t\"1\": 1,\n\t\t\"2\": 2,\n\t\t\"3\": 3,\n\t\t\"4\": 4,\n\t\t\"5\": 5,\n\t\t\"6\": 6,\n\t\t\"7\": 7,\n\t\t\"8\": 8,\n\t\t\"9\": 9,\n\t}\n\n\tcount := 0\n\tfor entry := range ctrie.Iterator(nil) {\n\t\texp, ok := expected[string(entry.Key)]\n\t\tif assert.True(ok) {\n\t\t\tassert.Equal(exp, entry.Value)\n\t\t}\n\t\tcount++\n\t}\n\tassert.Equal(len(expected), count)\n\n\t// Closing cancel channel should close iterator channel.\n\tcancel := make(chan struct{})\n\titer := ctrie.Iterator(cancel)\n\tentry := <-iter\n\texp, ok := expected[string(entry.Key)]\n\tif assert.True(ok) {\n\t\tassert.Equal(exp, entry.Value)\n\t}\n\tclose(cancel)\n\t// Drain anything already put on the channel. Since select chooses a\n\t// pseudo-random case, we must attempt to drain for every item.\n\tfor _ = range expected {\n\t\t<-iter\n\t}\n\t_, ok = <-iter\n\tassert.False(ok)\n}\n\n// TestIteratorCoversTNodes reproduces the scenario of a bug where tNodes weren't being traversed.\nfunc TestIteratorCoversTNodes(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(mockHashFactory)\n\t// Add a pair of keys that collide (because we're using the mock hash).\n\tctrie.Insert([]byte(\"a\"), true)\n\tctrie.Insert([]byte(\"b\"), true)\n\t// Delete one key, leaving exactly one sNode in the cNode. This will\n\t// trigger creation of a tNode.\n\tctrie.Remove([]byte(\"b\"))\n\tseenKeys := map[string]bool{}\n\tfor entry := range ctrie.Iterator(nil) {\n\t\tseenKeys[string(entry.Key)] = true\n\t}\n\tassert.Contains(seenKeys, \"a\", \"Iterator did not return 'a'.\")\n\tassert.Len(seenKeys, 1)\n}\n\nfunc TestSize(t *testing.T) {\n\tctrie := New(nil)\n\tfor i := 0; i < 10; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\tassert.Equal(t, uint(10), ctrie.Size())\n}\n\nfunc TestClear(t *testing.T) {\n\tassert := assert.New(t)\n\tctrie := New(nil)\n\tfor i := 0; i < 10; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\tassert.Equal(uint(10), ctrie.Size())\n\tsnapshot := ctrie.Snapshot()\n\n\tctrie.Clear()\n\n\tassert.Equal(uint(0), ctrie.Size())\n\tassert.Equal(uint(10), snapshot.Size())\n}\n\ntype fakehash struct{}\n\nfunc (h *fakehash) Sum32() uint32 {\n\treturn 42\n}\n\nfunc (h *fakehash) Sum(b []byte) []byte {\n\treturn nil\n}\n\nfunc (h *fakehash) Size() int {\n\treturn 0\n}\n\nfunc (h *fakehash) BlockSize() int {\n\treturn 0\n}\n\nfunc (h *fakehash) Reset() {\n\n}\n\nfunc (h *fakehash) Write(b []byte) (int, error) {\n\treturn 0, nil\n}\n\nfunc factory() hash.Hash32 {\n\treturn &fakehash{}\n}\n\nfunc TestHashCollision(t *testing.T) {\n\ttrie := New(factory)\n\ttrie.Insert([]byte(\"foobar\"), 1)\n\ttrie.Insert([]byte(\"zogzog\"), 2)\n\ttrie.Insert([]byte(\"foobar\"), 3)\n\tval, exists := trie.Lookup([]byte(\"foobar\"))\n\tassert.True(t, exists)\n\tassert.Equal(t, 3, val)\n\n\ttrie.Remove([]byte(\"foobar\"))\n\n\t_, exists = trie.Lookup([]byte(\"foobar\"))\n\tassert.False(t, exists)\n}\n\nfunc BenchmarkInsert(b *testing.B) {\n\tctrie := New(nil)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tctrie.Insert([]byte(\"foo\"), 0)\n\t}\n}\n\nfunc BenchmarkLookup(b *testing.B) {\n\tnumItems := 1000\n\tctrie := New(nil)\n\tfor i := 0; i < numItems; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\tkey := []byte(strconv.Itoa(numItems / 2))\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tctrie.Lookup(key)\n\t}\n}\n\nfunc BenchmarkRemove(b *testing.B) {\n\tnumItems := 1000\n\tctrie := New(nil)\n\tfor i := 0; i < numItems; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\tkey := []byte(strconv.Itoa(numItems / 2))\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tctrie.Remove(key)\n\t}\n}\n\nfunc BenchmarkSnapshot(b *testing.B) {\n\tnumItems := 1000\n\tctrie := New(nil)\n\tfor i := 0; i < numItems; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tctrie.Snapshot()\n\t}\n}\n\nfunc BenchmarkReadOnlySnapshot(b *testing.B) {\n\tnumItems := 1000\n\tctrie := New(nil)\n\tfor i := 0; i < numItems; i++ {\n\t\tctrie.Insert([]byte(strconv.Itoa(i)), i)\n\t}\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tctrie.ReadOnlySnapshot()\n\t}\n}\n" }, { "path": "trie/dtrie/dtrie.go", "content": "/*\nCopyright (c) 2016, Theodore Butler\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/\n\n// Package dtrie provides an implementation of the dtrie data structure, which\n// is a persistent hash trie that dynamically expands or shrinks to provide\n// efficient memory allocation. This data structure is based on the papers\n// Ideal Hash Trees by Phil Bagwell and Optimizing Hash-Array Mapped Tries for\n// Fast and Lean Immutable JVM Collections by Michael J. Steindorfer and\n// Jurgen J. Vinju\npackage dtrie\n\n// Dtrie is a persistent hash trie that dynamically expands or shrinks\n// to provide efficient memory allocation.\ntype Dtrie struct {\n\troot *node\n\thasher func(v interface{}) uint32\n}\n\ntype entry struct {\n\thash uint32\n\tkey interface{}\n\tvalue interface{}\n}\n\nfunc (e *entry) KeyHash() uint32 {\n\treturn e.hash\n}\n\nfunc (e *entry) Key() interface{} {\n\treturn e.key\n}\n\nfunc (e *entry) Value() interface{} {\n\treturn e.value\n}\n\n// New creates an empty DTrie with the given hashing function.\n// If nil is passed in, the default hashing function will be used.\nfunc New(hasher func(v interface{}) uint32) *Dtrie {\n\tif hasher == nil {\n\t\thasher = defaultHasher\n\t}\n\treturn &Dtrie{\n\t\troot: emptyNode(0, 32),\n\t\thasher: hasher,\n\t}\n}\n\n// Size returns the number of entries in the Dtrie.\nfunc (d *Dtrie) Size() (size int) {\n\tfor _ = range iterate(d.root, nil) {\n\t\tsize++\n\t}\n\treturn size\n}\n\n// Get returns the value for the associated key or returns nil if the\n// key does not exist.\nfunc (d *Dtrie) Get(key interface{}) interface{} {\n node := get(d.root, d.hasher(key), key)\n if node != nil {\n return node.Value()\n }\n\treturn nil\n}\n\n// Insert adds a key value pair to the Dtrie, replacing the existing value if\n// the key already exists and returns the resulting Dtrie.\nfunc (d *Dtrie) Insert(key, value interface{}) *Dtrie {\n\troot := insert(d.root, &entry{d.hasher(key), key, value})\n\treturn &Dtrie{root, d.hasher}\n}\n\n// Remove deletes the value for the associated key if it exists and returns\n// the resulting Dtrie.\nfunc (d *Dtrie) Remove(key interface{}) *Dtrie {\n\troot := remove(d.root, d.hasher(key), key)\n\treturn &Dtrie{root, d.hasher}\n}\n\n// Iterator returns a read-only channel of Entries from the Dtrie. If a stop\n// channel is provided, closing it will terminate and close the iterator\n// channel. Note that if a cancel channel is not used and not every entry is\n// read from the iterator, a goroutine will leak.\nfunc (d *Dtrie) Iterator(stop <-chan struct{}) <-chan Entry {\n\treturn iterate(d.root, stop)\n}\n" }, { "path": "trie/dtrie/dtrie_test.go", "content": "/*\nCopyright (c) 2016, Theodore Butler\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/\n\npackage dtrie\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestDefaultHasher(t *testing.T) {\n\tassert.Equal(t,\n\t\tdefaultHasher(map[int]string{11234: \"foo\"}),\n\t\tdefaultHasher(map[int]string{11234: \"foo\"}))\n\tassert.NotEqual(t, defaultHasher(\"foo\"), defaultHasher(\"bar\"))\n}\n\nfunc collisionHash(key interface{}) uint32 {\n\treturn uint32(0xffffffff) // for testing collisions\n}\n\nfunc TestInsert(t *testing.T) {\n\tinsertTest(t, defaultHasher, 10000)\n\tinsertTest(t, collisionHash, 1000)\n}\n\nfunc insertTest(t *testing.T, hashfunc func(interface{}) uint32, count int) *node {\n\tn := emptyNode(0, 32)\n\tfor i := 0; i < count; i++ {\n\t\tn = insert(n, &entry{hashfunc(i), i, i})\n\t}\n\treturn n\n}\n\nfunc TestGet(t *testing.T) {\n\tgetTest(t, defaultHasher, 10000)\n\tgetTest(t, collisionHash, 1000)\n}\n\nfunc getTest(t *testing.T, hashfunc func(interface{}) uint32, count int) {\n\tn := insertTest(t, hashfunc, count)\n\tfor i := 0; i < count; i++ {\n\t\tx := get(n, hashfunc(i), i)\n\t\tassert.Equal(t, i, x.Value())\n\t}\n}\n\nfunc TestRemove(t *testing.T) {\n\tremoveTest(t, defaultHasher, 10000)\n\tremoveTest(t, collisionHash, 1000)\n}\n\nfunc removeTest(t *testing.T, hashfunc func(interface{}) uint32, count int) {\n\tn := insertTest(t, hashfunc, count)\n\tfor i := 0; i < count; i++ {\n\t\tn = remove(n, hashfunc(i), i)\n\t}\n\tfor _, e := range n.entries {\n\t\tif e != nil {\n\t\t\tt.Fatal(\"final node is not empty\")\n\t\t}\n\t}\n}\n\nfunc TestUpdate(t *testing.T) {\n\tupdateTest(t, defaultHasher, 10000)\n\tupdateTest(t, collisionHash, 1000)\n}\n\nfunc updateTest(t *testing.T, hashfunc func(interface{}) uint32, count int) {\n\tn := insertTest(t, hashfunc, count)\n\tfor i := 0; i < count; i++ {\n\t\tn = insert(n, &entry{hashfunc(i), i, -i})\n\t}\n}\n\nfunc TestIterate(t *testing.T) {\n\tn := insertTest(t, defaultHasher, 10000)\n\techan := iterate(n, nil)\n\tc := 0\n\tfor _ = range echan {\n\t\tc++\n\t}\n\tassert.Equal(t, 10000, c)\n\t// test with stop chan\n\tc = 0\n\tstop := make(chan struct{})\n\techan = iterate(n, stop)\n\tfor _ = range echan {\n\t\tc++\n\t\tif c == 100 {\n\t\t\tclose(stop)\n\t\t\tbreak\n\t\t}\n\t}\n\tassert.True(t, c == 100)\n\t// test with collisions\n\tn = insertTest(t, collisionHash, 1000)\n\tc = 0\n\techan = iterate(n, nil)\n\tfor _ = range echan {\n\t\tc++\n\t}\n\tassert.Equal(t, 1000, c)\n}\n\nfunc TestSize(t *testing.T) {\n\tn := insertTest(t, defaultHasher, 10000)\n\td := &Dtrie{n, defaultHasher}\n\tassert.Equal(t, 10000, d.Size())\n}\n\nfunc BenchmarkInsert(b *testing.B) {\n\tb.ReportAllocs()\n\tn := emptyNode(0, 32)\n\tb.ResetTimer()\n\tfor i := b.N; i > 0; i-- {\n\t\tn = insert(n, &entry{defaultHasher(i), i, i})\n\t}\n}\n\nfunc BenchmarkGet(b *testing.B) {\n\tb.ReportAllocs()\n\tn := insertTest(nil, defaultHasher, b.N)\n\tb.ResetTimer()\n\tfor i := b.N; i > 0; i-- {\n\t\tget(n, defaultHasher(i), i)\n\t}\n}\n\nfunc BenchmarkRemove(b *testing.B) {\n\tb.ReportAllocs()\n\tn := insertTest(nil, defaultHasher, b.N)\n\tb.ResetTimer()\n\tfor i := b.N; i > 0; i-- {\n\t\tn = remove(n, defaultHasher(i), i)\n\t}\n}\n\nfunc BenchmarkUpdate(b *testing.B) {\n\tb.ReportAllocs()\n\tn := insertTest(nil, defaultHasher, b.N)\n\tb.ResetTimer()\n\tfor i := b.N; i > 0; i-- {\n\t\tn = insert(n, &entry{defaultHasher(i), i, -i})\n\t}\n}\n" }, { "path": "trie/dtrie/node.go", "content": "/*\nCopyright (c) 2016, Theodore Butler\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/\n\npackage dtrie\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\n\t\"github.com/Workiva/go-datastructures/bitarray\"\n)\n\ntype node struct {\n\tentries []Entry\n\tnodeMap bitarray.Bitmap32\n\tdataMap bitarray.Bitmap32\n\tlevel uint8 // level starts at 0\n}\n\nfunc (n *node) KeyHash() uint32 { return 0 }\nfunc (n *node) Key() interface{} { return nil }\nfunc (n *node) Value() interface{} { return nil }\n\nfunc (n *node) String() string {\n\treturn fmt.Sprint(n.entries)\n}\n\ntype collisionNode struct {\n\tentries []Entry\n}\n\nfunc (n *collisionNode) KeyHash() uint32 { return 0 }\nfunc (n *collisionNode) Key() interface{} { return nil }\nfunc (n *collisionNode) Value() interface{} { return nil }\n\nfunc (n *collisionNode) String() string {\n\treturn fmt.Sprintf(\"%v\", len(n.entries), n.entries)\n}\n\n// Entry defines anything held within the data structure\ntype Entry interface {\n\tKeyHash() uint32\n\tKey() interface{}\n\tValue() interface{}\n}\n\nfunc emptyNode(level uint8, capacity int) *node {\n\treturn &node{entries: make([]Entry, capacity), level: level}\n}\n\nfunc insert(n *node, entry Entry) *node {\n\tindex := uint(mask(entry.KeyHash(), n.level))\n\tnewNode := n\n\tif newNode.level == 6 { // handle hash collisions on 6th level\n\t\tif newNode.entries[index] == nil {\n\t\t\tnewNode.entries[index] = entry\n\t\t\tnewNode.dataMap = newNode.dataMap.SetBit(index)\n\t\t\treturn newNode\n\t\t}\n\t\tif newNode.dataMap.GetBit(index) {\n\t\t\tif newNode.entries[index].Key() == entry.Key() {\n\t\t\t\tnewNode.entries[index] = entry\n\t\t\t\treturn newNode\n\t\t\t}\n\t\t\tcNode := &collisionNode{entries: make([]Entry, 2)}\n\t\t\tcNode.entries[0] = newNode.entries[index]\n\t\t\tcNode.entries[1] = entry\n\t\t\tnewNode.entries[index] = cNode\n\t\t\tnewNode.dataMap = newNode.dataMap.ClearBit(index)\n\t\t\treturn newNode\n\t\t}\n\t\tcNode := newNode.entries[index].(*collisionNode)\n\t\tcNode.entries = append(cNode.entries, entry)\n\t\treturn newNode\n\t}\n\tif !newNode.dataMap.GetBit(index) && !newNode.nodeMap.GetBit(index) { // insert directly\n\t\tnewNode.entries[index] = entry\n\t\tnewNode.dataMap = newNode.dataMap.SetBit(index)\n\t\treturn newNode\n\t}\n\tif newNode.nodeMap.GetBit(index) { // insert into sub-node\n\t\tnewNode.entries[index] = insert(newNode.entries[index].(*node), entry)\n\t\treturn newNode\n\t}\n\tif newNode.entries[index].Key() == entry.Key() {\n\t\tnewNode.entries[index] = entry\n\t\treturn newNode\n\t}\n\t// create new node with the new and existing entries\n\tvar subNode *node\n\tif newNode.level == 5 { // only 2 bits left at level 6 (4 possible indices)\n\t\tsubNode = emptyNode(newNode.level+1, 4)\n\t} else {\n\t\tsubNode = emptyNode(newNode.level+1, 32)\n\t}\n\tsubNode = insert(subNode, newNode.entries[index])\n\tsubNode = insert(subNode, entry)\n\tnewNode.dataMap = newNode.dataMap.ClearBit(index)\n\tnewNode.nodeMap = newNode.nodeMap.SetBit(index)\n\tnewNode.entries[index] = subNode\n\treturn newNode\n}\n\n// returns nil if not found\nfunc get(n *node, keyHash uint32, key interface{}) Entry {\n\tindex := uint(mask(keyHash, n.level))\n\tif n.dataMap.GetBit(index) {\n\t\treturn n.entries[index]\n\t}\n\tif n.nodeMap.GetBit(index) {\n\t\treturn get(n.entries[index].(*node), keyHash, key)\n\t}\n\tif n.level == 6 { // get from collisionNode\n\t\tif n.entries[index] == nil {\n\t\t\treturn nil\n\t\t}\n\t\tcNode := n.entries[index].(*collisionNode)\n\t\tfor _, e := range cNode.entries {\n\t\t\tif e.Key() == key {\n\t\t\t\treturn e\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc remove(n *node, keyHash uint32, key interface{}) *node {\n\tindex := uint(mask(keyHash, n.level))\n\tnewNode := n\n\tif n.dataMap.GetBit(index) {\n\t\tnewNode.entries[index] = nil\n\t\tnewNode.dataMap = newNode.dataMap.ClearBit(index)\n\t\treturn newNode\n\t}\n\tif n.nodeMap.GetBit(index) {\n\t\tsubNode := newNode.entries[index].(*node)\n\t\tsubNode = remove(subNode, keyHash, key)\n\t\t// compress if only 1 entry exists in sub-node\n\t\tif subNode.nodeMap.PopCount() == 0 && subNode.dataMap.PopCount() == 1 {\n\t\t\tvar e Entry\n\t\t\tfor i := uint(0); i < 32; i++ {\n\t\t\t\tif subNode.dataMap.GetBit(i) {\n\t\t\t\t\te = subNode.entries[i]\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tnewNode.entries[index] = e\n\t\t\tnewNode.nodeMap = newNode.nodeMap.ClearBit(index)\n\t\t\tnewNode.dataMap = newNode.dataMap.SetBit(index)\n\t\t}\n\t\tnewNode.entries[index] = subNode\n\t\treturn newNode\n\t}\n\tif n.level == 6 { // delete from collisionNode\n\t\tcNode := newNode.entries[index].(*collisionNode)\n\t\tfor i, e := range cNode.entries {\n\t\t\tif e.Key() == key {\n\t\t\t\tcNode.entries = append(cNode.entries[:i], cNode.entries[i+1:]...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\t// compress if only 1 entry exists in collisionNode\n\t\tif len(cNode.entries) == 1 {\n\t\t\tnewNode.entries[index] = cNode.entries[0]\n\t\t\tnewNode.dataMap = newNode.dataMap.SetBit(index)\n\t\t}\n\t\treturn newNode\n\t}\n\treturn n\n}\n\nfunc iterate(n *node, stop <-chan struct{}) <-chan Entry {\n\tout := make(chan Entry)\n\tgo func() {\n\t\tdefer close(out)\n\t\tpushEntries(n, stop, out)\n\t}()\n\treturn out\n}\n\nfunc pushEntries(n *node, stop <-chan struct{}, out chan Entry) {\n\tvar wg sync.WaitGroup\n\tfor i, e := range n.entries {\n\t\tselect {\n\t\tcase <-stop:\n\t\t\treturn\n\t\tdefault:\n\t\t\tindex := uint(i)\n\t\t\tswitch {\n\t\t\tcase n.dataMap.GetBit(index):\n\t\t\t\tout <- e\n\t\t\tcase n.nodeMap.GetBit(index):\n\t\t\t\twg.Add(1)\n\t\t\t\tgo func() {\n\t\t\t\t\tdefer wg.Done()\n\t\t\t\t\tpushEntries(e.(*node), stop, out)\n\t\t\t\t}()\n\t\t\t\twg.Wait()\n\t\t\tcase n.level == 6 && e != nil:\n\t\t\t\tfor _, ce := range n.entries[index].(*collisionNode).entries {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-stop:\n\t\t\t\t\t\treturn\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tout <- ce\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "trie/dtrie/util.go", "content": "/*\nCopyright (c) 2016, Theodore Butler\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/\n\npackage dtrie\n\nimport (\n\t\"fmt\"\n\t\"hash/fnv\"\n)\n\nfunc mask(hash uint32, level uint8) uint32 {\n\treturn (hash >> (5 * level)) & 0x01f\n}\n\nfunc defaultHasher(value interface{}) uint32 {\n\tswitch v := value.(type) {\n\tcase uint8:\n\t\treturn uint32(v)\n\tcase uint16:\n\t\treturn uint32(v)\n\tcase uint32:\n\t\treturn v\n\tcase uint64:\n\t\treturn uint32(v)\n\tcase int8:\n\t\treturn uint32(v)\n\tcase int16:\n\t\treturn uint32(v)\n\tcase int32:\n\t\treturn uint32(v)\n\tcase int64:\n\t\treturn uint32(v)\n\tcase uint:\n\t\treturn uint32(v)\n\tcase int:\n\t\treturn uint32(v)\n\tcase uintptr:\n\t\treturn uint32(v)\n\tcase float32:\n\t\treturn uint32(v)\n\tcase float64:\n\t\treturn uint32(v)\n\t}\n\thasher := fnv.New32a()\n\thasher.Write([]byte(fmt.Sprintf(\"%#v\", value)))\n\treturn hasher.Sum32()\n}\n" }, { "path": "trie/xfast/iterator.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage xfast\n\n// Entries is a typed list of Entry interfaces.\ntype Entries []Entry\n\n// Iterator will iterate of the results of a query.\ntype Iterator struct {\n\tn *node\n\tfirst bool\n}\n\n// Next will return a bool indicating if another value exists\n// in the iterator.\nfunc (iter *Iterator) Next() bool {\n\tif iter.first {\n\t\titer.first = false\n\t\treturn iter.n != nil\n\t}\n\n\titer.n = iter.n.children[1]\n\treturn iter.n != nil\n}\n\n// Value will return the Entry representing the iterator's current position.\n// If no Entry exists at the present condition, the iterator is\n// exhausted and this method will return nil.\nfunc (iter *Iterator) Value() Entry {\n\tif iter.n == nil {\n\t\treturn nil\n\t}\n\n\treturn iter.n.entry\n}\n\n// exhaust is a helper function that will exhaust this iterator\n// and return a list of entries. This is for internal use only.\nfunc (iter *Iterator) exhaust() Entries {\n\tentries := make(Entries, 0, 100)\n\tfor it := iter; it.Next(); {\n\t\tentries = append(entries, it.Value())\n\t}\n\n\treturn entries\n}\n" }, { "path": "trie/xfast/iterator_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage xfast\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestIterator(t *testing.T) {\n\titer := &Iterator{\n\t\tfirst: true,\n\t}\n\n\tassert.False(t, iter.Next())\n\tassert.Nil(t, iter.Value())\n\n\te1 := newMockEntry(5)\n\tn1 := newNode(nil, e1)\n\titer = &Iterator{\n\t\tfirst: true,\n\t\tn: n1,\n\t}\n\n\tassert.True(t, iter.Next())\n\tassert.Equal(t, e1, iter.Value())\n\tassert.False(t, iter.Next())\n\tassert.Nil(t, iter.Value())\n\n\te2 := newMockEntry(10)\n\tn2 := newNode(nil, e2)\n\tn1.children[1] = n2\n\n\titer = &Iterator{\n\t\tfirst: true,\n\t\tn: n1,\n\t}\n\n\tassert.True(t, iter.Next())\n\tassert.True(t, iter.Next())\n\tassert.Equal(t, e2, iter.Value())\n\tassert.False(t, iter.Next())\n\tassert.Nil(t, iter.Value())\n}\n" }, { "path": "trie/xfast/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage xfast\n\nimport \"github.com/stretchr/testify/mock\"\n\ntype mockEntry struct {\n\tmock.Mock\n}\n\nfunc (me *mockEntry) Key() uint64 {\n\targs := me.Called()\n\treturn args.Get(0).(uint64)\n}\n\nfunc newMockEntry(key uint64) *mockEntry {\n\tme := new(mockEntry)\n\tme.On(`Key`).Return(key)\n\treturn me\n}\n" }, { "path": "trie/xfast/xfast.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage xfast provides access to a sorted tree that treats integers\nas if they were words of m bits, where m can be 8, 16, 32, or 64.\nThe advantage to storing integers as a trie of words is that operations\ncan be performed in constant time depending on the size of the\nuniverse and not on the number of items in the trie.\n\nThe time complexity is as follows:\nSpace: O(n log M)\nInsert: O(log M)\nDelete: O(log M)\nSearch: O(log log M)\nGet: O(1)\n\nwhere n is the number of items in the trie and M is the size of the\nuniverse, ie, 2^63-1 for 64 bit ints.\n\nAs you can see, for 64 bit ints, inserts and deletes can be performed\nin O(64), constant time which provides very predictable behavior\nin the best case.\n\nA get by key can be performed in O(1) time and searches can be performed\nin O(6) time for 64 bit integers.\n\nWhile x-tries have relatively slow insert, deletions, and consume a large\namount of space, they form the top half of a y-fast trie which can\ninsert and delete in O(log log M) time and consumes O(n) space.\n*/\npackage xfast\n\nimport \"fmt\"\n\n// isInternal returns a bool indicating if the provided\n// node is an internal node, that is, non-leaf node.\nfunc isInternal(n *node) bool {\n\tif n == nil {\n\t\treturn false\n\t}\n\treturn n.entry == nil\n}\n\n// hasInternal returns a bool indicating if the provided\n// node has a child that is an internal node.\nfunc hasInternal(n *node) bool {\n\treturn isInternal(n.children[0]) || isInternal(n.children[1])\n}\n\n// isLeaf returns a bool indicating if the provided node\n// is a leaf node, that is, has a valid entry value.\nfunc isLeaf(n *node) bool {\n\tif n == nil {\n\t\treturn false\n\t}\n\treturn !isInternal(n)\n}\n\n// Entry defines items that can be inserted into the x-fast\n// trie.\ntype Entry interface {\n\t// Key is the key for this entry. If the trie has been\n\t// given bit size n, only the last n bits of this key\n\t// will matter. Use a bit size of 64 to enable all\n\t// 2^64-1 keys.\n\tKey() uint64\n}\n\n// masks are used to determine the prefix of any given key. The masks\n// are stored in a [64] array where each position of the array represents\n// a bitmask to the ith bit. For example, if you wanted to mask the first\n// bit of a 64-bit int you'd and it with masks[0]. If you wanted to mask\n// the first bit of an 8 bit key, you'd have to shift 56 bits to the right\n// and perform the mask operation. This array is immutable and should not\n// be changed after initialization.\nvar masks = func() [64]uint64 { // we don't technically need the last mask, this is just to be consistent\n\tmasks := [64]uint64{}\n\tmask := uint64(0)\n\tfor i := uint64(0); i < 64; i++ {\n\t\tmask = mask | 1<<(63-i)\n\t\tmasks[i] = mask\n\t}\n\treturn masks\n}()\n\n// positions are similar to masks and that the positions array allows\n// us to determine if a node should go left or right at a specific bit\n// position of the key. Basically, this array stores every 2^n number\n// where n is in [0, 64). This array is immutable and should not be\n// changed after initialization.\nvar positions = func() [64]uint64 {\n\tpositions := [64]uint64{}\n\tfor i := uint64(0); i < 64; i++ {\n\t\tpositions[i] = uint64(1 << (63 - i))\n\t}\n\treturn positions\n}()\n\ntype node struct {\n\t// entry will store the entry for this node. Is nil for\n\t// every internal node and non-nil for all leaves. It is\n\t// how the internal/leaf function helpers determine the\n\t// position of this node.\n\tentry Entry\n\t// children stores the left and right child of this node.\n\t// At any time, and at any layer, it's possible for a pointer\n\t// to a child to point to a leaf due to threading.\n\tchildren [2]*node\n\t// i hate this, but it is really the best way\n\t// to walk up successor and predecessor threads\n\tparent *node\n}\n\n// newNode will allocate and initialize a newNode with the provided\n// parent and entry. Parent should never be nil, but entry may be\n// if constructing an internal node.\nfunc newNode(parent *node, entry Entry) *node {\n\treturn &node{\n\t\tchildren: [2]*node{},\n\t\tentry: entry,\n\t\tparent: parent,\n\t}\n}\n\n// binarySearchHashMaps will perform a binary search of the provided\n// maps to return a node that matches the longest prefix of the provided\n// key. This will return nil if a match could not be found, which would\n// also return layer 0. Layer information is useful when determining the\n// distance from the provided node to the leaves.\nfunc binarySearchHashMaps(layers []map[uint64]*node, key uint64) (int, *node) {\n\tlow, high := 0, len(layers)-1\n\tdiff := 64 - len(layers)\n\tvar mid int\n\tvar node *node\n\tfor low <= high {\n\t\tmid = (low + high) / 2\n\t\tn, ok := layers[mid][key&masks[diff+mid]]\n\t\tif ok {\n\t\t\tnode = n\n\t\t\tlow = mid + 1\n\t\t} else {\n\t\t\thigh = mid - 1\n\t\t}\n\t}\n\n\treturn low, node\n}\n\n// whichSide returns an int representing the side on which\n// the node resides in its parent. NOTE: this function will panic\n// if the child does not within the parent. This situation should\n// should be caught as early as possible as if it happens data\n// coming from the x-fast trie cannot be trusted.\nfunc whichSide(n, parent *node) int {\n\tif parent.children[0] == n {\n\t\treturn 0\n\t}\n\n\tif parent.children[1] == n {\n\t\treturn 1\n\t}\n\n\tpanic(fmt.Sprintf(`Node: %+v, %p not a child of: %+v, %p`, n, n, parent, parent))\n}\n\n// XFastTrie is a datastructure for storing integers in a known\n// universe, where universe size is determined by the bit size\n// of the desired keys. This structure should be faster than\n// binary search tries for very large datasets and slower for\n// smaller datasets.\ntype XFastTrie struct {\n\t// layers stores the hashmaps of the individual layers of the trie.\n\t// The hashmaps store prefixes, allowing use to do a binary search\n\t// of these maps before visiting the trie for successor/predecessor\n\t// queries.\n\tlayers []map[uint64]*node\n\t// root is a pointer to the first node of the trie, which actually\n\t// adds an additional layer, ie, instead of 64 layers for a\n\t// uint64, this will cause the number of layers to be 65.\n\troot *node\n\t// num is the number of items in the trie.\n\tnum uint64\n\t// bits represents the number of bits of the keys this trie\n\t// expects. Because the time complexity of operations is\n\t// dependent upon universe size, smaller sized keys will\n\t// actually cause the trie to be faster. Diff is the difference\n\t// between the desired bit size and 64 as we have to offset\n\t// in the position and mask arrays.\n\tbits, diff uint8\n\t// min and max index the lowest and highest seen keys respectively.\n\t// this immediately allows us to check a desired key against\n\t// constraints and allows min/max operations to be performed\n\t// in O(1) time.\n\tmin, max *node\n}\n\n// init will initialize the XFastTrie with the provided byte-size.\n// I'd prefer generics here, but it is what it is. We expect uints\n// here when ints would perform just as well, but the public methods\n// on the XFastTrie all expect uint64, so we expect a uint in the\n// constructor for consistency's sake.\nfunc (xft *XFastTrie) init(intType interface{}) {\n\tbits := uint8(0)\n\tswitch intType.(type) {\n\tcase uint8:\n\t\tbits = 8\n\tcase uint16:\n\t\tbits = 16\n\tcase uint32:\n\t\tbits = 32\n\tcase uint, uint64:\n\t\tbits = 64\n\tdefault:\n\t\t// we'll panic with a bad value to the constructor.\n\t\tpanic(`Invalid universe size provided.`)\n\t}\n\n\txft.layers = make([]map[uint64]*node, bits)\n\txft.bits = bits\n\txft.diff = 64 - bits\n\tfor i := uint8(0); i < bits; i++ {\n\t\txft.layers[i] = make(map[uint64]*node, 50) // we can obviously be more intelligent about this.\n\t}\n\txft.num = 0\n\txft.root = newNode(nil, nil)\n}\n\n// Exists returns a bool indicating if the provided\n// key exists in the trie. This is an O(1) operation.\nfunc (xft *XFastTrie) Exists(key uint64) bool {\n\t// the bottom hashmap of the trie has every entry\n\t// in it.\n\t_, ok := xft.layers[xft.bits-1][key]\n\treturn ok\n}\n\n// Len returns the number of items in this trie. This is an\n// O(1) operation.\nfunc (xft *XFastTrie) Len() uint64 {\n\treturn xft.num\n}\n\n// Max will return the highest keyed value in the trie. This is\n// an O(1) operation.\nfunc (xft *XFastTrie) Max() Entry {\n\tif xft.max == nil {\n\t\treturn nil\n\t}\n\n\treturn xft.max.entry\n}\n\n// Min will return the lowest keyed value in the trie. This is\n// an O(1) operation.\nfunc (xft *XFastTrie) Min() Entry {\n\tif xft.min == nil {\n\t\treturn nil\n\t}\n\n\treturn xft.min.entry\n}\n\n// insert will add the provided entry to the trie or overwrite the existing\n// entry if it exists.\nfunc (xft *XFastTrie) insert(entry Entry) {\n\tkey := entry.Key() // so we aren't calling this interface method over and over, fucking Go\n\tn := xft.layers[xft.bits-1][key]\n\tif n != nil {\n\t\tn.entry = entry\n\t\treturn\n\t}\n\n\t// we need to find a predecessor or successor if it exists\n\t// to help us set threads later in this method.\n\tvar predecessor, successor *node\n\tif xft.min != nil && key < xft.min.entry.Key() {\n\t\tsuccessor = xft.min\n\t} else {\n\t\tsuccessor = xft.successor(key)\n\t}\n\n\t// only need to find predecessor if successor is nil as otherwise\n\t// the successor will provide us is the predecessor if it exists.\n\tif successor == nil {\n\t\tif xft.max != nil && key > xft.max.entry.Key() {\n\t\t\tpredecessor = xft.max\n\t\t} else {\n\t\t\tpredecessor = xft.predecessor(key)\n\t\t}\n\t}\n\n\t// find the deepest root with a matching prefix, this should\n\t// save us some time, assuming the hashmap has perfect hashing.\n\tlayer, root := binarySearchHashMaps(xft.layers, key)\n\tif root == nil {\n\t\tn = xft.root\n\t\tlayer = 0\n\t} else {\n\t\tn = root\n\t}\n\n\tvar leftOrRight uint64\n\n\t// from the existing node, create new nodes.\n\tfor i := uint8(layer); i < xft.bits; i++ {\n\t\t// on 0th, this will be root\n\t\t// find out if we need to go left or right\n\t\tleftOrRight = (key & positions[xft.diff+i]) >> (xft.bits - 1 - i)\n\t\tif n.children[leftOrRight] == nil || isLeaf(n.children[leftOrRight]) {\n\t\t\tvar nn *node\n\t\t\tif i < xft.bits-1 {\n\t\t\t\tnn = newNode(n, nil)\n\t\t\t} else {\n\t\t\t\tnn = newNode(n, entry)\n\t\t\t\txft.num++\n\t\t\t}\n\n\t\t\tn.children[leftOrRight] = nn\n\t\t\txft.layers[i][key&masks[xft.diff+i]] = nn // prefix for this layer\n\t\t}\n\n\t\tn = n.children[leftOrRight]\n\t}\n\n\t// we need to put the new node where it belongs in the doubly-linked\n\t// list comprised of all the leaves.\n\tif successor != nil { // we have to walk predecessor and successor threads\n\t\tpredecessor = successor.children[0]\n\t\tif predecessor != nil {\n\t\t\tpredecessor.children[1] = n\n\t\t\tn.children[0] = predecessor\n\t\t}\n\t\tn.children[1] = successor\n\t\tsuccessor.children[0] = n\n\t} else if predecessor != nil {\n\t\tn.children[0] = predecessor\n\t\tpredecessor.children[1] = n\n\t}\n\n\t// walk up the successor if it exists to set that branch's new\n\t// predecessor.\n\tif successor != nil {\n\t\txft.walkUpSuccessor(root, n, successor)\n\t}\n\n\t// walk up the predecessor if it exists to set that branch's\n\t// new successor.\n\tif predecessor != nil {\n\t\txft.walkUpPredecessor(root, n, predecessor)\n\t}\n\n\t// finally, walk up our own branch to set both successors\n\t// and predecessors.\n\txft.walkUpNode(root, n, predecessor, successor)\n\n\t// and then do a final check against the min/max indicies.\n\tif xft.max == nil || key > xft.max.entry.Key() {\n\t\txft.max = n\n\t}\n\n\tif xft.min == nil || key < xft.min.entry.Key() {\n\t\txft.min = n\n\t}\n}\n\n// walkUpSuccessor will walk up the successor branch setting\n// the predecessor where possible. This breaks when a common\n// ancestor between successor and node is found, ie, the root.\nfunc (xft *XFastTrie) walkUpSuccessor(root, node, successor *node) {\n\tn := successor.parent\n\tfor n != nil && n != root {\n\t\t// we don't really want to overwrite existing internal nodes,\n\t\t// or where the child is a leaf that is the successor\n\t\tif !isInternal(n.children[0]) && n.children[0] != successor {\n\t\t\tn.children[0] = node\n\t\t}\n\t\tn = n.parent\n\t}\n}\n\n// walkUpPredecessor will walk up the predecessor branch setting\n// the successor where possible. This breaks when a common\n// ancestor between predecessor and node is found, ie, the root.\nfunc (xft *XFastTrie) walkUpPredecessor(root, node, predecessor *node) {\n\tn := predecessor.parent\n\tfor n != nil && n != root {\n\t\tif !isInternal(n.children[1]) && n.children[1] != predecessor {\n\t\t\tn.children[1] = node\n\t\t}\n\t\tn = n.parent\n\t}\n}\n\n// walkUpNode will walk up the newly created branch and set predecessor\n// and successor where possible. If predecessor or successor are nil,\n// this will set nil where possible.\nfunc (xft *XFastTrie) walkUpNode(root, node, predecessor, successor *node) {\n\tn := node.parent\n\tfor n != nil && n != root {\n\t\tif !isInternal(n.children[1]) && n.children[1] != successor && n.children[1] != node {\n\t\t\tn.children[1] = successor\n\t\t}\n\t\tif !isInternal(n.children[0]) && n.children[0] != predecessor && n.children[0] != node {\n\t\t\tn.children[0] = predecessor\n\t\t}\n\t\tn = n.parent\n\t}\n}\n\n// Insert will insert the provided entries into the trie. Any entry\n// with an existing key will cause an overwrite. This is an O(log M)\n// operation, for each entry.\nfunc (xft *XFastTrie) Insert(entries ...Entry) {\n\tfor _, e := range entries {\n\t\txft.insert(e)\n\t}\n}\n\nfunc (xft *XFastTrie) delete(key uint64) {\n\tn := xft.layers[xft.bits-1][key]\n\tif n == nil { // there's no matching k, v pair\n\t\treturn\n\t}\n\n\tsuccessor, predecessor := n.children[1], n.children[0]\n\n\ti := uint8(1)\n\tdelete(xft.layers[xft.bits-1], key)\n\tleftOrRight := whichSide(n, n.parent)\n\tn.parent.children[leftOrRight] = nil\n\tn.children[0], n.children[1] = nil, nil\n\tn = n.parent\n\thasImmediateSibling := false\n\tif successor != nil && successor.parent == n {\n\t\thasImmediateSibling = true\n\t}\n\tif predecessor != nil && predecessor.parent == n {\n\t\thasImmediateSibling = true\n\t}\n\n\t// this loop will kill any nodes that no longer link to internal\n\t// nodes\n\tfor n != nil && n.parent != nil {\n\t\t// if we have an internal node remaining we should abort\n\t\t// now as no further node will be removed. We should also\n\t\t// abort if the first parent of a leaf references the pre\n\t\tif hasInternal(n) || (i == 1 && hasImmediateSibling) {\n\t\t\tn = n.parent // we had one side deleted, need to set the other\n\t\t\tbreak\n\t\t}\n\n\t\tleftOrRight = whichSide(n, n.parent)\n\t\tn.parent.children[leftOrRight] = nil\n\t\tn.children[0], n.children[1] = nil, nil\n\t\tdelete(xft.layers[xft.bits-i-1], key&masks[len(masks)-1-int(i)])\n\t\tn = n.parent\n\t\ti++\n\t}\n\n\t// we need to check now and update threads, but in the leaves\n\t// and in their branches\n\tif predecessor != nil {\n\t\tpredecessor.children[1] = successor\n\t\txft.walkUpPredecessor(n, successor, predecessor)\n\t}\n\n\tif successor != nil {\n\t\tsuccessor.children[0] = predecessor\n\t\txft.walkUpSuccessor(n, predecessor, successor)\n\t}\n\n\t// check max/min indices\n\tif xft.max.entry.Key() == key {\n\t\txft.max = predecessor\n\t}\n\n\tif xft.min.entry.Key() == key {\n\t\txft.min = successor\n\t}\n\n\t// decrement number of nodes\n\txft.num--\n}\n\n// Delete will delete the provided keys from the trie. If an entry\n// associated with a provided key cannot be found, that deletion is\n// a no-op. Each deletion is an O(log M) operation.\nfunc (xft *XFastTrie) Delete(keys ...uint64) {\n\tfor _, key := range keys {\n\t\txft.delete(key)\n\t}\n}\n\n// predecessor will find the node equal to or immediately less\n// than the provided key.\nfunc (xft *XFastTrie) predecessor(key uint64) *node {\n\tif xft.root == nil || xft.max == nil { // no successor if no nodes\n\t\treturn nil\n\t}\n\n\tif key >= xft.max.entry.Key() {\n\t\treturn xft.max\n\t}\n\n\tif key < xft.min.entry.Key() {\n\t\treturn nil\n\t}\n\n\tn := xft.layers[xft.bits-1][key]\n\tif n != nil {\n\t\treturn n\n\t}\n\n\tlayer, n := binarySearchHashMaps(xft.layers, key)\n\tif n == nil && layer > 1 {\n\t\treturn nil\n\t} else if n == nil {\n\t\tn = xft.root\n\t}\n\n\tif isInternal(n.children[0]) && isLeaf(n.children[1]) {\n\t\treturn n.children[1].children[0]\n\t}\n\treturn n.children[0]\n}\n\n// successor will find the node equal to or immediately more\n// than the provided key.\nfunc (xft *XFastTrie) successor(key uint64) *node {\n\tif xft.root == nil || xft.min == nil { // no successor if no nodes\n\t\treturn nil\n\t}\n\n\tif key <= xft.min.entry.Key() {\n\t\treturn xft.min\n\t}\n\n\tif key > xft.max.entry.Key() {\n\t\treturn nil\n\t}\n\n\tn := xft.layers[xft.bits-1][key]\n\tif n != nil {\n\t\treturn n\n\t}\n\n\tlayer, n := binarySearchHashMaps(xft.layers, key)\n\tif n == nil && layer > 1 {\n\t\treturn nil\n\t} else if n == nil {\n\t\tn = xft.root\n\t}\n\n\tif isInternal(n.children[1]) && isLeaf(n.children[0]) {\n\t\treturn n.children[0].children[1]\n\t}\n\treturn n.children[1]\n}\n\n// Successor will return an Entry which matches the provided\n// key or its immediate successor. Will return nil if a successor\n// does not exist. This is an O(log log M) operation.\nfunc (xft *XFastTrie) Successor(key uint64) Entry {\n\tn := xft.successor(key)\n\tif n == nil {\n\t\treturn nil\n\t}\n\n\treturn n.entry\n}\n\n// Predecessor will return an Entry which matches the provided\n// key or its immediate predecessor. Will return nil if a predecessor\n// does not exist. This is an O(log log M) operation.\nfunc (xft *XFastTrie) Predecessor(key uint64) Entry {\n\tn := xft.predecessor(key)\n\tif n == nil {\n\t\treturn nil\n\t}\n\n\treturn n.entry\n}\n\n// Iter will return an iterator that will iterate over all values\n// equal to or immediately greater than the provided key. Iterator\n// will iterate successor relationships.\nfunc (xft *XFastTrie) Iter(key uint64) *Iterator {\n\treturn &Iterator{\n\t\tn: xft.successor(key),\n\t\tfirst: true,\n\t}\n}\n\n// Get will return a value in the trie associated with the provided\n// key if it exists. Returns nil if the key does not exist. This\n// is expected to take O(1) time.\nfunc (xft *XFastTrie) Get(key uint64) Entry {\n\t// only have to check the last hashmap for the provided\n\t// key.\n\tn := xft.layers[xft.bits-1][key]\n\tif n == nil {\n\t\treturn nil\n\t}\n\n\treturn n.entry\n}\n\n// New will construct a new X-Fast Trie with the given \"size,\"\n// that is the size of the universe of the trie. This expects\n// a uint of some sort, ie, uint8, uint16, etc. The size of the\n// universe will be 2^n-1 and will affect the speed of all operations.\n// IFC MUST be a uint type.\nfunc New(ifc interface{}) *XFastTrie {\n\txft := &XFastTrie{}\n\txft.init(ifc)\n\treturn xft\n}\n" }, { "path": "trie/xfast/xfast_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage xfast\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/Workiva/go-datastructures/slice\"\n)\n\nfunc checkTrie(t *testing.T, xft *XFastTrie) {\n\tcheckSuccessor(t, xft)\n\tcheckPredecessor(t, xft)\n\tcheckNodes(t, xft)\n}\n\nfunc checkSuccessor(t *testing.T, xft *XFastTrie) {\n\tn := xft.min\n\tvar side int\n\tvar successor *node\n\tfor n != nil {\n\t\tsuccessor = n.children[1]\n\t\thasSuccesor := successor != nil\n\t\timmediateSuccessor := false\n\t\tif hasSuccesor {\n\t\t\tassert.Equal(t, n, successor.children[0])\n\t\t\tif n.parent == successor.parent {\n\t\t\t\timmediateSuccessor = true\n\t\t\t}\n\t\t}\n\n\t\tfor n.parent != nil {\n\t\t\tside = whichSide(n, n.parent)\n\t\t\tif isInternal(n.parent.children[1]) && isInternal(n.parent.children[0]) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif immediateSuccessor && n.parent == successor.parent {\n\t\t\t\tassert.Equal(t, successor, n.parent.children[1])\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif side == 0 && !isInternal(n.parent.children[1]) && hasSuccesor {\n\t\t\t\tassert.Equal(t, successor, n.parent.children[1])\n\t\t\t}\n\t\t\tn = n.parent\n\t\t}\n\t\tn = successor\n\t}\n}\n\nfunc checkPredecessor(t *testing.T, xft *XFastTrie) {\n\tn := xft.max\n\tvar side int\n\tvar predecessor *node\n\tfor n != nil {\n\t\tpredecessor = n.children[0]\n\t\thasPredecessor := predecessor != nil\n\t\timmediatePredecessor := false\n\t\tif hasPredecessor {\n\t\t\tassert.Equal(t, n, predecessor.children[1])\n\t\t\tif n.parent == predecessor.parent {\n\t\t\t\timmediatePredecessor = true\n\t\t\t}\n\t\t}\n\t\tfor n.parent != nil {\n\t\t\tside = whichSide(n, n.parent)\n\t\t\tif isInternal(n.parent.children[0]) && isInternal(n.parent.children[1]) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif immediatePredecessor && n.parent == predecessor.parent {\n\t\t\t\tassert.Equal(t, predecessor, n.parent.children[0])\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif side == 1 && !isInternal(n.parent.children[0]) && hasPredecessor {\n\t\t\t\tassert.Equal(t, predecessor, n.parent.children[0])\n\t\t\t}\n\t\t\tn = n.parent\n\t\t}\n\t\tn = predecessor\n\t}\n}\n\nfunc checkNodes(t *testing.T, xft *XFastTrie) {\n\tcount := uint64(0)\n\tn := xft.min\n\tfor n != nil {\n\t\tcount++\n\t\tcheckNode(t, xft, n)\n\t\tn = n.children[1]\n\t}\n\n\tassert.Equal(t, count, xft.Len())\n}\n\nfunc checkNode(t *testing.T, xft *XFastTrie, n *node) {\n\tif n.entry == nil {\n\t\tassert.Fail(t, `Expected non-nil entry`)\n\t\treturn\n\t}\n\tkey := n.entry.Key()\n\tbits := make([]int, 0, xft.bits)\n\tfor i := uint8(0); i < xft.bits; i++ {\n\t\tleftOrRight := (key & positions[xft.diff+i]) >> (xft.bits - 1 - i)\n\t\tbits = append(bits, int(leftOrRight))\n\t}\n\n\tcheckPattern(t, n, bits)\n}\n\nfunc dumpNode(t *testing.T, n *node) {\n\tfor n != nil {\n\t\tt.Logf(`NODE: %+v, %p`, n, n)\n\t\tn = n.parent\n\t}\n}\n\nfunc checkPattern(t *testing.T, n *node, pattern []int) {\n\ti := len(pattern) - 1\n\tbottomNode := n\n\tfor n.parent != nil {\n\t\tif !assert.False(t, i < 0, fmt.Sprintf(`Too many parents. NODE: %+v, PATTERN: %+v`, bottomNode, pattern)) {\n\t\t\tdumpNode(t, bottomNode)\n\t\t\tbreak // so we don't panic on the next line\n\t\t}\n\t\tassert.Equal(t, pattern[i], whichSide(n, n.parent))\n\t\ti--\n\t\tn = n.parent\n\t}\n\n\tassert.Equal(t, -1, i)\n}\n\nfunc TestEmptyMinMax(t *testing.T) {\n\txft := New(uint8(0))\n\n\tassert.Nil(t, xft.Min())\n\tassert.Nil(t, xft.Max())\n}\n\nfunc TestMask(t *testing.T) {\n\tassert.Equal(t, uint64(math.MaxUint64), masks[63])\n}\n\nfunc TestInsert(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(5)\n\txft.Insert(e1)\n\n\tassert.True(t, xft.Exists(5))\n\tassert.Equal(t, e1, xft.Min())\n\tassert.Equal(t, e1, xft.Max())\n\tcheckTrie(t, xft)\n\n\te2 := newMockEntry(20)\n\txft.Insert(e2)\n\n\tassert.True(t, xft.Exists(20))\n\tassert.Equal(t, uint64(2), xft.Len())\n\tassert.Equal(t, e1, xft.Min())\n\tassert.Equal(t, e2, xft.Max())\n\tcheckTrie(t, xft)\n}\n\nfunc TestGet(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(5)\n\txft.Insert(e1)\n\n\tassert.Equal(t, e1, xft.Get(5))\n\tassert.Nil(t, xft.Get(6))\n}\n\nfunc TestInsertOverwrite(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(5)\n\txft.Insert(e1)\n\n\te2 := newMockEntry(5)\n\txft.Insert(e2)\n\tcheckTrie(t, xft)\n\n\titer := xft.Iter(5)\n\tassert.Equal(t, Entries{e2}, iter.exhaust())\n}\n\nfunc TestInsertBetween(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(10)\n\txft.Insert(e1)\n\n\tassert.True(t, xft.Exists(10))\n\tassert.Equal(t, e1, xft.Min())\n\tassert.Equal(t, e1, xft.Max())\n\tcheckTrie(t, xft)\n\n\te2 := newMockEntry(20)\n\txft.Insert(e2)\n\tcheckTrie(t, xft)\n\n\tassert.True(t, xft.Exists(20))\n\tassert.Equal(t, uint64(2), xft.Len())\n\tassert.Equal(t, e1, xft.Min())\n\tassert.Equal(t, e2, xft.Max())\n\n\tassert.Equal(t, e2, xft.Successor(15))\n\n\te3 := newMockEntry(15)\n\txft.Insert(e3)\n\n\tassert.True(t, xft.Exists(15))\n\tassert.Equal(t, uint64(3), xft.Len())\n\tassert.Equal(t, e1, xft.Min())\n\tassert.Equal(t, e2, xft.Max())\n\tcheckTrie(t, xft)\n\n\titer := xft.Iter(0)\n\tentries := iter.exhaust()\n\tassert.Equal(t, Entries{e1, e3, e2}, entries)\n\n\titer = xft.Iter(11)\n\tentries = iter.exhaust()\n\tassert.Equal(t, Entries{e3, e2}, entries)\n\n\titer = xft.Iter(16)\n\tentries = iter.exhaust()\n\tassert.Equal(t, Entries{e2}, entries)\n}\n\nfunc TestSuccessorDoesNotExist(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(5)\n\txft.Insert(e1)\n\n\tresult := xft.Successor(6)\n\tassert.Nil(t, result)\n}\n\nfunc TestSuccessorIsExactValue(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(5)\n\txft.Insert(e1)\n\n\tresult := xft.Successor(5)\n\tassert.Equal(t, e1, result)\n}\n\nfunc TestSuccessorGreaterThanKey(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(math.MaxUint8)\n\txft.Insert(e1)\n\n\tresult := xft.Successor(5)\n\tassert.Equal(t, e1, result)\n}\n\nfunc TestSuccessorCloseToKey(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(10)\n\txft.Insert(e1)\n\n\tresult := xft.Successor(5)\n\tassert.Equal(t, e1, result)\n}\n\nfunc TestSuccessorBetweenTwoKeys(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(10)\n\txft.Insert(e1)\n\n\te2 := newMockEntry(20)\n\txft.Insert(e2)\n\n\tfor i := uint64(11); i < 20; i++ {\n\t\tresult := xft.Successor(i)\n\t\tassert.Equal(t, e2, result)\n\t}\n\n\tfor i := uint64(21); i < 100; i++ {\n\t\tresult := xft.Successor(i)\n\t\tassert.Nil(t, result)\n\t}\n}\n\nfunc TestPredecessorDoesNotExist(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(5)\n\txft.Insert(e1)\n\n\tresult := xft.Predecessor(4)\n\tassert.Nil(t, result)\n}\n\nfunc TestPredecessorIsExactValue(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(5)\n\txft.Insert(e1)\n\n\tresult := xft.Predecessor(5)\n\tassert.Equal(t, e1, result)\n}\n\nfunc TestPredecessorLessThanKey(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(0)\n\txft.Insert(e1)\n\n\tresult := xft.Predecessor(math.MaxUint64)\n\tassert.Equal(t, e1, result)\n}\n\nfunc TestPredecessorCloseToKey(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(5)\n\txft.Insert(e1)\n\n\tresult := xft.Predecessor(10)\n\tassert.Equal(t, e1, result)\n}\n\nfunc TestPredecessorBetweenTwoKeys(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(10)\n\txft.Insert(e1)\n\n\te2 := newMockEntry(20)\n\txft.Insert(e2)\n\n\tfor i := uint64(11); i < 20; i++ {\n\t\tresult := xft.Predecessor(i)\n\t\tassert.Equal(t, e1, result)\n\t}\n\n\tfor i := uint64(0); i < 10; i++ {\n\t\tresult := xft.Predecessor(i)\n\t\tassert.Nil(t, result)\n\t}\n}\n\nfunc TestInsertPredecessor(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(10)\n\txft.Insert(e1)\n\n\te2 := newMockEntry(5)\n\txft.Insert(e2)\n\tcheckTrie(t, xft)\n\n\tassert.Equal(t, e2, xft.Min())\n\tassert.Equal(t, e1, xft.Max())\n\n\titer := xft.Iter(2)\n\tassert.Equal(t, Entries{e2, e1}, iter.exhaust())\n\n\titer = xft.Iter(5)\n\tassert.Equal(t, Entries{e2, e1}, iter.exhaust())\n\n\titer = xft.Iter(6)\n\tassert.Equal(t, Entries{e1}, iter.exhaust())\n\n\titer = xft.Iter(11)\n\tassert.Equal(t, Entries{}, iter.exhaust())\n}\n\nfunc TestDeleteOnlyBranch(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(10)\n\txft.Insert(e1)\n\n\txft.Delete(10)\n\tcheckTrie(t, xft)\n\tassert.Equal(t, uint64(0), xft.Len())\n\tassert.Nil(t, xft.Min())\n\tassert.Nil(t, xft.Max())\n\tfor _, hm := range xft.layers {\n\t\tassert.Len(t, hm, 0)\n\t}\n\n\tassert.NotNil(t, xft.root)\n\tassert.Nil(t, xft.root.children[0])\n\tassert.Nil(t, xft.root.children[1])\n\n\titer := xft.Iter(0)\n\tassert.False(t, iter.Next())\n}\n\nfunc TestDeleteLargeBranch(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(0)\n\te2 := newMockEntry(math.MaxUint8)\n\n\txft.Insert(e1, e2)\n\tcheckTrie(t, xft)\n\n\txft.Delete(0)\n\tassert.Equal(t, e2, xft.Min())\n\tassert.Equal(t, e2, xft.Max())\n\tcheckTrie(t, xft)\n\n\tassert.Nil(t, xft.root.children[0])\n\n\tn := xft.max\n\tfor n != nil {\n\t\tassert.Nil(t, n.children[0])\n\t\tn = n.parent\n\t}\n\n\titer := xft.Iter(0)\n\tassert.Equal(t, Entries{e2}, iter.exhaust())\n}\n\nfunc TestDeleteLateBranching(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(0)\n\te2 := newMockEntry(1)\n\n\txft.Insert(e1, e2)\n\tcheckTrie(t, xft)\n\n\txft.Delete(1)\n\tassert.Equal(t, e1, xft.Min())\n\tassert.Equal(t, e1, xft.Max())\n\tcheckTrie(t, xft)\n\n\tn := xft.min\n\tfor n != nil {\n\t\tassert.Nil(t, n.children[1])\n\t\tn = n.parent\n\t}\n\n\titer := xft.Iter(0)\n\tassert.Equal(t, Entries{e1}, iter.exhaust())\n}\n\nfunc TestDeleteLateBranchingMin(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(0)\n\te2 := newMockEntry(1)\n\n\txft.Insert(e1, e2)\n\tcheckTrie(t, xft)\n\n\txft.Delete(0)\n\tassert.Equal(t, e2, xft.Min())\n\tassert.Equal(t, e2, xft.Max())\n\tcheckTrie(t, xft)\n\n\tassert.Nil(t, xft.min.children[0])\n\tn := xft.min.parent\n\tassert.Nil(t, n.children[0])\n\tn = n.parent\n\tfor n != nil {\n\t\tassert.Nil(t, n.children[1])\n\t\tn = n.parent\n\t}\n\n\titer := xft.Iter(0)\n\tassert.Equal(t, Entries{e2}, iter.exhaust())\n}\n\nfunc TestDeleteMiddleBranch(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(0)\n\te2 := newMockEntry(math.MaxUint8)\n\te3 := newMockEntry(64) // [0, 1, 0, 0, 0, 0, 0, 0]\n\n\txft.Insert(e1, e2, e3)\n\tcheckTrie(t, xft)\n\n\txft.Delete(64)\n\tassert.Equal(t, e1, xft.Min())\n\tassert.Equal(t, e2, xft.Max())\n\tcheckTrie(t, xft)\n\n\titer := xft.Iter(0)\n\tassert.Equal(t, Entries{e1, e2}, iter.exhaust())\n}\n\nfunc TestDeleteMiddleBranchOtherSide(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(0)\n\te2 := newMockEntry(math.MaxUint8)\n\te3 := newMockEntry(128) // [1, 0, 0, 0, 0, 0, 0, 0]\n\n\txft.Insert(e1, e2, e3)\n\tcheckTrie(t, xft)\n\n\txft.Delete(128)\n\tassert.Equal(t, e1, xft.Min())\n\tassert.Equal(t, e2, xft.Max())\n\tcheckTrie(t, xft)\n\n\titer := xft.Iter(0)\n\tassert.Equal(t, Entries{e1, e2}, iter.exhaust())\n}\n\nfunc TestDeleteNotFound(t *testing.T) {\n\txft := New(uint8(0))\n\te1 := newMockEntry(64)\n\txft.Insert(e1)\n\tcheckTrie(t, xft)\n\n\txft.Delete(128)\n\tassert.Equal(t, e1, xft.Max())\n\tassert.Equal(t, e1, xft.Min())\n\tcheckTrie(t, xft)\n}\n\nfunc BenchmarkSuccessor(b *testing.B) {\n\tnumItems := 10000\n\txft := New(uint64(0))\n\n\tfor i := uint64(0); i < uint64(numItems); i++ {\n\t\txft.Insert(newMockEntry(i))\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\txft.Successor(uint64(i))\n\t}\n}\n\nfunc BenchmarkDelete(b *testing.B) {\n\txs := make([]*XFastTrie, 0, b.N)\n\n\tfor i := 0; i < b.N; i++ {\n\t\tx := New(uint8(0))\n\t\tx.Insert(newMockEntry(0))\n\t\txs = append(xs, x)\n\t}\n\n\t// this is actually a pretty bad case for the x-fast\n\t// trie as the entire branch will have to be walked.\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\txs[i].Delete(0)\n\t}\n}\n\nfunc BenchmarkInsert(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\txft := New(uint64(0))\n\t\te := newMockEntry(uint64(i))\n\t\txft.Insert(e)\n\t}\n}\n\n// benchmarked against a flat list\nfunc BenchmarkListInsert(b *testing.B) {\n\tnumItems := 100000\n\n\ts := make(slice.Int64Slice, 0, numItems)\n\tfor j := int64(0); j < int64(numItems); j++ {\n\t\ts = append(s, j)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ts.Insert(int64(i))\n\t}\n}\n\nfunc BenchmarkListSearch(b *testing.B) {\n\tnumItems := 1000000\n\n\ts := make(slice.Int64Slice, 0, numItems)\n\tfor j := int64(0); j < int64(numItems); j++ {\n\t\ts = append(s, j)\n\t}\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\ts.Search(int64(i))\n\t}\n}\n" }, { "path": "trie/yfast/entries.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage yfast\n\nimport \"sort\"\n\ntype entriesWrapper struct {\n\tkey uint64\n\tentries Entries\n}\n\n// Key will return the key of the highest entry in this list.\n// This is required by the x-fast trie Entry interface. This\n// returns 0 if this list is empty.\nfunc (ew *entriesWrapper) Key() uint64 {\n\treturn ew.key\n}\n\n// Entries is a typed list of Entry. The size of entries\n// will be limited to 1/2log M to 2log M where M is the size\n// of the universe.\ntype Entries []Entry\n\n// search will perform a sort package search on this list\n// of entries and return an index indicating position.\n// If the returned index is >= len(entries) then a suitable\n// position could not be found. The index does not guarantee\n// equality, just indicates where the key would be inserted.\nfunc (entries Entries) search(key uint64) int {\n\treturn sort.Search(len(entries), func(i int) bool {\n\t\treturn entries[i].Key() >= key\n\t})\n}\n\n// insert will insert the provided entry into this list of\n// entries. Returned is an entry if an entry already exists\n// for the provided key. If nothing is overwritten, Entry\n// will be nil.\nfunc (entries *Entries) insert(entry Entry) Entry {\n\ti := entries.search(entry.Key())\n\n\tif i == len(*entries) {\n\t\t*entries = append(*entries, entry)\n\t\treturn nil\n\t}\n\n\tif (*entries)[i].Key() == entry.Key() {\n\t\toldEntry := (*entries)[i]\n\t\t(*entries)[i] = entry\n\t\treturn oldEntry\n\t}\n\n\t(*entries) = append(*entries, nil)\n\tcopy((*entries)[i+1:], (*entries)[i:])\n\t(*entries)[i] = entry\n\treturn nil\n}\n\n// delete will remove the provided key from this list of entries.\n// Returned is a deleted Entry. This will be nil if the key\n// cannot be found.\nfunc (entries *Entries) delete(key uint64) Entry {\n\ti := entries.search(key)\n\tif i == len(*entries) { // key not found\n\t\treturn nil\n\t}\n\n\tif (*entries)[i].Key() != key {\n\t\treturn nil\n\t}\n\n\toldEntry := (*entries)[i]\n\tcopy((*entries)[i:], (*entries)[i+1:])\n\t(*entries)[len(*entries)-1] = nil // GC\n\t*entries = (*entries)[:len(*entries)-1]\n\treturn oldEntry\n}\n\n// max returns the value of the highest key in this list\n// of entries. The bool indicates if it's a valid key, that\n// is if there is more than zero entries in this list.\nfunc (entries Entries) max() (uint64, bool) {\n\tif len(entries) == 0 {\n\t\treturn 0, false\n\t}\n\n\treturn entries[len(entries)-1].Key(), true\n}\n\n// get will perform a lookup over this list of entries\n// and return an Entry if it exists. Returns nil if the\n// entry does not exist.\nfunc (entries Entries) get(key uint64) Entry {\n\ti := entries.search(key)\n\tif i == len(entries) {\n\t\treturn nil\n\t}\n\n\tif entries[i].Key() == key {\n\t\treturn entries[i]\n\t}\n\n\treturn nil\n}\n\n// successor will return the first entry that has a key\n// greater than or equal to provided key. Also returned\n// is the index of the find. Returns nil, -1 if a successor does\n// not exist.\nfunc (entries Entries) successor(key uint64) (Entry, int) {\n\ti := entries.search(key)\n\tif i == len(entries) {\n\t\treturn nil, -1\n\t}\n\n\treturn entries[i], i\n}\n\n// predecessor will return the first entry that has a key\n// less than or equal to the provided key. Also returned\n// is the index of the find. Returns nil, -1 if a predecessor\n// does not exist.\nfunc (entries Entries) predecessor(key uint64) (Entry, int) {\n\tif len(entries) == 0 {\n\t\treturn nil, -1\n\t}\n\n\ti := entries.search(key)\n\tif i == len(entries) {\n\t\treturn entries[i-1], i - 1\n\t}\n\n\tif entries[i].Key() == key {\n\t\treturn entries[i], i\n\t}\n\n\ti--\n\n\tif i < 0 {\n\t\treturn nil, -1\n\t}\n\n\treturn entries[i], i\n}\n" }, { "path": "trie/yfast/entries_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage yfast\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestEntriesInsert(t *testing.T) {\n\tes := Entries{}\n\n\te1 := newMockEntry(5)\n\te2 := newMockEntry(1)\n\n\tes.insert(e1)\n\tes.insert(e2)\n\n\tassert.Equal(t, Entries{e2, e1}, es)\n\n\te3 := newMockEntry(3)\n\tes.insert(e3)\n\n\tassert.Equal(t, Entries{e2, e3, e1}, es)\n}\n\nfunc TestEntriesDelete(t *testing.T) {\n\tes := Entries{}\n\n\te1 := newMockEntry(5)\n\te2 := newMockEntry(1)\n\tes.insert(e1)\n\tes.insert(e2)\n\n\tes.delete(5)\n\tassert.Equal(t, Entries{e2}, es)\n\n\tes.delete(1)\n\tassert.Equal(t, Entries{}, es)\n}\n\nfunc TestEntriesMax(t *testing.T) {\n\tes := Entries{}\n\tmax, ok := es.max()\n\tassert.Equal(t, uint64(0), max)\n\tassert.False(t, ok)\n\n\te2 := newMockEntry(1)\n\tes.insert(e2)\n\tmax, ok = es.max()\n\tassert.Equal(t, uint64(1), max)\n\tassert.True(t, ok)\n\n\te1 := newMockEntry(5)\n\tes.insert(e1)\n\tmax, ok = es.max()\n\tassert.Equal(t, uint64(5), max)\n\tassert.True(t, ok)\n}\n\nfunc TestEntriesGet(t *testing.T) {\n\tes := Entries{}\n\n\te1 := newMockEntry(5)\n\te2 := newMockEntry(1)\n\tes.insert(e1)\n\tes.insert(e2)\n\n\tresult := es.get(5)\n\tassert.Equal(t, e1, result)\n\n\tresult = es.get(1)\n\tassert.Equal(t, e2, result)\n\n\tresult = es.get(10)\n\tassert.Nil(t, result)\n}\n\nfunc TestEntriesSuccessor(t *testing.T) {\n\tes := Entries{}\n\n\tsuccessor, i := es.successor(5)\n\tassert.Equal(t, -1, i)\n\tassert.Nil(t, successor)\n\n\te1 := newMockEntry(5)\n\te2 := newMockEntry(1)\n\tes.insert(e1)\n\tes.insert(e2)\n\n\tsuccessor, i = es.successor(0)\n\tassert.Equal(t, 0, i)\n\tassert.Equal(t, e2, successor)\n\n\tsuccessor, i = es.successor(2)\n\tassert.Equal(t, 1, i)\n\tassert.Equal(t, e1, successor)\n\n\tsuccessor, i = es.successor(5)\n\tassert.Equal(t, 1, i)\n\tassert.Equal(t, e1, successor)\n\n\tsuccessor, i = es.successor(10)\n\tassert.Equal(t, -1, i)\n\tassert.Nil(t, successor)\n}\n\nfunc TestEntriesPredecessor(t *testing.T) {\n\tes := Entries{}\n\n\tpredecessor, i := es.predecessor(5)\n\tassert.Equal(t, -1, i)\n\tassert.Nil(t, predecessor)\n\n\te1 := newMockEntry(5)\n\te2 := newMockEntry(1)\n\tes.insert(e1)\n\tes.insert(e2)\n\n\tpredecessor, i = es.predecessor(0)\n\tassert.Equal(t, -1, i)\n\tassert.Nil(t, predecessor)\n\n\tpredecessor, i = es.predecessor(2)\n\tassert.Equal(t, 0, i)\n\tassert.Equal(t, e2, predecessor)\n\n\tpredecessor, i = es.predecessor(5)\n\tassert.Equal(t, 1, i)\n\tassert.Equal(t, e1, predecessor)\n\n\tpredecessor, i = es.predecessor(10)\n\tassert.Equal(t, 1, i)\n\tassert.Equal(t, e1, predecessor)\n}\n" }, { "path": "trie/yfast/interface.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage yfast\n\n// Entry defines items that can be inserted into the y-fast\n// trie.\ntype Entry interface {\n\t// Key is the key for this entry. If the trie has been\n\t// given bit size n, only the last n bits of this key\n\t// will matter. Use a bit size of 64 to enable all\n\t// 2^64-1 keys.\n\tKey() uint64\n}\n" }, { "path": "trie/yfast/iterator.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage yfast\n\nimport \"github.com/Workiva/go-datastructures/trie/xfast\"\n\n// iteratorExhausted is a magic value for an index to tell us\n// that the iterator has been exhausted.\nconst iteratorExhausted = -2\n\n// iterExhausted is a helper function to tell us if an iterator\n// has been exhausted.\nfunc iterExhausted(iter *Iterator) bool {\n\treturn iter.index == iteratorExhausted\n}\n\n// Iterator will iterate of the results of a query.\ntype Iterator struct {\n\txfastIterator *xfast.Iterator\n\tindex int\n\tentries *entriesWrapper\n}\n\n// Next will return a bool indicating if another value exists\n// in the iterator.\nfunc (iter *Iterator) Next() bool {\n\tif iterExhausted(iter) {\n\t\treturn false\n\t}\n\titer.index++\n\tif iter.index >= len(iter.entries.entries) {\n\t\tnext := iter.xfastIterator.Next()\n\t\tif !next {\n\t\t\titer.index = iteratorExhausted\n\t\t\treturn false\n\t\t}\n\t\tvar ok bool\n\t\titer.entries, ok = iter.xfastIterator.Value().(*entriesWrapper)\n\t\tif !ok {\n\t\t\titer.index = iteratorExhausted\n\t\t\treturn false\n\t\t}\n\t\titer.index = 0\n\t}\n\n\treturn true\n}\n\n// Value will return the Entry representing the iterator's current position.\n// If no Entry exists at the present condition, the iterator is\n// exhausted and this method will return nil.\nfunc (iter *Iterator) Value() Entry {\n\tif iterExhausted(iter) {\n\t\treturn nil\n\t}\n\n\tif iter.entries == nil || iter.index < 0 || iter.index >= len(iter.entries.entries) {\n\t\treturn nil\n\t}\n\n\treturn iter.entries.entries[iter.index]\n}\n\n// exhaust is a helper function that will exhaust this iterator\n// and return a list of entries. This is for internal use only.\nfunc (iter *Iterator) exhaust() Entries {\n\tentries := make(Entries, 0, 100)\n\tfor it := iter; it.Next(); {\n\t\tentries = append(entries, it.Value())\n\t}\n\n\treturn entries\n}\n\n// nilIterator is an iterator that will always return false\n// from Next() and nil for Value().\nfunc nilIterator() *Iterator {\n\treturn &Iterator{\n\t\tindex: iteratorExhausted,\n\t}\n}\n" }, { "path": "trie/yfast/mock_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage yfast\n\ntype mockEntry struct {\n\t// not going to use mock here as it skews benchmarks\n\tkey uint64\n}\n\nfunc (me *mockEntry) Key() uint64 {\n\treturn me.key\n}\n\nfunc newMockEntry(key uint64) *mockEntry {\n\treturn &mockEntry{key}\n}\n" }, { "path": "trie/yfast/yfast.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n/*\nPackage yfast implements a y-fast trie. Instead of a red-black BBST\nfor the leaves, this implementation uses a simple ordered list. This\npackage should have searches that are as performant as the x-fast\ntrie while having faster inserts/deletes and linear space consumption.\n\nPerformance characteristics:\nSpace: O(n)\nGet: O(log log M)\nSearch: O(log log M)\nInsert: O(log log M)\nDelete: O(log log M)\n\nwhere n is the number of items in the trie and M is the size of the\nuniverse, ie, 2^m where m is the number of bits in the specified key\nsize.\n\nThis particular implementation also uses fixed bucket sizes as this should\naid in multithreading these functions for performance optimization.\n*/\npackage yfast\n\nimport \"github.com/Workiva/go-datastructures/trie/xfast\"\n\n// YFastTrie implements all the methods available to the y-fast\n// trie datastructure. The top half is composed of an x-fast trie\n// while the leaves are composed of ordered lists of type Entry,\n// an interface found in this package.\ntype YFastTrie struct {\n\tnum uint64\n\txfast *xfast.XFastTrie\n\tbits uint8\n}\n\nfunc (yfast *YFastTrie) init(intType interface{}) {\n\tswitch intType.(type) {\n\tcase uint8:\n\t\tyfast.bits = 8\n\tcase uint16:\n\t\tyfast.bits = 16\n\tcase uint32:\n\t\tyfast.bits = 32\n\tcase uint, uint64:\n\t\tyfast.bits = 64\n\tdefault:\n\t\t// we'll panic with a bad value to the constructor.\n\t\tpanic(`Invalid universe size provided.`)\n\t}\n\n\tyfast.xfast = xfast.New(intType)\n}\n\n// getBucketKey finds the largest possible value in this key's bucket.\n// This is the representative value for the entry in the x-fast trie.\nfunc (yfast *YFastTrie) getBucketKey(key uint64) uint64 {\n\ti := key/uint64(yfast.bits) + 1\n\treturn uint64(yfast.bits)*i - 1\n}\n\nfunc (yfast *YFastTrie) insert(entry Entry) Entry {\n\t// first, we need to determine if we have a node in the x-trie\n\t// that already matches for the key\n\tbundleKey := yfast.getBucketKey(entry.Key())\n\tbundle := yfast.xfast.Get(bundleKey)\n\n\tif bundle != nil {\n\t\toverwritten := bundle.(*entriesWrapper).entries.insert(entry)\n\t\tif overwritten == nil {\n\t\t\tyfast.num++\n\t\t\treturn nil\n\t\t}\n\n\t\treturn overwritten\n\t}\n\n\tyfast.num++\n\tentries := make(Entries, 0, yfast.bits)\n\tentries.insert(entry)\n\n\tew := &entriesWrapper{\n\t\tkey: bundleKey,\n\t\tentries: entries,\n\t}\n\n\tyfast.xfast.Insert(ew)\n\treturn nil\n}\n\n// Insert will insert the provided entries into the y-fast trie\n// and return a list of entries that were overwritten.\nfunc (yfast *YFastTrie) Insert(entries ...Entry) Entries {\n\toverwritten := make(Entries, 0, len(entries))\n\tfor _, e := range entries {\n\t\toverwritten = append(overwritten, yfast.insert(e))\n\t}\n\n\treturn overwritten\n}\n\nfunc (yfast *YFastTrie) delete(key uint64) Entry {\n\tbundleKey := yfast.getBucketKey(key)\n\n\tbundle := yfast.xfast.Get(bundleKey)\n\tif bundle == nil {\n\t\treturn nil\n\t}\n\n\tew := bundle.(*entriesWrapper)\n\tentry := ew.entries.delete(key)\n\tif entry == nil {\n\t\treturn nil\n\t}\n\n\tyfast.num--\n\n\tif len(ew.entries) == 0 {\n\t\tyfast.xfast.Delete(bundleKey)\n\t}\n\n\treturn entry\n}\n\n// Delete will delete the provided keys from the y-fast trie\n// and return a list of entries that were deleted.\nfunc (yfast *YFastTrie) Delete(keys ...uint64) Entries {\n\tentries := make(Entries, 0, len(keys))\n\tfor _, key := range keys {\n\t\tentries = append(entries, yfast.delete(key))\n\t}\n\n\treturn entries\n}\n\nfunc (yfast *YFastTrie) get(key uint64) Entry {\n\tbundleKey := yfast.getBucketKey(key)\n\tbundle := yfast.xfast.Get(bundleKey)\n\tif bundle == nil {\n\t\treturn nil\n\t}\n\n\tentry := bundle.(*entriesWrapper).entries.get(key)\n\tif entry == nil { // go interfaces :(\n\t\treturn nil\n\t}\n\n\treturn entry\n}\n\n// Get will look for the provided key in the y-fast trie and return\n// the associated value if it is found. If it is not found, this\n// method returns nil.\nfunc (yfast *YFastTrie) Get(key uint64) Entry {\n\tentry := yfast.get(key)\n\tif entry == nil {\n\t\treturn nil\n\t}\n\n\treturn entry\n}\n\n// Len returns the number of items in the y-fast trie.\nfunc (yfast *YFastTrie) Len() uint64 {\n\treturn yfast.num\n}\n\nfunc (yfast *YFastTrie) successor(key uint64) Entry {\n\tbundle := yfast.xfast.Successor(key)\n\tif bundle == nil {\n\t\treturn nil\n\t}\n\n\tentry, _ := bundle.(*entriesWrapper).entries.successor(key)\n\tif entry == nil {\n\t\treturn nil\n\t}\n\n\treturn entry\n}\n\n// Successor returns an Entry with a key equal to or immediately\n// greater than the provided key. If such an Entry does not exist\n// this returns nil.\nfunc (yfast *YFastTrie) Successor(key uint64) Entry {\n\tentry := yfast.successor(key)\n\tif entry == nil {\n\t\treturn nil\n\t}\n\n\treturn entry\n}\n\nfunc (yfast *YFastTrie) predecessor(key uint64) Entry {\n\t// harder case because our representative value in the\n\t// x-fast trie is the a max\n\tbundleKey := yfast.getBucketKey(key)\n\tbundle := yfast.xfast.Predecessor(bundleKey)\n\tif bundle == nil {\n\t\treturn nil\n\t}\n\n\tew := bundle.(*entriesWrapper)\n\tentry, _ := ew.entries.predecessor(key)\n\tif entry != nil {\n\t\treturn entry\n\t}\n\n\t// it's possible we do exist somewhere earlier in the x-fast trie\n\tbundle = yfast.xfast.Predecessor(bundleKey - 1)\n\tif bundle == nil {\n\t\treturn nil\n\t}\n\n\tew = bundle.(*entriesWrapper)\n\n\tentry, _ = ew.entries.predecessor(key)\n\tif entry == nil {\n\t\treturn nil\n\t}\n\n\treturn entry\n}\n\n// Predecessor returns an Entry with a key equal to or immediately\n// preceding than the provided key. If such an Entry does not exist\n// this returns nil.\nfunc (yfast *YFastTrie) Predecessor(key uint64) Entry {\n\tentry := yfast.predecessor(key)\n\tif entry == nil {\n\t\treturn nil\n\t}\n\n\treturn entry\n}\n\nfunc (yfast *YFastTrie) iter(key uint64) *Iterator {\n\txfastIter := yfast.xfast.Iter(key)\n\txfastIter.Next()\n\tbundle := xfastIter.Value()\n\tif bundle == nil {\n\t\treturn nilIterator()\n\t}\n\n\ti := bundle.(*entriesWrapper).entries.search(key)\n\treturn &Iterator{\n\t\tindex: i - 1,\n\t\txfastIterator: xfastIter,\n\t\tentries: bundle.(*entriesWrapper),\n\t}\n}\n\n// Iter will return an iterator that will iterate across all values\n// that start or immediately proceed the provided key. Iteration\n// happens in ascending order.\nfunc (yfast *YFastTrie) Iter(key uint64) *Iterator {\n\treturn yfast.iter(key)\n}\n\n// New constructs, initializes, and returns a new y-fast trie.\n// Provided should be a uint type that specifies the number\n// of bits in the desired universe. This will affect the time\n// complexity of all lookup and mutate operations.\nfunc New(ifc interface{}) *YFastTrie {\n\tyfast := &YFastTrie{}\n\tyfast.init(ifc)\n\treturn yfast\n}\n" }, { "path": "trie/yfast/yfast_test.go", "content": "/*\nCopyright 2014 Workiva, LLC\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage yfast\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc generateEntries(num int) Entries {\n\tentries := make(Entries, 0, num)\n\tfor i := uint64(0); i < uint64(num); i++ {\n\t\tentries = append(entries, newMockEntry(i))\n\t}\n\n\treturn entries\n}\n\nfunc TestTrieSimpleInsert(t *testing.T) {\n\tyfast := New(uint8(0))\n\n\te1 := newMockEntry(3)\n\te2 := newMockEntry(7)\n\te3 := newMockEntry(8)\n\n\tyfast.Insert(e1, e2, e3)\n\n\tresult := yfast.get(3)\n\tassert.Equal(t, e1, result)\n\n\tresult = yfast.get(7)\n\tassert.Equal(t, e2, result)\n\n\tresult = yfast.get(8)\n\tassert.Equal(t, e3, result)\n\n\tresult = yfast.get(250)\n\tassert.Nil(t, result)\n\n\tassert.Equal(t, uint64(3), yfast.Len())\n}\n\nfunc TestTrieOverwriteInsert(t *testing.T) {\n\tyfast := New(uint8(0))\n\n\te1 := newMockEntry(3)\n\te2 := newMockEntry(3)\n\tyfast.Insert(e1)\n\n\tyfast.Insert(e2)\n\tassert.Equal(t, e2, yfast.Get(3))\n\tassert.Equal(t, uint64(1), yfast.Len())\n}\n\nfunc TestTrieDelete(t *testing.T) {\n\tyfast := New(uint8(0))\n\n\te1 := newMockEntry(3)\n\te2 := newMockEntry(7)\n\te3 := newMockEntry(8)\n\n\tyfast.Insert(e1, e2, e3)\n\n\tresult := yfast.Delete(3)\n\tassert.Equal(t, Entries{e1}, result)\n\tassert.Nil(t, yfast.Get(3))\n\tassert.Equal(t, uint64(2), yfast.Len())\n\n\tresult = yfast.Delete(7)\n\tassert.Equal(t, Entries{e2}, result)\n\tassert.Nil(t, yfast.Get(7))\n\tassert.Equal(t, uint64(1), yfast.Len())\n\n\tresult = yfast.Delete(8)\n\tassert.Equal(t, Entries{e3}, result)\n\tassert.Nil(t, yfast.Get(8))\n\tassert.Equal(t, uint64(0), yfast.Len())\n\n\tresult = yfast.Delete(5)\n\tassert.Equal(t, Entries{nil}, result)\n\tassert.Equal(t, uint64(0), yfast.Len())\n}\n\nfunc TestTrieSuccessor(t *testing.T) {\n\tyfast := New(uint8(0))\n\n\te3 := newMockEntry(13)\n\tyfast.Insert(e3)\n\n\tsuccessor := yfast.Successor(0)\n\tassert.Equal(t, e3, successor)\n\n\te1 := newMockEntry(3)\n\te2 := newMockEntry(7)\n\n\tyfast.Insert(e1, e2)\n\n\tsuccessor = yfast.Successor(0)\n\tassert.Equal(t, e1, successor)\n\n\tsuccessor = yfast.Successor(3)\n\tassert.Equal(t, e1, successor)\n\n\tsuccessor = yfast.Successor(4)\n\tassert.Equal(t, e2, successor)\n\n\tsuccessor = yfast.Successor(8)\n\tassert.Equal(t, e3, successor)\n\n\tsuccessor = yfast.Successor(14)\n\tassert.Nil(t, successor)\n\n\tsuccessor = yfast.Successor(100)\n\tassert.Nil(t, successor)\n}\n\nfunc TestTriePredecessor(t *testing.T) {\n\tyfast := New(uint8(0))\n\n\tpredecessor := yfast.Predecessor(5)\n\tassert.Nil(t, predecessor)\n\n\te1 := newMockEntry(5)\n\tyfast.Insert(e1)\n\n\tpredecessor = yfast.Predecessor(13)\n\tassert.Equal(t, e1, predecessor)\n\n\te2 := newMockEntry(12)\n\tyfast.Insert(e2)\n\n\tpredecessor = yfast.Predecessor(11)\n\tassert.Equal(t, e1, predecessor)\n\n\tpredecessor = yfast.Predecessor(5)\n\tassert.Equal(t, e1, predecessor)\n\n\tpredecessor = yfast.Predecessor(4)\n\tassert.Nil(t, predecessor)\n\n\tpredecessor = yfast.Predecessor(100)\n\tassert.Equal(t, e2, predecessor)\n}\n\nfunc TestTrieIterator(t *testing.T) {\n\tyfast := New(uint8(0))\n\n\titer := yfast.Iter(5)\n\tassert.Equal(t, Entries{}, iter.exhaust())\n\n\te1 := newMockEntry(5)\n\tyfast.Insert(e1)\n\n\titer = yfast.Iter(5)\n\tassert.Equal(t, Entries{e1}, iter.exhaust())\n\n\te2 := newMockEntry(12)\n\tyfast.Insert(e2)\n\n\titer = yfast.Iter(5)\n\tassert.Equal(t, Entries{e1, e2}, iter.exhaust())\n\n\titer = yfast.Iter(6)\n\tassert.Equal(t, Entries{e2}, iter.exhaust())\n\n\te3 := newMockEntry(6)\n\tyfast.Insert(e3)\n\n\titer = yfast.Iter(7)\n\tassert.Equal(t, Entries{e2}, iter.exhaust())\n\n\titer = yfast.Iter(0)\n\tassert.Equal(t, Entries{e1, e3, e2}, iter.exhaust())\n\n\titer = yfast.Iter(13)\n\tassert.Equal(t, Entries{}, iter.exhaust())\n}\n\nfunc BenchmarkInsert(b *testing.B) {\n\tyfast := New(uint64(0))\n\tentries := generateEntries(b.N)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tyfast.Insert(entries[i])\n\t}\n}\n\nfunc BenchmarkGet(b *testing.B) {\n\tnumItems := 1000\n\n\tentries := generateEntries(numItems)\n\n\tyfast := New(uint32(0))\n\tyfast.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tyfast.Get(uint64(numItems / 2))\n\t}\n}\n\nfunc BenchmarkDelete(b *testing.B) {\n\tentries := generateEntries(b.N)\n\tyfast := New(uint64(0))\n\tyfast.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tyfast.Delete(uint64(i))\n\t}\n}\n\nfunc BenchmarkSuccessor(b *testing.B) {\n\tnumItems := 100000\n\n\tentries := make(Entries, 0, numItems)\n\tfor i := uint64(0); i < uint64(numItems); i++ {\n\t\tentries = append(entries, newMockEntry(i+uint64(b.N/2)))\n\t}\n\n\tyfast := New(uint64(0))\n\tyfast.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tyfast.Successor(uint64(i))\n\t}\n}\n\nfunc BenchmarkPredecessor(b *testing.B) {\n\tnumItems := 100000\n\n\tentries := make(Entries, 0, numItems)\n\tfor i := uint64(0); i < uint64(numItems); i++ {\n\t\tentries = append(entries, newMockEntry(i+uint64(b.N/2)))\n\t}\n\n\tyfast := New(uint64(0))\n\tyfast.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tyfast.Predecessor(uint64(i))\n\t}\n}\n\nfunc BenchmarkIterator(b *testing.B) {\n\tnumItems := 1000\n\tentries := generateEntries(numItems)\n\n\tyfast := New(uint64(0))\n\tyfast.Insert(entries...)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tfor iter := yfast.Iter(0); iter.Next(); {\n\t\t\titer.Value()\n\t\t}\n\t}\n}\n" } ]