Repository: tidwall/tile38 Branch: master Commit: 5b8ce586ab16 Files: 152 Total size: 1.0 MB Directory structure: gitextract_ljil7_jn/ ├── .github/ │ ├── CONTRIBUTING.md │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ └── config.yml │ ├── PULL_REQUEST_TEMPLATE.md │ └── workflows/ │ └── main.yml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd/ │ ├── tile38-benchmark/ │ │ ├── az/ │ │ │ └── az.go │ │ └── main.go │ ├── tile38-cli/ │ │ └── main.go │ ├── tile38-luamemtest/ │ │ └── main.go │ └── tile38-server/ │ └── main.go ├── core/ │ ├── commands.go │ ├── commands.json │ ├── commands_gen.go │ ├── commands_test.go │ ├── gen.sh │ └── version.go ├── go.mod ├── go.sum ├── internal/ │ ├── bing/ │ │ ├── bing.go │ │ ├── bing_test.go │ │ ├── ext.go │ │ └── ext_test.go │ ├── buffer/ │ │ ├── buffer.go │ │ └── buffer_test.go │ ├── clip/ │ │ ├── clip.go │ │ ├── clip_test.go │ │ ├── collection.go │ │ ├── feature.go │ │ ├── linestring.go │ │ ├── point.go │ │ ├── polygon.go │ │ └── rect.go │ ├── collection/ │ │ ├── collection.go │ │ ├── collection_test.go │ │ ├── geodesic.go │ │ └── string.go │ ├── deadline/ │ │ └── deadline.go │ ├── endpoint/ │ │ ├── amqp.go │ │ ├── cfqueue.go │ │ ├── disque.go │ │ ├── endpoint.go │ │ ├── eventHub.go │ │ ├── grpc.go │ │ ├── http.go │ │ ├── kafka.go │ │ ├── local.go │ │ ├── mqtt.go │ │ ├── nats.go │ │ ├── pubsub.go │ │ ├── redis.go │ │ ├── scram_client.go │ │ └── sqs.go │ ├── field/ │ │ ├── field.go │ │ ├── field_test.go │ │ ├── list_binary.go │ │ ├── list_struct.go │ │ └── list_test.go │ ├── glob/ │ │ ├── glob.go │ │ ├── glob_test.go │ │ └── match.go │ ├── hservice/ │ │ ├── gen.sh │ │ ├── hservice.pb.go │ │ └── hservice.proto │ ├── log/ │ │ ├── log.go │ │ └── log_test.go │ ├── object/ │ │ ├── object_binary.go │ │ ├── object_struct.go │ │ └── object_test.go │ ├── server/ │ │ ├── aof.go │ │ ├── aofmigrate.go │ │ ├── aofshrink.go │ │ ├── bson.go │ │ ├── bson_test.go │ │ ├── checksum.go │ │ ├── client.go │ │ ├── config.go │ │ ├── crud.go │ │ ├── dev.go │ │ ├── expire.go │ │ ├── expr.go │ │ ├── expression.go │ │ ├── fence.go │ │ ├── follow.go │ │ ├── group.go │ │ ├── hooks.go │ │ ├── json.go │ │ ├── json_test.go │ │ ├── keys.go │ │ ├── live.go │ │ ├── metrics.go │ │ ├── monitor.go │ │ ├── must.go │ │ ├── must_test.go │ │ ├── mvt.go │ │ ├── mvt_test.go │ │ ├── output.go │ │ ├── pubqueue.go │ │ ├── pubsub.go │ │ ├── readonly.go │ │ ├── respconn.go │ │ ├── scan.go │ │ ├── scanner.go │ │ ├── scanner_test.go │ │ ├── scripts.go │ │ ├── search.go │ │ ├── server.go │ │ ├── stats.go │ │ ├── stats_cpu.go │ │ ├── stats_cpu_darlin.go │ │ ├── test.go │ │ ├── token.go │ │ └── token_test.go │ ├── sstring/ │ │ ├── sstring.go │ │ └── sstring_test.go │ └── viewer/ │ ├── index.html │ └── viewer.go ├── scripts/ │ ├── RELEASE.md │ ├── build.sh │ ├── docker-push.sh │ ├── package.sh │ └── test.sh └── tests/ ├── 107/ │ ├── .gitignore │ ├── LINK │ └── main.go ├── 616/ │ └── main.go ├── README.md ├── aof_legacy ├── aof_test.go ├── client_test.go ├── fence_roaming_test.go ├── fence_test.go ├── follower_test.go ├── json_test.go ├── keys_search_test.go ├── keys_test.go ├── metrics_test.go ├── mock_io_test.go ├── mock_test.go ├── monitor_test.go ├── proto_test.go ├── scripts_test.go ├── stats_test.go ├── testcmd_test.go ├── tests_test.go └── timeout_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/CONTRIBUTING.md ================================================ ## How to contribute to Tile38 Before getting starting with contributing, please know that we currently use [Tile38 Slack](https://tile38.com/slack) channel for casual questions and user chat. ### Did you find a bug? - **Do not open up a GitHub issue if the bug is a security vulnerability in Tile38**. Sensitive security-related issues should be reported to [security@tile38.com](mailto:security@tile38.com). - **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/tidwall/tile38/issues). - If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/tidwall/tile38/issues/new). Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring. ### Did you fix whitespace, format code, or make a purely cosmetic patch? Changes that are cosmetic in nature and do not add anything substantial to the stability, functionality, or testability of Tile38 will generally not be accepted. ### Do you intend to add a new feature or change an existing one? - New features will probably not be approved without prior discussion. If you need a specialized feature, make sure to express your willingness to fund the work and maintenance. - Please do not open a pull request without filing an issue and/or discussing it with a maintainer beforehand. Thanks! ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: '' labels: '' assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: **Expected behavior** A clear and concise description of what you expected to happen. **Logs** If applicable, provide logs, panics, system messages to help explain your problem. **Operating System (please complete the following information):** - OS: [e.g. Linux / Windows / Mac OS] - CPU: [e.g. amd64 / arm64 / Apple Silicon / Intel] - Version: [e.g. 1.19.0] - Container: [e.g. Docker / None] **Additional context** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: true contact_links: - name: Community Support url: https://tile38.com/slack/ about: Please ask and answer questions here. - name: Documenation Issues url: https://github.com/tile38/tile38.github.io/issues about: Please documenation related issues here. ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ Please do not open a pull request without first filing an issue and/or discussing the feature directly with the project maintainer. ### Please ensure you adhere to every item in this list - [ ] This PR was pre-approved by the project maintainer - [ ] I have self-reviewed the code - [ ] I have added all necessary tests ### Describe your changes Please provide detailed description of the changes. ### Issue number and link Pull request require a prior issue with discussion. Include the issue number of link here. ================================================ FILE: .github/workflows/main.yml ================================================ name: Go on: push: branches: [master] pull_request: branches: [master] jobs: build: name: Build runs-on: ubuntu-latest steps: - name: Set up Go 1.x uses: actions/setup-go@v2 with: go-version: ^1.25 - name: Check out code uses: actions/checkout@v2 with: fetch-depth: 0 - name: Test run: make test - name: Package run: make package - name: Docker push env: DOCKER_LOGIN: tidwall DOCKER_USER: tile38 DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: ./scripts/docker-push.sh ================================================ FILE: .gitignore ================================================ .DS_Store tile38-* !cmd/tile38-* *.test data*/ coverage.out packages/ # Ignore IDE folders .idea/ .vscode/ ================================================ FILE: CHANGELOG.md ================================================ # Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). ## [1.37.0] = 2026-01-06 ### Added - #791: Add regexp to where expressions using '=~' (@TomDeVito) ### Fixed - #793: Avoid NaN points and rects for insertions and searches (@krkeshav) - #789: Fix LineString feature encoding to use all points (@metaxasa) - 45496a0: Ensure strict resp clients for pubsub - f6e6fae: Ignore -o json flag for HELLO command ### Updated - 653e7a0: Bumped github.com/eclipse/paho.mqtt.golang - 2a86b4b: Bumped golang.org/x/crypto ## [1.36.5] = 2025-10-05 ### Added - 5462de9: Add -o flag to auto set client output to json or resp #779 (@huangpeizhi2018) ### Fixed - 8c27bd7: Fix 'outside' detect not firing when 'cross' is present ## [1.36.4] = 2025-10-03 ### Fixed - 8c9f56c: Fix leader hanging on sigterm #783 (@dobiadi) ## [1.36.3] = 2025-09-26 ### Fixed - 684ad73: Fix panic, move pubq init cond #782 (@ayaIbrahimm) - cde0ef7: Do not throw Lua nil access error during scans ## [1.36.2] = 2025-09-04 ### Fixed - 653aea6: Add GeoJSON "properties" member for Lua filtering - 70c244f: Make the String.match() function case insensitive ## [1.36.1] = 2025-08-28 ### Updated - 53bed30: Update lock strategy for faster writes - 40f58b0: Kafka Endpoint Improvement #778 (@ifiok) - 82f4d24: Smaller vector tile sizes for polygons - 5b21c24: Auto clip geometries at low zoom levels ## [1.36.0] = 2025-07-14 ### Added - 35bde95: Support for Cloudflare Queues endpoint #773 (@tobilg) - a2afb21: Vector tile support (Mapbox vector tiles) #775 ### Updated - 56c70a1: Bumped github.com/golang-jwt/jwt/v4 - 86b698d: Updated Go dependencies - f95fcab: Go 1.25 ## [1.35.0] = 2025-06-16 ### Added - 4638279: Added NATS Jetstream acks, user credentials, and tls #770 (@VeryStrongFingers) ### Updated - 3fe57a7; Use atomics for HEALTHZ command to remove contention. - 3085316: Expose the 'this' property to WHERE clauses. ## [1.34.4] = 2025-05-08 ### Fixed - 977bf25: Fix issue with some startup flags not being read (@salilgupta1) ### Updated - 86333cd: Bump golang.org/x/net #767 ## [1.34.3] = 2025-04-16 ### Fixed - dd98481: Fix channel message delay #763 (@txtsd) ## [1.34.2] = 2025-04-01 ### Updated - a80eaf2: Upgrade to Go 1.24 ### Fixed - 556390e: Fix equality tests with WHERE clause for nested values - 42e17a1: Fix channel test that sometimes stalls ### Security - 0ecf097: Bump golang.org/x/net #765 ## [1.34.1] = 2025-01-13 ### Security - 927f382: CVE-2024-45338 golang.org/x/net #762 (@tduong2049) - 07389d8: CVE-2024-45337 golang.org/x/crypto #760 ## [1.34.0] = 2024-12-09 ### Added - 459b3e6: Added fifo support for SQS webhooks #759 (@crankycookie) ### Updated - bed590b: Upgrade to Alpine 3.20 #757 (@tduong2049) - 2b09508: Upgrade to Go 1.23 ## [1.33.4] = 2024-11-05 ### Fixed - aa1caa6: Use zero for undefined fields in where expressions #754 (@unendingblue) ## [1.33.3] = 2024-09-29 ### Fixed - 2b080f4: Include field to INFO replication command output #752 (@Kilowhisky) ## [1.33.2] = 2024-08-02 ### Fixed - 2e3eaa7: Remove extra quote in ROLE command with JSON output #749 (@Kilowhisky) ## [1.33.1] = 2024-07-02 ### Fixed - 193bce1: Fix followers not receiving channel messages #468 (@hibooboo2, @trendstate, @DoisKoh) ## [1.33.0] = 2024-05-03 ## Added - #726: Add EXIST and FEXIST command (@Kilowhisky) ## Fixed - #738: Add support for CORS in http requests (@Kilowhisky) - #741: FSET transforms field names to lowercase (@unendingblue, @iwpnd) - #736: Fix field floating point parsing misrepresentation (@Kilowhisky) ## Updated - #733: golang.org/x/net - #724: google.golang.org/protobuf ## [1.32.2] = 2024-02-14 ### Fixed - #714: Fix crash when mixing z-coord dimensionality in a geometry (@prathik) - #717: Metric expired_keys never incremented (@undeadcat) - Updated Go runtime to 1.22 ## [1.32.1] = 2023-11-20 ### Fixed - #711: Updated dependencies to address security vulnerabilities (@hcmf-wice) - #706: Add support for 'none' authentication for kafka while still allowing SSL (@Kilowhisky) - #702: Fix AWS SQS base domain parsing for China region (@LLluma) ## [1.32.0] = 2023-07-31 ### Added - #686: Support the ROLE command (@Kilowhisky) ### Fixed - #698: Allow AUTH while loading data (@Kilowhisky) - #694: Allow PING in pubsub (@Kilowhisky) - #692: Properly support replica_announce properties (@Kilowhisky) - #691: HEALTHZ should not be AUTH protected (@Kilowhisky) - #685: Heap size not coming down after objects are removed (@Mukund2900, @iwpnd) - 0144ca6: Fix missing lock ## [1.31.0] = 2023-05-09 ### Added - #682: Enables cross platform building and pushing of docker images (arm64/amd64) (@eelcocramer) - #680: Add hostname, port, output, and password env variables to tile38-cli (@ptsilva) ### Fixed - #606: Only create AMQP queue and bindings for non-topic exchanges (@pacaj2am, @uwer) - #672: Add graceful shutdown on SIGTERM (@dmitri-zganiaiko) ### Updated - e9a0500: Upgrade to Go 1.20 - 05b2fb9: Security updates ## [1.30.2] = 2022-12-29 ### Fixed - #668: Fixed fields not persisting (@DucPhan2997) ## [1.30.1] = 2022-12-14 ### Fixed - a8c92a0: Speed up leader/follower replication - e60ea70: Fix field names converting to lowercase ## [1.30.0] = 2022-11-22 ### Added - bdc80a7: Add WHERE expressions ([more info](https://tile38.com/topics/filter-expressions)) - f24c251: Allow for multiple MATCH patterns - #652: Allow WHERE for geofence detection - #657: Add distance to NEARBY IDS response (@iwpnd) - #663: Lua Sanitization (@program--) ### Fixed - 023433a: Fix server hang on shared address - #655: fix: allow host ca sets for SASL and TLS connections (@iwpnd) ### Updated - 7f2ce23: Upgrade to Go 1.19 - cbfb271: Updated data structures to use Go generics ## [1.29.2] = 2022-11-11 ### Fixed - #664: Fix bad line in inner ring response ## [1.29.1] = 2022-09-21 ### Fixed - fe180dc: Fix follower not authenticating after aofshink ## [1.29.0] = 2022-07-14 ### Added - b883f35: Add pending_events stat - #643: Expose config and INFO response for replia-priorty (@rave-eserating) ### Fixed - 8e61f81: Fixed test on Apple silicon ## [1.28.0] = 2022-04-12 ### Added - 10f8564: Added option to "not found" for DEL - #633: Added "clear" command in the tile38-cli (@CaioDallaqua) - #634: Added -x flag to tile38-cli (@sign0) ### Fixed - #636: Workaround for the RESP3 Java lettuce client (@rave-eserating) - a1cc8e6: Fix eof error for incomplete commands (Theresa D) ### Updated - fcdb469: Security updates - #638: Upgrade alpine in Dockerfile (@bb) - a124738: Upgrade to Go 1.18 - 38ea913: Upgrade prometheous client - 45fde6a: Upgraded nats dependencies ## [1.27.1] = 2021-01-04 ### Fix - b6833a2: Auto assign server_id for bootstrapped config files ## [1.27.0] = 2021-12-28 ### Added - #629: JSON logging (@iwpnd) - 241117c: BUFFER option for WITHIN and INTERSECTS, see #79 ## [1.26.4] = 2021-10-25 ### Hotfix - a7592f7: Bump version to match changelog ## [1.26.3] = 2021-10-25 ### Updated - a47443a: Upgrade tidwall modules ## [1.26.2] = 2021-10-22 ### Added - #625: Azure EventHub hook support ### Changed - 11cea4d: Removed vendor directory ## [1.26.1] = 2021-10-01 ### Updated - 9e552c3: Allow some basic client commands before AOF data loads ## [1.26.0] = 2021-09-29 ### Added - #623: Added SECTOR type to spatial searches (@iwpnd, @gmonk) ### Fixed - #624: AOFSHRINK causes panic on server (@saques) ## [1.25.5] = 2021-09-26 ### Fixed - 8ebcbeb: Fixed Z not matching on where clause for Feature/Point. (@tomquas) ## [1.25.4] = 2021-09-14 ### Added - a737a78: Add unix socket support ### Updated - 8829b8f: Change hooks collection type from hashmap to btree - 83094b2: Update hook expiration logic - c686b87: Return hook ttl with HOOKS request - 06a92d8: Increase the precision of TIMEOUT - Upgrade to Go 1.17.1 ## [1.25.3] = 2021-08-23 ### Fixed - #621: Fixed a memory leak (@Morgiflute) ### Updated - Update B-tree library - Upgrade to Go 1.17 ## [1.25.2] = 2021-08-10 ### Fixed - #620: Fixed kafka authentication methods ### Updated - Upgraded various dependencies ## [1.25.1] = 2021-07-22 ### Fixed - #618: Fixed NEARBY with SPARSE returning too many results. (@nesjett) ## [1.25.0] = 2021-07-12 ### Added - #504: Added TLS support for Nats webhook provider. - #552: Add CLIPBY subcommand to INTERSECTS/WITHIN. (@rshura) - #561: Added geofence webhook for GCP Pubsub. (@mscno) - #615: Add SASL to Kafka provider. (@mathieux51, @iwpnd) ### Updated - #551: Optimize field value access. (@mpoindexter) - #554: Improved kNN using geodesic algorithm for NEARBY command. (@mpoindexter) ### Fixed - #611: Close follower files before finishing aofshrink. (@mzbrau) - #613: Fix Memory Leak in Kafka Producer. (@iwpnd) - #616: Fixed expiration logic issue. (@Neuintown) ## [1.24.3] = 2021-06-09 ### Fixed - af43d5a: Hotfix. Fixed invalid healthz output. ## [1.24.2] = 2021-06-07 ### Updated - b610633: Update Go to 1.16 ## [1.24.1] = 2021-06-07 ### Added - #609: Added HEALTHZ command (@iwpnd, @stevelacy) ## [1.24.0] = 2021-05-19 ### Added - #604: Added Prometheus metrics (@oliver006) ### Fixed - #605: Remove deprecated threads flag (@cep-ter) ## [1.23.0] = 2021-04-01 ### Updated - #598: Added TLS Config to Kafka (@iwpnd) - #599: Include "distance" to output when user specifically requests (@iwpnd) - #597: Allow for all command types for roaming event (@johnpmayer) - 31a0fbd: Upgraded dependencies and moved to Go 1.16 ### Fixed - #600: Fix invalid queue.db error (@lokisisland) - #603: Fix tile38-cli output showing protocol size when piping (@bb) ## [1.22.6] = 2021-02-07 ### Updated - 72dfaae: Updated various dependencies - 016f397: Updated btree library, optimization - 4f8bc05: Updated rtree library, optimization ### Fixed - 6092f73: Better handle connection errors in tile38-cli ## [1.22.5] = 2020-11-09 ### Fixed - 9ce2033: Fixed fields being shuffled after AOFSHRINK ## [1.22.4] = 2020-11-07 ### Updated - 1a7d8d6: Added ENV var for 500 http errors ## [1.22.3] = 2020-10-28 ### Updated - #583: Optimization for non-"cross" based geofence detection (@cliedeman) - 79bee85: Replaced the underlying B-tree structure. ## [1.22.2] = 2020-10-07 ### Fixed - #230: Fix trailing zeros in AOF at startup ## [1.22.1] = 2020-09-22 ### Updated - 9a34a37: Updated Go version to 1.15 - b1dc463: Updated outdated dependencies (40 in total) ### Added - #578 Fix "cross" detection not firing in some cases (@feichler-or) ## [1.22.0] = 2020-08-12 ### Added - #571 Added MONITOR command (@tomquas) ### Fixed - #566: Fixed crash in fenceMatchRoam causing an index out of range panic (@larsw) - #569: Fixed wrong order for fields with SCAN (@ipsusila) - #573: Fixed crash with geohash precision above 12 (@superloach) - 68e2b6d: Updated Kafka client to support (@LeonardoBonacci) ## [1.21.1] = 2020-06-04 ### Fixed - #564: Fix OUTPUT client command requiring authentication. (@LeonardoBonacci) ## [1.20.0] = 2020-05-20 ### Updated - #534: Avoid sorting fields for each written object. (@rshura) - #544: Match geometry indexing to server config - b3dc025: Optimize point in ring - 3718cd7: Added priority option for AMQP endpoints ### Fixed - #538: DEL geofence notifications are missing the "key" field - #539: Fixed issue with some features not working with WITHIN (@rshura) - #540: Fix a concurrent write/read on the server conn map (@mpoindexter) - #543: Fix clipping empty rings (@rshura) - #558: Fixed clip test (@mmcloughlin) - #562: Crashes under go1.14 runtime - ff48054: Fixed a missing faraway event for roaming geofences - 5162ac5: Stable sort roam notifications ## [1.19.5] = 2020-02-11 ### Fixed - c567512: Fix packages not vendoring on build ## [1.19.4] = 2020-02-10 ### Fixed - #529: Fix linestring features behave diffrent with CIRCLE (@spierepf) ## [1.19.3] = 2019-12-11 ### Fixed - #513: Fix tile38-cli from freezing with non-quoted geojson (@duartejc) ## [1.19.2] = 2019-11-28 ### Fixed - 6f3716a: Fix false negative for intersecting rings (@thomascoquet) ## [1.19.1] = 2019-11-18 ### Updated - cfc65a1: Refactored repo, moved to Go modules, updated vendor dependencies. ### Fixed - 9d27533: Fix infinite loop on tile38-cli connection failure. - #509: Fixed panic on AOFSHRINK. (@jordanferenz) ## [1.19.0] = 2019-11-02 ### Added - #464: Add area expressions TEST command. (@rshura) ### Fixed - #493: Fix invalid JSON when JSET strings that look like numbers. (@spierepf, @JordanArmstrong) - #499: Fix invalid PubSub format when output is set to JSON. (@dmvass) - #500: Fix Tile38-cli not propertly handling quotes. (@vthorsell) - #502: Fix excessive memory usage for objects with TTLs. commit 23b016d. (@FreakyBytes) - #503: Fix fprintf type error in stats_cpu.go for non-linux/darwin builds. (@JordanArmstrong) ### Changed - #505: Update Travi-ci to use Go 1.13.x ## [1.18.0] = 2019-10-09 ### Updated - 639f6e2: Updated the spatial index (R-tree) implementation. ### Fixed - b092cea: Fixed MQTT blocking on publish/wait. - #496: Fixed MQTT client ID uniqueness. (@neterror) - #497: Fixed data race on webhook map with TTLs. (@belek) - #498: Fixed JSET cancels objects TTL expiry value. (@belek) ## [1.17.6] - 2019-08-22 ### Fixed - 3d96b17: Fixed periodic stop-the-world pauses for systems with large heaps. ## [1.17.5] - 2019-08-22 ### Fixed - #489: Fixed nearby count always one (@jkarjala) ## [1.17.4] - 2019-08-09 ### Fixed - #486: Fixed data condition on connections map (@saltatory) ## [1.17.3] - 2019-08-03 ### Fixed - #483: Fixed lua pool pruning (@rshura) - f7888c1: Fixed malformed json for chans command ## [1.17.2] - 2019-06-28 ### Fixed - #422: Fixes NEARBY command distance normalization issue (@TrivikrAm-Pamarthi, @melbania) ## [1.17.1] - 2019-05-04 ### Fixed - #448: Fixed missing commands for unsubscribing from active channel (@githubfr) - #454: Fixed colored output for fatalf (@olevole) - #453: Fixed nearby json field results showing wrong data (@melbania) ## [1.17.0] - 2019-04-26 ### Added - #446: Added timeouts to allow prepending commands with a TIMEOUT option. (@rshura) ### Fixed - #440: Fixed crash with fence ROAM (@githubfr) ### Changed - 3ae5927: Removed experimental evio option ## [1.16.4] - 2019-03-19 ### Fixed - e1a7145: Hotfix. Do not ignore SIGHUP. Use the `--nohup` flag or `nohup` command. ## [1.16.3] - 2019-03-19 ### Fixed - #437: Fixed clients blocking while webook sending. (@tesujiro) ### Added - #430: Support more SQS credential providers. (@tobilg) - #435: Added pprof flags for optional memory and cpu diagnostics. - e47540b: Added auth flag to tile38-benchmark. - 5335aec: Allow for standard SQS URLs. (@tobilg) ## [1.16.2] - 2019-03-12 ### Fixed - #432: Ignore SIGHUP signals. (@abhit011) - #433: Fixed nearby inaccuracy with geofence. (@stcktrce) - #429: Memory optimization, recycle AOF buffer. - 95a5556: Added periodic yielding to iterators. (@rshura) ## [1.16.1] - 2019-03-01 ### Fixed - #421: Nearby with MATCH is returning invalid results (@nithinkota) ## [1.16.0] - 2019-02-25 ### Fixed - #415: Fixed overlapping geofences sending notifcation to wrong endpoint. (@belek, @s32x) - #412: Allow SERVER command for Lua scripts. (@1995parham) - #410: Allow slashes in MQTT Topics (@pstuifzand) - #409: Fixed bug in polygon clipping. (@rshura) - 30f903b: Require properties member for geojson features. (@rshura) ### Added - #409: Added TEST command for executing WITHIN and INTERSECTS on two objects. (@rshura) - #407: Allow 201 & 202 status code on webhooks. (@s32x) - #404: Adding more replication data to INFO response. (@s32x) ## [1.15.0] - 2019-01-16 ### Fixed - #403: JSON Output for INFO and CLIENT (@s32x) - #401: Fixing KEYS command (@s32x) - #398: Ensuring channel publish order (@s32x) - d7d0baa: Fix roam fence missing ### Added - #402: Adding ARM and ARM64 packages (@s32x) - #399: Add RequireValid and update geojson dependency (@stevelacy) - #396: Add distance_to function to the tile38 namespace in lua. (@rshura) - #395: Add RENAME and RENAMENX commands. (@rshura) ## [1.14.4] - 2018-12-03 ### Fixed - #394: Hotfix MultiPolygon intersect failure. (@contra) - #392: Fix TLS certs missing in Docker. (@vziukas, @s32x) ### Added - Add extended server stats with SERVER EXT. (@s32x) - Add Kafka key to match notication key. (@Joey92) - Add optimized spatial index for fences ## [1.14.3] - 2018-11-20 ### Fixed - Hotfix SCRIPT LOAD not executing from cli. (@rshura) ## [1.14.2] - 2018-11-15 ### Fixed - #386: Fix version not being set at build. (@stevelacy) ## [1.14.1] - 2018-11-15 ### Fixed - #385: Add `version` to SERVER command response (@stevelacy) - Hotfix replica sync needs flushing (@rshura) - Fixed a bug where some AOF commands where corrupted during reload ## [1.14.0] - 2018-11-11 ### Added - INTERSECT/WITHIN optimization that may drastically improve searching polygons that have lots of points. - Faster responses for write operations such as SET/DEL - NEARBY now always returns objects from nearest to farthest (@rshura) - kNN haversine distance optimization (@rshura) - Evio networking beta using the "-evio yes" and "-threads num" flags ### Fixed - #369: Fix poly in hole query ## [1.13.0] - 2018-08-29 ### Added - eef5f3c: Add geofence notifications over pub/sub channels - 3a6f366: Add NODWELL keyword to roaming geofences - #343: Add Nats endpoints (@lennycampino) - #340: Add MQTT tls/cert options (@tobilg) - #314: Add CLIP subcommand to INTERSECTS (@rshura) ### Changed - 3ae26e3: Updated B-tree implementation - 1d78a41: Updated R-tree implementation ## [1.12.3] - 2018-06-16 ### Fixed - #316: Fix AMQP and AMQPS webhook endpoints to support namespaces (@DeadWisdom) - #318: Fix aofshrink crash on windows (@abhit011) - #326: Fix sporadic kNN results when TTL is used (@pmaseberg) ## [1.12.2] - 2018-05-10 ### Fixed - #313: Hotfix intersect returning incorrect results (@stevelacy) ## [1.12.1] - 2018-04-30 ### Fixed - #300: Fix pdelhooks not persisting (@tobilg) - #293: Fix kafka lockup issue (@Joey92) - #301: Fix AMQP uri custom params not working (@tobilg) - #302: Fix tile with zoom level over 63 panics (@rshura) - b99cd39: Fix Sync hook msg ttl with server time ## [1.12.0] - 2018-04-12 ### Added - 11b42c0: Option to disable AOF or to use a custom path: #220 #223 #297 (@sign0, @umpc, @fmr683, @zhangfeng158) - #296: Add Meta data to hooks command (@tobilg) ### Changed - 11b42c0: Updated help menu and show more options ### Fixed - #295: Intersects returning nothing in some cases (@fils) - #294: HTTP requests stopped working (@zhangfeng158) - 0aa04a1: Lotsa package not vendored ## [1.11.1] - 2018-03-16 ### Added - #272: Preserve Docker image tag history (@gechr) - 9428b84: Added cpu and threads to SERVER stats ### Fixed - #281: Linestring intersection failure (@contra) - #280: Filter id match before kNN results (@sweco-semtne) - #269: Safe atomic ints for arm32 (@gmonk63) - #267: Optimization for multiploygons intersect queries (@contra) ## [1.11.0] - 2018-03-05 ### Added - #221: Add WHEREEVAL clause to scan/search commands (@rshura) ### Fixed - #254: Add maxmemory protection to FSET (@rshura) - #258: Clear expires on reset (@zycbobby) - #268: Avoid bbox intersect for non-bbox objects (@contra) ## [1.10.1] - 2018-01-17 ### Fixed - #244: Fix issue with points not being detected inside MultiPolygons (@fazlul3003) - #245: Precalculate and store bboxes for complex objects (@huangpeizhi) - #246: Fix server crash when receiving zero arg commands (@behrad) ## [1.10.0] - 2017-12-18 ### Added - #221: Sqs endpoint (@lennycampino) - #226: Lua scripting (@rshura) - #231: Allow setting multiple fields in a single fset command (@rshura) - #235: Add json library (encode/decode methods) to lua. (@rshura) - 26d0083: Update vendoring to use golang/dep - c8ed7ca: Add WHEREIN command (@rshura) - d817814: Optimized network pipelining ### Fixed - #237: Flush to file periodically (@rshura) - #241: Point match on interior hole (@genesor) - 920dc3a: Use atomic ints/bools - 730502d: Set keepalive default to 300 seconds - 1084c60: Apply limit on top of cursor (@rshura) ## [1.9.1] - 2017-08-16 ### Added - cd05708: Spatial index optimizations - #208: Debug message for failed webhook notifications (@karnivas) - #201: New ECHO command (@yorkxiao) - #183: Include tile38-cli in Docker image (@jchamberlain) - #121: Allow reads for disconnected followers (@octete) ### Fixed - 3fae3f7: Allow cursors for kNN queries - #211: Crash when shrinking AOF on Windows (@icewukong) - #203: Lifted LIMIT restriction all queries and COUNT keyword (@yorkxiao, @FX-HAO) - #207: Send empty results for queries on nonexistent keys (@FX-HAO) - #195: Added kNN overscan ordering (@rshura) - #199: Apply LIMIT after WHERE clause (@rshura) - #199: Require Go 1.7 (@rshura) - #198: Omit fields for Resp when NOFIELDS is used (@rshura) ## [1.9.0] - 2017-04-13 ### Added - #159: AMQP/RabbitMQ webhook support (@m1ome, @paavalan) - #152: Kafka webhook support (@m1ome) - #141: Add distances to Geofence notifications - #54: New benchmark tool (@literadix, @Lars-Meijer, @m1ome) - #20: Ability to specify pidfile via args (@olevole) ### Fixed - b1c76d7: tile38-cli auto doesn't auto reconnect - #156: Use redis-style TTL implementation (@Lars-Meijer, @m1ome) - #150: Live "inside" fence event not triggering for new object (@phulst) ## [1.8.0] - 2017-02-21 ### Added - #145: TCP Keepalives option (@UriHendler) - #136: K nearest neighbors for NEARBY command (@m1ome, @tomquas, @joernroeder) - #139: Added CLIENT command (@UriHendler) - #133: AutoGC config option (@m1ome, @amorskoy) ### Fixed - #147: Leaking http hook connections (@mkabischev) - #143: Duplicate data in hook data (@mkabischev) ## [1.7.5] - 2017-01-13 ### Added - Performance bump for all SET commands, ~10% faster - Lower memory footprint for large datasets - #112: Added distance to NEARBY command (@m1ome, @auselen) - #123: Redis endpoint for webhooks (@m1ome) - #128: Allow disabling HTTP & WebSocket transport (@m1ome) ### Fixed - #116: Missing response in TTL json command (@phulst) - #117: Error in command documentation (@juanpabloaj) - #118: Unexpected EOF bug with websockets (@m1ome) - #122: Disque typo timeout handling (@m1ome) - #127: 3d object searches with 2d geojson area (@damariei) ## [1.7.0] - 2016-12-29 ### Added - #104: PDEL command - Selete objects that match a pattern (@GameFreedom) - #99: COMMAND keyword for masking geofences by command type (@amorskoy) - #96: SCAN keyword for roaming geofences - fba34a9: JSET, JGET, JDEL commands ### Fixed - #107: Memory leak (@amorskoy) - #98: Output json fix ## [1.6.0] - 2016-12-11 ### Added - #87: Fencing event grouping (@huangpeizhi) ### Fixed - #91: Wrong winding order for CirclePolygon function (@antonioromano) - #73: Corruption for AOFSHRINK (@huangpeizhi) - #71: Lower memory usage. About 25% savings (@thisisaaronland, @umpc) - Polygon raycast bug. tidwall/poly#1 (@drewlesueur) - Added black-box testing ## [1.5.4] - 2016-11-17 ### Fixed - #84: Hotfix - roaming fence deadlock (@tomquas) ## [1.5.3] - 2016-11-16 ### Added - #4: Official docker support (@gordysc) ### Fixed - #77: NX/XX bug (@damariei) - #76: Match on prefix star (@GameFreedom, @icewukong) - #82: Allow for precise search for strings (@GameFreedom) - #83: Faster congruent modulo for points (@icewukong, @umpc) ## [1.5.2] - 2016-10-20 ### Fixed - #70: Invalid results for INTERSECTS query (@thisisaaronland) ## [1.5.1] - 2016-10-19 ### Fixed - #67: Call the EXPIRE command hangs the server (@PapaStifflera) - #64: Missing points in 'Nearby' queries (@umpc) ## [1.5.0] - 2016-10-03 ### Added - #61: Optimized queries on 3d objects (@damariei) - #60: Added [NX|XX] keywords to SET command (@damariei) - #29: Generalized hook interface (@jeremytregunna) - GRPC geofence hook support ### Fixed - #62: Potential Replace Bug Corrupting the Index (@umpc) - #57: CRLF codes in info after bump from 1.3.0 to 1.4.2 (@olevole) ## [1.4.2] - 2016-08-26 ### Fixed - #49. Allow fragmented pipeline requests (@owaaa) - #51: Allow multispace delim in native proto (@huangpeizhi) - #50: MATCH with slashes (@huangpeizhi) - #43: Linestring nearby search correction (@owaaa) ## [1.4.1] - 2016-08-26 ### Added - #34: Added "BOUNDS key" command (@icewukong) ### Fixed - #38: Allow for nginx support (@GameFreedom) - #39: Reset requirepass (@GameFreedom) ## [1.3.0] - 2016-07-22 ### Added - New EXPIRE, PERSISTS, TTL commands. New EX keyword to SET command - Support for plain strings using `SET ... STRING value.` syntax - New SEARCH command for finding strings - Scans can now order descending ### Fixed - #28: fix windows cli issue (@zhangkaizhao) ## [1.2.0] - 2016-05-24 ### Added - #17: Roaming Geofences for NEARBY command (@ElectroCamel, @davidxv) - #15: maxmemory config setting (@jrots) ## [1.1.4] - 2016-04-19 ### Fixed - #12: Issue where a newline was being added to HTTP POST requests (@davidxv) - #13: OBJECT keyword not accepted for WITHIN command (@ray93) - Panic on missing key for search requests ## [1.1.2] - 2016-04-12 ### Fixed - A glob suffix wildcard can result in extra hits - The native live geofence sometimes fails connections ## [1.1.0] - 2016-04-02 ### Added - Resp client support. All major programming languages now supported - Added WITHFIELDS option to GET - Added OUTPUT command to allow for outputing JSON when using RESP - Added DETECT option to geofences ### Changed - New AOF file structure. - Quicker and safer AOFSHRINK. ### Deprecation Warning - Native protocol support is being deprecated in a future release in favor of RESP ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing Thanks for your interest in improving Tile38! ## Issues - **Found a bug?** Please open an issue with a clear description and precise steps to reproduce. ## Pull Requests - **Scope.** PRs are welcome for bug fixes and small, low-risk changes. - **Please coordinate first.** For large changes, such as new features, please file an issue and/or discuss it with a maintainer beforehand. - **Quality.** Keep PRs focused, include tests (where applicable), and update docs. ## When Not to use a Pull Request Do NOT submit a Pull Request for: - Security-related changes. - Works in progress. - Changes that require design discussion, or are likely to be controversial. - Changes needing specialized or cross-subsystem review. - Large refactors or mechanical tree-wide changes. - Changes generated by AI tools. ## AI Assisted Contributions We are not accepting AI assisted code into Tile38 at this time. This policy may change in the future. Please reach out if you have any questions or concerns regarding AI and Tile38. ### Contributor Responsibilities - Monitor your Pull Request and respond to review feedback promptly. - Pull Requests may be closed if there is no response for **one month**. By submitting a contribution, you confirm you have the right to do so and grant Tile38 LLC the rights needed to use, modify, and redistribute your contribution as part of the project. If you have any questions or concerns regarding licensing, please contact us at licensing@tile38.com. For all other questions, contributing@tile38.com. ================================================ FILE: Dockerfile ================================================ FROM alpine:3.20 ARG VERSION ARG TARGETOS ARG TARGETARCH RUN apk add --no-cache ca-certificates ADD packages/tile38-$VERSION-$TARGETOS-$TARGETARCH/tile38-server /usr/local/bin ADD packages/tile38-$VERSION-$TARGETOS-$TARGETARCH/tile38-cli /usr/local/bin ADD packages/tile38-$VERSION-$TARGETOS-$TARGETARCH/tile38-benchmark /usr/local/bin RUN addgroup -S tile38 && \ adduser -S -G tile38 tile38 && \ mkdir /data && chown tile38:tile38 /data VOLUME /data EXPOSE 9851 CMD ["tile38-server", "-d", "/data"] ================================================ FILE: LICENSE ================================================ Copyright (c) 2016 Josh Baker Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ all: tile38-server tile38-cli tile38-benchmark tile38-luamemtest .PHONY: tile38-server tile38-server: @./scripts/build.sh tile38-server .PHONY: tile38-cli tile38-cli: @./scripts/build.sh tile38-cli .PHONY: tile38-benchmark tile38-benchmark: @./scripts/build.sh tile38-benchmark .PHONY: tile38-luamemtest tile38-luamemtest: @./scripts/build.sh tile38-luamemtest test: all @./scripts/test.sh package: @rm -rf packages/ @scripts/package.sh Windows windows amd64 @scripts/package.sh Mac darwin amd64 @scripts/package.sh Linux linux amd64 @scripts/package.sh FreeBSD freebsd amd64 @scripts/package.sh ARM linux arm @scripts/package.sh ARM64 linux arm64 clean: rm -rf tile38-server tile38-cli tile38-benchmark tile38-luamemtest distclean: clean rm -rf packages/ install: all cp tile38-server /usr/local/bin cp tile38-cli /usr/local/bin cp tile38-benchmark /usr/local/bin uninstall: rm -f /usr/local/bin/tile38-server rm -f /usr/local/bin/tile38-cli rm -f /usr/local/bin/tile38-benchmark ================================================ FILE: README.md ================================================

Tile38

Slack Discord

Tile38 is an open source (MIT licensed), in-memory geolocation data store, spatial index, and realtime geofencing server. It supports a variety of object types including lat/lon points, bounding boxes, XYZ tiles, Geohashes, and GeoJSON.

This README is quick start document. You can find detailed documentation at https://tile38.com.

Nearby Within Intersects Geofencing Roaming Geofences

## Features - Spatial index with [search](#searching) methods such as Nearby, Within, and Intersects. - Realtime [geofencing](#geofencing) through [webhooks](https://tile38.com/commands/sethook) or [pub/sub channels](#pubsub-channels). - Object types of [lat/lon](#latlon-point), [bbox](#bounding-box), [Geohash](#geohash), [GeoJSON](#geojson), [QuadKey](#quadkey), and [XYZ tile](#xyz-tile). - Support for lots of [Clients Libraries](#tile38-client-libraries) written in many different languages. - Variety of protocols, including [http](#http) (curl), [websockets](#websockets), [telnet](#telnet), and the [Redis RESP](https://redis.io/topics/protocol). - Server responses are [RESP](https://redis.io/topics/protocol) or [JSON](https://www.json.org). - Full [command line interface](#cli). - Leader / follower [replication](#replication). - In-memory database that persists on disk. ## Components - tile38-server: The server - tile38-cli: Command line interface tool - tile38-benchmark: Server benchmark tool ## Getting Started ### Getting Tile38 Perhaps the easiest way to get the latest Tile38 is to use one of the pre-built release binaries which are available for OSX, Linux, FreeBSD, and Windows. Instructions for using these binaries are on the GitHub [releases page](https://github.com/tidwall/tile38/releases). ### Docker To run the latest stable version of Tile38: ``` docker pull tile38/tile38 docker run -p 9851:9851 tile38/tile38 ``` Visit the [Tile38 hub page](https://hub.docker.com/r/tile38/tile38/) for more information. ### Homebrew (macOS) Install Tile38 using [Homebrew](https://brew.sh/) ```sh brew install tile38 tile38-server ``` ### Building Tile38 Tile38 can be compiled and used on Linux, OSX, Windows, FreeBSD, and probably others since the codebase is 100% Go. We support both 32 bit and 64 bit systems. [Go](https://golang.org/dl/) must be installed on the build machine. To build everything simply: ``` $ make ``` To test: ``` $ make test ``` ### Running For command line options invoke: ``` $ ./tile38-server -h ``` To run a single server: ``` $ ./tile38-server # The tile38 shell connects to localhost:9851 $ ./tile38-cli > help ``` #### Prometheus Metrics Tile38 can natively export Prometheus metrics by setting the `--metrics-addr` command line flag (disabled by default). This example exposes the HTTP metrics server on port 4321: ``` # start server and enable Prometheus metrics, listen on local interface only ./tile38-server --metrics-addr=127.0.0.1:4321 # access metrics curl http://127.0.0.1:4321/metrics ``` If you need to access the `/metrics` endpoint from a different host you'll have to set the flag accordingly, e.g. set it to `0.0.0.0:<>` to listen on all interfaces. Use the [redis_exporter](https://github.com/oliver006/redis_exporter) for more advanced use cases like extracting key values or running a lua script. ## Playing with Tile38 Basic operations: ``` $ ./tile38-cli # add a couple of points named 'truck1' and 'truck2' to a collection named 'fleet'. > set fleet truck1 point 33.5123 -112.2693 # on the Loop 101 in Phoenix > set fleet truck2 point 33.4626 -112.1695 # on the I-10 in Phoenix # search the 'fleet' collection. > scan fleet # returns both trucks in 'fleet' > nearby fleet point 33.462 -112.268 6000 # search 6 kilometers around a point. returns one truck. # key value operations > get fleet truck1 # returns 'truck1' > del fleet truck2 # deletes 'truck2' > drop fleet # removes all ``` Tile38 has a ton of [great commands](https://tile38.com/commands). ## Fields Fields are extra data that belongs to an object. A field is always a double precision floating point. There is no limit to the number of fields that an object can have. To set a field when setting an object: ``` > set fleet truck1 field speed 90 point 33.5123 -112.2693 > set fleet truck1 field speed 90 field age 21 point 33.5123 -112.2693 ``` To set a field when an object already exists: ``` > fset fleet truck1 speed 90 ``` To get a field when an object already exists: ``` > fget fleet truck1 speed ``` ## Searching Tile38 has support to search for objects and points that are within or intersects other objects. All object types can be searched including Polygons, MultiPolygons, GeometryCollections, etc. Search Within #### Within WITHIN searches a collection for objects that are fully contained inside a specified bounding area.
Search Intersects #### Intersects INTERSECTS searches a collection for objects that intersect a specified bounding area.
Search Nearby #### Nearby NEARBY searches a collection for objects that intersect a specified radius.
### Search options **WHERE** - This option allows for filtering out results based on [field](#fields) values. For example
```nearby fleet where speed 70 +inf point 33.462 -112.268 6000``` will return only the objects in the 'fleet' collection that are within the 6 km radius **and** have a field named `speed` that is greater than `70`.

Multiple WHEREs are concatenated as **and** clauses. ```WHERE speed 70 +inf WHERE age -inf 24``` would be interpreted as *speed is over 70 and age is less than 24.*

The default value for a field is always `0`. Thus if you do a WHERE on the field `speed` and an object does not have that field set, the server will pretend that the object does and that the value is Zero.

WHERE expressions support the `=~` operator for regex matching. It uses Go's re2 regular expression engine to match string values in fields or GeoJSON properties within objects. For example, `WHERE properties.name =~ 'truck.*'` filters objects where the 'name' property matches the pattern, and `WHERE field_name =~ 'value.*'` works similarly for fields. **MATCH** - MATCH is similar to WHERE except that it works on the object id instead of fields.
```nearby fleet match truck* point 33.462 -112.268 6000``` will return only the objects in the 'fleet' collection that are within the 6 km radius **and** have an object id that starts with `truck`. There can be multiple MATCH options in a single search. The MATCH value is a simple [glob pattern](https://en.wikipedia.org/wiki/Glob_(programming)). **CURSOR** - CURSOR is used to iterate though many objects from the search results. An iteration begins when the CURSOR is set to Zero or not included with the request, and completes when the cursor returned by the server is Zero. **NOFIELDS** - NOFIELDS tells the server that you do not want field values returned with the search results. **LIMIT** - LIMIT can be used to limit the number of objects returned for a single search request. ## Geofencing Geofence animation A geofence is a virtual boundary that can detect when an object enters or exits the area. This boundary can be a radius, bounding box, or a polygon. Tile38 can turn any standard search into a geofence monitor by adding the FENCE keyword to the search. *Tile38 also allows for [Webhooks](https://tile38.com/commands/sethook) to be assigned to Geofences.*
A simple example: ``` > nearby fleet fence point 33.462 -112.268 6000 ``` This command opens a geofence that monitors the 'fleet' collection. The server will respond with: ``` {"ok":true,"live":true} ``` And the connection will be kept open. If any object enters or exits the 6 km radius around `33.462,-112.268` the server will respond in realtime with a message such as: ``` {"command":"set","detect":"enter","id":"truck02","object":{"type":"Point","coordinates":[-112.2695,33.4626]}} ``` The server will notify the client if the `command` is `del | set | drop`. - `del` notifies the client that an object has been deleted from the collection that is being fenced. - `drop` notifies the client that the entire collection is dropped. - `set` notifies the client that an object has been added or updated, and when it's position is detected by the fence. The `detect` may be one of the following values. - `inside` is when an object is inside the specified area. - `outside` is when an object is outside the specified area. - `enter` is when an object that **was not** previously in the fence has entered the area. - `exit` is when an object that **was** previously in the fence has exited the area. - `cross` is when an object that **was not** previously in the fence has entered **and** exited the area. These can be used when establishing a geofence, to pre-filter responses. For instance, to limit responses to `enter` and `exit` detections: ``` > nearby fleet fence detect enter,exit point 33.462 -112.268 6000 ``` ### Pub/sub channels Tile38 supports delivering geofence notifications over pub/sub channels. To create a static geofence that sends notifications when a bus is within 200 meters of a point and sends to the `busstop` channel: ``` > setchan busstop nearby buses fence point 33.5123 -112.2693 200 ``` Subscribe on the `busstop` channel: ``` > subscribe busstop ``` ## Object types All object types except for XYZ Tiles and QuadKeys can be stored in a collection. XYZ Tiles and QuadKeys are reserved for the SEARCH keyword only. #### Lat/lon point The most basic object type is a point that is composed of a latitude and a longitude. There is an optional `z` member that may be used for auxiliary data such as elevation or a timestamp. ``` set fleet truck1 point 33.5123 -112.2693 # plain lat/lon set fleet truck1 point 33.5123 -112.2693 225 # lat/lon with z member ``` #### Bounding box A bounding box consists of two points. The first being the southwestern most point and the second is the northeastern most point. ``` set fleet truck1 bounds 30 -110 40 -100 ``` #### Geohash A [geohash](https://en.wikipedia.org/wiki/Geohash) is a string representation of a point. With the length of the string indicating the precision of the point. ``` set fleet truck1 hash 9tbnthxzr # this would be equivalent to 'point 33.5123 -112.2693' ``` #### GeoJSON [GeoJSON](https://tools.ietf.org/html/rfc7946) is an industry standard format for representing a variety of object types including a point, multipoint, linestring, multilinestring, polygon, multipolygon, geometrycollection, feature, and featurecollection. * All ignored members will not persist. **Important to note that all coordinates are in Longitude, Latitude order.** ``` set city tempe object {"type":"Polygon","coordinates":[[[0,0],[10,10],[10,0],[0,0]]]} ``` #### XYZ Tile An XYZ tile is rectangle bounding area on earth that is represented by an X, Y coordinate and a Z (zoom) level. Check out [maptiler.org](http://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/) for an interactive example. #### QuadKey A QuadKey used the same coordinate system as an XYZ tile except that the string representation is a string characters composed of 0, 1, 2, or 3. For a detailed explanation checkout [The Bing Maps Tile System](https://msdn.microsoft.com/en-us/library/bb259689.aspx). ## Network protocols It's recommended to use a [client library](#tile38-client-libraries) or the [Tile38 CLI](#running), but there are times when only HTTP is available or when you need to test from a remote terminal. In those cases we provide an HTTP and telnet options. #### HTTP One of the simplest ways to call a tile38 command is to use HTTP. From the command line you can use [curl](https://curl.haxx.se/). For example: ``` # call with request in the body curl --data "set fleet truck3 point 33.4762 -112.10923" localhost:9851 # call with request in the url path curl localhost:9851/set+fleet+truck3+point+33.4762+-112.10923 ``` #### Websockets Websockets can be used when you need to Geofence and keep the connection alive. It works just like the HTTP example above, with the exception that the connection stays alive and the data is sent from the server as text websocket messages. #### Telnet There is the option to use a plain telnet connection. The default output through telnet is [RESP](https://redis.io/topics/protocol). ``` telnet localhost 9851 set fleet truck3 point 33.4762 -112.10923 +OK ``` The server will respond in [JSON](https://json.org) or [RESP](https://redis.io/topics/protocol) depending on which protocol is used when initiating the first command. - HTTP and Websockets use JSON. - Telnet and RESP clients use RESP. ## Tile38 Client Libraries The following clients are built specifically for Tile38. Clients that support most Tile38 features are marked with a ⭐️. - ⭐️ Go: [xjem/t38c](https://github.com/xjem/t38c) - ⭐️ Node.js: [node-tile38](https://github.com/phulst/node-tile38) ([example code](https://github.com/tidwall/tile38/wiki/Node.js-example-(node-tile38))) - ⭐️ Python: [pyle38](https://github.com/iwpnd/pyle38) - ⭐️ TypeScript: [tile38-ts](https://github.com/iwpnd/tile38-ts) - Go: [cjkreklow/t38c](https://github.com/cjkreklow/t38c) - Python: [pytile38](https://github.com/mitghi/pytile38) - Rust: [nazar](https://github.com/younisshah/nazar) - Swift: [Talon](https://github.com/mikekinney/Talon) - Java: [tile38-client-java](https://github.com/jamshidrostami/tile38-client-java) - Java: [tile38-client](https://github.com/HkMoyun/tile38-client) ## Redis Client Libraries Tile38 uses the [Redis RESP](https://redis.io/topics/protocol) protocol natively. Therefore most clients that support basic Redis commands will also support Tile38. - C: [hiredis](https://github.com/redis/hiredis) - C#: [StackExchange.Redis](https://github.com/StackExchange/StackExchange.Redis) - C++: [redox](https://github.com/hmartiro/redox) - Clojure: [carmine](https://github.com/ptaoussanis/carmine) - Common Lisp: [CL-Redis](https://github.com/vseloved/cl-redis) - Erlang: [Eredis](https://github.com/wooga/eredis) - Go: [go-redis](https://github.com/go-redis/redis) ([example code](https://github.com/tidwall/tile38/wiki/Go-example-(go-redis))) - Go: [redigo](https://github.com/gomodule/redigo) ([example code](https://github.com/tidwall/tile38/wiki/Go-example-(redigo))) - Haskell: [hedis](https://github.com/informatikr/hedis) - Java: [lettuce](https://github.com/mp911de/lettuce) ([example code](https://github.com/tidwall/tile38/wiki/Java-example-(lettuce))) - Node.js: [node_redis](https://github.com/NodeRedis/node_redis) ([example code](https://github.com/tidwall/tile38/wiki/Node.js-example-(node-redis))) - Perl: [perl-redis](https://github.com/PerlRedis/perl-redis) - PHP: [tinyredisclient](https://github.com/ptrofimov/tinyredisclient) ([example code](https://github.com/tidwall/tile38/wiki/PHP-example-(tinyredisclient))) - PHP: [phpredis](https://github.com/phpredis/phpredis) - Python: [redis-py](https://github.com/andymccurdy/redis-py) ([example code](https://github.com/tidwall/tile38/wiki/Python-example)) - Ruby: [redic](https://github.com/amakawa/redic) ([example code](https://github.com/tidwall/tile38/wiki/Ruby-example-(redic))) - Ruby: [redis-rb](https://github.com/redis/redis-rb) ([example code](https://github.com/tidwall/tile38/wiki/Ruby-example-(redis-rb))) - Rust: [redis-rs](https://github.com/mitsuhiko/redis-rs) - Scala: [scala-redis](https://github.com/debasishg/scala-redis) - Swift: [Redbird](https://github.com/czechboy0/Redbird) ## Contact Josh Baker [@tidwall](https://twitter.com/tidwall) ## License Tile38 source code is available under the MIT [License](/LICENSE). ================================================ FILE: cmd/tile38-benchmark/az/az.go ================================================ package az // JSON is GeoJSON var JSON = ` { "type": "MultiPolygon", "coordinates": [ [ [ [ -114.635458, 34.876902 ], [ -114.636768000000103, 34.885705 ], [ -114.636725, 34.889107 ], [ -114.635425, 34.895192 ], [ -114.63185, 34.903942 ], [ -114.630877, 34.907263 ], [ -114.630552, 34.911852 ], [ -114.631537, 34.916153 ], [ -114.633237, 34.92123 ], [ -114.633253, 34.924608 ], [ -114.632196, 34.930628 ], [ -114.629753, 34.938684 ], [ -114.629811, 34.94481 ], [ -114.631681, 34.95131 ], [ -114.634274, 34.956662 ], [ -114.634953, 34.958918 ], [ -114.635237000000103, 34.965149 ], [ -114.634607, 34.96906 ], [ -114.629907, 34.980791 ], [ -114.629129, 34.986132 ], [ -114.629443, 34.991825 ], [ -114.630244000000104, 34.99464 ], [ -114.631807, 34.998632 ], [ -114.632665, 34.999806 ], [ -114.635570000000101, 35.005933 ], [ -114.637071, 35.010371 ], [ -114.637769, 35.014948 ], [ -114.63819, 35.022069 ], [ -114.637524, 35.027053 ], [ -114.633715, 35.035602 ], [ -114.629027, 35.042531 ], [ -114.625799, 35.045834 ], [ -114.615902, 35.05272 ], [ -114.610701000000105, 35.055458 ], [ -114.606694, 35.058941 ], [ -114.604715, 35.061744 ], [ -114.603619, 35.064226 ], [ -114.602908, 35.068588 ], [ -114.603175, 35.070445 ], [ -114.604736, 35.07483 ], [ -114.607701, 35.078533 ], [ -114.613132, 35.083097 ], [ -114.61842, 35.086539 ], [ -114.622517, 35.088703 ], [ -114.632053, 35.092559 ], [ -114.63937, 35.094733 ], [ -114.642831, 35.096503 ], [ -114.646579, 35.10082 ], [ -114.646764, 35.101868 ], [ -114.645152, 35.104995 ], [ -114.644354, 35.105903 ], [ -114.641116, 35.108401 ], [ -114.637432000000103, 35.112489 ], [ -114.632282, 35.117088 ], [ -114.628427, 35.118943 ], [ -114.623761, 35.120602 ], [ -114.628993, 35.119411 ], [ -114.624954000000102, 35.120742 ], [ -114.618697, 35.121749 ], [ -114.604007, 35.121252 ], [ -114.60274, 35.121666 ], [ -114.597794, 35.121735 ], [ -114.589787, 35.123522 ], [ -114.584877, 35.125194 ], [ -114.579882, 35.127506 ], [ -114.578263, 35.12881 ], [ -114.577146, 35.130982 ], [ -114.574411, 35.13495 ], [ -114.572597, 35.139557 ], [ -114.573706, 35.142698 ], [ -114.573879, 35.145351 ], [ -114.569529, 35.162317 ], [ -114.56876, 35.172195 ], [ -114.569214, 35.17289 ], [ -114.568989, 35.175085 ], [ -114.569258, 35.183424 ], [ -114.569653, 35.186267 ], [ -114.571404, 35.191026 ], [ -114.572084, 35.200794 ], [ -114.574037, 35.20379 ], [ -114.574233, 35.205481 ], [ -114.574958, 35.206714 ], [ -114.578581, 35.208113 ], [ -114.579535, 35.208911 ], [ -114.579897000000102, 35.21097 ], [ -114.580312, 35.220095 ], [ -114.583523, 35.230348 ], [ -114.58248, 35.233173 ], [ -114.582842, 35.238703 ], [ -114.584993, 35.242717 ], [ -114.586053, 35.248891 ], [ -114.585714, 35.253145 ], [ -114.585768, 35.257743 ], [ -114.586604, 35.262386 ], [ -114.587497, 35.265473 ], [ -114.590513, 35.272334 ], [ -114.593247, 35.284361 ], [ -114.595705, 35.289939 ], [ -114.596682, 35.294557 ], [ -114.597268, 35.299565 ], [ -114.59721, 35.303223 ], [ -114.595163, 35.321883 ], [ -114.595553, 35.326547 ], [ -114.599771, 35.34111 ], [ -114.604607, 35.355239 ], [ -114.606173, 35.359651 ], [ -114.611206, 35.370119 ], [ -114.617698, 35.380131 ], [ -114.618257, 35.382646 ], [ -114.618984, 35.389391 ], [ -114.620887, 35.396867 ], [ -114.621783, 35.39945 ], [ -114.625702, 35.407976 ], [ -114.626765, 35.409644 ], [ -114.629061000000107, 35.411175 ], [ -114.65208, 35.430134 ], [ -114.653817, 35.432853 ], [ -114.654295, 35.436854 ], [ -114.658105, 35.441835 ], [ -114.661747, 35.444735 ], [ -114.662896, 35.446449 ], [ -114.663934, 35.449466 ], [ -114.664215, 35.451707 ], [ -114.663880000000105, 35.454657 ], [ -114.664217, 35.455845 ], [ -114.665142, 35.457331 ], [ -114.666151, 35.458198 ], [ -114.667217, 35.46037 ], [ -114.666769000000102, 35.462085 ], [ -114.665790000000101, 35.463915 ], [ -114.665651, 35.466911 ], [ -114.665988, 35.467985 ], [ -114.667389, 35.469904 ], [ -114.67235, 35.47374 ], [ -114.673164, 35.474814 ], [ -114.673585, 35.475843 ], [ -114.673473000000101, 35.476849 ], [ -114.672074, 35.479709 ], [ -114.671794, 35.480806 ], [ -114.671907, 35.482087 ], [ -114.673534, 35.485675 ], [ -114.676815000000104, 35.489787 ], [ -114.6767040000001, 35.491845 ], [ -114.676257, 35.493103 ], [ -114.677743, 35.495182 ], [ -114.678642, 35.497628 ], [ -114.678587, 35.499846 ], [ -114.678892000000104, 35.501276 ], [ -114.67748, 35.510948 ], [ -114.677143, 35.512945 ], [ -114.675685, 35.51563 ], [ -114.672767, 35.518428 ], [ -114.66954, 35.52079 ], [ -114.668586, 35.521225 ], [ -114.666565, 35.520993 ], [ -114.664601000000104, 35.521519 ], [ -114.663983000000101, 35.522161 ], [ -114.661682, 35.526682 ], [ -114.659886, 35.527919 ], [ -114.657753, 35.530741 ], [ -114.657163, 35.532301 ], [ -114.65677, 35.534964 ], [ -114.657809, 35.536963 ], [ -114.660335, 35.540433 ], [ -114.661457, 35.544062 ], [ -114.66157, 35.545692 ], [ -114.66112, 35.549021 ], [ -114.661963, 35.552604 ], [ -114.661963, 35.555887 ], [ -114.663451, 35.559884 ], [ -114.663535, 35.560963 ], [ -114.662805, 35.564268 ], [ -114.6639, 35.56629 ], [ -114.664433000000102, 35.568426 ], [ -114.666231, 35.571642 ], [ -114.668393, 35.574331 ], [ -114.670022, 35.575596 ], [ -114.671567, 35.576217 ], [ -114.674881, 35.578379 ], [ -114.675751, 35.579459 ], [ -114.675667, 35.580033 ], [ -114.670191, 35.583471 ], [ -114.664209, 35.585944 ], [ -114.660558, 35.586583 ], [ -114.659238, 35.587477 ], [ -114.654518, 35.596609 ], [ -114.653900000000107, 35.598491 ], [ -114.653731, 35.600373 ], [ -114.654489, 35.605173 ], [ -114.653618, 35.607192 ], [ -114.653534, 35.609672 ], [ -114.653927, 35.611739 ], [ -114.655219, 35.614059 ], [ -114.657241, 35.617046 ], [ -114.659461, 35.619552 ], [ -114.660641, 35.620334 ], [ -114.663647, 35.620773 ], [ -114.665389, 35.621556 ], [ -114.666682, 35.623073 ], [ -114.668087, 35.627115 ], [ -114.669015000000101, 35.628861 ], [ -114.672134, 35.633365 ], [ -114.675001, 35.638304 ], [ -114.677615, 35.641774 ], [ -114.679415000000105, 35.643429 ], [ -114.686133, 35.647522 ], [ -114.689001, 35.65028 ], [ -114.689507, 35.651429 ], [ -114.689226, 35.652898 ], [ -114.690494000000101, 35.662657 ], [ -114.690214, 35.665159 ], [ -114.686055, 35.670642 ], [ -114.682317, 35.677825 ], [ -114.680827, 35.682255 ], [ -114.680631000000105, 35.684046 ], [ -114.680997000000104, 35.685929 ], [ -114.682657000000106, 35.688571 ], [ -114.691263, 35.693125 ], [ -114.696214, 35.69655 ], [ -114.701416, 35.701084 ], [ -114.703608, 35.703922 ], [ -114.704501, 35.705993 ], [ -114.704959, 35.706366 ], [ -114.704842, 35.706744 ], [ -114.705597, 35.708274 ], [ -114.705347000000103, 35.708344 ], [ -114.705447, 35.711757 ], [ -114.699405, 35.726929 ], [ -114.697859, 35.731657 ], [ -114.69654, 35.738934 ], [ -114.6964, 35.742653 ], [ -114.696655, 35.746143 ], [ -114.697585, 35.748417 ], [ -114.697726, 35.750966 ], [ -114.696854, 35.752756 ], [ -114.696546, 35.754638 ], [ -114.694267, 35.756633 ], [ -114.694717, 35.757897 ], [ -114.69742, 35.760677 ], [ -114.700266, 35.766879 ], [ -114.701027, 35.76968 ], [ -114.70117, 35.774112 ], [ -114.699036000000106, 35.788046 ], [ -114.699318000000105, 35.79048 ], [ -114.703178, 35.794685 ], [ -114.705827, 35.798889 ], [ -114.71149, 35.80438 ], [ -114.712026, 35.805529 ], [ -114.710534, 35.807525 ], [ -114.709324, 35.81005 ], [ -114.70634, 35.812022 ], [ -114.703665, 35.814614 ], [ -114.700654, 35.822004 ], [ -114.697276, 35.826776 ], [ -114.69553, 35.829897 ], [ -114.695277000000104, 35.831091 ], [ -114.695249, 35.832285 ], [ -114.695757, 35.833387 ], [ -114.701478, 35.839316 ], [ -114.702293, 35.840792 ], [ -114.702339, 35.842151 ], [ -114.703527, 35.841845 ], [ -114.704173, 35.842669 ], [ -114.704203000000106, 35.844274 ], [ -114.706288, 35.846218 ], [ -114.706532, 35.849027 ], [ -114.705856, 35.850508 ], [ -114.703599, 35.852595 ], [ -114.701904, 35.853223 ], [ -114.696581, 35.853727 ], [ -114.69437, 35.854463 ], [ -114.693446, 35.855125 ], [ -114.691456, 35.858661 ], [ -114.6877980000001, 35.860728 ], [ -114.68205, 35.86295 ], [ -114.678186, 35.863311 ], [ -114.672289, 35.865011 ], [ -114.66968, 35.865036 ], [ -114.667471000000106, 35.867061 ], [ -114.662623, 35.869213 ], [ -114.661636, 35.870545 ], [ -114.661636, 35.871233 ], [ -114.663214, 35.873692 ], [ -114.668145, 35.875201 ], [ -114.672009, 35.878018 ], [ -114.678972, 35.88551 ], [ -114.693602, 35.895311 ], [ -114.69454, 35.896587 ], [ -114.696064, 35.896464 ], [ -114.694928, 35.897594 ], [ -114.696132000000105, 35.898662 ], [ -114.697558, 35.89936 ], [ -114.700258000000105, 35.901757 ], [ -114.700769, 35.903064 ], [ -114.703538, 35.906707 ], [ -114.705119, 35.907637 ], [ -114.705991, 35.908598 ], [ -114.7057140000001, 35.909316 ], [ -114.706767, 35.90895 ], [ -114.708112, 35.909933 ], [ -114.709187, 35.916827 ], [ -114.707784, 35.916993 ], [ -114.707398, 35.918057 ], [ -114.70788, 35.919207 ], [ -114.707329000000101, 35.926177 ], [ -114.707603, 35.92795 ], [ -114.712965, 35.932159 ], [ -114.712756, 35.932639 ], [ -114.713413, 35.9319 ], [ -114.713312, 35.933844 ], [ -114.729762, 35.959895 ], [ -114.728496000000106, 35.960395 ], [ -114.728666, 35.961757 ], [ -114.730090000000104, 35.962691 ], [ -114.732456, 35.965891 ], [ -114.736195, 35.969421 ], [ -114.740536, 35.975545 ], [ -114.743494, 35.983553 ], [ -114.743638, 35.985785 ], [ -114.743117, 35.987387 ], [ -114.740043, 35.990534 ], [ -114.739318, 35.991804 ], [ -114.740544, 35.994853 ], [ -114.740815, 35.997464 ], [ -114.741536, 35.99969 ], [ -114.741679, 36.002283 ], [ -114.743163, 36.006722 ], [ -114.743005, 36.00845 ], [ -114.740866, 36.012928 ], [ -114.738555, 36.015223 ], [ -114.728874000000104, 36.021387 ], [ -114.723324, 36.026588 ], [ -114.722214, 36.027964 ], [ -114.722096, 36.028952 ], [ -114.722742, 36.030286 ], [ -114.723673, 36.03123 ], [ -114.727602, 36.033099 ], [ -114.730563, 36.036207 ], [ -114.733417000000102, 36.037913 ], [ -114.735739, 36.038033 ], [ -114.740018, 36.037467 ], [ -114.741262000000106, 36.038044 ], [ -114.742105, 36.039792 ], [ -114.742661, 36.042573 ], [ -114.742479, 36.045697 ], [ -114.741677, 36.047877 ], [ -114.735701, 36.053393 ], [ -114.73508, 36.054435 ], [ -114.735285, 36.056648 ], [ -114.736023000000102, 36.059063 ], [ -114.74006, 36.062437 ], [ -114.7422, 36.067833 ], [ -114.742138, 36.068676 ], [ -114.743542, 36.071037 ], [ -114.748891, 36.074981 ], [ -114.75057, 36.08033 ], [ -114.754032, 36.083093 ], [ -114.754681, 36.085052 ], [ -114.754508, 36.086171 ], [ -114.752836, 36.089393 ], [ -114.750095, 36.092275 ], [ -114.748913000000101, 36.095183 ], [ -114.737497000000104, 36.103102 ], [ -114.734857, 36.104426 ], [ -114.718257, 36.107164 ], [ -114.709269, 36.107396 ], [ -114.706091, 36.108239 ], [ -114.703737, 36.108348 ], [ -114.696981, 36.110297 ], [ -114.693655000000106, 36.112482 ], [ -114.691631, 36.112535 ], [ -114.6880740000001, 36.111457 ], [ -114.684426000000101, 36.109472 ], [ -114.681847000000104, 36.109192 ], [ -114.679775000000106, 36.109874 ], [ -114.678375, 36.110815 ], [ -114.675106, 36.114111 ], [ -114.671867, 36.115964 ], [ -114.664343000000102, 36.1163 ], [ -114.662144, 36.117742 ], [ -114.660448, 36.119999 ], [ -114.658131, 36.124127 ], [ -114.655512, 36.126187 ], [ -114.645728, 36.131995 ], [ -114.641976, 36.13373 ], [ -114.640125, 36.135126 ], [ -114.636862, 36.135552 ], [ -114.635809, 36.13617 ], [ -114.630474, 36.142218 ], [ -114.628462, 36.141822 ], [ -114.627079, 36.140761 ], [ -114.623837, 36.137144 ], [ -114.620605, 36.131759 ], [ -114.618429, 36.130328 ], [ -114.615455, 36.129653 ], [ -114.61324, 36.130266 ], [ -114.609288, 36.132229 ], [ -114.596474, 36.141537 ], [ -114.5930350000001, 36.142674 ], [ -114.589828, 36.143192 ], [ -114.583716, 36.14556 ], [ -114.580707, 36.145987 ], [ -114.578828, 36.147175 ], [ -114.57706, 36.148845 ], [ -114.57109, 36.151099 ], [ -114.561173, 36.150921 ], [ -114.556162, 36.15247 ], [ -114.548742000000104, 36.150697 ], [ -114.543232, 36.151871 ], [ -114.539233, 36.151764 ], [ -114.534478, 36.15023 ], [ -114.532924, 36.149282 ], [ -114.532308, 36.14804 ], [ -114.531091, 36.147644 ], [ -114.52621, 36.148177 ], [ -114.51428, 36.150795 ], [ -114.511218, 36.150576 ], [ -114.508104000000102, 36.149713 ], [ -114.501049, 36.144516 ], [ -114.500236, 36.143226 ], [ -114.499992, 36.141594 ], [ -114.500339, 36.1407 ], [ -114.50108, 36.14006 ], [ -114.50515, 36.138078 ], [ -114.507175, 36.13634 ], [ -114.50921, 36.133247 ], [ -114.508467, 36.129913 ], [ -114.507201, 36.128484 ], [ -114.504715, 36.127188 ], [ -114.501798, 36.126556 ], [ -114.498849, 36.126612 ], [ -114.487635, 36.128656 ], [ -114.483827, 36.12972 ], [ -114.478248, 36.132683 ], [ -114.468674, 36.138889 ], [ -114.465579, 36.139496 ], [ -114.4626, 36.139644 ], [ -114.458945, 36.139214 ], [ -114.456487, 36.138032 ], [ -114.45511, 36.136372 ], [ -114.453798, 36.133586 ], [ -114.451331, 36.129831 ], [ -114.447135, 36.126022 ], [ -114.445042, 36.125346 ], [ -114.443736, 36.125593 ], [ -114.435507, 36.130057 ], [ -114.423114, 36.13735 ], [ -114.418193, 36.142771 ], [ -114.415253, 36.145123 ], [ -114.412491000000102, 36.146511 ], [ -114.40914, 36.147 ], [ -114.405624, 36.146983 ], [ -114.398373, 36.145799 ], [ -114.381479, 36.141349 ], [ -114.379976, 36.141388 ], [ -114.375278, 36.143592 ], [ -114.373745, 36.143722 ], [ -114.370181, 36.142624 ], [ -114.368551, 36.140892 ], [ -114.367381, 36.13852 ], [ -114.365529, 36.136306 ], [ -114.364499, 36.134072 ], [ -114.358968, 36.127795 ], [ -114.348592, 36.121147 ], [ -114.3451, 36.118556 ], [ -114.342601, 36.115878 ], [ -114.34095, 36.113457 ], [ -114.338815, 36.111309 ], [ -114.337264, 36.110428 ], [ -114.334632, 36.106784 ], [ -114.333587, 36.106342 ], [ -114.328801, 36.105902 ], [ -114.325814, 36.103933 ], [ -114.325539, 36.102989 ], [ -114.323458000000102, 36.101186 ], [ -114.320866, 36.096463 ], [ -114.316983, 36.093409 ], [ -114.313086, 36.088816 ], [ -114.306939, 36.082487 ], [ -114.304171, 36.07558 ], [ -114.304384, 36.074019 ], [ -114.305853, 36.071478 ], [ -114.307485, 36.069672 ], [ -114.31242, 36.066117 ], [ -114.3136, 36.064148 ], [ -114.314328, 36.062016 ], [ -114.314427, 36.060523 ], [ -114.313591, 36.059048 ], [ -114.311904, 36.057661 ], [ -114.308624, 36.056976 ], [ -114.300971000000104, 36.05746 ], [ -114.298593, 36.057263 ], [ -114.295941, 36.056168 ], [ -114.293435000000102, 36.0545 ], [ -114.290867, 36.050511 ], [ -114.287992, 36.04907 ], [ -114.284006, 36.048242 ], [ -114.279637, 36.046103 ], [ -114.278166, 36.045819 ], [ -114.273911, 36.046529 ], [ -114.272299, 36.046289 ], [ -114.270862, 36.045523 ], [ -114.269548, 36.043769 ], [ -114.268896, 36.04094 ], [ -114.26922, 36.036807 ], [ -114.268586, 36.035034 ], [ -114.26438, 36.027911 ], [ -114.262388, 36.026107 ], [ -114.259518, 36.024206 ], [ -114.251633, 36.019886 ], [ -114.248419, 36.018556 ], [ -114.246111, 36.017164 ], [ -114.243865, 36.015266 ], [ -114.240439, 36.015245 ], [ -114.238154, 36.014473 ], [ -114.236892, 36.013247 ], [ -114.233443, 36.012835 ], [ -114.231854, 36.013147 ], [ -114.228015, 36.014731 ], [ -114.226459, 36.014606 ], [ -114.224798, 36.013699 ], [ -114.218759, 36.014511 ], [ -114.216609, 36.014336 ], [ -114.214679, 36.014806 ], [ -114.213549, 36.014615 ], [ -114.211932, 36.014834 ], [ -114.206052, 36.016634 ], [ -114.204156, 36.016575 ], [ -114.201227, 36.017751 ], [ -114.200066, 36.017743 ], [ -114.191221, 36.020019 ], [ -114.185860000000105, 36.022266 ], [ -114.179438, 36.024313 ], [ -114.176304, 36.026129 ], [ -114.174683, 36.02667 ], [ -114.164402, 36.026852 ], [ -114.161237, 36.026279 ], [ -114.157344, 36.024966 ], [ -114.1534, 36.02317 ], [ -114.15139, 36.023133 ], [ -114.150225, 36.023515 ], [ -114.145907, 36.027229 ], [ -114.145637, 36.028559 ], [ -114.145672, 36.03297 ], [ -114.144666, 36.034272 ], [ -114.143153, 36.035295 ], [ -114.13826, 36.03719 ], [ -114.137112, 36.038491 ], [ -114.135721, 36.041238 ], [ -114.134841, 36.043873 ], [ -114.134824, 36.045343 ], [ -114.135927, 36.050358 ], [ -114.136206, 36.053232 ], [ -114.1352, 36.056946 ], [ -114.133389, 36.061665 ], [ -114.129768, 36.068484 ], [ -114.125891, 36.072935 ], [ -114.124019, 36.075563 ], [ -114.121186, 36.082755 ], [ -114.119648, 36.085822 ], [ -114.112297, 36.09405 ], [ -114.111998, 36.09491 ], [ -114.1119, 36.095845 ], [ -114.115208, 36.099878 ], [ -114.11707, 36.101177 ], [ -114.119329, 36.10193 ], [ -114.121033, 36.103885 ], [ -114.121779, 36.105699 ], [ -114.12167, 36.108294 ], [ -114.120865, 36.11085 ], [ -114.118497, 36.1139 ], [ -114.116061, 36.115471 ], [ -114.108381, 36.119154 ], [ -114.107419, 36.119401 ], [ -114.100433, 36.119359 ], [ -114.097707, 36.120213 ], [ -114.096994, 36.120823 ], [ -114.092753, 36.132356 ], [ -114.092366, 36.135331 ], [ -114.091701, 36.137303 ], [ -114.089279, 36.140326 ], [ -114.087899, 36.142923 ], [ -114.081234, 36.150208 ], [ -114.07945, 36.154625 ], [ -114.078832, 36.157434 ], [ -114.075641, 36.162523 ], [ -114.071652, 36.170921 ], [ -114.066798, 36.179087 ], [ -114.058662, 36.187835 ], [ -114.052743, 36.190919 ], [ -114.049484, 36.192134 ], [ -114.043944, 36.19335 ], [ -114.043849, 36.245114 ], [ -114.045518, 36.27439 ], [ -114.045559, 36.288837 ], [ -114.045033, 36.30305 ], [ -114.044345, 36.310234 ], [ -114.044051, 36.317628 ], [ -114.044776, 36.331969 ], [ -114.044702, 36.346298 ], [ -114.043034, 36.38587 ], [ -114.042843000000104, 36.448175 ], [ -114.043133, 36.469716 ], [ -114.044816, 36.491343 ], [ -114.045647, 36.521095 ], [ -114.04632, 36.564615 ], [ -114.049935, 36.709521 ], [ -114.049973, 36.738672 ], [ -114.050327, 36.752899 ], [ -114.049879, 36.781909 ], [ -114.050502, 36.895232 ], [ -114.049995, 36.957769 ], [ -114.050600000000102, 37.000396 ], [ -114.0008, 37.000448 ], [ -113.96266, 36.999973 ], [ -113.052912, 36.999983 ], [ -112.875756, 37.000533 ], [ -112.538546, 37.000652 ], [ -112.529846, 37.000899 ], [ -112.36102, 37.001114 ], [ -112.36037, 37.000912 ], [ -112.359329, 37.001117 ], [ -112.125741, 37.001237 ], [ -112.000735, 37.000959 ], [ -111.62572, 37.001401 ], [ -111.616249, 37.001647 ], [ -111.406146, 37.001481 ], [ -111.405895, 37.001702 ], [ -111.313211, 37.000894 ], [ -111.312169, 37.001193 ], [ -111.305843, 37.000776 ], [ -111.278221, 37.000467 ], [ -111.254853, 37.001076 ], [ -111.133718, 37.000779 ], [ -111.081493, 37.002261 ], [ -111.052354, 37.00246 ], [ -111.00182, 37.002293 ], [ -110.625691, 37.003725 ], [ -110.625605, 37.003416 ], [ -110.599512, 37.003448 ], [ -110.509004, 37.003985 ], [ -110.50069, 37.00426 ], [ -110.490908, 37.003566 ], [ -110.478446, 36.999996 ], [ -110.47729, 36.999997 ], [ -110.47019, 36.997997 ], [ -110.023043, 36.998601 ], [ -110.000876, 36.998502 ], [ -110.000677, 36.997968 ], [ -109.969958, 36.997949 ], [ -109.938511, 36.998491 ], [ -109.750669, 36.99816 ], [ -109.743284, 36.998453 ], [ -109.625658, 36.998308 ], [ -109.495338, 36.999105 ], [ -109.362565, 36.999304 ], [ -109.125691, 36.999389 ], [ -109.045223, 36.999084 ], [ -109.045554, 36.645013 ], [ -109.04539, 36.503241 ], [ -109.045946, 36.375002 ], [ -109.045637, 36.374625 ], [ -109.045744, 36.257214 ], [ -109.046024, 36.247197 ], [ -109.045877, 36.188719 ], [ -109.046183, 36.181751 ], [ -109.045726, 36.116908 ], [ -109.045767, 36.033679 ], [ -109.046124, 35.990618 ], [ -109.046009, 35.875012 ], [ -109.046423, 35.624911 ], [ -109.046181, 35.614569 ], [ -109.046795, 35.379918 ], [ -109.046084, 35.249986 ], [ -109.046256, 35.125041 ], [ -109.045842, 34.966076 ], [ -109.046136, 34.875006 ], [ -109.046072, 34.828566 ], [ -109.045626, 34.814226 ], [ -109.046104, 34.799981 ], [ -109.045363, 34.785406 ], [ -109.046087, 34.770963 ], [ -109.046175, 34.520102 ], [ -109.046561, 34.379479 ], [ -109.046337, 34.283639 ], [ -109.046664, 34.250046 ], [ -109.04696, 34.068968 ], [ -109.047006, 34.00005 ], [ -109.046426, 33.875052 ], [ -109.046869, 33.844183 ], [ -109.047145, 33.74001 ], [ -109.046662, 33.625055 ], [ -109.046825, 33.469389 ], [ -109.047309, 33.462131 ], [ -109.046928, 33.4428 ], [ -109.047304, 33.439442 ], [ -109.047298, 33.409774 ], [ -109.046564, 33.375059 ], [ -109.047045, 33.36928 ], [ -109.046827, 33.365271 ], [ -109.047104, 33.27046 ], [ -109.04747, 33.250168 ], [ -109.047122, 33.2408 ], [ -109.047324, 33.18408 ], [ -109.047208, 33.107377 ], [ -109.046905, 33.091931 ], [ -109.047513, 33.059137 ], [ -109.047382, 33.000311 ], [ -109.04711, 32.99225 ], [ -109.047117, 32.777569 ], [ -109.047518, 32.749997 ], [ -109.047796, 32.68263 ], [ -109.047912, 32.500261 ], [ -109.047629, 32.413987 ], [ -109.048323, 32.070887 ], [ -109.048731, 32.028174 ], [ -109.048465, 32.000089 ], [ -109.048738, 31.876905 ], [ -109.049048, 31.870689 ], [ -109.049298, 31.796742 ], [ -109.04899, 31.721922 ], [ -109.049311, 31.544932 ], [ -109.050173, 31.480004 ], [ -109.049934, 31.437907 ], [ -109.050044, 31.332502 ], [ -109.1256, 31.332685 ], [ -109.271744, 31.333942 ], [ -109.49449, 31.334125 ], [ -109.500621, 31.333911 ], [ -109.875628, 31.33405 ], [ -110.000613, 31.333145 ], [ -110.140512, 31.333965 ], [ -110.375635, 31.332896 ], [ -110.460172, 31.332827 ], [ -110.68143, 31.33309 ], [ -110.750638, 31.333636 ], [ -110.795467, 31.33363 ], [ -110.94232, 31.332833 ], [ -111.000643, 31.332177 ], [ -111.074825, 31.332239 ], [ -111.125646, 31.348978 ], [ -111.129451, 31.349979 ], [ -111.357436, 31.423346 ], [ -111.500659, 31.468862 ], [ -111.560194, 31.488138 ], [ -111.659998, 31.519448 ], [ -111.738873, 31.544718 ], [ -111.875674, 31.587657 ], [ -111.979304, 31.620648 ], [ -112.200717, 31.690033 ], [ -112.365328, 31.741078 ], [ -112.375759, 31.743987 ], [ -112.399254, 31.751638 ], [ -112.433246, 31.762162 ], [ -112.737399, 31.855527 ], [ -112.800213, 31.87507 ], [ -112.834233, 31.885137 ], [ -112.871505, 31.896838 ], [ -113.125961, 31.97278 ], [ -113.21163, 32.000061 ], [ -113.211365, 32.000061 ], [ -113.217307, 32.002106 ], [ -113.250731, 32.012405 ], [ -113.493196, 32.088943 ], [ -113.750756, 32.169005 ], [ -113.78168, 32.179034 ], [ -114.250775, 32.323909 ], [ -114.625785, 32.43789 ], [ -114.790245, 32.487505 ], [ -114.813613, 32.494276 ], [ -114.813991000000101, 32.497231 ], [ -114.812316, 32.500054 ], [ -114.813402, 32.501764 ], [ -114.813753000000105, 32.50426 ], [ -114.815185, 32.506023 ], [ -114.81651, 32.506963 ], [ -114.816591000000102, 32.507696 ], [ -114.815591, 32.508612 ], [ -114.814321000000106, 32.509023 ], [ -114.812942, 32.509116 ], [ -114.810159, 32.508383 ], [ -114.807726, 32.508726 ], [ -114.804076, 32.510375 ], [ -114.802833, 32.511749 ], [ -114.802211, 32.513191 ], [ -114.802238, 32.515206 ], [ -114.80367, 32.516374 ], [ -114.807753, 32.516925 ], [ -114.809672, 32.517567 ], [ -114.810374, 32.518391 ], [ -114.809969, 32.520291 ], [ -114.810482, 32.521758 ], [ -114.810969, 32.522444 ], [ -114.812888, 32.52359 ], [ -114.813348000000104, 32.524186 ], [ -114.812645, 32.525399 ], [ -114.811293, 32.526429 ], [ -114.810563, 32.527666 ], [ -114.808617, 32.529017 ], [ -114.80640000000011, 32.531191 ], [ -114.804858, 32.533689 ], [ -114.802559, 32.535521 ], [ -114.802181, 32.536414 ], [ -114.802018, 32.53946 ], [ -114.80237, 32.540078 ], [ -114.804776000000103, 32.541659 ], [ -114.805966, 32.545346 ], [ -114.8058300000001, 32.546354 ], [ -114.803883, 32.548001 ], [ -114.795635, 32.550956 ], [ -114.793769, 32.552329 ], [ -114.792065, 32.555009 ], [ -114.791551, 32.557023 ], [ -114.791523, 32.558602 ], [ -114.792955000000106, 32.562085 ], [ -114.792088, 32.568497 ], [ -114.792358000000107, 32.569091 ], [ -114.793224, 32.569459 ], [ -114.794684, 32.568703 ], [ -114.795253, 32.56662 ], [ -114.79766, 32.564444 ], [ -114.801311, 32.562865 ], [ -114.803664, 32.560689 ], [ -114.806830000000105, 32.55888 ], [ -114.808885, 32.558467 ], [ -114.810318, 32.558628 ], [ -114.812914, 32.560049 ], [ -114.813995, 32.562201 ], [ -114.814212, 32.56369 ], [ -114.813968, 32.566209 ], [ -114.812995, 32.568706 ], [ -114.81148, 32.569781 ], [ -114.804421, 32.572941 ], [ -114.803474, 32.573628 ], [ -114.801877, 32.576009 ], [ -114.801471, 32.578255 ], [ -114.80193, 32.579194 ], [ -114.803879, 32.580889 ], [ -114.803987, 32.582652 ], [ -114.802823, 32.585079 ], [ -114.800441, 32.588079 ], [ -114.799737, 32.592177 ], [ -114.799683, 32.593621 ], [ -114.801251, 32.596232 ], [ -114.801548, 32.598591 ], [ -114.802361, 32.59937 ], [ -114.805932, 32.600721 ], [ -114.8069050000001, 32.60143 ], [ -114.808041, 32.603172 ], [ -114.807879, 32.605416 ], [ -114.809042, 32.608806 ], [ -114.808906, 32.612951 ], [ -114.809555, 32.616203 ], [ -114.808662, 32.619157 ], [ -114.80739, 32.621332 ], [ -114.806821, 32.621721 ], [ -114.799302, 32.625115 ], [ -114.797564, 32.624578 ], [ -114.794102, 32.622475 ], [ -114.792640000000105, 32.621948 ], [ -114.791179, 32.621833 ], [ -114.787715, 32.623573 ], [ -114.782573, 32.624304 ], [ -114.781896, 32.624702 ], [ -114.781766, 32.625613 ], [ -114.782518, 32.628625 ], [ -114.782235, 32.630215 ], [ -114.779215, 32.633578 ], [ -114.77457, 32.63593 ], [ -114.771978, 32.637954 ], [ -114.768199, 32.639874 ], [ -114.764382, 32.642666 ], [ -114.76331, 32.644616 ], [ -114.763512, 32.645995 ], [ -114.764917, 32.648079 ], [ -114.76495, 32.649391 ], [ -114.75831, 32.655178 ], [ -114.751079, 32.659789 ], [ -114.749480000000105, 32.66178 ], [ -114.748000000000104, 32.664184 ], [ -114.748183, 32.665098 ], [ -114.747817, 32.667777 ], [ -114.746383, 32.669853 ], [ -114.745344, 32.67219 ], [ -114.7449, 32.677231 ], [ -114.744349, 32.678935 ], [ -114.740541, 32.684196 ], [ -114.739405, 32.686385 ], [ -114.730453, 32.698844 ], [ -114.72981, 32.700282 ], [ -114.72974, 32.703121 ], [ -114.730086, 32.704298 ], [ -114.728408, 32.706648 ], [ -114.726974, 32.707875 ], [ -114.72534, 32.710369 ], [ -114.72241, 32.713597 ], [ -114.719938, 32.71829 ], [ -114.717695, 32.721547 ], [ -114.715788, 32.727758 ], [ -114.714522, 32.73039 ], [ -114.712629, 32.732678 ], [ -114.710615, 32.733936 ], [ -114.709074, 32.735456 ], [ -114.706114, 32.740986 ], [ -114.70294, 32.744793 ], [ -114.701582000000101, 32.745632 ], [ -114.699247, 32.745098 ], [ -114.695387, 32.742244 ], [ -114.691801, 32.740147 ], [ -114.689282, 32.737927 ], [ -114.688230000000104, 32.73753 ], [ -114.682614, 32.737348 ], [ -114.672025, 32.734951 ], [ -114.665921, 32.734028 ], [ -114.654247, 32.73357 ], [ -114.645353, 32.732139 ], [ -114.635006000000104, 32.731372 ], [ -114.629299, 32.729908 ], [ -114.617479, 32.728243 ], [ -114.61567, 32.728454 ], [ -114.61587, 32.729717 ], [ -114.615501, 32.730044 ], [ -114.615504, 32.731449 ], [ -114.614786, 32.732846 ], [ -114.614787000000106, 32.734076 ], [ -114.615112, 32.734515 ], [ -114.581784, 32.734946 ], [ -114.581736, 32.74232 ], [ -114.564508, 32.742274 ], [ -114.564447, 32.749554 ], [ -114.539224, 32.749812 ], [ -114.539092, 32.756949 ], [ -114.526856, 32.757094 ], [ -114.528443, 32.767276 ], [ -114.529264, 32.769484 ], [ -114.531831, 32.774264 ], [ -114.532432, 32.776922 ], [ -114.532426, 32.778644 ], [ -114.531746, 32.782503 ], [ -114.531669, 32.791185 ], [ -114.529633000000103, 32.795477 ], [ -114.522031, 32.801675 ], [ -114.520385, 32.803576 ], [ -114.520363000000103, 32.804385 ], [ -114.519758, 32.805676 ], [ -114.515389, 32.811439 ], [ -114.510327, 32.816488 ], [ -114.494116, 32.823287 ], [ -114.475892, 32.838693 ], [ -114.468971, 32.845155 ], [ -114.465711, 32.873681 ], [ -114.465172, 32.885295 ], [ -114.463307, 32.899116 ], [ -114.462929, 32.907944 ], [ -114.46365, 32.911682 ], [ -114.464448, 32.913128 ], [ -114.473713, 32.920594 ], [ -114.47664, 32.923628 ], [ -114.477952000000101, 32.925706 ], [ -114.479005, 32.928291 ], [ -114.480783, 32.933678 ], [ -114.480925, 32.936276 ], [ -114.48074, 32.937027 ], [ -114.478456, 32.940555 ], [ -114.474042, 32.94515 ], [ -114.470768, 32.949424 ], [ -114.468536, 32.953922 ], [ -114.467624, 32.956663 ], [ -114.467274, 32.960172 ], [ -114.467367, 32.965384 ], [ -114.468379, 32.970745 ], [ -114.468995, 32.972239 ], [ -114.470511, 32.973858 ], [ -114.472606, 32.974654 ], [ -114.475171, 32.975154 ], [ -114.477308, 32.975023 ], [ -114.479477, 32.974189 ], [ -114.480831, 32.973362 ], [ -114.481315, 32.972064 ], [ -114.484806, 32.971339 ], [ -114.488625, 32.969946 ], [ -114.490129, 32.969884 ], [ -114.492184, 32.971021 ], [ -114.492938, 32.971781 ], [ -114.494212, 32.974262 ], [ -114.495712, 32.980075 ], [ -114.496798, 32.986534 ], [ -114.497052, 32.990206 ], [ -114.49941, 33.00004 ], [ -114.499797, 33.003905 ], [ -114.50287, 33.011154 ], [ -114.506129, 33.017009 ], [ -114.507956, 33.019708 ], [ -114.511343, 33.023455 ], [ -114.5149, 33.026524 ], [ -114.52013, 33.029984 ], [ -114.523578, 33.03096 ], [ -114.538459, 33.033422 ], [ -114.553189, 33.033974 ], [ -114.56085, 33.035285 ], [ -114.5648, 33.035077 ], [ -114.571653, 33.036624 ], [ -114.575161, 33.036541 ], [ -114.578287, 33.035375 ], [ -114.581404, 33.032545 ], [ -114.584765, 33.02823 ], [ -114.586982, 33.026944 ], [ -114.589778, 33.026228 ], [ -114.598093, 33.025384 ], [ -114.601014, 33.02541 ], [ -114.611584, 33.026221 ], [ -114.618788, 33.027202 ], [ -114.625787, 33.029435 ], [ -114.628294, 33.03105 ], [ -114.629732, 33.032546 ], [ -114.63419, 33.039024 ], [ -114.639552, 33.04529 ], [ -114.641621, 33.046894 ], [ -114.64482, 33.048644 ], [ -114.645979, 33.048902 ], [ -114.647049, 33.048416 ], [ -114.649001, 33.046762 ], [ -114.650999, 33.044131 ], [ -114.655038000000104, 33.037106 ], [ -114.657827, 33.033824 ], [ -114.659832, 33.032664 ], [ -114.662317, 33.03267 ], [ -114.66506, 33.033906 ], [ -114.670803, 33.037983 ], [ -114.673659, 33.041896 ], [ -114.67483, 33.045507 ], [ -114.675103000000107, 33.04753 ], [ -114.674295, 33.057169 ], [ -114.679114, 33.061966 ], [ -114.686991, 33.070968 ], [ -114.68912, 33.076121 ], [ -114.689307, 33.079179 ], [ -114.688597000000101, 33.082869 ], [ -114.68902, 33.084035 ], [ -114.692548000000102, 33.085786 ], [ -114.694628, 33.086226 ], [ -114.701165000000103, 33.086368 ], [ -114.70473, 33.087051 ], [ -114.706488, 33.08816 ], [ -114.707819, 33.091102 ], [ -114.708133000000103, 33.094022 ], [ -114.707896000000105, 33.097431 ], [ -114.706175, 33.105334 ], [ -114.703682, 33.113768 ], [ -114.696914, 33.131119 ], [ -114.694858, 33.13346 ], [ -114.690246, 33.137724 ], [ -114.687405, 33.141983 ], [ -114.684907, 33.147823 ], [ -114.682759, 33.154808 ], [ -114.679945, 33.159059 ], [ -114.67935, 33.162433 ], [ -114.68089, 33.169074 ], [ -114.680237, 33.169637 ], [ -114.679115, 33.174608 ], [ -114.675830000000104, 33.18152 ], [ -114.6753590000001, 33.185488 ], [ -114.675189, 33.188178 ], [ -114.678163, 33.199488 ], [ -114.678749, 33.203448 ], [ -114.676072, 33.210835 ], [ -114.673715, 33.219245 ], [ -114.673626, 33.223121 ], [ -114.674479000000105, 33.225504 ], [ -114.678097, 33.2303 ], [ -114.682731, 33.234918 ], [ -114.689421, 33.24525 ], [ -114.689541, 33.246428 ], [ -114.688205, 33.247965 ], [ -114.683253, 33.250034 ], [ -114.67766, 33.254426 ], [ -114.674491000000103, 33.255597 ], [ -114.672924, 33.257042 ], [ -114.672088, 33.258499 ], [ -114.672401, 33.260469 ], [ -114.677032, 33.270169 ], [ -114.680507, 33.273576 ], [ -114.684363000000104, 33.276023 ], [ -114.688599, 33.277861 ], [ -114.694449, 33.279785 ], [ -114.702873, 33.281916 ], [ -114.711197, 33.283341 ], [ -114.717875, 33.285156 ], [ -114.72167, 33.286982 ], [ -114.723259, 33.288079 ], [ -114.731223, 33.302433 ], [ -114.731222000000102, 33.304039 ], [ -114.729904000000104, 33.305745 ], [ -114.726484, 33.308273 ], [ -114.724665000000101, 33.310097 ], [ -114.723623000000103, 33.312109 ], [ -114.71861, 33.315761 ], [ -114.710627, 33.3205 ], [ -114.70787, 33.323316 ], [ -114.705186, 33.327709 ], [ -114.700938, 33.337014 ], [ -114.69935, 33.345692 ], [ -114.699124, 33.349258 ], [ -114.698035, 33.352442 ], [ -114.69817, 33.356575 ], [ -114.699056, 33.361148 ], [ -114.701959, 33.367134 ], [ -114.704201, 33.371238 ], [ -114.706722, 33.37503 ], [ -114.707348, 33.376627 ], [ -114.707485000000105, 33.378375 ], [ -114.707009, 33.380633 ], [ -114.707309, 33.38254 ], [ -114.708407, 33.384142 ], [ -114.713602, 33.388256 ], [ -114.72425, 33.40042 ], [ -114.725292, 33.402341 ], [ -114.725535, 33.404055 ], [ -114.725282, 33.405048 ], [ -114.723829, 33.406531 ], [ -114.722201, 33.407384 ], [ -114.720065000000105, 33.407891 ], [ -114.710878, 33.407254 ], [ -114.701788, 33.408377 ], [ -114.697708, 33.410942 ], [ -114.696805, 33.412087 ], [ -114.696507, 33.414063 ], [ -114.695658, 33.415128 ], [ -114.68795, 33.417934 ], [ -114.673691, 33.419157 ], [ -114.658254, 33.413021 ], [ -114.656735, 33.412813 ], [ -114.652828, 33.412923 ], [ -114.64954, 33.413633 ], [ -114.643302, 33.416746 ], [ -114.635183, 33.422725 ], [ -114.633262, 33.425024 ], [ -114.630903, 33.426754 ], [ -114.62964, 33.428137 ], [ -114.627479, 33.432307 ], [ -114.622283, 33.447558 ], [ -114.622519, 33.450879 ], [ -114.623395000000102, 33.45449 ], [ -114.622918, 33.456561 ], [ -114.618354, 33.462708 ], [ -114.614331, 33.467315 ], [ -114.6137820000001, 33.469049 ], [ -114.612472, 33.470768 ], [ -114.607843000000102, 33.474834 ], [ -114.603396000000103, 33.480631 ], [ -114.601694, 33.481396 ], [ -114.599712, 33.484316 ], [ -114.597283000000104, 33.490653 ], [ -114.593721, 33.495932 ], [ -114.592369, 33.498675 ], [ -114.589246, 33.501813 ], [ -114.580468, 33.506465 ], [ -114.573757, 33.507543 ], [ -114.569533, 33.509219 ], [ -114.560963, 33.516739 ], [ -114.560552, 33.518272 ], [ -114.560835, 33.524334 ], [ -114.560098, 33.526663 ], [ -114.559507, 33.530724 ], [ -114.558898, 33.531819 ], [ -114.542011, 33.542481 ], [ -114.531802, 33.547862 ], [ -114.530401, 33.550099 ], [ -114.525997000000103, 33.551457 ], [ -114.524599, 33.552231 ], [ -114.524215, 33.553068 ], [ -114.52822, 33.559318 ], [ -114.531613, 33.561702 ], [ -114.532333, 33.562879 ], [ -114.533192, 33.565823 ], [ -114.535965000000104, 33.569154 ], [ -114.536784, 33.570959 ], [ -114.537801, 33.575555 ], [ -114.538983, 33.576792 ], [ -114.5403, 33.580615 ], [ -114.540652, 33.582872 ], [ -114.540111, 33.588354 ], [ -114.540664, 33.589789 ], [ -114.540617, 33.591412 ], [ -114.537493, 33.594895 ], [ -114.536777, 33.596394 ], [ -114.531051, 33.604482 ], [ -114.529186, 33.60665 ], [ -114.526782, 33.608831 ], [ -114.523994, 33.60999 ], [ -114.522071, 33.611277 ], [ -114.521845, 33.612544 ], [ -114.522367, 33.614172 ], [ -114.527378, 33.617828 ], [ -114.528578, 33.619994 ], [ -114.52908, 33.621711 ], [ -114.531215, 33.623913 ], [ -114.531034, 33.628213 ], [ -114.530311, 33.629037 ], [ -114.52637, 33.630259 ], [ -114.523802, 33.6347 ], [ -114.525394, 33.640669 ], [ -114.529549000000102, 33.643861 ], [ -114.533215, 33.648443 ], [ -114.533194, 33.65166 ], [ -114.532164, 33.653194 ], [ -114.530583, 33.654461 ], [ -114.525163, 33.655939 ], [ -114.518337, 33.655927 ], [ -114.514559000000105, 33.658014 ], [ -114.514057, 33.660179 ], [ -114.515336, 33.662033 ], [ -114.517112, 33.662877 ], [ -114.520671, 33.662681 ], [ -114.526439, 33.66388 ], [ -114.530267, 33.666821 ], [ -114.532123, 33.669702 ], [ -114.531523, 33.675108 ], [ -114.530348, 33.679245 ], [ -114.527782, 33.682684 ], [ -114.523959, 33.685879 ], [ -114.519113, 33.688473 ], [ -114.512409, 33.691282 ], [ -114.507996, 33.692018 ], [ -114.504993, 33.693022 ], [ -114.502899, 33.694255 ], [ -114.496489, 33.696901 ], [ -114.495719, 33.698454 ], [ -114.495537, 33.701506 ], [ -114.494407, 33.705395 ], [ -114.494197, 33.707922 ], [ -114.494901, 33.71443 ], [ -114.496565, 33.719155 ], [ -114.498133, 33.720634 ], [ -114.500788, 33.722204 ], [ -114.502661, 33.724584 ], [ -114.504176, 33.728055 ], [ -114.506799, 33.730518 ], [ -114.510265, 33.732146 ], [ -114.512348000000102, 33.734214 ], [ -114.510777, 33.737574 ], [ -114.508206, 33.741587 ], [ -114.506, 33.746344 ], [ -114.504483, 33.750998 ], [ -114.50434, 33.756381 ], [ -114.504863, 33.760465 ], [ -114.507089, 33.76793 ], [ -114.516734, 33.788345 ], [ -114.518942, 33.797302 ], [ -114.521555, 33.801982 ], [ -114.524682, 33.808961 ], [ -114.527188, 33.812639 ], [ -114.52805, 33.814963 ], [ -114.527886, 33.815617 ], [ -114.527161, 33.816191 ], [ -114.522714, 33.818979 ], [ -114.520733, 33.822031 ], [ -114.51997, 33.825381 ], [ -114.520465, 33.827778 ], [ -114.523409, 33.835323 ], [ -114.525539, 33.838614 ], [ -114.529597, 33.848063 ], [ -114.530607, 33.85544 ], [ -114.529883, 33.857563 ], [ -114.527069, 33.859429 ], [ -114.525666, 33.860003 ], [ -114.524292, 33.860133 ], [ -114.52287, 33.859965 ], [ -114.518998, 33.858563 ], [ -114.516811000000104, 33.85812 ], [ -114.514673, 33.858638 ], [ -114.511346, 33.86157 ], [ -114.506635, 33.863484 ], [ -114.505638, 33.864276 ], [ -114.503887, 33.865754 ], [ -114.503104, 33.867166 ], [ -114.503017, 33.867998 ], [ -114.503860000000103, 33.871234 ], [ -114.503395, 33.875018 ], [ -114.50434, 33.876882 ], [ -114.510138, 33.880777 ], [ -114.51866, 33.888263 ], [ -114.522768, 33.892583 ], [ -114.524813, 33.895684 ], [ -114.525872, 33.901008 ], [ -114.52569, 33.901428 ], [ -114.524289, 33.901587 ], [ -114.516344, 33.897918 ], [ -114.513715, 33.897959 ], [ -114.510944, 33.899099 ], [ -114.508708, 33.90064 ], [ -114.507988, 33.901813 ], [ -114.50792, 33.903807 ], [ -114.508558, 33.906098 ], [ -114.511511, 33.911092 ], [ -114.514503, 33.914214 ], [ -114.518434, 33.917518 ], [ -114.523393, 33.921221 ], [ -114.525361, 33.922272 ], [ -114.528385, 33.923674 ], [ -114.531107, 33.924633 ], [ -114.534146, 33.925187 ], [ -114.534951, 33.9257 ], [ -114.535853, 33.928103 ], [ -114.535478, 33.934651 ], [ -114.530566, 33.943629 ], [ -114.52868, 33.947817 ], [ -114.526353, 33.950917 ], [ -114.522002, 33.955623 ], [ -114.51586, 33.958106 ], [ -114.51497, 33.958149 ], [ -114.511231, 33.95704 ], [ -114.509568, 33.957264 ], [ -114.499883, 33.961789 ], [ -114.496042, 33.96589 ], [ -114.490398, 33.97062 ], [ -114.488459, 33.972832 ], [ -114.484784, 33.975519 ], [ -114.483097, 33.977745 ], [ -114.482333, 33.980181 ], [ -114.481455, 33.981261 ], [ -114.475907, 33.984424 ], [ -114.471138, 33.98804 ], [ -114.467932, 33.992877 ], [ -114.466187, 33.993465 ], [ -114.461133, 33.993541 ], [ -114.46012, 33.993888 ], [ -114.459258, 33.994711 ], [ -114.458028, 33.997158 ], [ -114.458026000000103, 33.99782 ], [ -114.459184, 34.000016 ], [ -114.460689, 34.001128 ], [ -114.46628, 34.003885 ], [ -114.46731, 34.00519 ], [ -114.467404, 34.00745 ], [ -114.465867000000102, 34.010987 ], [ -114.464525, 34.011982 ], [ -114.463336, 34.012259 ], [ -114.454807, 34.010968 ], [ -114.450206, 34.012574 ], [ -114.446815, 34.01421 ], [ -114.443821, 34.016176 ], [ -114.44054, 34.019329 ], [ -114.438266, 34.022609 ], [ -114.436171, 34.028083 ], [ -114.434949, 34.037784 ], [ -114.435816, 34.04373 ], [ -114.438602, 34.050205 ], [ -114.439406, 34.05381 ], [ -114.43934, 34.057893 ], [ -114.437683, 34.071937 ], [ -114.435907, 34.077491 ], [ -114.434181, 34.087379 ], [ -114.43338, 34.088413 ], [ -114.428026, 34.092787 ], [ -114.426168, 34.097042 ], [ -114.422899, 34.099661 ], [ -114.420499, 34.103466 ], [ -114.415908, 34.107636 ], [ -114.41168, 34.110031 ], [ -114.405916, 34.111468 ], [ -114.401336, 34.111638 ], [ -114.390565, 34.110084 ], [ -114.379223, 34.11599 ], [ -114.369292000000101, 34.117519 ], [ -114.366517, 34.118577 ], [ -114.360402, 34.123577 ], [ -114.359389, 34.125016 ], [ -114.358358, 34.127617 ], [ -114.356372, 34.130428 ], [ -114.35303, 34.13312 ], [ -114.350478, 34.134107 ], [ -114.348051, 34.134457 ], [ -114.336112, 34.134034 ], [ -114.324576, 34.136759 ], [ -114.320777, 34.138635 ], [ -114.312206, 34.144776 ], [ -114.307802, 34.150574 ], [ -114.298168, 34.160321 ], [ -114.292806, 34.166725 ], [ -114.287294, 34.170529 ], [ -114.275267, 34.172149 ], [ -114.26846, 34.170177 ], [ -114.257034, 34.172837 ], [ -114.253444000000101, 34.174129 ], [ -114.244421, 34.179403 ], [ -114.240712000000102, 34.183232 ], [ -114.229715, 34.186928 ], [ -114.227034, 34.188866 ], [ -114.225814, 34.191238 ], [ -114.224941, 34.193896 ], [ -114.225075, 34.196814 ], [ -114.22579, 34.199236 ], [ -114.225861, 34.201774 ], [ -114.225194, 34.203642 ], [ -114.223384, 34.205136 ], [ -114.215454, 34.208956 ], [ -114.211761, 34.211539 ], [ -114.208253, 34.215505 ], [ -114.190876, 34.230858 ], [ -114.17805, 34.239969 ], [ -114.176403, 34.241512 ], [ -114.175948, 34.242695 ], [ -114.175906, 34.245587 ], [ -114.174597, 34.247303 ], [ -114.166536, 34.249647 ], [ -114.166124, 34.250015 ], [ -114.164476, 34.251667 ], [ -114.163867, 34.253349 ], [ -114.163959, 34.255377 ], [ -114.165335, 34.258486 ], [ -114.165249, 34.259125 ], [ -114.164648, 34.259699 ], [ -114.156853, 34.258415 ], [ -114.153346, 34.258289 ], [ -114.147159, 34.259564 ], [ -114.144779, 34.259623 ], [ -114.13545, 34.257886 ], [ -114.133264, 34.258462 ], [ -114.131489000000101, 34.260387 ], [ -114.131211, 34.26273 ], [ -114.134768, 34.268965 ], [ -114.136671, 34.274377 ], [ -114.137045, 34.277018 ], [ -114.13605, 34.280833 ], [ -114.136677, 34.283936 ], [ -114.138365, 34.288564 ], [ -114.139534, 34.295844 ], [ -114.139187, 34.298074 ], [ -114.138167, 34.300936 ], [ -114.138282, 34.30323 ], [ -114.14093, 34.305919 ], [ -114.157206, 34.317862 ], [ -114.157939, 34.320277 ], [ -114.164249, 34.330816 ], [ -114.168807, 34.339513 ], [ -114.172845, 34.344979 ], [ -114.176909, 34.349306 ], [ -114.181145, 34.352186 ], [ -114.185556, 34.354386 ], [ -114.191094, 34.356125 ], [ -114.19648, 34.359187 ], [ -114.199482, 34.361373 ], [ -114.213774, 34.36246 ], [ -114.226107, 34.365916 ], [ -114.229686, 34.368908 ], [ -114.233065, 34.375013 ], [ -114.234275, 34.376662 ], [ -114.245261, 34.385659 ], [ -114.248649, 34.388113 ], [ -114.252739, 34.3901 ], [ -114.25822, 34.395046 ], [ -114.262909, 34.400373 ], [ -114.264317000000105, 34.401329 ], [ -114.267521, 34.402486 ], [ -114.272184, 34.402961 ], [ -114.280108, 34.403147 ], [ -114.282261, 34.403641 ], [ -114.286802, 34.40534 ], [ -114.288663, 34.406623 ], [ -114.290219, 34.408291 ], [ -114.291751, 34.411104 ], [ -114.291903, 34.416231 ], [ -114.292226, 34.417606 ], [ -114.294836, 34.421389 ], [ -114.301016000000104, 34.426807 ], [ -114.308659, 34.430485 ], [ -114.312251, 34.432726 ], [ -114.319054, 34.435831 ], [ -114.32613, 34.437251 ], [ -114.32688, 34.438048 ], [ -114.330669, 34.445295 ], [ -114.332991, 34.448082 ], [ -114.335372, 34.450038 ], [ -114.339627, 34.451435 ], [ -114.342615, 34.451442 ], [ -114.348974, 34.450166 ], [ -114.356025, 34.449744 ], [ -114.363404000000102, 34.447773 ], [ -114.373719, 34.446938 ], [ -114.375789, 34.447798 ], [ -114.378852, 34.450376 ], [ -114.382985, 34.453971 ], [ -114.386699, 34.457911 ], [ -114.387407, 34.460492 ], [ -114.387187, 34.462021 ], [ -114.383525, 34.470405 ], [ -114.381701, 34.47604 ], [ -114.381555, 34.477883 ], [ -114.383038, 34.488903 ], [ -114.382358, 34.495758 ], [ -114.381402, 34.499245 ], [ -114.378124, 34.507288 ], [ -114.378223, 34.516521 ], [ -114.380838, 34.529724 ], [ -114.389603, 34.542982 ], [ -114.398847, 34.559149 ], [ -114.405228, 34.569637 ], [ -114.422382, 34.580711 ], [ -114.435671, 34.593841 ], [ -114.43681, 34.596074 ], [ -114.436363, 34.596797 ], [ -114.427502, 34.599227 ], [ -114.425338, 34.600842 ], [ -114.424326, 34.602338 ], [ -114.424202, 34.610453 ], [ -114.428648, 34.614641 ], [ -114.438739, 34.621455 ], [ -114.439495, 34.625858 ], [ -114.441398, 34.630171 ], [ -114.441525, 34.631529 ], [ -114.440685, 34.634739 ], [ -114.440294, 34.63824 ], [ -114.440519, 34.640066 ], [ -114.441465, 34.64253 ], [ -114.444276, 34.646542 ], [ -114.445664, 34.647542 ], [ -114.449549, 34.651423 ], [ -114.457985, 34.657113 ], [ -114.45821, 34.657994 ], [ -114.457702, 34.659328 ], [ -114.457185, 34.659992 ], [ -114.451785, 34.663891 ], [ -114.450614, 34.665793 ], [ -114.450506, 34.666836 ], [ -114.451532, 34.668605 ], [ -114.454305, 34.671234 ], [ -114.45491, 34.673092 ], [ -114.454881, 34.675639 ], [ -114.455536, 34.677335 ], [ -114.458163, 34.681161 ], [ -114.462178, 34.6858 ], [ -114.463633, 34.68794 ], [ -114.465246, 34.691202 ], [ -114.46809, 34.701786 ], [ -114.46813, 34.704445 ], [ -114.46862, 34.707573 ], [ -114.470477, 34.711368 ], [ -114.47162, 34.712966 ], [ -114.473682, 34.713964 ], [ -114.477297, 34.714514 ], [ -114.482779, 34.714511 ], [ -114.487508, 34.716626 ], [ -114.489287, 34.720155 ], [ -114.490971, 34.724848 ], [ -114.492017, 34.725702 ], [ -114.495858, 34.727956 ], [ -114.499007, 34.729096 ], [ -114.500795, 34.730418 ], [ -114.510292, 34.733582 ], [ -114.514178, 34.735288 ], [ -114.516619, 34.736745 ], [ -114.521048, 34.741173 ], [ -114.522619, 34.74373 ], [ -114.525611, 34.747005 ], [ -114.529079, 34.750006 ], [ -114.529615, 34.750822 ], [ -114.540306, 34.757109 ], [ -114.546884, 34.761802 ], [ -114.552682, 34.766871 ], [ -114.558653, 34.773852 ], [ -114.563979, 34.782597 ], [ -114.565184, 34.785976 ], [ -114.569383, 34.791568 ], [ -114.571010000000101, 34.794294 ], [ -114.574474, 34.804214 ], [ -114.574694, 34.807471 ], [ -114.576452, 34.8153 ], [ -114.578681000000103, 34.820977 ], [ -114.581126, 34.826115 ], [ -114.586842, 34.835672 ], [ -114.592339, 34.841153 ], [ -114.600653, 34.847361 ], [ -114.604255, 34.849573 ], [ -114.619878, 34.856873 ], [ -114.623939, 34.859738 ], [ -114.628276, 34.863596 ], [ -114.630682000000107, 34.866352 ], [ -114.633051, 34.869971 ], [ -114.634382, 34.87289 ], [ -114.635176, 34.875003 ], [ -114.635458, 34.876902 ] ] ] ] } ` ================================================ FILE: cmd/tile38-benchmark/main.go ================================================ package main import ( "fmt" "log" "math" "math/rand" "net" "os" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/tidwall/redbench" "github.com/tidwall/redcon" "github.com/tidwall/tile38/cmd/tile38-benchmark/az" "github.com/tidwall/tile38/core" ) var ( hostname = "127.0.0.1" port = 9851 auth = "" clients = 50 requests = 100000 quiet = false pipeline = 1 csv = false json = false allTests = "PING,SET,GET,INTERSECTS,WITHIN,NEARBY,EVAL" tests = allTests redis = false ) var addr string func showHelp() bool { gitsha := "" if core.GitSHA == "" || core.GitSHA == "0000000" { gitsha = "" } else { gitsha = " (git:" + core.GitSHA + ")" } fmt.Fprintf(os.Stdout, "tile38-benchmark %s%s\n\n", core.Version, gitsha) fmt.Fprintf(os.Stdout, "Usage: tile38-benchmark [-h ] [-p ] [-c ] [-n ]\n") fmt.Fprintf(os.Stdout, " -h Server hostname (default: %s)\n", hostname) fmt.Fprintf(os.Stdout, " -p Server port (default: %d)\n", port) fmt.Fprintf(os.Stdout, " -a Password for Tile38 Auth\n") fmt.Fprintf(os.Stdout, " -c Number of parallel connections (default %d)\n", clients) fmt.Fprintf(os.Stdout, " -n Total number or requests (default %d)\n", requests) fmt.Fprintf(os.Stdout, " -q Quiet. Just show query/sec values\n") fmt.Fprintf(os.Stdout, " -P Pipeline requests. Default 1 (no pipeline).\n") fmt.Fprintf(os.Stdout, " -t Only run the comma separated list of tests. The test\n") fmt.Fprintf(os.Stdout, " names are the same as the ones produced as output.\n") fmt.Fprintf(os.Stdout, " --csv Output in CSV format.\n") fmt.Fprintf(os.Stdout, " --json Request JSON responses (default is RESP output)\n") fmt.Fprintf(os.Stdout, " --redis Runs against a Redis server\n") fmt.Fprintf(os.Stdout, "\n") return false } func parseArgs() bool { defer func() { if v := recover(); v != nil { if v, ok := v.(string); ok && v == "bad arg" { showHelp() } } }() args := os.Args[1:] readArg := func(arg string) string { if len(args) == 0 { panic("bad arg") } var narg = args[0] args = args[1:] return narg } readIntArg := func(arg string) int { n, err := strconv.ParseUint(readArg(arg), 10, 64) if err != nil { panic("bad arg") } return int(n) } badArg := func(arg string) bool { fmt.Fprintf(os.Stderr, "Unrecognized option or bad number of args for: '%s'\n", arg) return false } for len(args) > 0 { arg := readArg("") if arg == "--help" || arg == "-?" { return showHelp() } if !strings.HasPrefix(arg, "-") { args = append([]string{arg}, args...) break } switch arg { default: return badArg(arg) case "-h": hostname = readArg(arg) case "-p": port = readIntArg(arg) case "-a": auth = readArg(arg) case "-c": clients = readIntArg(arg) if clients <= 0 { clients = 1 } case "-n": requests = readIntArg(arg) if requests <= 0 { requests = 0 } case "-q": quiet = true case "-P": pipeline = readIntArg(arg) if pipeline <= 0 { pipeline = 1 } case "-t": tests = readArg(arg) case "--csv": csv = true case "--json": json = true case "--redis": redis = true } } return true } func fillOpts() *redbench.Options { opts := *redbench.DefaultOptions opts.CSV = csv opts.Clients = clients opts.Pipeline = pipeline opts.Quiet = quiet opts.Requests = requests opts.Stderr = os.Stderr opts.Stdout = os.Stdout return &opts } func randPoint() (lat, lon float64) { return rand.Float64()*180 - 90, rand.Float64()*360 - 180 } func isValidRect(minlat, minlon, maxlat, maxlon float64) bool { return minlat > -90 && maxlat < 90 && minlon > -180 && maxlon < 180 } func randRect(meters float64) (minlat, minlon, maxlat, maxlon float64) { for { lat, lon := randPoint() maxlat, _ = destinationPoint(lat, lon, meters, 0) _, maxlon = destinationPoint(lat, lon, meters, 90) minlat, _ = destinationPoint(lat, lon, meters, 180) _, minlon = destinationPoint(lat, lon, meters, 270) if isValidRect(minlat, minlon, maxlat, maxlon) { return } } } func prepFn(conn net.Conn) bool { var resp [64]byte conn.Write([]byte("CONFIG GET requirepass\r\n")) n, err := conn.Read(resp[:]) if err != nil { log.Fatal(err) } if string(resp[:n]) == "-ERR authentication required\r\n" { if auth == "" { log.Fatal("invalid auth") } else { cmd := redcon.AppendArray(nil, 2) cmd = redcon.AppendBulkString(cmd, "AUTH") cmd = redcon.AppendBulkString(cmd, auth) conn.Write(cmd) n, err := conn.Read(resp[:]) if err != nil || string(resp[:n]) != "+OK\r\n" { log.Fatal("invalid auth") } } } else if auth != "" { log.Fatal("invalid auth") } if json { conn.Write([]byte("output json\r\n")) conn.Read(make([]byte, 64)) } return true } func main() { rand.Seed(time.Now().UnixNano()) if !parseArgs() { return } opts := fillOpts() addr = fmt.Sprintf("%s:%d", hostname, port) testsArr := strings.Split(allTests, ",") var subtract bool var add bool for _, test := range strings.Split(tests, ",") { if strings.HasPrefix(test, "-") { if add { os.Stderr.Write([]byte("test flag cannot mix add and subtract\n")) os.Exit(1) } subtract = true for i := range testsArr { if strings.EqualFold(testsArr[i], test[1:]) { testsArr = append(testsArr[:i], testsArr[i+1:]...) break } } } else if subtract { add = true os.Stderr.Write([]byte("test flag cannot mix add and subtract\n")) os.Exit(1) } } if !subtract { testsArr = strings.Split(tests, ",") } for _, test := range testsArr { switch strings.ToUpper(strings.TrimSpace(test)) { case "PING": redbench.Bench("PING", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "PING") }, ) case "GEOADD": //GEOADD key longitude latitude member if redis { var i int64 redbench.Bench("GEOADD", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) lat, lon := randPoint() return redbench.AppendCommand(buf, "GEOADD", "key:bench", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "id:"+strconv.FormatInt(i, 10), ) }, ) } case "SET", "SET-POINT", "SET-RECT", "SET-LINE", "SET-STRING": if redis { redbench.Bench("SET", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "SET", "key:__rand_int__", "xxx") }, ) } else { var i int64 switch strings.ToUpper(strings.TrimSpace(test)) { case "SET", "SET-POINT": redbench.Bench("SET (point)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) lat, lon := randPoint() return redbench.AppendCommand(buf, "SET", "key:bench", "id:"+strconv.FormatInt(i, 10), "POINT", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), ) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "SET", "SET-RECT": redbench.Bench("SET (rect)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) minlat, minlon, maxlat, maxlon := randRect(10000) return redbench.AppendCommand(buf, "SET", "key:bench", "id:"+strconv.FormatInt(i, 10), "BOUNDS", strconv.FormatFloat(minlat, 'f', 5, 64), strconv.FormatFloat(minlon, 'f', 5, 64), strconv.FormatFloat(maxlat, 'f', 5, 64), strconv.FormatFloat(maxlon, 'f', 5, 64), ) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "SET", "SET-LINE": redbench.Bench("SET (line)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) alat, alon, blat, blon := randRect(10000) if rand.Int()%2 == 0 { alat, alon, blat, blon = blat, blon, alat, alon } linestring := fmt.Sprintf(`{"type":"LineString","coordinates":[[%f,%f],[%f,%f]]}`, alon, alat, blon, blat) return redbench.AppendCommand(buf, "SET", "key:bench", "id:"+strconv.FormatInt(i, 10), "OBJECT", linestring, ) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "SET", "SET-STRING": redbench.Bench("SET (string)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) return redbench.AppendCommand(buf, "SET", "key:bench", "id:"+strconv.FormatInt(i, 10), "STRING", "xxx") }, ) } } case "GET": if redis { redbench.Bench("GET", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "GET", "key:__rand_int__") }, ) } else { var i int64 redbench.Bench("GET (point)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) return redbench.AppendCommand(buf, "GET", "key:bench", "id:"+strconv.FormatInt(i, 10), "POINT") }, ) redbench.Bench("GET (rect)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) return redbench.AppendCommand(buf, "GET", "key:bench", "id:"+strconv.FormatInt(i, 10), "BOUNDS") }, ) redbench.Bench("GET (string)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) return redbench.AppendCommand(buf, "GET", "key:bench", "id:"+strconv.FormatInt(i, 10), "OBJECT") }, ) } case "INTERSECTS", "INTERSECTS-BOUNDS", "INTERSECTS-BOUNDS-1000", "INTERSECTS-BOUNDS-10000", "INTERSECTS-BOUNDS-100000", "INTERSECTS-CIRCLE", "INTERSECTS-CIRCLE-1000", "INTERSECTS-CIRCLE-10000", "INTERSECTS-CIRCLE-100000", "INTERSECTS-AZ": if redis { break } switch strings.ToUpper(strings.TrimSpace(test)) { case "INTERSECTS", "INTERSECTS-CIRCLE", "INTERSECTS-CIRCLE-1000": redbench.Bench("INTERSECTS (intersects-circle 1km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "INTERSECTS", "key:bench", "COUNT", "CIRCLE", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "1000") }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "INTERSECTS", "INTERSECTS-CIRCLE", "INTERSECTS-CIRCLE-10000": redbench.Bench("INTERSECTS (intersects-circle 10km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "INTERSECTS", "key:bench", "COUNT", "CIRCLE", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "10000") }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "INTERSECTS", "INTERSECTS-CIRCLE", "INTERSECTS-CIRCLE-100000": redbench.Bench("INTERSECTS (intersects-circle 100km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "INTERSECTS", "key:bench", "COUNT", "CIRCLE", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "100000") }, ) } // INTERSECTS-BOUNDS switch strings.ToUpper(strings.TrimSpace(test)) { case "INTERSECTS", "INTERSECTS-BOUNDS", "INTERSECTS-BOUNDS-1000": minlat, minlon, maxlat, maxlon := randRect(1000) redbench.Bench("INTERSECTS (intersects-bounds 1km)", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "INTERSECTS", "key:bench", "COUNT", "BOUNDS", strconv.FormatFloat(minlat, 'f', 5, 64), strconv.FormatFloat(minlon, 'f', 5, 64), strconv.FormatFloat(maxlat, 'f', 5, 64), strconv.FormatFloat(maxlon, 'f', 5, 64)) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "INTERSECTS", "INTERSECTS-BOUNDS", "INTERSECTS-BOUNDS-10000": minlat, minlon, maxlat, maxlon := randRect(10000) redbench.Bench("INTERSECTS (intersects-bounds 10km)", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "INTERSECTS", "key:bench", "COUNT", "BOUNDS", strconv.FormatFloat(minlat, 'f', 5, 64), strconv.FormatFloat(minlon, 'f', 5, 64), strconv.FormatFloat(maxlat, 'f', 5, 64), strconv.FormatFloat(maxlon, 'f', 5, 64)) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "INTERSECTS", "INTERSECTS-BOUNDS", "INTERSECTS-BOUNDS-100000": minlat, minlon, maxlat, maxlon := randRect(10000) redbench.Bench("INTERSECTS (intersects-bounds 100km)", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "INTERSECTS", "key:bench", "COUNT", "BOUNDS", strconv.FormatFloat(minlat, 'f', 5, 64), strconv.FormatFloat(minlon, 'f', 5, 64), strconv.FormatFloat(maxlat, 'f', 5, 64), strconv.FormatFloat(maxlon, 'f', 5, 64)) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "INTERSECTS", "INTERSECTS-AZ": var mu sync.Mutex var loaded bool redbench.Bench("INTERSECTS (intersects-az limit 5)", addr, opts, func(conn net.Conn) bool { func() { mu.Lock() defer mu.Unlock() if loaded { return } loaded = true p := make([]byte, 0xFF) conn.Write([]byte("GET keys:bench:geo az point\r\n")) n, err := conn.Read(p) if err != nil { panic(err) } if string(p[:n]) != "$-1\r\n" { return } args := []string{"SET", "key:bench:geo", "az", "object", az.JSON} out := redcon.AppendArray(nil, len(args)) for _, arg := range args { out = redcon.AppendBulkString(out, arg) } conn.Write(out) n, err = conn.Read(p) if err != nil { panic(err) } if string(p[:n]) != "+OK\r\n" { panic("expected OK") } }() return prepFn(conn) }, func(buf []byte) []byte { args := []string{"INTERSECTS", "key:bench", "LIMIT", "5", "COUNT", "GET", "key:bench:geo", "az"} return redbench.AppendCommand(buf, args...) }, ) } case "WITHIN", "WITHIN-RECT", "WITHIN-RECT-1000", "WITHIN-RECT-10000", "WITHIN-RECT-100000", "WITHIN-CIRCLE", "WITHIN-CIRCLE-1000", "WITHIN-CIRCLE-10000", "WITHIN-CIRCLE-100000": if redis { break } switch strings.ToUpper(strings.TrimSpace(test)) { case "WITHIN", "WITHIN-CIRCLE", "WITHIN-CIRCLE-1000": redbench.Bench("WITHIN (within-circle 1km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "WITHIN", "key:bench", "COUNT", "CIRCLE", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "1000") }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "WITHIN", "WITHIN-CIRCLE", "WITHIN-CIRCLE-10000": redbench.Bench("WITHIN (within-circle 10km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "WITHIN", "key:bench", "COUNT", "CIRCLE", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "10000") }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "WITHIN", "WITHIN-CIRCLE", "WITHIN-CIRCLE-100000": redbench.Bench("WITHIN (within-circle 100km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "WITHIN", "key:bench", "COUNT", "CIRCLE", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "100000") }, ) } // WITHIN-BOUNDS switch strings.ToUpper(strings.TrimSpace(test)) { case "WITHIN", "WITHIN-BOUNDS", "WITHIN-BOUNDS-1000": minlat, minlon, maxlat, maxlon := randRect(1000) redbench.Bench("WITHIN (within-bounds 1km)", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "WITHIN", "key:bench", "COUNT", "BOUNDS", strconv.FormatFloat(minlat, 'f', 5, 64), strconv.FormatFloat(minlon, 'f', 5, 64), strconv.FormatFloat(maxlat, 'f', 5, 64), strconv.FormatFloat(maxlon, 'f', 5, 64)) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "WITHIN", "WITHIN-BOUNDS", "WITHIN-BOUNDS-10000": minlat, minlon, maxlat, maxlon := randRect(10000) redbench.Bench("WITHIN (within-bounds 10km)", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "WITHIN", "key:bench", "COUNT", "BOUNDS", strconv.FormatFloat(minlat, 'f', 5, 64), strconv.FormatFloat(minlon, 'f', 5, 64), strconv.FormatFloat(maxlat, 'f', 5, 64), strconv.FormatFloat(maxlon, 'f', 5, 64)) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "WITHIN", "WITHIN-BOUNDS", "WITHIN-BOUNDS-100000": minlat, minlon, maxlat, maxlon := randRect(10000) redbench.Bench("WITHIN (within-bounds 100km)", addr, opts, prepFn, func(buf []byte) []byte { return redbench.AppendCommand(buf, "WITHIN", "key:bench", "COUNT", "BOUNDS", strconv.FormatFloat(minlat, 'f', 5, 64), strconv.FormatFloat(minlon, 'f', 5, 64), strconv.FormatFloat(maxlat, 'f', 5, 64), strconv.FormatFloat(maxlon, 'f', 5, 64)) }, ) } case "NEARBY", "NEARBY-KNN", "NEARBY-KNN-1", "NEARBY-KNN-10", "NEARBY-KNN-100", "NEARBY-POINT", "NEARBY-POINT-1000", "NEARBY-POINT-10000", "NEARBY-POINT-100000": if redis { break } switch strings.ToUpper(strings.TrimSpace(test)) { case "NEARBY", "NEARBY-KNN", "NEARBY-KNN-1": redbench.Bench("NEARBY (limit 1)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "NEARBY", "key:bench", "LIMIT", "1", "COUNT", "POINT", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), ) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "NEARBY", "NEARBY-KNN", "NEARBY-KNN-10": redbench.Bench("NEARBY (limit 10)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "NEARBY", "key:bench", "LIMIT", "10", "COUNT", "POINT", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), ) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "NEARBY", "NEARBY-KNN", "NEARBY-KNN-100": redbench.Bench("NEARBY (limit 100)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "NEARBY", "key:bench", "LIMIT", "100", "COUNT", "POINT", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), ) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "NEARBY", "NEARBY-POINT", "NEARBY-POINT-1000": redbench.Bench("NEARBY (point 1km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "NEARBY", "key:bench", "COUNT", "POINT", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "1000", ) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "NEARBY", "NEARBY-POINT", "NEARBY-POINT-10000": redbench.Bench("NEARBY (point 10km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "NEARBY", "key:bench", "COUNT", "POINT", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "10000", ) }, ) } switch strings.ToUpper(strings.TrimSpace(test)) { case "NEARBY", "NEARBY-POINT", "NEARBY-POINT-100000": redbench.Bench("NEARBY (point 100km)", addr, opts, prepFn, func(buf []byte) []byte { lat, lon := randPoint() return redbench.AppendCommand(buf, "NEARBY", "key:bench", "COUNT", "POINT", strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), "100000", ) }, ) } case "EVAL": if !redis { var i int64 getScript := "return tile38.call('GET', KEYS[1], ARGV[1], 'point')" get4Script := "local a = tile38.call('GET', KEYS[1], ARGV[1], 'point');" + "local b = tile38.call('GET', KEYS[1], ARGV[2], 'point');" + "local c = tile38.call('GET', KEYS[1], ARGV[3], 'point');" + "local d = tile38.call('GET', KEYS[1], ARGV[4], 'point');" + "return d" setScript := "return tile38.call('SET', KEYS[1], ARGV[1], 'point', ARGV[2], ARGV[3])" if !opts.Quiet { fmt.Println("Scripts to run:") fmt.Println("GET SCRIPT: " + getScript) fmt.Println("GET FOUR SCRIPT: " + get4Script) fmt.Println("SET SCRIPT: " + setScript) } redbench.Bench("EVAL (set point)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) lat, lon := randPoint() return redbench.AppendCommand(buf, "EVAL", setScript, "1", "key:bench", "id:"+strconv.FormatInt(i, 10), strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), ) }, ) redbench.Bench("EVALNA (set point)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) lat, lon := randPoint() return redbench.AppendCommand(buf, "EVALNA", setScript, "1", "key:bench", "id:"+strconv.FormatInt(i, 10), strconv.FormatFloat(lat, 'f', 5, 64), strconv.FormatFloat(lon, 'f', 5, 64), ) }, ) redbench.Bench("EVALRO (get point)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) return redbench.AppendCommand(buf, "EVALRO", getScript, "1", "key:bench", "id:"+strconv.FormatInt(i, 10)) }, ) redbench.Bench("EVALRO (get 4 points)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) return redbench.AppendCommand(buf, "EVALRO", get4Script, "1", "key:bench", "id:"+strconv.FormatInt(i, 10), "id:"+strconv.FormatInt(i+1, 10), "id:"+strconv.FormatInt(i+2, 10), "id:"+strconv.FormatInt(i+3, 10), ) }, ) redbench.Bench("EVALNA (get point)", addr, opts, prepFn, func(buf []byte) []byte { i := atomic.AddInt64(&i, 1) return redbench.AppendCommand(buf, "EVALNA", getScript, "1", "key:bench", "id:"+strconv.FormatInt(i, 10)) }, ) } } } } const earthRadius = 6371e3 func toRadians(deg float64) float64 { return deg * math.Pi / 180 } func toDegrees(rad float64) float64 { return rad * 180 / math.Pi } // destinationPoint return the destination from a point based on a distance and bearing. func destinationPoint(lat, lon, meters, bearingDegrees float64) (destLat, destLon float64) { // see http://williams.best.vwh.net/avform.htm#LL δ := meters / earthRadius // angular distance in radians θ := toRadians(bearingDegrees) φ1 := toRadians(lat) λ1 := toRadians(lon) φ2 := math.Asin(math.Sin(φ1)*math.Cos(δ) + math.Cos(φ1)*math.Sin(δ)*math.Cos(θ)) λ2 := λ1 + math.Atan2(math.Sin(θ)*math.Sin(δ)*math.Cos(φ1), math.Cos(δ)-math.Sin(φ1)*math.Sin(φ2)) λ2 = math.Mod(λ2+3*math.Pi, 2*math.Pi) - math.Pi // normalise to -180..+180° return toDegrees(φ2), toDegrees(λ2) } ================================================ FILE: cmd/tile38-cli/main.go ================================================ package main import ( "bufio" "bytes" "encoding/json" "errors" "fmt" "io" "net" "os" "os/exec" "path/filepath" "runtime" "sort" "strconv" "strings" "github.com/peterh/liner" "github.com/tidwall/gjson" "github.com/tidwall/resp" "github.com/tidwall/tile38/core" ) func getEnv(name string, defaultValue string) string { val, exists := os.LookupEnv(name) if !exists { return defaultValue } return val } func userHomeDir() string { if runtime.GOOS == "windows" { home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") if home == "" { home = os.Getenv("USERPROFILE") } return home } return os.Getenv("HOME") } var ( historyFile = filepath.Join(userHomeDir(), ".liner_example_history") ) type connError struct { OK bool `json:"ok"` Err string `json:"err"` } var ( hostname = "127.0.0.1" output = "json" port = 9851 oneCommand string raw bool noprompt bool tty bool stdin bool ) func showHelp() bool { gitsha := "" if core.GitSHA == "" || core.GitSHA == "0000000" { gitsha = "" } else { gitsha = " (git:" + core.GitSHA + ")" } fmt.Fprintf(os.Stdout, "tile38-cli %s%s\n\n", core.Version, gitsha) fmt.Fprintf(os.Stdout, "Usage: tile38-cli [OPTIONS] [cmd [arg [arg ...]]]\n") fmt.Fprintf(os.Stdout, " --raw Use raw formatting for replies\n") fmt.Fprintf(os.Stdout, " --noprompt Do not display a prompt\n") fmt.Fprintf(os.Stdout, " --resp Use RESP output formatting (default is JSON output)\n") fmt.Fprintf(os.Stdout, " --json Use JSON output formatting (default is JSON output)\n") fmt.Fprintf(os.Stdout, " -h Server hostname (default: %s)\n", hostname) fmt.Fprintf(os.Stdout, " -p Server port (default: %d)\n", port) fmt.Fprintf(os.Stdout, " -x Read last argument from STDIN.\n") fmt.Fprintf(os.Stdout, "\n") return false } func parseArgs() bool { defer func() { if v := recover(); v != nil { if v, ok := v.(string); ok && v == "bad arg" { showHelp() } } }() hostname = getEnv("TILE38_HOSTNAME", hostname) output = getEnv("TILE38_OUTPUT", output) portStr := getEnv("TILE38_PORT", "") if portStr != "" { tempPort, err := strconv.Atoi(portStr) if err == nil { port = tempPort } } args := os.Args[1:] readArg := func(arg string) string { if len(args) == 0 { panic("bad arg") } var narg = args[0] args = args[1:] return narg } badArg := func(arg string) bool { fmt.Fprintf(os.Stderr, "Unrecognized option or bad number of args for: '%s'\n", arg) return false } for len(args) > 0 { arg := readArg("") if arg == "--help" || arg == "-?" { return showHelp() } if !strings.HasPrefix(arg, "-") { args = append([]string{arg}, args...) break } switch arg { default: return badArg(arg) case "--raw": raw = true case "--tty": tty = true case "--noprompt": noprompt = true case "--resp": output = "resp" case "--json": output = "json" case "-x": stdin = true case "-h": hostname = readArg(arg) case "-p": n, err := strconv.ParseUint(readArg(arg), 10, 16) if err != nil { return badArg(arg) } port = int(n) } } oneCommand = strings.Join(args, " ") if stdin { data, err := io.ReadAll(os.Stdin) if err != nil { println(err) } if !gjson.ValidBytes(data) { fmt.Fprintf(os.Stderr, "Invalid STDIN: Not JSON\n") return false } arg := strings.Replace(string(data), "\r", "", -1) arg = strings.Replace(arg, "\n", "", -1) arg = strings.Replace(arg, "'", "\\'", -1) oneCommand += " '" + arg + "'" } return true } func refusedErrorString(addr string) string { return fmt.Sprintf("Could not connect to Tile38 at %s: Connection refused", addr) } var groupsM = make(map[string][]string) func jsonOK(msg []byte) bool { return gjson.GetBytes(msg, "ok").Bool() } func main() { if !parseArgs() { return } if len(oneCommand) > 0 && strings.Split(strings.ToLower(oneCommand), " ")[0] == "help" { showHelp() return } addr := fmt.Sprintf("%s:%d", hostname, port) var conn *client connDial := func() { var err error conn, err = clientDial("tcp", addr) if err != nil { if _, ok := err.(net.Error); ok { fmt.Fprintln(os.Stderr, refusedErrorString(addr)) } else { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } if oneCommand != "" { os.Exit(1) } } else if _, err := conn.Do("output " + output); err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } } connDial() monitor := false livemode := false aof := false defer func() { if livemode { var err error if aof { _, err = io.Copy(os.Stdout, conn.Reader()) fmt.Fprintln(os.Stderr, "") } else { var msg []byte for { msg, err = conn.readLiveResp() if err != nil { break } if !raw { if output == "resp" { msg = convert2termresp(msg) } else { msg = convert2termjson(msg) } } fmt.Fprintln(os.Stderr, string(msg)) } } if err != nil && err != io.EOF { fmt.Fprintln(os.Stderr, err.Error()) } } }() line := liner.NewLiner() defer line.Close() var commands []string for name, command := range core.Commands { commands = append(commands, name) groupsM[command.Group] = append(groupsM[command.Group], name) } sort.Strings(commands) var groups []string for group, arr := range groupsM { groups = append(groups, "@"+group) sort.Strings(arr) groupsM[group] = arr } sort.Strings(groups) line.SetMultiLineMode(false) line.SetCtrlCAborts(true) if !(noprompt && tty) { line.SetCompleter(func(line string) (c []string) { if strings.HasPrefix(strings.ToLower(line), "help ") { var nitems []string nline := strings.TrimSpace(line[5:]) if nline == "" || nline[0] == '@' { for _, n := range groups { if strings.HasPrefix(strings.ToLower(n), strings.ToLower(nline)) { nitems = append(nitems, line[:len(line)-len(nline)]+strings.ToLower(n)) } } } else { for _, n := range commands { if strings.HasPrefix(strings.ToLower(n), strings.ToLower(nline)) { nitems = append(nitems, line[:len(line)-len(nline)]+strings.ToUpper(n)) } } } for _, n := range nitems { if strings.HasPrefix(strings.ToLower(n), strings.ToLower(line)) { c = append(c, n) } } } else { for _, n := range commands { if strings.HasPrefix(strings.ToLower(n), strings.ToLower(line)) { c = append(c, n) } } } return }) } if f, err := os.Open(historyFile); err == nil { line.ReadHistory(f) f.Close() } defer func() { if f, err := os.Create(historyFile); err != nil { fmt.Fprintln(os.Stderr, err.Error()) } else { line.WriteHistory(f) f.Close() } }() password := getEnv("TILE38_PASSWORD", "") if conn != nil && password != "" { conn.Do(fmt.Sprintf("auth %s", password)) } for { var command string var err error if oneCommand == "" { if raw || noprompt { command, err = line.Prompt("") } else { if conn == nil { command, err = line.Prompt("not connected> ") } else { command, err = line.Prompt(addr + "> ") } } } else { command = oneCommand } if err == nil { nohist := strings.HasPrefix(command, " ") command = strings.TrimSpace(command) if command == "" { if conn != nil { _, err := conn.Do("pInG") if err != nil { if err != io.EOF && !strings.Contains(err.Error(), "broken pipe") { fmt.Fprintln(os.Stderr, err.Error()) } else { fmt.Fprintln(os.Stderr, refusedErrorString(addr)) } conn.wr.Close() conn = nil continue } } } else { if !nohist { line.AppendHistory(command) } if strings.ToLower(command) == "exit" { return } if strings.ToLower(command) == "quit" { return } if strings.ToLower(command) == "clear" { clearScreen() continue } if strings.ToLower(command) == "help" || strings.HasPrefix(strings.ToLower(command), "help") { err = help(strings.TrimSpace(command[4:])) if err != nil { return } continue } aof = strings.HasPrefix(strings.ToLower(command), "aof ") tryAgain: if conn == nil { connDial() if conn == nil { continue } } msg, err := conn.Do(command) if err != nil { if err != io.EOF { fmt.Fprintln(os.Stderr, err.Error()) } conn.wr.Close() conn = nil goto tryAgain } switch strings.ToLower(command) { case "output resp": if string(msg) == "+OK\r\n" { output = "resp" } case "output json": if jsonOK(msg) { output = "json" } case "monitor": monitor = true livemode = true output = "resp" } if output == "resp" && (strings.HasPrefix(string(msg), "*3\r\n$10\r\npsubscribe\r\n") || strings.HasPrefix(string(msg), "*3\r\n$9\r\nsubscribe\r\n")) { livemode = true } if !raw { if output == "resp" { msg = convert2termresp(msg) } else { msg = convert2termjson(msg) } } if !livemode && output == "json" { if gjson.GetBytes(msg, "command").String() == "psubscribe" || gjson.GetBytes(msg, "command").String() == "subscribe" || string(msg) == liveJSON { livemode = true } } mustOutput := true if !monitor && oneCommand == "" && output == "json" && !jsonOK(msg) { var cerr connError if err := json.Unmarshal(msg, &cerr); err == nil { fmt.Fprintln(os.Stderr, "(error) "+cerr.Err) mustOutput = false } } else if livemode { fmt.Fprintln(os.Stderr, string(msg)) break // break out of prompt and just feed data to screen } if mustOutput { fmt.Fprintln(os.Stdout, string(msg)) } } } else if err == liner.ErrPromptAborted { return } else if err == io.EOF { os.Exit(0) } else { fmt.Fprintf(os.Stderr, "Error reading line: %s", err.Error()) } if oneCommand != "" { return } } } func convert2termresp(msg []byte) []byte { rd := resp.NewReader(bytes.NewBuffer(msg)) out := "" for { v, _, err := rd.ReadValue() if err != nil { break } out += convert2termrespval(v, 0) } return []byte(strings.TrimSpace(out)) } func convert2termjson(msg []byte) []byte { if msg[0] == '{' { return msg } return bytes.TrimSpace(msg[bytes.IndexByte(msg, '\n')+1:]) } func convert2termrespval(v resp.Value, spaces int) string { switch v.Type() { default: return v.String() case resp.BulkString: if v.IsNull() { return "(nil)" } return "\"" + v.String() + "\"" case resp.Integer: return "(integer) " + v.String() case resp.Error: return "(error) " + v.String() case resp.Array: arr := v.Array() if len(arr) == 0 { return "(empty list or set)" } out := "" nspaces := spaces + numlen(len(arr)) for i, v := range arr { if i > 0 { out += strings.Repeat(" ", spaces) } iout := strings.TrimSpace(convert2termrespval(v, nspaces+2)) out += fmt.Sprintf("%d) %s\n", i+1, iout) } return out } } func numlen(n int) int { l := 1 if n < 0 { l++ n = n * -1 } for i := 0; i < 1000; i++ { if n < 10 { break } l++ n = n / 10 } return l } func help(arg string) error { var groupsA []string for group := range groupsM { groupsA = append(groupsA, "@"+group) } groups := "Groups: " + strings.Join(groupsA, ", ") + "\n" if arg == "" { fmt.Fprintf(os.Stderr, "tile38-cli %s (git:%s)\n", core.Version, core.GitSHA) fmt.Fprintf(os.Stderr, `Type: "help @" to get a list of commands in `+"\n") fmt.Fprintf(os.Stderr, ` "help " for help on `+"\n") if !(noprompt && tty) { fmt.Fprintf(os.Stderr, ` "help " to get a list of possible help topics`+"\n") } fmt.Fprintf(os.Stderr, ` "quit" to exit`+"\n") if noprompt && tty { fmt.Fprint(os.Stderr, groups) } return nil } showGroups := false found := false if strings.HasPrefix(arg, "@") { for _, command := range groupsM[arg[1:]] { fmt.Fprintf(os.Stderr, "%s\n", core.Commands[command].TermOutput(" ")) found = true } if !found { showGroups = true } } else { if command, ok := core.Commands[strings.ToUpper(arg)]; ok { fmt.Fprintf(os.Stderr, "%s\n", command.TermOutput(" ")) found = true } } if showGroups { if noprompt && tty { fmt.Fprint(os.Stderr, groups) } } else if !found { if noprompt && tty { help("") } } return nil } const liveJSON = `{"ok":true,"live":true}` type client struct { wr net.Conn rd *bufio.Reader } func clientDial(network, addr string) (*client, error) { conn, err := net.Dial(network, addr) if err != nil { return nil, err } return &client{wr: conn, rd: bufio.NewReader(conn)}, nil } func (c *client) Do(command string) ([]byte, error) { _, err := c.wr.Write(plainToCompat(command)) if err != nil { return nil, err } return c.readResp() } func (c *client) readResp() ([]byte, error) { ch, err := c.rd.Peek(1) if err != nil { return nil, err } switch ch[0] { case ':', '+', '-', '{': return c.readLine() case '$': return c.readBulk() case '*': return c.readArray() default: return nil, fmt.Errorf("invalid response character '%c", ch[0]) } } func (c *client) readArray() ([]byte, error) { out, err := c.readLine() if err != nil { return nil, err } n, err := strconv.ParseUint(string(bytes.TrimSpace(out[1:])), 10, 64) if err != nil { return nil, err } for i := 0; i < int(n); i++ { resp, err := c.readResp() if err != nil { return nil, err } out = append(out, resp...) } return out, nil } func (c *client) readBulk() ([]byte, error) { line, err := c.readLine() if err != nil { return nil, err } x, err := strconv.ParseInt(string(bytes.TrimSpace(line[1:])), 10, 64) if err != nil { return nil, err } if x < 0 { return line, nil } out := make([]byte, len(line)+int(x)+2) if _, err := io.ReadFull(c.rd, out[len(line):]); err != nil { return nil, err } if !bytes.HasSuffix(out, []byte{'\r', '\n'}) { return nil, errors.New("invalid response") } copy(out, line) return out, nil } func (c *client) readLine() ([]byte, error) { line, err := c.rd.ReadBytes('\r') if err != nil { return nil, err } ch, err := c.rd.ReadByte() if err != nil { return nil, err } if ch != '\n' { return nil, errors.New("invalid response") } return append(line, '\n'), nil } func (c *client) Reader() io.Reader { return c.rd } func (c *client) readLiveResp() (message []byte, err error) { return c.readResp() } // plainToCompat converts a plain message like "SET fleet truck1 ..." into a // Tile38 compatible blob. func plainToCompat(message string) []byte { var args []string // search for the beginning of the first argument for i := 0; i < len(message); i++ { if message[i] != ' ' { // first argument found if message[i] == '"' || message[i] == '\'' { // using a string caps s := i cap := message[i] for ; i < len(message); i++ { if message[i] == cap { if message[i-1] == '\\' { continue } if i == len(message)-1 || message[i+1] == ' ' { args = append(args, message[s:i+1]) i++ break } } } } else { // using plain string, terminated by a space s := i var quotes bool for ; i < len(message); i++ { if message[i] == '"' || message[i] == '\'' { quotes = true } if i == len(message)-1 || message[i+1] == ' ' { arg := message[s : i+1] if quotes { arg = strconv.Quote(arg) } args = append(args, arg) i++ break } } } } } return []byte(strings.Join(args, " ") + "\r\n") } func clearScreen() { var cmd *exec.Cmd if runtime.GOOS == "windows" { cmd = exec.Command("cmd", "/c", "cls") } else { cmd = exec.Command("clear") } cmd.Stdout = os.Stdout cmd.Run() } ================================================ FILE: cmd/tile38-luamemtest/main.go ================================================ package main import ( "crypto/sha1" "encoding/hex" "errors" "fmt" "math" "runtime" "runtime/debug" "strings" "github.com/tidwall/resp" lua "github.com/yuin/gopher-lua" ) func Sha1Sum(s string) string { h := sha1.New() h.Write([]byte(s)) return hex.EncodeToString(h.Sum(nil)) } // Convert lua LValue to RESP value func ConvertToResp(val lua.LValue) resp.Value { switch val.Type() { case lua.LTNil: return resp.NullValue() case lua.LTBool: if val == lua.LTrue { return resp.IntegerValue(1) } else { return resp.NullValue() } case lua.LTNumber: if float := float64(val.(lua.LNumber)); math.IsNaN(float) || math.IsInf(float, 0) { return resp.FloatValue(float) } else { return resp.IntegerValue(int(math.Floor(float))) } case lua.LTString: return resp.StringValue(val.String()) case lua.LTTable: var values []resp.Value var specialValues []resp.Value var cb func(lk lua.LValue, lv lua.LValue) tbl := val.(*lua.LTable) if tbl.Len() != 0 { // list cb = func(lk lua.LValue, lv lua.LValue) { values = append(values, ConvertToResp(lv)) } } else { // map cb = func(lk lua.LValue, lv lua.LValue) { if lk.Type() == lua.LTString { lks := lk.String() switch lks { case "ok": specialValues = append(specialValues, resp.SimpleStringValue(lv.String())) case "err": specialValues = append(specialValues, resp.ErrorValue(errors.New(lv.String()))) } } values = append(values, resp.ArrayValue( []resp.Value{ConvertToResp(lk), ConvertToResp(lv)})) } } tbl.ForEach(cb) if len(values) == 1 && len(specialValues) == 1 { return specialValues[0] } return resp.ArrayValue(values) } return resp.ErrorValue(errors.New("Unsupported lua type: " + val.Type().String())) } // Convert RESP value to lua LValue func ConvertToLua(L *lua.LState, val resp.Value) lua.LValue { if val.IsNull() { return lua.LFalse } switch val.Type() { case resp.Integer: return lua.LNumber(val.Integer()) case resp.BulkString: return lua.LString(val.String()) case resp.Error: tbl := L.CreateTable(0, 1) tbl.RawSetString("err", lua.LString(val.String())) return tbl case resp.SimpleString: tbl := L.CreateTable(0, 1) tbl.RawSetString("ok", lua.LString(val.String())) return tbl case resp.Array: tbl := L.CreateTable(len(val.Array()), 0) for _, item := range val.Array() { tbl.Append(ConvertToLua(L, item)) } return tbl } return lua.LString("ERR: unknown RESP type: " + val.Type().String()) } func luaTile38Call(evalcmd string, cmd string, args ...string) (resp.Value, error) { var values []resp.Value values = append(values, resp.StringValue("RUNNING:")) values = append(values, resp.StringValue(evalcmd)) values = append(values, resp.StringValue(cmd)) for _, arg := range args { values = append(values, resp.StringValue(arg)) } return resp.ArrayValue(values), nil } func NewLuaState() *lua.LState { L := lua.NewState() get_args := func(ls *lua.LState) (evalCmd string, args []string) { evalCmd = ls.GetGlobal("EVAL_CMD").String() //log.Debugf("EVAL_CMD %s\n", evalCmd) // Trying to work with unknown number of args. // When we see empty arg we call it enough. for i := 1; ; i++ { if arg := ls.ToString(i); arg == "" { break } else { args = append(args, arg) } } return } call := func(ls *lua.LState) int { evalCmd, args := get_args(ls) if res, err := luaTile38Call(evalCmd, args[0], args[1:]...); err != nil { //log.Debugf("RES type: %s value: %s ERR %s\n", res.Type(), res.String(), err); ls.RaiseError("ERR %s", err.Error()) return 0 } else { //log.Debugf("RES type: %s value: %s\n", res.Type(), res.String()); ls.Push(ConvertToLua(ls, res)) return 1 } } pcall := func(ls *lua.LState) int { evalCmd, args := get_args(ls) if res, err := luaTile38Call(evalCmd, args[0], args[1:]...); err != nil { //log.Debugf("RES type: %s value: %s ERR %s\n", res.Type(), res.String(), err); ls.Push(ConvertToLua(ls, resp.ErrorValue(err))) return 1 } else { //log.Debugf("RES type: %s value: %s\n", res.Type(), res.String()); ls.Push(ConvertToLua(ls, res)) return 1 } } error_reply := func(ls *lua.LState) int { tbl := L.CreateTable(0, 1) tbl.RawSetString("err", lua.LString(ls.ToString(1))) ls.Push(tbl) return 1 } status_reply := func(ls *lua.LState) int { tbl := L.CreateTable(0, 1) tbl.RawSetString("ok", lua.LString(ls.ToString(1))) ls.Push(tbl) return 1 } sha1hex := func(ls *lua.LState) int { sha_sum := Sha1Sum(ls.ToString(1)) ls.Push(lua.LString(sha_sum)) return 1 } var exports = map[string]lua.LGFunction{ "call": call, "pcall": pcall, "error_reply": error_reply, "status_reply": status_reply, "sha1hex": sha1hex, } L.SetGlobal("tile38", L.SetFuncs(L.NewTable(), exports)) return L } func makeSafeErr(err error) error { return errors.New(strings.Replace(err.Error(), "\n", `\n`, -1)) } func runLuaFunc(luaState *lua.LState, script string, name string) resp.Value { luaState.SetGlobal("EVAL_CMD", lua.LString("FAKE_EVAL")) fn, err := luaState.Load(strings.NewReader(script), name) if err != nil { return resp.ErrorValue(makeSafeErr(err)) } luaState.Push(fn) if err := luaState.PCall(0, 1, nil); err != nil { return resp.ErrorValue(makeSafeErr(err)) } ret := luaState.Get(-1) // returned value luaState.Pop(1) luaState.SetGlobal("EVAL_CMD", lua.LNil) return ConvertToResp(ret) } func runMany(luaState *lua.LState, start int, num int) int { fmt.Printf("\nRunning %d lua calls... ", num) for i := 0; i < num; i++ { script := fmt.Sprintf("return tile38.call('foo', 'bar', %d)", i) name := fmt.Sprintf("f_%020d", i) ret := runLuaFunc(luaState, script, name) if ret.Type() == resp.Error { panic(ret.String()) } } fmt.Printf("done.\n") return start + num } func printMemStats() { var mem runtime.MemStats runtime.GC() debug.FreeOSMemory() runtime.GC() debug.FreeOSMemory() runtime.GC() debug.FreeOSMemory() runtime.GC() debug.FreeOSMemory() runtime.ReadMemStats(&mem) fmt.Printf("MemStats: Alloc %d, HeapAlloc %d, HeapSys %d, GCSys %d, HeapObjects %d.\n", mem.Alloc, mem.HeapAlloc, mem.HeapSys, mem.GCSys, mem.HeapObjects) } func testLua() { var luaState *lua.LState start := 12345 luaState = NewLuaState() printMemStats() fmt.Printf("\nRunning single call as a test\n") ret := runLuaFunc(luaState, "return tile38.call('fake_cmd', 'a', 'b')", "test_call") fmt.Printf("Result: %s\n", ret.String()) printMemStats() start = runMany(luaState, start, 100) printMemStats() start = runMany(luaState, start, 100) printMemStats() start = runMany(luaState, start, 100) printMemStats() start = runMany(luaState, start, 100) printMemStats() start = runMany(luaState, start, 1000) printMemStats() start = runMany(luaState, start, 10000) printMemStats() start = runMany(luaState, start, 1000) printMemStats() start = runMany(luaState, start, 100) printMemStats() _ = runMany(luaState, start, 1000) printMemStats() luaState.Close() } func main() { fmt.Printf("Starting memtest.\n") testLua() } ================================================ FILE: cmd/tile38-server/main.go ================================================ package main import ( "flag" "fmt" "io" "net" "net/http" _ "net/http/pprof" "os" "os/signal" "path/filepath" "runtime" "runtime/pprof" "strconv" "strings" "sync" "syscall" "github.com/tidwall/gjson" "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/hservice" "github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/server" "golang.org/x/net/context" "google.golang.org/grpc" ) // TODO: Set to false in 2.* var httpTransport = true //////////////////////////////////////////////////////////////////////////////// // // Fire up a webhook test server by using the --webhook-http-consumer-port // for example // $ ./tile38-server --webhook-http-consumer-port 9999 // // The create hooks like such... // SETHOOK myhook http://localhost:9999/myhook NEARBY mykey FENCE POINT 33.5 -115.5 1000 // //////////////////////////////////////////////////////////////////////////////// // // Memory profiling - start the server with the -pprofport flag // // $ ./tile38-server -pprofport 6060 // // Then, at any point, from a different terminal execute: // $ go tool pprof -svg http://localhost:6060/debug/pprof/heap > out.svg // // Load the SVG into a web browser to visualize the memory usage // //////////////////////////////////////////////////////////////////////////////// type hserver struct{} func (s *hserver) Send(ctx context.Context, in *hservice.MessageRequest) (*hservice.MessageReply, error) { return &hservice.MessageReply{Ok: true}, nil } func main() { gitsha := " (" + core.GitSHA + ")" if gitsha == " (0000000)" { gitsha = "" } versionLine := `tile38-server version: ` + core.Version + gitsha output := os.Stderr flag.Usage = func() { fmt.Fprintf(output, "%s", versionLine+` Usage: tile38-server [-p port] Basic Options: -h hostname : listening host -p port : listening port (default: 9851) -d path : data directory (default: data) -s socket : listen on unix socket file -l encoding : set log encoding to json or text (default: text) -o output : auto set client output to json or resp (default: resp) -q : no logging. totally silent output -v : enable verbose logging -vv : enable very verbose logging Advanced Options: --pidfile path : file that contains the pid --appendonly yes/no : AOF persistence (default: yes) --appendfilename path : AOF path (default: data/appendonly.aof) --queuefilename path : Event queue path (default:data/queue.db) --http-transport yes/no : HTTP transport (default: yes) --protected-mode yes/no : protected mode (default: yes) --nohup : do not exit on SIGHUP --spinlock : use a spinlock. For very write-heavy workloads Developer Options: --dev : enable developer mode --webhook-http-consumer-port port : Start a test HTTP webhook server --webhook-grpc-consumer-port port : Start a test GRPC webhook server `, ) } if len(os.Args) == 3 && os.Args[1] == "--webhook-http-consumer-port" { log.SetOutput(os.Stderr) port, err := strconv.ParseUint(os.Args[2], 10, 16) if err != nil { log.Fatal(err) } http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { data, err := io.ReadAll(r.Body) if err != nil { log.Fatal(err) } log.HTTPf("http: %s : %s", r.URL.Path, string(data)) }) log.Infof("webhook server http://localhost:%d/", port) if err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil); err != nil { log.Fatal(err) } return } if len(os.Args) == 3 && os.Args[1] == "--webhook-grpc-consumer-port" { log.SetOutput(os.Stderr) port, err := strconv.ParseUint(os.Args[2], 10, 16) if err != nil { log.Fatal(err) } lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { log.Fatal(err) } s := grpc.NewServer() hservice.RegisterHookServiceServer(s, &hserver{}) log.Infof("webhook server grpc://localhost:%d/", port) if err := s.Serve(lis); err != nil { log.Fatal(err) } return } var ( nohup bool showEvioDisabled bool showThreadsDisabled bool spinlock bool ) var ( // use to be in core/options.go // DevMode puts application in to dev mode devMode = false // ShowDebugMessages allows for log.Debug to print to console. showDebugMessages = false // ProtectedMode forces Tile38 to default in protected mode. protectedMode = "no" // AppendOnly allows for disabling the appendonly file. appendOnly = true // AppendFileName allows for custom appendonly file path appendFileName = "" // QueueFileName allows for custom queue.db file path queueFileName = "" // ClientOutput for auto assigning the output for client. clientOutput = "" ) // parse non standard args. nargs := []string{os.Args[0]} for i := 1; i < len(os.Args); i++ { switch os.Args[i] { case "--help": output = os.Stdout flag.Usage() return case "--version": fmt.Fprintf(os.Stdout, "%s\n", versionLine) return case "--protected-mode", "-protected-mode": i++ if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "no": protectedMode = "no" continue case "yes": protectedMode = "yes" continue } } fmt.Fprintf(os.Stderr, "protected-mode must be 'yes' or 'no'\n") os.Exit(1) case "--dev", "-dev": devMode = true continue case "--nohup", "-nohup": nohup = true continue case "--spinlock", "-spinlock": spinlock = true continue case "--appendonly", "-appendonly": i++ if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "no": appendOnly = false continue case "yes": appendOnly = true continue } } fmt.Fprintf(os.Stderr, "appendonly must be 'yes' or 'no'\n") os.Exit(1) case "--appendfilename", "-appendfilename": i++ if i == len(os.Args) || os.Args[i] == "" { fmt.Fprintf(os.Stderr, "appendfilename must have a value\n") os.Exit(1) } appendFileName = os.Args[i] case "--queuefilename", "-queuefilename": i++ if i == len(os.Args) || os.Args[i] == "" { fmt.Fprintf(os.Stderr, "queuefilename must have a value\n") os.Exit(1) } queueFileName = os.Args[i] case "-o": i++ if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "resp", "json": clientOutput = strings.ToLower(os.Args[i]) continue } } fmt.Fprintf(os.Stderr, "output must be 'resp' or 'json'\n") os.Exit(1) case "--http-transport", "-http-transport": i++ if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "1", "true", "yes": httpTransport = true continue case "0", "false", "no": httpTransport = false continue } } fmt.Fprintf(os.Stderr, "http-transport must be 'yes' or 'no'\n") os.Exit(1) case "--threads", "-threads": i++ if i < len(os.Args) { _, err := strconv.ParseUint(os.Args[i], 10, 16) if err != nil { fmt.Fprintf(os.Stderr, "threads must be a valid number\n") os.Exit(1) } showThreadsDisabled = true continue } fmt.Fprintf(os.Stderr, "threads must be a valid number \n") os.Exit(1) case "--evio", "-evio": i++ if i < len(os.Args) { switch strings.ToLower(os.Args[i]) { case "no", "yes": showEvioDisabled = true continue } } fmt.Fprintf(os.Stderr, "evio must be 'yes' or 'no'\n") os.Exit(1) default: nargs = append(nargs, os.Args[i]) } } os.Args = nargs metricsAddr := flag.String("metrics-addr", "", "The listening addr for Prometheus metrics.") var ( dir string port int host string unixSocket string verbose bool veryVerbose bool logEncoding string quiet bool pidfile string cpuprofile string memprofile string pprofport int ) flag.IntVar(&port, "p", 9851, "The listening port") flag.StringVar(&pidfile, "pidfile", "", "A file that contains the pid") flag.StringVar(&host, "h", "", "The listening host") flag.StringVar(&unixSocket, "s", "", "Listen on a unix socket") flag.StringVar(&dir, "d", "data", "The data directory") flag.StringVar(&logEncoding, "l", "text", "The log encoding json or text (default: text)") flag.BoolVar(&verbose, "v", false, "Enable verbose logging") flag.BoolVar(&quiet, "q", false, "Quiet logging. Totally silent") flag.BoolVar(&veryVerbose, "vv", false, "Enable very verbose logging") flag.IntVar(&pprofport, "pprofport", 0, "pprofport http at port") flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to `file`") flag.StringVar(&memprofile, "memprofile", "", "write memory profile to `file`") flag.Parse() if logEncoding == "json" { log.SetLogJSON(true) data, _ := os.ReadFile(filepath.Join(dir, "config")) if gjson.GetBytes(data, "logconfig.encoding").String() == "json" { c := gjson.GetBytes(data, "logconfig").String() log.Build(c) } else { log.Build("") } } var logw io.Writer = os.Stderr if quiet { logw = io.Discard } log.SetOutput(logw) if quiet { log.SetLevel(0) } else if veryVerbose { log.SetLevel(3) } else if verbose { log.SetLevel(2) } else { log.SetLevel(1) } showDebugMessages = veryVerbose hostd := "" if host != "" { hostd = "Addr: " + host + ", " } // pprof if cpuprofile != "" { log.Debugf("cpuprofile active") f, err := os.Create(cpuprofile) if err != nil { log.Fatal("could not create CPU profile: ", err) } if err := pprof.StartCPUProfile(f); err != nil { log.Fatal("could not start CPU profile: ", err) } } if memprofile != "" { log.Debug("memprofile active") } var pprofcleanedup bool var pprofcleanupMu sync.Mutex pprofcleanup := func() { pprofcleanupMu.Lock() defer pprofcleanupMu.Unlock() if pprofcleanedup { return } // cleanup code if cpuprofile != "" { pprof.StopCPUProfile() } if memprofile != "" { f, err := os.Create(memprofile) if err != nil { log.Fatal("could not create memory profile: ", err) } runtime.GC() // get up-to-date statistics if err := pprof.WriteHeapProfile(f); err != nil { log.Fatal("could not write memory profile: ", err) } f.Close() } pprofcleanedup = true } defer pprofcleanup() if pprofport != 0 { log.Debugf("pprof http at port %d", pprofport) go func() { log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", pprofport), nil)) }() } if unixSocket != "" { port = 0 } // pid file var pidferr error var pidcleanedup bool var pidcleanupMu sync.Mutex pidcleanup := func() { if pidfile != "" { pidcleanupMu.Lock() defer pidcleanupMu.Unlock() if pidcleanedup { return } // cleanup code if pidfile != "" { os.Remove(pidfile) } pidcleanedup = true } } defer pidcleanup() if pidfile != "" { os.WriteFile(pidfile, []byte(fmt.Sprintf("%d\n", os.Getpid())), 0666) } c := make(chan os.Signal, 1) shutdown := make(chan bool, 1) signal.Notify(c, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) go func() { for s := range c { if s == syscall.SIGHUP && nohup { continue } log.Warnf("signal: %v", s) pidcleanup() pprofcleanup() switch { default: os.Exit(-1) case s == syscall.SIGHUP: os.Exit(1) case s == syscall.SIGINT: os.Exit(2) case s == syscall.SIGQUIT: os.Exit(3) case s == syscall.SIGTERM: shutdown <- true } } }() var saddr string if unixSocket != "" { saddr = fmt.Sprintf("Socket: %s", unixSocket) } else { saddr = fmt.Sprintf("Port: %d", port) } if log.LogJSON() { log.Printf(`Tile38 %s%s %d bit (%s/%s) %s%s, PID: %d. Visit tile38.com/sponsor to support the project`, core.Version, gitsha, strconv.IntSize, runtime.GOARCH, runtime.GOOS, hostd, saddr, os.Getpid()) } else { fmt.Fprintf(logw, ` _____ _ _ ___ ___ |_ _|_| |___|_ | . | Tile38 %s%s %d bit (%s/%s) | | | | | -_|_ | . | %s%s, PID: %d |_| |_|_|___|___|___| tile38.com `, core.Version, gitsha, strconv.IntSize, runtime.GOARCH, runtime.GOOS, hostd, saddr, os.Getpid()) } if pidferr != nil { log.Warnf("pidfile: %v", pidferr) } if showEvioDisabled { log.Warnf("evio is not currently supported") } if showThreadsDisabled { log.Warnf("thread flag is deprecated use GOMAXPROCS to set number of threads instead") } opts := server.Options{ Host: host, Port: port, Dir: dir, UseHTTP: httpTransport, MetricsAddr: *metricsAddr, UnixSocketPath: unixSocket, DevMode: devMode, ShowDebugMessages: showDebugMessages, ProtectedMode: protectedMode, AppendOnly: appendOnly, AppendFileName: appendFileName, QueueFileName: queueFileName, Shutdown: shutdown, Spinlock: spinlock, ClientOutput: clientOutput, } if err := server.Serve(opts); err != nil { log.Fatal(err) } } ================================================ FILE: core/commands.go ================================================ //go:build ignore package core import ( "encoding/json" "strings" ) const ( clear = "\x1b[0m" bright = "\x1b[1m" gray = "\x1b[90m" yellow = "\x1b[33m" ) // Command represents a Tile38 command. type Command struct { Name string `json:"-"` Summary string `json:"summary"` Complexity string `json:"complexity"` Arguments []Argument `json:"arguments"` Since string `json:"since"` Group string `json:"group"` DevOnly bool `json:"dev"` } // String returns a string representation of the command. func (c Command) String() string { var s = c.Name for _, arg := range c.Arguments { s += " " + arg.String() } return s } // TermOutput returns a string representation of the command suitable for displaying in a terminal. func (c Command) TermOutput(indent string) string { line := c.String() var line1 string if strings.HasPrefix(line, c.Name) { line1 = bright + c.Name + clear + gray + line[len(c.Name):] + clear } else { line1 = bright + strings.Replace(c.String(), " ", " "+clear+gray, 1) + clear } line2 := yellow + "summary: " + clear + c.Summary //line3 := yellow + "since: " + clear + c.Since return indent + line1 + "\n" + indent + line2 + "\n" //+ indent + line3 + "\n" } // EnumArg represents a enum arguments. type EnumArg struct { Name string `json:"name"` Arguments []Argument `json:"arguments"` } // String returns a string representation of an EnumArg. func (a EnumArg) String() string { var s = a.Name for _, arg := range a.Arguments { s += " " + arg.String() } return s } // Argument represents a command argument. type Argument struct { Command string `json:"command"` NameAny interface{} `json:"name"` TypeAny interface{} `json:"type"` Optional bool `json:"optional"` Multiple bool `json:"multiple"` Variadic bool `json:"variadic"` Enum []string `json:"enum"` EnumArgs []EnumArg `json:"enumargs"` } // String returns a string representation of an Argument. func (a Argument) String() string { var s string if a.Command != "" { s += " " + a.Command } if len(a.EnumArgs) > 0 { eargs := "" for _, arg := range a.EnumArgs { v := arg.String() if strings.Contains(v, " ") { v = "(" + v + ")" } eargs += v + "|" } if len(eargs) > 0 { eargs = eargs[:len(eargs)-1] } s += " " + eargs } else if len(a.Enum) > 0 { s += " " + strings.Join(a.Enum, "|") } else { names, _ := a.NameTypes() subs := "" for _, name := range names { subs += " " + name } subs = strings.TrimSpace(subs) s += " " + subs if a.Variadic { if len(names) == 0 { s += " [" + subs + " ...]" } else { s += " [" + names[len(names)-1] + " ...]" } } if a.Multiple { s += " ..." } } s = strings.TrimSpace(s) if a.Optional { s = "[" + s + "]" } return s } func parseAnyStringArray(any interface{}) []string { if str, ok := any.(string); ok { return []string{str} } else if any, ok := any.([]interface{}); ok { arr := []string{} for _, any := range any { if str, ok := any.(string); ok { arr = append(arr, str) } } return arr } return []string{} } // NameTypes returns the types and names of an argument as separate arrays. func (a Argument) NameTypes() (names, types []string) { names = parseAnyStringArray(a.NameAny) types = parseAnyStringArray(a.TypeAny) if len(types) > len(names) { types = types[:len(names)] } else { for len(types) < len(names) { types = append(types, "") } } return } // Commands is a map of all of the commands. var Commands = func() map[string]Command { var commands map[string]Command if err := json.Unmarshal([]byte(commandsJSON), &commands); err != nil { panic(err.Error()) } for name, command := range commands { command.Name = strings.ToUpper(name) commands[name] = command } return commands }() var commandsJSON = `{{.CommandsJSON}}` ================================================ FILE: core/commands.json ================================================ { "SET": { "summary": "Sets the value of an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "command": "FIELD", "name": ["name", "value"], "type": ["string", "double"], "optional": true, "multiple": true }, { "command": "EX", "name": ["seconds"], "type": ["double"], "optional": true, "multiple": false }, { "name": "type", "optional": true, "enumargs": [ { "name": "NX" }, { "name": "XX" } ] }, { "name": "value", "enumargs": [ { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "POINT", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "z", "type": "double", "optional": true } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] }, { "name": "STRING", "arguments": [ { "name": "value", "type": "string" } ] } ] } ], "since": "1.0.0", "group": "keys" }, "EXPIRE": { "summary": "Set a timeout on an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "seconds", "type": "double" } ], "since": "1.0.0", "group": "keys" }, "TTL": { "summary": "Get a timeout on an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ], "since": "1.0.0", "group": "keys" }, "EXISTS": { "summary": "Checks to see if a id exists", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ], "since": "1.33.0", "group": "keys" }, "FEXISTS": { "summary": "Checks to see if a field exists on a id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "field", "type": "string" } ], "since": "1.33.0", "group": "keys" }, "PERSIST": { "summary": "Remove the existing timeout on an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ], "since": "1.0.0", "group": "keys" }, "FSET": { "summary": "Set the value for one or more fields of an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "command": "XX", "name": [], "type": [], "optional": true }, { "name": ["field", "value"], "type": ["string", "double"] }, { "name": ["field", "value"], "type": ["string", "double"], "multiple": true, "optional": true } ], "since": "1.0.0", "group": "keys" }, "FGET": { "summary": "Gets the value for the field of an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "field", "type": "string" } ], "since": "1.33.0", "group": "keys" }, "BOUNDS": { "summary": "Get the combined bounds of all the objects in a key", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" } ], "since": "1.3.0", "group": "keys" }, "GET": { "summary": "Get the object of an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "command": "WITHFIELDS", "name": [], "type": [], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "OBJECT" }, { "name": "POINT" }, { "name": "BOUNDS" }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] } ] } ], "since": "1.0.0", "group": "keys" }, "DEL": { "summary": "Delete an id from a key", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "command": "ERRON404", "name": [], "type": [], "optional": true } ], "since": "1.0.0", "group": "keys" }, "DROP": { "summary": "Remove a key from the database", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" } ], "since": "1.0.0", "group": "keys" }, "RENAME": { "summary": "Rename a key to be stored under a different name.", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "newkey", "type": "string" } ], "since": "1.14.5", "group": "keys" }, "RENAMENX": { "summary": "Rename a key to be stored under a different name, if a new key does not exist.", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "newkey", "type": "string" } ], "since": "1.14.5", "group": "keys" }, "KEYS": { "summary": "Finds all keys matching the given pattern", "complexity": "O(N) where N is the number of keys in the database", "arguments": [ { "name": "pattern", "type": "pattern" } ], "since": "1.0.0", "group": "keys" }, "STATS": { "summary": "Show stats for one or more keys", "complexity": "O(N) where N is the number of keys being requested", "arguments": [ { "name": "key", "type": "string", "variadic": true } ], "since": "1.0.0", "group": "keys" }, "SEARCH": { "summary": "Search for string values in a key", "complexity": "O(N) where N is the number of values in the key", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "name": "order", "optional": true, "enumargs": [ { "name": "ASC" }, { "name": "DESC" } ] }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" } ] } ], "since": "1.4.2", "group": "search" }, "SCAN": { "summary": "Incrementally iterate though a key", "complexity": "O(N) where N is the number of ids in the key", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "name": "order", "optional": true, "enumargs": [ { "name": "ASC" }, { "name": "DESC" } ] }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" }, { "name": "OBJECTS" }, { "name": "POINTS" }, { "name": "BOUNDS" }, { "name": "HASHES", "arguments": [ { "name": "precision", "type": "integer" } ] } ] } ], "since": "1.0.0", "group": "search" }, "NEARBY": { "summary": "Searches for ids that are nearby a point", "complexity": "O(log(N)) where N is the number of ids in the area", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "command": "DISTANCE", "name": [], "type": [], "optional": true }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "command": "FENCE", "name": [], "type": [], "optional": true }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" }, { "name": "OBJECTS" }, { "name": "POINTS" }, { "name": "BOUNDS" }, { "name": "HASHES", "arguments": [ { "name": "precision", "type": "integer" } ] } ] }, { "name": "area", "enumargs": [ { "name": "POINT", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "ROAM", "arguments": [ { "name": "key", "type": "string" }, { "name": "pattern", "type": "pattern" }, { "name": "meters", "type": "double" } ] } ] } ], "since": "1.0.0", "group": "search" }, "WITHIN": { "summary": "Searches for ids that completely within the area", "complexity": "O(log(N)) where N is the number of ids in the area", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "command": "FENCE", "name": [], "type": [], "optional": true }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" }, { "name": "OBJECTS" }, { "name": "POINTS" }, { "name": "BOUNDS" }, { "name": "HASHES", "arguments": [ { "name": "precision", "type": "integer" } ] } ] }, { "name": "area", "enumargs": [ { "name": "GET", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "CIRCLE", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "TILE", "arguments": [ { "name": "x", "type": "double" }, { "name": "y", "type": "double" }, { "name": "z", "type": "double" } ] }, { "name": "QUADKEY", "arguments": [ { "name": "quadkey", "type": "string" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] }, { "name": "SECTOR", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "radius", "type": "double" }, { "name": "startBearing", "type": "double" }, { "name": "endBearing", "type": "double" } ] } ] } ], "since": "1.0.0", "group": "search" }, "INTERSECTS": { "summary": "Searches for ids that intersect an area", "complexity": "O(log(N)) where N is the number of ids in the area", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "CLIP", "name": [], "type": [], "optional": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "command": "FENCE", "name": [], "type": [], "optional": true }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" }, { "name": "OBJECTS" }, { "name": "POINTS" }, { "name": "BOUNDS" }, { "name": "HASHES", "arguments": [ { "name": "precision", "type": "integer" } ] } ] }, { "name": "area", "enumargs": [ { "name": "GET", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "CIRCLE", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "TILE", "arguments": [ { "name": "x", "type": "double" }, { "name": "y", "type": "double" }, { "name": "z", "type": "double" } ] }, { "name": "QUADKEY", "arguments": [ { "name": "quadkey", "type": "string" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] }, { "name": "SECTOR", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "radius", "type": "double" }, { "name": "startBearing", "type": "double" }, { "name": "endBearing", "type": "double" } ] } ] } ], "since": "1.0.0", "group": "search" }, "CONFIG GET": { "summary": "Get the value of a configuration parameter", "arguments": [ { "name": "parameter", "type": "string" } ], "group": "server" }, "CONFIG SET": { "summary": "Set a configuration parameter to the given value", "arguments": [ { "name": "parameter", "type": "string" }, { "name": "value", "type": "string", "optional": true } ], "group": "server" }, "CONFIG REWRITE": { "summary": "Rewrite the configuration file with the in memory configuration", "arguments": [], "group": "server" }, "SERVER": { "summary": "Show server stats and details", "complexity": "O(1)", "arguments": [], "since": "1.0.0", "group": "server" }, "GC": { "summary": "Forces a garbage collection", "complexity": "O(1)", "arguments": [], "since": "1.0.0", "group": "server" }, "READONLY": { "summary": "Turns on or off readonly mode", "complexity": "O(1)", "arguments": [ { "enum": ["yes", "no"] } ], "since": "1.0.0", "group": "server" }, "FLUSHDB": { "summary": "Removes all keys", "complexity": "O(1)", "arguments": [], "since": "1.0.0", "group": "server" }, "FOLLOW": { "summary": "Follows a leader host", "complexity": "O(1)", "arguments": [ { "name": "host", "type": "string" }, { "name": "port", "type": "integer" } ], "since": "1.0.0", "group": "replication" }, "AOF": { "summary": "Downloads the AOF starting from pos and keeps the connection alive", "complexity": "O(1)", "arguments": [ { "name": "pos", "type": "integer" } ], "since": "1.0.0", "group": "replication" }, "AOFMD5": { "summary": "Performs a checksum on a portion of the aof", "complexity": "O(1)", "arguments": [ { "name": "pos", "type": "integer" }, { "name": "size", "type": "integer" } ], "since": "1.0.0", "group": "replication" }, "AOFSHRINK": { "summary": "Shrinks the aof in the background", "group": "replication" }, "PING": { "summary": "Ping the server", "group": "connection" }, "QUIT": { "summary": "Close the connection", "group": "connection" }, "AUTH": { "summary": "Authenticate to the server", "arguments": [ { "name": "password", "type": "string" } ], "group": "connection" }, "OUTPUT": { "summary": "Gets or sets the output format for the current connection.", "arguments": [ { "name": "format", "optional": true, "enumargs": [ { "name": "json" }, { "name": "resp" } ] } ], "group": "connection" }, "TIMEOUT": { "summary": "Runs the following command with the timeout", "arguments": [ { "name": "seconds", "type": "double" }, { "name": "COMMAND", "type": "string" }, { "command": "arg", "type": "string", "multiple": true, "optional": true } ], "group": "connection" }, "SETHOOK": { "summary": "Creates a webhook which points to geofenced search", "arguments": [ { "name": "name", "type": "string" }, { "name": "endpoint", "type": "string" }, { "command": "META", "name": ["name", "value"], "type": ["string", "string"], "optional": true, "multiple": true }, { "command": "EX", "name": ["seconds"], "type": ["double"], "optional": true, "multiple": false }, { "enum": ["NEARBY", "WITHIN", "INTERSECTS"] }, { "name": "key", "type": "string" }, { "command": "FENCE", "name": [], "type": [] }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "param", "type": "string", "variadic": true } ], "group": "webhook" }, "DELHOOK": { "summary": "Removes a webhook", "arguments": [ { "name": "name", "type": "string" } ], "group": "webhook" }, "HOOKS": { "summary": "Finds all hooks matching a pattern", "arguments": [ { "name": "pattern", "type": "pattern" } ], "group": "webhook" }, "PDELHOOK": { "summary": "Removes all hooks matching a pattern", "arguments": [ { "name": "pattern", "type": "pattern" } ], "group": "webhook" }, "SETCHAN": { "summary": "Creates a pubsub channel which points to geofenced search", "arguments": [ { "name": "name", "type": "string" }, { "command": "META", "name": ["name", "value"], "type": ["string", "string"], "optional": true, "multiple": true }, { "command": "EX", "name": ["seconds"], "type": ["double"], "optional": true, "multiple": false }, { "enum": ["NEARBY", "WITHIN", "INTERSECTS"] }, { "name": "key", "type": "string" }, { "command": "FENCE", "name": [], "type": [] }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "param", "type": "string", "variadic": true } ], "group": "pubsub" }, "DELCHAN": { "summary": "Removes a channel", "arguments": [ { "name": "name", "type": "string" } ], "group": "pubsub" }, "CHANS": { "summary": "Finds all channels matching a pattern", "arguments": [ { "name": "pattern", "type": "pattern" } ], "group": "pubsub" }, "PDELCHAN": { "summary": "Removes all channels matching a pattern", "arguments": [ { "name": "pattern", "type": "pattern" } ], "group": "pubsub" }, "SUBSCRIBE": { "summary": "Subscribe to a geofence channel", "arguments": [ { "name": "channel", "type": "string", "variadic": true } ], "group": "pubsub" }, "PSUBSCRIBE": { "summary": "Subscribes the client to the given patterns", "arguments": [ { "name": "pattern", "type": "pattern", "variadic": true } ], "group": "pubsub" }, "PDEL": { "summary": "Removes all objects matching a pattern", "arguments": [ { "name": "key", "type": "string" }, { "name": "pattern", "type": "pattern" } ], "group": "keys" }, "JGET": { "summary": "Get a value from a JSON document", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "path", "type": "string" }, { "command": "RAW", "name": [], "type": [], "optional": true } ], "group": "keys" }, "JSET": { "summary": "Set a value in a JSON document", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "path", "type": "string" }, { "name": "value", "type": "string" }, { "name": [], "optional": true, "enumargs": [ { "name": "RAW" }, { "name": "STR" } ] } ], "group": "keys" }, "JDEL": { "summary": "Delete a value from a JSON document", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "path", "type": "string" } ], "group": "keys" }, "EVAL": { "summary": "Evaluates a Lua script", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "script", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALSHA": { "summary": "Evaluates a Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "sha1", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALRO": { "summary": "Evaluates a read-only Lua script", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "script", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALROSHA": { "summary": "Evaluates a read-only Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "script", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALNA": { "summary": "Evaluates a Lua script in a non-atomic fashion", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "script", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALNASHA": { "summary": "Evaluates, in a non-atomic fashion, a Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "sha1", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "SCRIPT EXISTS": { "summary": "Returns information about the existence of the scripts in server cache", "complexity": "O(N) where N is the number of provided sha1 arguments", "arguments": [ { "name": "sha1", "type": "string", "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "SCRIPT LOAD": { "summary": "Loads the compiled version of a script into the server cache, without executing", "complexity": "O(N) where N is the number of bytes in the script", "arguments": [ { "name": "script", "type": "string" } ], "since": "1.10.0", "group": "scripting" }, "SCRIPT FLUSH": { "summary": "Flushes the server cache of Lua scripts", "complexity": "O(1)", "since": "1.10.0", "group": "scripting" }, "TEST": { "summary": "Performs spatial test", "complexity": "One test per command, complexity depends on the test", "arguments": [ { "name": "area1", "enumargs": [ { "name": "POINT", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" } ] }, { "name": "GET", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "CIRCLE", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "TILE", "arguments": [ { "name": "x", "type": "double" }, { "name": "y", "type": "double" }, { "name": "z", "type": "double" } ] }, { "name": "QUADKEY", "arguments": [ { "name": "quadkey", "type": "string" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] } ] }, { "name": "test", "enumargs": [ { "name": "INTERSECTS" }, { "name": "WITHIN" } ] }, { "command": "CLIP", "name": [], "type": [], "optional": true }, { "name": "area2", "enumargs": [ { "name": "POINT", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" } ] }, { "name": "GET", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "CIRCLE", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "TILE", "arguments": [ { "name": "x", "type": "double" }, { "name": "y", "type": "double" }, { "name": "z", "type": "double" } ] }, { "name": "QUADKEY", "arguments": [ { "name": "quadkey", "type": "string" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] } ] } ], "since": "1.16.0", "group": "tests" } } ================================================ FILE: core/commands_gen.go ================================================ // This file was autogenerated. DO NOT EDIT. package core import ( "encoding/json" "strings" ) const ( clear = "\x1b[0m" bright = "\x1b[1m" gray = "\x1b[90m" yellow = "\x1b[33m" ) // Command represents a Tile38 command. type Command struct { Name string `json:"-"` Summary string `json:"summary"` Complexity string `json:"complexity"` Arguments []Argument `json:"arguments"` Since string `json:"since"` Group string `json:"group"` DevOnly bool `json:"dev"` } // String returns a string representation of the command. func (c Command) String() string { var s = c.Name for _, arg := range c.Arguments { s += " " + arg.String() } return s } // TermOutput returns a string representation of the command suitable for displaying in a terminal. func (c Command) TermOutput(indent string) string { line := c.String() var line1 string if strings.HasPrefix(line, c.Name) { line1 = bright + c.Name + clear + gray + line[len(c.Name):] + clear } else { line1 = bright + strings.Replace(c.String(), " ", " "+clear+gray, 1) + clear } line2 := yellow + "summary: " + clear + c.Summary //line3 := yellow + "since: " + clear + c.Since return indent + line1 + "\n" + indent + line2 + "\n" //+ indent + line3 + "\n" } // EnumArg represents a enum arguments. type EnumArg struct { Name string `json:"name"` Arguments []Argument `json:"arguments"` } // String returns a string representation of an EnumArg. func (a EnumArg) String() string { var s = a.Name for _, arg := range a.Arguments { s += " " + arg.String() } return s } // Argument represents a command argument. type Argument struct { Command string `json:"command"` NameAny interface{} `json:"name"` TypeAny interface{} `json:"type"` Optional bool `json:"optional"` Multiple bool `json:"multiple"` Variadic bool `json:"variadic"` Enum []string `json:"enum"` EnumArgs []EnumArg `json:"enumargs"` } // String returns a string representation of an Argument. func (a Argument) String() string { var s string if a.Command != "" { s += " " + a.Command } if len(a.EnumArgs) > 0 { eargs := "" for _, arg := range a.EnumArgs { v := arg.String() if strings.Contains(v, " ") { v = "(" + v + ")" } eargs += v + "|" } if len(eargs) > 0 { eargs = eargs[:len(eargs)-1] } s += " " + eargs } else if len(a.Enum) > 0 { s += " " + strings.Join(a.Enum, "|") } else { names, _ := a.NameTypes() subs := "" for _, name := range names { subs += " " + name } subs = strings.TrimSpace(subs) s += " " + subs if a.Variadic { if len(names) == 0 { s += " [" + subs + " ...]" } else { s += " [" + names[len(names)-1] + " ...]" } } if a.Multiple { s += " ..." } } s = strings.TrimSpace(s) if a.Optional { s = "[" + s + "]" } return s } func parseAnyStringArray(any interface{}) []string { if str, ok := any.(string); ok { return []string{str} } else if any, ok := any.([]interface{}); ok { arr := []string{} for _, any := range any { if str, ok := any.(string); ok { arr = append(arr, str) } } return arr } return []string{} } // NameTypes returns the types and names of an argument as separate arrays. func (a Argument) NameTypes() (names, types []string) { names = parseAnyStringArray(a.NameAny) types = parseAnyStringArray(a.TypeAny) if len(types) > len(names) { types = types[:len(names)] } else { for len(types) < len(names) { types = append(types, "") } } return } // Commands is a map of all of the commands. var Commands = func() map[string]Command { var commands map[string]Command if err := json.Unmarshal([]byte(commandsJSON), &commands); err != nil { panic(err.Error()) } for name, command := range commands { command.Name = strings.ToUpper(name) commands[name] = command } return commands }() var commandsJSON = `{ "SET": { "summary": "Sets the value of an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "command": "FIELD", "name": ["name", "value"], "type": ["string", "double"], "optional": true, "multiple": true }, { "command": "EX", "name": ["seconds"], "type": ["double"], "optional": true, "multiple": false }, { "name": "type", "optional": true, "enumargs": [ { "name": "NX" }, { "name": "XX" } ] }, { "name": "value", "enumargs": [ { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "POINT", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "z", "type": "double", "optional": true } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] }, { "name": "STRING", "arguments": [ { "name": "value", "type": "string" } ] } ] } ], "since": "1.0.0", "group": "keys" }, "EXPIRE": { "summary": "Set a timeout on an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "seconds", "type": "double" } ], "since": "1.0.0", "group": "keys" }, "TTL": { "summary": "Get a timeout on an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ], "since": "1.0.0", "group": "keys" }, "EXISTS": { "summary": "Checks to see if a id exists", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ], "since": "1.33.0", "group": "keys" }, "FEXISTS": { "summary": "Checks to see if a field exists on a id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "field", "type": "string" } ], "since": "1.33.0", "group": "keys" }, "PERSIST": { "summary": "Remove the existing timeout on an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ], "since": "1.0.0", "group": "keys" }, "FSET": { "summary": "Set the value for one or more fields of an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "command": "XX", "name": [], "type": [], "optional": true }, { "name": ["field", "value"], "type": ["string", "double"] }, { "name": ["field", "value"], "type": ["string", "double"], "multiple": true, "optional": true } ], "since": "1.0.0", "group": "keys" }, "FGET": { "summary": "Gets the value for the field of an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "field", "type": "string" } ], "since": "1.33.0", "group": "keys" }, "BOUNDS": { "summary": "Get the combined bounds of all the objects in a key", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" } ], "since": "1.3.0", "group": "keys" }, "GET": { "summary": "Get the object of an id", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "command": "WITHFIELDS", "name": [], "type": [], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "OBJECT" }, { "name": "POINT" }, { "name": "BOUNDS" }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] } ] } ], "since": "1.0.0", "group": "keys" }, "DEL": { "summary": "Delete an id from a key", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "command": "ERRON404", "name": [], "type": [], "optional": true } ], "since": "1.0.0", "group": "keys" }, "DROP": { "summary": "Remove a key from the database", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" } ], "since": "1.0.0", "group": "keys" }, "RENAME": { "summary": "Rename a key to be stored under a different name.", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "newkey", "type": "string" } ], "since": "1.14.5", "group": "keys" }, "RENAMENX": { "summary": "Rename a key to be stored under a different name, if a new key does not exist.", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "newkey", "type": "string" } ], "since": "1.14.5", "group": "keys" }, "KEYS": { "summary": "Finds all keys matching the given pattern", "complexity": "O(N) where N is the number of keys in the database", "arguments": [ { "name": "pattern", "type": "pattern" } ], "since": "1.0.0", "group": "keys" }, "STATS": { "summary": "Show stats for one or more keys", "complexity": "O(N) where N is the number of keys being requested", "arguments": [ { "name": "key", "type": "string", "variadic": true } ], "since": "1.0.0", "group": "keys" }, "SEARCH": { "summary": "Search for string values in a key", "complexity": "O(N) where N is the number of values in the key", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "name": "order", "optional": true, "enumargs": [ { "name": "ASC" }, { "name": "DESC" } ] }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" } ] } ], "since": "1.4.2", "group": "search" }, "SCAN": { "summary": "Incrementally iterate though a key", "complexity": "O(N) where N is the number of ids in the key", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "name": "order", "optional": true, "enumargs": [ { "name": "ASC" }, { "name": "DESC" } ] }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" }, { "name": "OBJECTS" }, { "name": "POINTS" }, { "name": "BOUNDS" }, { "name": "HASHES", "arguments": [ { "name": "precision", "type": "integer" } ] } ] } ], "since": "1.0.0", "group": "search" }, "NEARBY": { "summary": "Searches for ids that are nearby a point", "complexity": "O(log(N)) where N is the number of ids in the area", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "command": "DISTANCE", "name": [], "type": [], "optional": true }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "command": "FENCE", "name": [], "type": [], "optional": true }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" }, { "name": "OBJECTS" }, { "name": "POINTS" }, { "name": "BOUNDS" }, { "name": "HASHES", "arguments": [ { "name": "precision", "type": "integer" } ] } ] }, { "name": "area", "enumargs": [ { "name": "POINT", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "ROAM", "arguments": [ { "name": "key", "type": "string" }, { "name": "pattern", "type": "pattern" }, { "name": "meters", "type": "double" } ] } ] } ], "since": "1.0.0", "group": "search" }, "WITHIN": { "summary": "Searches for ids that completely within the area", "complexity": "O(log(N)) where N is the number of ids in the area", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "command": "FENCE", "name": [], "type": [], "optional": true }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" }, { "name": "OBJECTS" }, { "name": "POINTS" }, { "name": "BOUNDS" }, { "name": "HASHES", "arguments": [ { "name": "precision", "type": "integer" } ] } ] }, { "name": "area", "enumargs": [ { "name": "GET", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "CIRCLE", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "TILE", "arguments": [ { "name": "x", "type": "double" }, { "name": "y", "type": "double" }, { "name": "z", "type": "double" } ] }, { "name": "QUADKEY", "arguments": [ { "name": "quadkey", "type": "string" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] }, { "name": "SECTOR", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "radius", "type": "double" }, { "name": "startBearing", "type": "double" }, { "name": "endBearing", "type": "double" } ] } ] } ], "since": "1.0.0", "group": "search" }, "INTERSECTS": { "summary": "Searches for ids that intersect an area", "complexity": "O(log(N)) where N is the number of ids in the area", "arguments": [ { "name": "key", "type": "string" }, { "command": "CURSOR", "name": "start", "type": "integer", "optional": true }, { "command": "LIMIT", "name": "count", "type": "integer", "optional": true }, { "command": "MATCH", "name": "pattern", "type": "pattern", "optional": true }, { "command": "WHERE", "name": ["field", "min", "max"], "type": ["string", "double", "double"], "optional": true, "multiple": true }, { "command": "WHEREIN", "name": ["field", "count", "value"], "type": ["string", "integer", "double"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVAL", "name": ["script", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "WHEREEVALSHA", "name": ["sha1", "numargs", "arg"], "type": ["string", "integer", "string"], "optional": true, "multiple": true, "variadic": true }, { "command": "CLIP", "name": [], "type": [], "optional": true }, { "command": "NOFIELDS", "name": [], "type": [], "optional": true }, { "command": "FENCE", "name": [], "type": [], "optional": true }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "type", "optional": true, "enumargs": [ { "name": "COUNT" }, { "name": "IDS" }, { "name": "OBJECTS" }, { "name": "POINTS" }, { "name": "BOUNDS" }, { "name": "HASHES", "arguments": [ { "name": "precision", "type": "integer" } ] } ] }, { "name": "area", "enumargs": [ { "name": "GET", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "CIRCLE", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "TILE", "arguments": [ { "name": "x", "type": "double" }, { "name": "y", "type": "double" }, { "name": "z", "type": "double" } ] }, { "name": "QUADKEY", "arguments": [ { "name": "quadkey", "type": "string" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] }, { "name": "SECTOR", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "radius", "type": "double" }, { "name": "startBearing", "type": "double" }, { "name": "endBearing", "type": "double" } ] } ] } ], "since": "1.0.0", "group": "search" }, "CONFIG GET": { "summary": "Get the value of a configuration parameter", "arguments": [ { "name": "parameter", "type": "string" } ], "group": "server" }, "CONFIG SET": { "summary": "Set a configuration parameter to the given value", "arguments": [ { "name": "parameter", "type": "string" }, { "name": "value", "type": "string", "optional": true } ], "group": "server" }, "CONFIG REWRITE": { "summary": "Rewrite the configuration file with the in memory configuration", "arguments": [], "group": "server" }, "SERVER": { "summary": "Show server stats and details", "complexity": "O(1)", "arguments": [], "since": "1.0.0", "group": "server" }, "GC": { "summary": "Forces a garbage collection", "complexity": "O(1)", "arguments": [], "since": "1.0.0", "group": "server" }, "READONLY": { "summary": "Turns on or off readonly mode", "complexity": "O(1)", "arguments": [ { "enum": ["yes", "no"] } ], "since": "1.0.0", "group": "server" }, "FLUSHDB": { "summary": "Removes all keys", "complexity": "O(1)", "arguments": [], "since": "1.0.0", "group": "server" }, "FOLLOW": { "summary": "Follows a leader host", "complexity": "O(1)", "arguments": [ { "name": "host", "type": "string" }, { "name": "port", "type": "integer" } ], "since": "1.0.0", "group": "replication" }, "AOF": { "summary": "Downloads the AOF starting from pos and keeps the connection alive", "complexity": "O(1)", "arguments": [ { "name": "pos", "type": "integer" } ], "since": "1.0.0", "group": "replication" }, "AOFMD5": { "summary": "Performs a checksum on a portion of the aof", "complexity": "O(1)", "arguments": [ { "name": "pos", "type": "integer" }, { "name": "size", "type": "integer" } ], "since": "1.0.0", "group": "replication" }, "AOFSHRINK": { "summary": "Shrinks the aof in the background", "group": "replication" }, "PING": { "summary": "Ping the server", "group": "connection" }, "QUIT": { "summary": "Close the connection", "group": "connection" }, "AUTH": { "summary": "Authenticate to the server", "arguments": [ { "name": "password", "type": "string" } ], "group": "connection" }, "OUTPUT": { "summary": "Gets or sets the output format for the current connection.", "arguments": [ { "name": "format", "optional": true, "enumargs": [ { "name": "json" }, { "name": "resp" } ] } ], "group": "connection" }, "TIMEOUT": { "summary": "Runs the following command with the timeout", "arguments": [ { "name": "seconds", "type": "double" }, { "name": "COMMAND", "type": "string" }, { "command": "arg", "type": "string", "multiple": true, "optional": true } ], "group": "connection" }, "SETHOOK": { "summary": "Creates a webhook which points to geofenced search", "arguments": [ { "name": "name", "type": "string" }, { "name": "endpoint", "type": "string" }, { "command": "META", "name": ["name", "value"], "type": ["string", "string"], "optional": true, "multiple": true }, { "command": "EX", "name": ["seconds"], "type": ["double"], "optional": true, "multiple": false }, { "enum": ["NEARBY", "WITHIN", "INTERSECTS"] }, { "name": "key", "type": "string" }, { "command": "FENCE", "name": [], "type": [] }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "param", "type": "string", "variadic": true } ], "group": "webhook" }, "DELHOOK": { "summary": "Removes a webhook", "arguments": [ { "name": "name", "type": "string" } ], "group": "webhook" }, "HOOKS": { "summary": "Finds all hooks matching a pattern", "arguments": [ { "name": "pattern", "type": "pattern" } ], "group": "webhook" }, "PDELHOOK": { "summary": "Removes all hooks matching a pattern", "arguments": [ { "name": "pattern", "type": "pattern" } ], "group": "webhook" }, "SETCHAN": { "summary": "Creates a pubsub channel which points to geofenced search", "arguments": [ { "name": "name", "type": "string" }, { "command": "META", "name": ["name", "value"], "type": ["string", "string"], "optional": true, "multiple": true }, { "command": "EX", "name": ["seconds"], "type": ["double"], "optional": true, "multiple": false }, { "enum": ["NEARBY", "WITHIN", "INTERSECTS"] }, { "name": "key", "type": "string" }, { "command": "FENCE", "name": [], "type": [] }, { "command": "DETECT", "name": ["what"], "type": ["string"], "optional": true }, { "command": "COMMANDS", "name": ["which"], "type": ["string"], "optional": true }, { "name": "param", "type": "string", "variadic": true } ], "group": "pubsub" }, "DELCHAN": { "summary": "Removes a channel", "arguments": [ { "name": "name", "type": "string" } ], "group": "pubsub" }, "CHANS": { "summary": "Finds all channels matching a pattern", "arguments": [ { "name": "pattern", "type": "pattern" } ], "group": "pubsub" }, "PDELCHAN": { "summary": "Removes all channels matching a pattern", "arguments": [ { "name": "pattern", "type": "pattern" } ], "group": "pubsub" }, "SUBSCRIBE": { "summary": "Subscribe to a geofence channel", "arguments": [ { "name": "channel", "type": "string", "variadic": true } ], "group": "pubsub" }, "PSUBSCRIBE": { "summary": "Subscribes the client to the given patterns", "arguments": [ { "name": "pattern", "type": "pattern", "variadic": true } ], "group": "pubsub" }, "PDEL": { "summary": "Removes all objects matching a pattern", "arguments": [ { "name": "key", "type": "string" }, { "name": "pattern", "type": "pattern" } ], "group": "keys" }, "JGET": { "summary": "Get a value from a JSON document", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "path", "type": "string" }, { "command": "RAW", "name": [], "type": [], "optional": true } ], "group": "keys" }, "JSET": { "summary": "Set a value in a JSON document", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "path", "type": "string" }, { "name": "value", "type": "string" }, { "name": [], "optional": true, "enumargs": [ { "name": "RAW" }, { "name": "STR" } ] } ], "group": "keys" }, "JDEL": { "summary": "Delete a value from a JSON document", "complexity": "O(1)", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" }, { "name": "path", "type": "string" } ], "group": "keys" }, "EVAL": { "summary": "Evaluates a Lua script", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "script", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALSHA": { "summary": "Evaluates a Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "sha1", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALRO": { "summary": "Evaluates a read-only Lua script", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "script", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALROSHA": { "summary": "Evaluates a read-only Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "script", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALNA": { "summary": "Evaluates a Lua script in a non-atomic fashion", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "script", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "EVALNASHA": { "summary": "Evaluates, in a non-atomic fashion, a Lua script cached on the server by its SHA1 digest", "complexity": "Depends on the evaluated script", "arguments": [ { "name": "sha1", "type": "string" }, { "name": "numkeys", "type": "integer" }, { "name": "key", "type": "string", "optional": true, "multiple": true }, { "name": "arg", "type": "string", "optional": true, "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "SCRIPT EXISTS": { "summary": "Returns information about the existence of the scripts in server cache", "complexity": "O(N) where N is the number of provided sha1 arguments", "arguments": [ { "name": "sha1", "type": "string", "multiple": true } ], "since": "1.10.0", "group": "scripting" }, "SCRIPT LOAD": { "summary": "Loads the compiled version of a script into the server cache, without executing", "complexity": "O(N) where N is the number of bytes in the script", "arguments": [ { "name": "script", "type": "string" } ], "since": "1.10.0", "group": "scripting" }, "SCRIPT FLUSH": { "summary": "Flushes the server cache of Lua scripts", "complexity": "O(1)", "since": "1.10.0", "group": "scripting" }, "TEST": { "summary": "Performs spatial test", "complexity": "One test per command, complexity depends on the test", "arguments": [ { "name": "area1", "enumargs": [ { "name": "POINT", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" } ] }, { "name": "GET", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "CIRCLE", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "TILE", "arguments": [ { "name": "x", "type": "double" }, { "name": "y", "type": "double" }, { "name": "z", "type": "double" } ] }, { "name": "QUADKEY", "arguments": [ { "name": "quadkey", "type": "string" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] } ] }, { "name": "test", "enumargs": [ { "name": "INTERSECTS" }, { "name": "WITHIN" } ] }, { "command": "CLIP", "name": [], "type": [], "optional": true }, { "name": "area2", "enumargs": [ { "name": "POINT", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" } ] }, { "name": "GET", "arguments": [ { "name": "key", "type": "string" }, { "name": "id", "type": "string" } ] }, { "name": "BOUNDS", "arguments": [ { "name": "minlat", "type": "double" }, { "name": "minlon", "type": "double" }, { "name": "maxlat", "type": "double" }, { "name": "maxlon", "type": "double" } ] }, { "name": "OBJECT", "arguments": [ { "name": "geojson", "type": "geojson" } ] }, { "name": "CIRCLE", "arguments": [ { "name": "lat", "type": "double" }, { "name": "lon", "type": "double" }, { "name": "meters", "type": "double" } ] }, { "name": "TILE", "arguments": [ { "name": "x", "type": "double" }, { "name": "y", "type": "double" }, { "name": "z", "type": "double" } ] }, { "name": "QUADKEY", "arguments": [ { "name": "quadkey", "type": "string" } ] }, { "name": "HASH", "arguments": [ { "name": "geohash", "type": "geohash" } ] } ] } ], "since": "1.16.0", "group": "tests" } }` ================================================ FILE: core/commands_test.go ================================================ package core import ( "fmt" "sort" "testing" ) func TestCommands(t *testing.T) { var names []string for name := range Commands { names = append(names, name) } sort.Strings(names) for _, name := range names { cmd := Commands[name] if cmd.Group == "server" { fmt.Printf("%v\n", cmd.String()) } } } ================================================ FILE: core/gen.sh ================================================ #!/bin/sh set -e cd $(dirname $0) export CommandsJSON="$(cat commands.json)" # replace out the json perl -pe ' while (($i = index($_, "{{.CommandsJSON}}")) != -1) { substr($_, $i, length("{{.CommandsJSON}}")) = $ENV{"CommandsJSON"}; } ' commands.go > commands_gen.go # remove the ignore sed -i -e 's/\/\/go:build ignore/\/\/ This file was autogenerated. DO NOT EDIT./g' commands_gen.go rm -rf commands_gen.go-e ================================================ FILE: core/version.go ================================================ package core // Build variables var ( Version = "0.0.0" // Placeholder for the version BuildTime = "" // Placeholder for the build time GitSHA = "0000000" // Placeholder for the git sha ) ================================================ FILE: go.mod ================================================ module github.com/tidwall/tile38 go 1.24.0 require ( github.com/tidwall/assert v0.1.0 github.com/tidwall/btree v1.8.1 github.com/tidwall/buntdb v1.3.2 github.com/tidwall/expr v0.14.0 github.com/tidwall/geojson v1.4.6 github.com/tidwall/gjson v1.18.0 github.com/tidwall/hashmap v1.8.1 github.com/tidwall/limiter v0.4.0 github.com/tidwall/match v1.2.0 github.com/tidwall/mvt v0.2.1 github.com/tidwall/pretty v1.2.1 github.com/tidwall/redbench v0.1.0 github.com/tidwall/redcon v1.6.2 github.com/tidwall/resp v0.1.1 github.com/tidwall/rtree v1.10.0 github.com/tidwall/sjson v1.2.5 github.com/tidwall/tinylru v1.2.1 ) require ( cloud.google.com/go/pubsub v1.50.0 github.com/Azure/azure-event-hubs-go/v3 v3.6.2 github.com/IBM/sarama v1.46.0 github.com/aws/aws-sdk-go v1.55.8 github.com/cloudflare/cloudflare-go/v4 v4.6.0 github.com/eclipse/paho.mqtt.golang v1.5.1 github.com/golang/protobuf v1.5.4 github.com/gomodule/redigo v1.9.2 github.com/iwpnd/sectr v0.1.2 github.com/mmcloughlin/geohash v0.10.0 github.com/nats-io/nats.go v1.44.0 github.com/peterh/liner v1.2.2 github.com/prometheus/client_golang v1.23.0 github.com/streadway/amqp v1.1.0 github.com/xdg-go/scram v1.1.2 github.com/yuin/gopher-lua v1.1.1 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/net v0.47.0 golang.org/x/term v0.37.0 google.golang.org/api v0.246.0 google.golang.org/grpc v1.74.2 layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf ) require ( cloud.google.com/go v0.121.4 // indirect cloud.google.com/go/auth v0.16.3 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.7.0 // indirect cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/pubsub/v2 v2.0.0 // indirect github.com/Azure/azure-amqp-common-go/v4 v4.2.0 // indirect github.com/Azure/azure-sdk-for-go v65.0.0+incompatible // indirect github.com/Azure/go-amqp v1.0.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/devigned/tab v0.1.1 // indirect github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nats-io/nkeys v0.4.11 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.65.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/tidwall/conv v0.1.0 // indirect github.com/tidwall/geoindex v1.7.0 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.12.0 // indirect google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250721164621-a45f3dfb1074 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 // indirect google.golang.org/protobuf v1.36.6 // indirect ) ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.121.4 h1:cVvUiY0sX0xwyxPwdSU2KsF9knOVmtRyAMt8xou0iTs= cloud.google.com/go v0.121.4/go.mod h1:XEBchUiHFJbz4lKBZwYBDHV/rSyfFktk737TLDU089s= cloud.google.com/go/auth v0.16.3 h1:kabzoQ9/bobUmnseYnBO6qQG7q4a/CffFRlJSxv2wCc= cloud.google.com/go/auth v0.16.3/go.mod h1:NucRGjaXfzP1ltpcQ7On/VTZ0H4kWB5Jy+Y9Dnm76fA= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= cloud.google.com/go/kms v1.22.0 h1:dBRIj7+GDeeEvatJeTB19oYZNV0aj6wEqSIT/7gLqtk= cloud.google.com/go/kms v1.22.0/go.mod h1:U7mf8Sva5jpOb4bxYZdtw/9zsbIjrklYwPcvMk34AL8= cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= cloud.google.com/go/pubsub v1.50.0 h1:hnYpOIxVlgVD1Z8LN7est4DQZK3K6tvZNurZjIVjUe0= cloud.google.com/go/pubsub v1.50.0/go.mod h1:Di2Y+nqXBpIS+dXUEJPQzLh8PbIQZMLE9IVUFhf2zmM= cloud.google.com/go/pubsub/v2 v2.0.0 h1:0qS6mRJ41gD1lNmM/vdm6bR7DQu6coQcVwD+VPf0Bz0= cloud.google.com/go/pubsub/v2 v2.0.0/go.mod h1:0aztFxNzVQIRSZ8vUr79uH2bS3jwLebwK6q1sgEub+E= github.com/Azure/azure-amqp-common-go/v4 v4.2.0 h1:q/jLx1KJ8xeI8XGfkOWMN9XrXzAfVTkyvCxPvHCjd2I= github.com/Azure/azure-amqp-common-go/v4 v4.2.0/go.mod h1:GD3m/WPPma+621UaU6KNjKEo5Hl09z86viKwQjTpV0Q= github.com/Azure/azure-event-hubs-go/v3 v3.6.2 h1:7rNj1/iqS/i3mUKokA2n2eMYO72TB7lO7OmpbKoakKY= github.com/Azure/azure-event-hubs-go/v3 v3.6.2/go.mod h1:n+ocYr9j2JCLYqUqz9eI+lx/TEAtL/g6rZzyTFSuIpc= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-amqp v1.0.0 h1:QfCugi1M+4F2JDTRgVnRw7PYXLXZ9hmqk3+9+oJh3OA= github.com/Azure/go-amqp v1.0.0/go.mod h1:+bg0x3ce5+Q3ahCEXnCsGG3ETpDQe3MEVnOuT2ywPwc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwpKNMpV+/FTWjwEbuPD495Tk= github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U= github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/IBM/sarama v1.46.0 h1:+YTM1fNd6WKMchlnLKRUB5Z0qD4M8YbvwIIPLvJD53s= github.com/IBM/sarama v1.46.0/go.mod h1:0lOcuQziJ1/mBGHkdp5uYrltqQuKQKM5O5FOWUQVVvo= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go/v4 v4.6.0 h1:ZaWwXjHFR5NoY8UEf4QFY0g3KTi72kqqEXpajV610/o= github.com/cloudflare/cloudflare-go/v4 v4.6.0/go.mod h1:XcYpLe7Mf6FN87kXzEWVnJ6z+vskW/k6eUqgqfhFE9k= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/devigned/tab v0.1.1 h1:3mD6Kb1mUOYeLpJvTVSDwSg5ZsfSxfvxGRTxRsJsITA= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE= github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/iwpnd/sectr v0.1.2 h1:FauaPRn5C2tC42HTF7gM3FJZXvGXWc6jabBbIxzTMag= github.com/iwpnd/sectr v0.1.2/go.mod h1:Dm6YXDJCRx1NTfMX/1RMIkGfVp2ORjCY/cQGfbknz4c= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/geohash v0.10.0 h1:9w1HchfDfdeLc+jFEf/04D27KP7E2QmpDu52wPbJWRE= github.com/mmcloughlin/geohash v0.10.0/go.mod h1:oNZxQo5yWJh0eMQEP/8hwQuVx9Z9tjwFUqcTB1SmG0c= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nats-io/nats.go v1.44.0 h1:ECKVrDLdh/kDPV1g0gAQ+2+m2KprqZK5O/eJAyAnH2M= github.com/nats-io/nats.go v1.44.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM= github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA= github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A= github.com/tidwall/buntdb v1.3.2 h1:qd+IpdEGs0pZci37G4jF51+fSKlkuUTMXuHhXL1AkKg= github.com/tidwall/buntdb v1.3.2/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/cities v0.1.0 h1:CVNkmMf7NEC9Bvokf5GoSsArHCKRMTgLuubRTHnH0mE= github.com/tidwall/cities v0.1.0/go.mod h1:lV/HDp2gCcRcHJWqgt6Di54GiDrTZwh1aG2ZUPNbqa4= github.com/tidwall/conv v0.1.0 h1:hieA69RrgNAIpirLIymxM1+v0TmuijwU4w9qfeyApHo= github.com/tidwall/conv v0.1.0/go.mod h1:jvlaM2rq1CHtNznWb3A5i+tkG6k6XYeMUNwPfyEGxIo= github.com/tidwall/expr v0.14.0 h1:tb2MDhay/Qtorm+UXv3DXdCpMlpsE8Phg1bWhXMM3Dw= github.com/tidwall/expr v0.14.0/go.mod h1:R5XZxQS2HA/teLqU67CqLZX78FyfvcFoDBDne6VRlaA= github.com/tidwall/geoindex v1.4.4/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= github.com/tidwall/geoindex v1.7.0 h1:jtk41sfgwIt8MEDyC3xyKSj75iXXf6rjReJGDNPtR5o= github.com/tidwall/geoindex v1.7.0/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= github.com/tidwall/geojson v1.4.6 h1:HpEGer4tc5ieFn8Ts8aTG9fo+hgFJkqfql4O9cgphmg= github.com/tidwall/geojson v1.4.6/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/hashmap v1.8.1 h1:hXNzBfSJ2Jwvt0lbkWD59O/r3OfatSIcbuWT0VKEVns= github.com/tidwall/hashmap v1.8.1/go.mod h1:v+0qJrJn7l+l2dB8+fAFpC62p2G0SMP2Teu8ejkebg8= github.com/tidwall/limiter v0.4.0 h1:nj+7mS6aMDRzp15QTVDrgkun0def5/PfB4ogs5NlIVQ= github.com/tidwall/limiter v0.4.0/go.mod h1:n+qBGuSOgAvgcq1xUvo+mXWg8oBLQC8wkkheN9KZou0= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/mvt v0.2.1 h1:pVoD4/INMeXk8VF8yfMOzn534W4u/0vhSnqyi+/FfG4= github.com/tidwall/mvt v0.2.1/go.mod h1:RLOVf5y8yfDR4vyA636MiY0kBJeP/i7RxxpZzygREI8= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/redbench v0.1.0 h1:UZYUMhwMMObQRq5xU4SA3lmlJRztXzqtushDii+AmPo= github.com/tidwall/redbench v0.1.0/go.mod h1:zxcRGCq/JcqV48YjK9WxBNJL7JSpMzbLXaHvMcnanKQ= github.com/tidwall/redcon v1.6.2 h1:5qfvrrybgtO85jnhSravmkZyC0D+7WstbfCs3MmPhow= github.com/tidwall/redcon v1.6.2/go.mod h1:p5Wbsgeyi2VSTBWOcA5vRXrOb9arFTcU2+ZzFjqV75Y= github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE= github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0= github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/rtree v1.3.1/go.mod h1:S+JSsqPTI8LfWA4xHBo5eXzie8WJLVFeppAutSegl6M= github.com/tidwall/rtree v1.10.0 h1:+EcI8fboEaW1L3/9oW/6AMoQ8HiEIHyR7bQOGnmz4Mg= github.com/tidwall/rtree v1.10.0/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tidwall/tinylru v1.2.1 h1:VgBr72c2IEr+V+pCdkPZUwiQ0KJknnWIYbhxAVkYfQk= github.com/tidwall/tinylru v1.2.1/go.mod h1:9bQnEduwB6inr2Y7AkBP7JPgCkyrhTV/ZpX0oOOpBI4= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps= go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.246.0 h1:H0ODDs5PnMZVZAEtdLMn2Ul2eQi7QNjqM2DIFp8TlTM= google.golang.org/api v0.246.0/go.mod h1:dMVhVcylamkirHdzEBAIQWUCgqY885ivNeZYd7VAVr8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= google.golang.org/genproto/googleapis/api v0.0.0-20250721164621-a45f3dfb1074 h1:mVXdvnmR3S3BQOqHECm9NGMjYiRtEvDYcqAqedTXY6s= google.golang.org/genproto/googleapis/api v0.0.0-20250721164621-a45f3dfb1074/go.mod h1:vYFwMYFbmA8vl6Z/krj/h7+U/AqpHknwJX4Uqgfyc7I= google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 h1:MAKi5q709QWfnkkpNQ0M12hYJ1+e8qYVDyowc4U1XZM= google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:rRz0YsF7VXj9fXRF6yQgFI7DzST+hsI3TeFSGupntu0= layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:ivKkcY8Zxw5ba0jldhZCYYQfGdb2K6u9tbYK1AwMIBc= ================================================ FILE: internal/bing/bing.go ================================================ // https://msdn.microsoft.com/en-us/library/bb259689.aspx package bing import "math" const ( // EarthRadius is the radius of the earth EarthRadius = 6378137.0 // MinLatitude is the min lat MinLatitude = -85.05112878 // MaxLatitude is the max lat MaxLatitude = 85.05112878 // MinLongitude is the min lon MinLongitude = -180.0 // MaxLongitude is the max lon MaxLongitude = 180.0 // TileSize is the size of a tile TileSize = 256 // MaxLevelOfDetail is the max level of detail MaxLevelOfDetail = 38 ) // Clips a number to the specified minimum and maximum values. // Param 'n' is the number to clip. // Param 'minValue' is the minimum allowable value. // Param 'maxValue' is the maximum allowable value. // Returns the clipped value. func clip(n, minValue, maxValue float64) float64 { if n < minValue { return minValue } if n > maxValue { return maxValue } return n } // MapSize determines the map width and height (in pixels) at a specified level of detail. // Param 'levelOfDetail' is the level of detail, from 1 (lowest detail) to N (highest detail). // Returns the map width and height in pixels. func MapSize(levelOfDetail uint64) uint64 { return TileSize << levelOfDetail } // // Determines the ground resolution (in meters per pixel) at a specified latitude and level of detail. // // Param 'latitude' is the Latitude (in degrees) at which to measure the ground resolution. // // Param 'levelOfDetail' is the Level of detail, from 1 (lowest detail) to N (highest detail). // // Returns the ground resolution, in meters per pixel. // func GroundResolution(latitude float64, levelOfDetail uint64) float64 { // latitude = clip(latitude, MinLatitude, MaxLatitude) // return math.Cos(latitude*math.Pi/180) * 2 * math.Pi * EarthRadius / float64(MapSize(levelOfDetail)) // } // // Determines the map scale at a specified latitude, level of detail, and screen resolution. // // Param 'latitude' is the latitude (in degrees) at which to measure the map scale. // // Param 'levelOfDetail' is the level of detail, from 1 (lowest detail) to N (highest detail). // // Param 'screenDpi' is the resolution of the screen, in dots per inch. // // Returns the map scale, expressed as the denominator N of the ratio 1 : N. // func MapScale(latitude float64, levelOfDetail, screenDpi uint64) float64 { // return GroundResolution(latitude, levelOfDetail) * float64(screenDpi) / 0.0254 // } // LatLongToPixelXY converts a point from latitude/longitude WGS-84 coordinates (in degrees) into pixel XY coordinates at a specified level of detail. // Param 'latitude' is the latitude of the point, in degrees. // Param 'longitude' is the longitude of the point, in degrees. // Param 'levelOfDetail' is the level of detail, from 1 (lowest detail) to N (highest detail). // Return value 'pixelX' is the output parameter receiving the X coordinate in pixels. // Return value 'pixelY' is the output parameter receiving the Y coordinate in pixels. func LatLongToPixelXY(latitude, longitude float64, levelOfDetail uint64) (pixelX, pixelY int64) { latitude = clip(latitude, MinLatitude, MaxLatitude) longitude = clip(longitude, MinLongitude, MaxLongitude) x := (longitude + 180) / 360 sinLatitude := math.Sin(latitude * math.Pi / 180) y := 0.5 - math.Log((1+sinLatitude)/(1-sinLatitude))/(4*math.Pi) mapSize := float64(MapSize(levelOfDetail)) pixelX = int64(clip(x*mapSize+0.5, 0, mapSize-1)) pixelY = int64(clip(y*mapSize+0.5, 0, mapSize-1)) return } // PixelXYToLatLong converts a pixel from pixel XY coordinates at a specified level of detail into latitude/longitude WGS-84 coordinates (in degrees). // Param 'pixelX' is the X coordinate of the point, in pixels. // Param 'pixelY' is the Y coordinates of the point, in pixels. // Param 'levelOfDetail' is the level of detail, from 1 (lowest detail) to N (highest detail). // Return value 'latitude' is the output parameter receiving the latitude in degrees. // Return value 'longitude' is the output parameter receiving the longitude in degrees. func PixelXYToLatLong(pixelX, pixelY int64, levelOfDetail uint64) (latitude, longitude float64) { mapSize := float64(MapSize(levelOfDetail)) x := (clip(float64(pixelX), 0, mapSize-1) / mapSize) - 0.5 y := 0.5 - (clip(float64(pixelY), 0, mapSize-1) / mapSize) latitude = 90 - 360*math.Atan(math.Exp(-y*2*math.Pi))/math.Pi longitude = 360 * x return } // PixelXYToTileXY converts pixel XY coordinates into tile XY coordinates of the tile containing the specified pixel. // Param 'pixelX' is the pixel X coordinate. // Param 'pixelY' is the pixel Y coordinate. // Return value 'tileX' is the output parameter receiving the tile X coordinate. // Return value 'tileY' is the output parameter receiving the tile Y coordinate. func PixelXYToTileXY(pixelX, pixelY int64) (tileX, tileY int64) { return pixelX >> 8, pixelY >> 8 } // TileXYToPixelXY converts tile XY coordinates into pixel XY coordinates of the upper-left pixel of the specified tile. // Param 'tileX' is the tile X coordinate. // Param 'tileY' is the tile Y coordinate. // Return value 'pixelX' is the output parameter receiving the pixel X coordinate. // Return value 'pixelY' is the output parameter receiving the pixel Y coordinate. func TileXYToPixelXY(tileX, tileY int64) (pixelX, pixelY int64) { return tileX << 8, tileY << 8 } // TileXYToQuadKey converts tile XY coordinates into a QuadKey at a specified level of detail. // Param 'tileX' is the tile X coordinate. // Param 'tileY' is the tile Y coordinate. // Param 'levelOfDetail' is the Level of detail, from 1 (lowest detail) to N (highest detail). // Returns a string containing the QuadKey. func TileXYToQuadKey(tileX, tileY int64, levelOfDetail uint64) string { quadKey := make([]byte, levelOfDetail) for i, j := levelOfDetail, 0; i > 0; i, j = i-1, j+1 { mask := int64(1 << (i - 1)) if (tileX & mask) != 0 { if (tileY & mask) != 0 { quadKey[j] = '3' } else { quadKey[j] = '1' } } else if (tileY & mask) != 0 { quadKey[j] = '2' } else { quadKey[j] = '0' } } return string(quadKey) } // QuadKeyToTileXY converts a QuadKey into tile XY coordinates. // Param 'quadKey' is the quadKey of the tile. // Return value 'tileX' is the output parameter receiving the tile X coordinate. // Return value 'tileY is the output parameter receiving the tile Y coordinate. // Return value 'levelOfDetail' is the output parameter receiving the level of detail. func QuadKeyToTileXY(quadKey string) (tileX, tileY int64, levelOfDetail uint64) { levelOfDetail = uint64(len(quadKey)) for i := levelOfDetail; i > 0; i-- { mask := int64(1 << (i - 1)) switch quadKey[levelOfDetail-i] { case '0': case '1': tileX |= mask case '2': tileY |= mask case '3': tileX |= mask tileY |= mask default: panic("Invalid QuadKey digit sequence.") } } return } ================================================ FILE: internal/bing/bing_test.go ================================================ package bing import ( "math/rand" "testing" "time" ) func TestLevelFuzz(t *testing.T) { rand.Seed(time.Now().UnixNano()) for i := 0; i < 10000; i++ { level := (rand.Int() % MaxLevelOfDetail) + 1 quad := "" for j := 0; j < level; j++ { quad += string(byte(rand.Int()%4) + '0') } tileX, tileY, levelOfDetail := QuadKeyToTileXY(quad) if levelOfDetail != uint64(len(quad)) { t.Fatalf("[%d,%d] levelOfDetail == %d, expect %d", i, level, levelOfDetail, len(quad)) } pixelX, pixelY := TileXYToPixelXY(tileX, tileY) latitude, longitude := PixelXYToLatLong(pixelX, pixelY, levelOfDetail) pixelX2, pixelY2 := LatLongToPixelXY(latitude, longitude, levelOfDetail) if pixelX2 != pixelX { t.Fatalf("[%d,%d] pixelX2 == %d, expect %d", i, level, pixelX2, pixelX) } if pixelY2 != pixelY { t.Fatalf("[%d,%d] pixelY2 == %d, expect %d", i, level, pixelY2, pixelY) } tileX2, tileY2 := PixelXYToTileXY(pixelX2, pixelY2) if tileX2 != tileX { t.Fatalf("[%d,%d] tileX2 == %d, expect %d", i, level, tileX2, tileX) } if tileY2 != tileY { t.Fatalf("[%d,%d] tileY2 == %d, expect %d", i, level, tileY2, tileY) } quad2 := TileXYToQuadKey(tileX2, tileY2, levelOfDetail) if quad2 != quad { t.Fatalf("[%d,%d] quad2 == %s, expect %s", i, level, quad2, quad) } } } func TestInvalidQuadKeyFuzz(t *testing.T) { rand.Seed(time.Now().UnixNano()) for i := 0; i < 10000; i++ { func() { defer func() { var s string if v := recover(); v != nil { s = v.(string) } if s != "Invalid QuadKey digit sequence." { t.Fatalf("s == '%s', expect '%s", s, "Invalid QuadKey digit sequence.") } }() level := (rand.Int() % MaxLevelOfDetail) + 1 valid := true quad := "" for valid { quad = "" for j := 0; j < level; j++ { c := byte(rand.Int()%5) + '0' quad += string(c) if c < '0' || c > '3' { valid = false } } } QuadKeyToTileXY(quad) }() } } func TestLatLonClippingFuzz(t *testing.T) { rand.Seed(time.Now().UnixNano()) for i := 0; i < 10000; i++ { lat := clip(rand.Float64()*180.0-90.0, MinLatitude, MaxLatitude) lon := clip(rand.Float64()*380.0-180.0, MinLongitude, MaxLongitude) if lat < MinLatitude { t.Fatalf("lat == %f, expect < %f", lat, MinLatitude) } if lat > MaxLatitude { t.Fatalf("lat == %f, expect > %f", lat, MaxLatitude) } if lon < MinLongitude { t.Fatalf("lon == %f, expect < %f", lon, MinLongitude) } if lon > MaxLongitude { t.Fatalf("lon == %f, expect > %f", lon, MaxLongitude) } } } func TestIssue302(t *testing.T) { // Requesting tile with zoom level > 63 crashes the server #302 for z := uint64(0); z < 256; z++ { tileX, tileY := PixelXYToTileXY(LatLongToPixelXY(33, -115, z)) TileXYToBounds(tileX, tileY, z) } } ================================================ FILE: internal/bing/ext.go ================================================ package bing import "errors" // LatLongToQuad iterates through all of the quads parts until levelOfDetail is reached. func LatLongToQuad(latitude, longitude float64, levelOfDetail uint64, iterator func(part int) bool) { pixelX, pixelY := LatLongToPixelXY(latitude, longitude, levelOfDetail) tileX, tileY := PixelXYToTileXY(pixelX, pixelY) for i := levelOfDetail; i > 0; i-- { if !iterator(partForTileXY(tileX, tileY, i)) { break } } } func partForTileXY(tileX, tileY int64, levelOfDetail uint64) int { mask := int64(1 << (levelOfDetail - 1)) if (tileX & mask) != 0 { if (tileY & mask) != 0 { return 3 } return 1 } else if (tileY & mask) != 0 { return 2 } return 0 } // TileXYToBounds returns the bounds around a tile. func TileXYToBounds(tileX, tileY int64, levelOfDetail uint64) (minLat, minLon, maxLat, maxLon float64) { size := int64(1 << levelOfDetail) pixelX, pixelY := TileXYToPixelXY(tileX, tileY) maxLat, minLon = PixelXYToLatLong(pixelX, pixelY, levelOfDetail) pixelX, pixelY = TileXYToPixelXY(tileX+1, tileY+1) minLat, maxLon = PixelXYToLatLong(pixelX, pixelY, levelOfDetail) if size == 0 || tileX%size == 0 { minLon = MinLongitude } if size == 0 || tileX%size == size-1 { maxLon = MaxLongitude } if tileY <= 0 { maxLat = MaxLatitude } if tileY >= size-1 { minLat = MinLatitude } return } // QuadKeyToBounds converts a quadkey to bounds func QuadKeyToBounds(quadkey string) (minLat, minLon, maxLat, maxLon float64, err error) { for i := 0; i < len(quadkey); i++ { switch quadkey[i] { case '0', '1', '2', '3': default: err = errors.New("invalid quadkey") return } } minLat, minLon, maxLat, maxLon = TileXYToBounds(QuadKeyToTileXY(quadkey)) return } ================================================ FILE: internal/bing/ext_test.go ================================================ package bing import ( "math/rand" "testing" "time" ) func TestIteratorFuzz(t *testing.T) { rand.Seed(time.Now().UnixNano()) for i := 0; i < 10000; i++ { latitude := rand.Float64()*180.0 - 90.0 longitude := rand.Float64()*380.0 - 180.0 levelOfDetail := uint64((rand.Int() % MaxLevelOfDetail) + 1) pixelX, pixelY := LatLongToPixelXY(latitude, longitude, levelOfDetail) tileX, tileY := PixelXYToTileXY(pixelX, pixelY) quad1 := TileXYToQuadKey(tileX, tileY, levelOfDetail) l := rand.Int() % len(quad1) i := 0 quad2 := "" LatLongToQuad(latitude, longitude, levelOfDetail, func(part int) bool { if i == l { return false } quad2 += string(byte(part) + '0') i++ return true }) if quad2 != quad1[:l] { t.Fatalf("[%d,%d] quad2 == %s, expect %s", i, levelOfDetail, quad2, quad1[:l]) } } } func TestExt(t *testing.T) { // tileX, tileY, levelOfDetail := int64(0), int64(0), uint64(0) // parts := strings.Split(os.Getenv("TEST_TILE"), ",") // if len(parts) == 3 { // tileX, _ = strconv.ParseInt(parts[0], 10, 64) // tileY, _ = strconv.ParseInt(parts[1], 10, 64) // levelOfDetail, _ = strconv.ParseUint(parts[2], 10, 64) // } // minLat, minLon, maxLat, maxLon := TileXYToBounds(tileX, tileY, levelOfDetail) // fmt.Printf("\x1b[32m== Tile Boundaries ==\x1b[0m\n") // fmt.Printf("\x1b[31m%d,%d,%d\x1b[0m\n", tileX, tileY, levelOfDetail) // fmt.Printf("\x1b[31mWGS84 datum (longitude/latitude):\x1b[0m\n") // fmt.Printf("%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) //fmt.Printf("\x1b[32m\x1b[0m\n%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) // minLat, minLon, maxLat, maxLon = TileXYToBounds(1, 0, 1) // fmt.Printf("\x1b[32m1,0\x1b[0m\n%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) // minLat, minLon, maxLat, maxLon = TileXYToBounds(0, 1, 1) // fmt.Printf("\x1b[32m0,1\x1b[0m\n%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) // minLat, minLon, maxLat, maxLon = TileXYToBounds(1, 1, 1) // fmt.Printf("\x1b[32m1,1\x1b[0m\n%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) // minLat, minLon, maxLat, maxLon = TileXYToBounds(1, 0, 1) // fmt.Printf("1,0: %f,%f %f,%f\n", minLat, minLon, maxLat, maxLon) // minLat, minLon, maxLat, maxLon = TileXYToBounds(0, 1, 1) // fmt.Printf("0,1: %f,%f %f,%f\n", minLat, minLon, maxLat, maxLon) // minLat, minLon, maxLat, maxLon = TileXYToBounds(1, 1, 1) // fmt.Printf("1,1: %f,%f %f,%f\n", minLat, minLon, maxLat, maxLon) } ================================================ FILE: internal/buffer/buffer.go ================================================ package buffer import ( "errors" "math" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geo" "github.com/tidwall/geojson/geometry" "github.com/tidwall/gjson" ) // TODO: detect of pole and antimeridian crossing and generate // valid multigeometries const bufferSteps = 15 // Simple performs a very simple buffer operation on a geojson object. func Simple(g geojson.Object, meters float64) (geojson.Object, error) { if meters <= 0 { return g, nil } if math.IsInf(meters, 0) || math.IsNaN(meters) { return g, errors.New("invalid meters") } switch g := g.(type) { case *geojson.Point: return bufferSimplePoint(g.Base(), meters), nil case *geojson.SimplePoint: return bufferSimplePoint(g.Base(), meters), nil case *geojson.MultiPoint: return bufferSimpleGeometries(g.Base(), meters) case *geojson.LineString: return bufferSimpleLineString(g, meters) case *geojson.MultiLineString: return bufferSimpleGeometries(g.Base(), meters) case *geojson.Polygon: return bufferSimplePolygon(g, meters) case *geojson.MultiPolygon: return bufferSimpleGeometries(g.Base(), meters) case *geojson.FeatureCollection: return bufferSimpleFeatures(g.Base(), meters) case *geojson.Feature: bg, err := Simple(g.Base(), meters) if err != nil { return nil, err } return geojson.NewFeature(bg, g.Members()), nil case *geojson.Circle: return Simple(g.Polygon(), meters) case nil: return nil, errors.New("cannot buffer nil object") default: typ := gjson.Get(g.JSON(), "type").String() return nil, errors.New("cannot buffer " + typ + " type") } } func bufferSimplePoint(p geometry.Point, meters float64) *geojson.Polygon { meters = geo.NormalizeDistance(meters) points := make([]geometry.Point, 0, bufferSteps+1) // calc the four corners maxY, _ := geo.DestinationPoint(p.Y, p.X, meters, 0) _, maxX := geo.DestinationPoint(p.Y, p.X, meters, 90) minY, _ := geo.DestinationPoint(p.Y, p.X, meters, 180) _, minX := geo.DestinationPoint(p.Y, p.X, meters, 270) // use the half width of the lat and lon lons := (maxX - minX) / 2 lats := (maxY - minY) / 2 // generate the circle polygon for th := 0.0; th <= 360.0; th += 360.0 / float64(bufferSteps) { radians := (math.Pi / 180) * th x := p.X + lons*math.Cos(radians) y := p.Y + lats*math.Sin(radians) points = append(points, geometry.Point{X: x, Y: y}) } // add last connecting point, make a total of steps+1 points = append(points, points[0]) poly := geojson.NewPolygon( geometry.NewPoly(points, nil, &geometry.IndexOptions{ Kind: geometry.None, }), ) return poly } func bufferSimpleGeometries(objs []geojson.Object, meters float64, ) (*geojson.GeometryCollection, error) { geoms := make([]geojson.Object, len(objs)) for i := 0; i < len(objs); i++ { g, err := Simple(objs[i], meters) if err != nil { return nil, err } geoms[i] = g } return geojson.NewGeometryCollection(geoms), nil } func bufferSimpleFeatures(objs []geojson.Object, meters float64, ) (*geojson.FeatureCollection, error) { geoms := make([]geojson.Object, len(objs)) for i := 0; i < len(objs); i++ { g, err := Simple(objs[i], meters) if err != nil { return nil, err } geoms[i] = g } return geojson.NewFeatureCollection(geoms), nil } // appendBufferSimpleSeries buffers a series and appends its parts to dst func appendBufferSimpleSeries(dst []geojson.Object, s geometry.Series, meters float64) []geojson.Object { nsegs := s.NumSegments() for i := 0; i < nsegs; i++ { dst = appendSimpleBufferSegment(dst, s.SegmentAt(i), meters, i == 0) } return dst } // appendSimpleBufferSegment buffers a segment and appends its parts to dst func appendSimpleBufferSegment(dst []geojson.Object, seg geometry.Segment, meters float64, first bool, ) []geojson.Object { if first { // endcap A dst = append(dst, bufferSimplePoint(seg.A, meters)) } // line polygon bear1 := geo.BearingTo(seg.A.Y, seg.A.X, seg.B.Y, seg.B.X) lat1, lon1 := geo.DestinationPoint(seg.A.Y, seg.A.X, meters, bear1-90) lat2, lon2 := geo.DestinationPoint(seg.A.Y, seg.A.X, meters, bear1+90) bear2 := geo.BearingTo(seg.B.Y, seg.B.X, seg.A.Y, seg.A.X) lat3, lon3 := geo.DestinationPoint(seg.B.Y, seg.B.X, meters, bear2-90) lat4, lon4 := geo.DestinationPoint(seg.B.Y, seg.B.X, meters, bear2+90) dst = append(dst, geojson.NewPolygon( geometry.NewPoly([]geometry.Point{ {X: lon1, Y: lat1}, {X: lon2, Y: lat2}, {X: lon3, Y: lat3}, {X: lon4, Y: lat4}, {X: lon1, Y: lat1}, }, nil, nil))) // endcap B dst = append(dst, bufferSimplePoint(seg.B, meters)) return dst } func bufferSimplePolygon(p *geojson.Polygon, meters float64, ) (*geojson.GeometryCollection, error) { var geoms []geojson.Object b := p.Base() geoms = appendBufferSimpleSeries(geoms, b.Exterior, meters) for _, hole := range b.Holes { geoms = appendBufferSimpleSeries(geoms, hole, meters) } geoms = append(geoms, p) return geojson.NewGeometryCollection(geoms), nil } func bufferSimpleLineString(l *geojson.LineString, meters float64, ) (*geojson.GeometryCollection, error) { geoms := appendBufferSimpleSeries(nil, l.Base(), meters) return geojson.NewGeometryCollection(geoms), nil } ================================================ FILE: internal/buffer/buffer_test.go ================================================ package buffer import ( "testing" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) const lineString = `{"type":"LineString","coordinates":[ [-116.40289306640624,34.125447565116126], [-116.36444091796875,34.14818102254435], [-116.0980224609375,34.15045403191448], [-115.74920654296874,34.127721186043985], [-115.54870605468749,34.075412438417395], [-115.5267333984375,34.11407854333859], [-115.21911621093749,34.048108084909835], [-115.25207519531249,33.8339199536547], [-115.40588378906249,33.71748624018193] ]}` var lineInPoints = []geometry.Point{ {X: -115.64363479614258, Y: 34.108251327293296}, {X: -115.54355621337892, Y: 34.07199987534163}, {X: -115.21482467651367, Y: 34.051237154976164}, {X: -115.4110336303711, Y: 33.715201644740844}, {X: -116.40701293945311, Y: 34.12345809664606}, } func TestBufferLineString(t *testing.T) { g, err := geojson.Parse(lineString, nil) if err != nil { t.Fatal(err) } g2, err := Simple(g, 1000) if err != nil { t.Fatal(err) } for _, pt := range lineInPoints { ok := g2.Contains(geojson.NewPoint(pt)) if !ok { t.Fatalf("!ok") } } } const polygon = `{"type": "Polygon","coordinates":[ [ [116.46881103515624,34.277644878733824], [115.87280273437499,34.20953080048952], [115.70251464843749,34.397844946449865], [115.9881591796875,34.61286625296406], [116.46881103515624,34.277644878733824] ], [ [115.90438842773436,34.38651267795365], [116.05270385742188,34.35023911062779], [115.99914550781249,34.44655621402982], [115.90438842773436,34.38651267795365] ] ]}` var polyInPoints = []geometry.Point{ {X: 115.95837593078612, Y: 34.59887847065301}, {X: 115.98755836486816, Y: 34.61879975173954}, {X: 115.98833084106445, Y: 34.59795999847678}, {X: 116.04536533355714, Y: 34.58082509817638}, {X: 116.47567749023438, Y: 34.27651009584797}, {X: 116.42005920410155, Y: 34.32018817684490}, {X: 116.33216857910156, Y: 34.25948651450623}, {X: 115.89340209960939, Y: 34.24132422972854}, {X: 115.95588684082033, Y: 34.42786803680155}, {X: 115.97236633300783, Y: 34.42107129982385}, {X: 115.99639892578125, Y: 34.43579686485573}, {X: 116.04652404785155, Y: 34.35364042469895}, {X: 115.92155456542967, Y: 34.38877925439021}, {X: 115.96755981445311, Y: 34.37687904351907}, {X: 115.88859558105467, Y: 34.42956713470528}, {X: 115.97511291503906, Y: 34.36327673174518}, {X: 115.69564819335938, Y: 34.39784494644986}, {X: 115.87005615234375, Y: 34.20385213966983}, {X: 115.76980590820312, Y: 34.31678550602221}, } var polyOutPoints = []geometry.Point{ {X: 115.68534851074217, Y: 34.40917568058836}, {X: 115.98953247070312, Y: 34.63038297923298}, {X: 115.98541259765624, Y: 34.39671178864245}, {X: 116.31500244140626, Y: 34.22145474280257}, {X: 115.85426330566406, Y: 34.18510984477340}, } func TestBufferPolygon(t *testing.T) { g, err := geojson.Parse(polygon, nil) if err != nil { t.Fatal(err) } g2, err := Simple(g, 1000) if err != nil { t.Fatal(err) } for _, pt := range polyInPoints { ok := g2.Contains(geojson.NewPoint(pt)) if !ok { t.Fatalf("!ok") } } for _, pt := range polyOutPoints { ok := g2.Contains(geojson.NewPoint(pt)) if ok { t.Fatalf("ok") } } } ================================================ FILE: internal/clip/clip.go ================================================ package clip import ( "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) // Clip clips the contents of a geojson object and return func Clip( obj geojson.Object, clipper geojson.Object, opts *geometry.IndexOptions, ) (clipped geojson.Object) { switch obj := obj.(type) { case *geojson.Point: return clipPoint(obj, clipper, opts) case *geojson.Rect: return clipRect(obj, clipper, opts) case *geojson.LineString: return clipLineString(obj, clipper, opts) case *geojson.Polygon: return clipPolygon(obj, clipper, opts) case *geojson.Feature: return clipFeature(obj, clipper, opts) case geojson.Collection: return clipCollection(obj, clipper, opts) } return obj } // clipSegment is Cohen-Sutherland Line Clipping // https://www.cs.helsinki.fi/group/goa/viewing/leikkaus/lineClip.html func clipSegment(seg geometry.Segment, rect geometry.Rect) ( res geometry.Segment, rejected bool, ) { startCode := getCode(rect, seg.A) endCode := getCode(rect, seg.B) if (startCode | endCode) == 0 { // trivially accept res = seg } else if (startCode & endCode) != 0 { // trivially reject rejected = true } else if startCode != 0 { // start is outside. get new start. newStart := intersect(rect, startCode, seg.A, seg.B) res, rejected = clipSegment(geometry.Segment{A: newStart, B: seg.B}, rect) } else { // end is outside. get new end. newEnd := intersect(rect, endCode, seg.A, seg.B) res, rejected = clipSegment(geometry.Segment{A: seg.A, B: newEnd}, rect) } return } // clipRing is Sutherland-Hodgman Polygon Clipping // https://www.cs.helsinki.fi/group/goa/viewing/leikkaus/intro2.html func clipRing(ring []geometry.Point, bbox geometry.Rect) ( resRing []geometry.Point, ) { if len(ring) < 4 { // under 4 elements this is not a polygon ring! return } var edge uint8 var inside, prevInside bool var prev geometry.Point for edge = 1; edge <= 8; edge *= 2 { prev = ring[len(ring)-2] prevInside = (getCode(bbox, prev) & edge) == 0 for _, p := range ring { inside = (getCode(bbox, p) & edge) == 0 if prevInside && inside { // Staying inside resRing = append(resRing, p) } else if prevInside && !inside { // Leaving resRing = append(resRing, intersect(bbox, edge, prev, p)) } else if !prevInside && inside { // Entering resRing = append(resRing, intersect(bbox, edge, prev, p)) resRing = append(resRing, p) } /* else { // Stay outside } */ prev, prevInside = p, inside } if len(resRing) > 0 && resRing[0] != resRing[len(resRing)-1] { resRing = append(resRing, resRing[0]) } ring, resRing = resRing, []geometry.Point{} if len(ring) == 0 { break } } resRing = ring return } func getCode(bbox geometry.Rect, point geometry.Point) (code uint8) { code = 0 if point.X < bbox.Min.X { code |= 1 // left } else if point.X > bbox.Max.X { code |= 2 // right } if point.Y < bbox.Min.Y { code |= 4 // bottom } else if point.Y > bbox.Max.Y { code |= 8 // top } return } func intersect(bbox geometry.Rect, code uint8, start, end geometry.Point) ( new geometry.Point, ) { if (code & 8) != 0 { // top new = geometry.Point{ X: start.X + (end.X-start.X)*(bbox.Max.Y-start.Y)/(end.Y-start.Y), Y: bbox.Max.Y, } } else if (code & 4) != 0 { // bottom new = geometry.Point{ X: start.X + (end.X-start.X)*(bbox.Min.Y-start.Y)/(end.Y-start.Y), Y: bbox.Min.Y, } } else if (code & 2) != 0 { //right new = geometry.Point{ X: bbox.Max.X, Y: start.Y + (end.Y-start.Y)*(bbox.Max.X-start.X)/(end.X-start.X), } } else if (code & 1) != 0 { // left new = geometry.Point{ X: bbox.Min.X, Y: start.Y + (end.Y-start.Y)*(bbox.Min.X-start.X)/(end.X-start.X), } } /* else { // should not call intersect with the zero code } */ return } ================================================ FILE: internal/clip/clip_test.go ================================================ package clip import ( "testing" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) func LO(points []geometry.Point) *geojson.LineString { return geojson.NewLineString(geometry.NewLine(points, nil)) } func RO(minX, minY, maxX, maxY float64) *geojson.Rect { return geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: minX, Y: minY}, Max: geometry.Point{X: maxX, Y: maxY}, }) } func PPO(exterior []geometry.Point, holes [][]geometry.Point) *geojson.Polygon { return geojson.NewPolygon(geometry.NewPoly(exterior, holes, nil)) } func TestClipLineStringSimple(t *testing.T) { ls := LO([]geometry.Point{ {X: 1, Y: 1}, {X: 2, Y: 2}, {X: 3, Y: 1}}) clipped := Clip(ls, RO(1.5, 0.5, 2.5, 1.8), nil) cl, ok := clipped.(*geojson.MultiLineString) if !ok { t.Fatal("wrong type") } if len(cl.Children()) != 2 { t.Fatal("result must have two parts in MultiString") } } func TestClipPolygonSimple(t *testing.T) { exterior := []geometry.Point{ {X: 2, Y: 2}, {X: 1, Y: 2}, {X: 1.5, Y: 1.5}, {X: 1, Y: 1}, {X: 2, Y: 1}, {X: 2, Y: 2}, } holes := [][]geometry.Point{ { {X: 1.9, Y: 1.9}, {X: 1.2, Y: 1.9}, {X: 1.45, Y: 1.65}, {X: 1.9, Y: 1.5}, {X: 1.9, Y: 1.9}, }, } polygon := PPO(exterior, holes) clipped := Clip(polygon, RO(1.3, 1.3, 1.4, 2.15), nil) cp, ok := clipped.(*geojson.Polygon) if !ok { t.Fatal("wrong type") } if cp.Base().Exterior.Empty() { t.Fatal("Empty result.") } if len(cp.Base().Holes) != 1 { t.Fatal("result must be a two-ring Polygon") } } func TestClipPolygon2(t *testing.T) { exterior := []geometry.Point{ {X: 2, Y: 2}, {X: 1, Y: 2}, {X: 1.5, Y: 1.5}, {X: 1, Y: 1}, {X: 2, Y: 1}, {X: 2, Y: 2}, } holes := [][]geometry.Point{ { {X: 1.9, Y: 1.9}, {X: 1.2, Y: 1.9}, {X: 1.45, Y: 1.65}, {X: 1.9, Y: 1.5}, {X: 1.9, Y: 1.9}, }, } polygon := PPO(exterior, holes) clipped := Clip(polygon, RO(1.1, 0.8, 1.15, 2.1), nil) cp, ok := clipped.(*geojson.Polygon) if !ok { t.Fatal("wrong type") } if cp.Base().Exterior.Empty() { t.Fatal("Empty result.") } if len(cp.Base().Holes) != 0 { t.Fatal("result must be a single-ring Polygon") } } // func TestClipLineString(t *testing.T) { // featuresJSON := ` // {"type": "FeatureCollection","features": [ // {"type": "Feature","properties":{},"geometry": {"type": "LineString","coordinates": [[-71.46537780761717,42.594290856363344],[-71.37714385986328,42.600861802789524],[-71.37508392333984,42.538156868495555],[-71.43756866455078,42.535374141307415],[-71.44683837890625,42.466018925787495],[-71.334228515625,42.465005871175755],[-71.32736206054688,42.52424199254517]]}}, // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.49284362792969,42.527784255084676],[-71.35791778564453,42.527784255084676],[-71.35791778564453,42.61096959812047],[-71.49284362792969,42.61096959812047],[-71.49284362792969,42.527784255084676]]]}}, // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.47396087646484,42.48247876554176],[-71.30744934082031,42.48247876554176],[-71.30744934082031,42.576596402826894],[-71.47396087646484,42.576596402826894],[-71.47396087646484,42.48247876554176]]]}}, // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.33491516113281,42.613496290695196],[-71.29920959472656,42.613496290695196],[-71.29920959472656,42.643556064374536],[-71.33491516113281,42.643556064374536],[-71.33491516113281,42.613496290695196]]]}}, // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.37130737304686,42.530061317794775],[-71.3287353515625,42.530061317794775],[-71.3287353515625,42.60414701616359],[-71.37130737304686,42.60414701616359],[-71.37130737304686,42.530061317794775]]]}}, // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.52889251708984,42.564460160624115],[-71.45713806152342,42.54043355305221],[-71.53266906738281,42.49969365675931],[-71.36547088623047,42.508552415528634],[-71.43962860107422,42.58999409368092],[-71.52889251708984,42.564460160624115]]]}}, // {"type": "Feature","properties": {},"geometry": {"type": "Point","coordinates": [-71.33079528808594,42.55940269610327]}}, // {"type": "Feature","properties": {},"geometry": {"type": "Point","coordinates": [-71.27208709716797,42.53107331902133]}} // ]} // ` // rectJSON := `{"type": "Feature","properties": {},"geometry": {"type": "Polygon","coordinates": [[[-71.44065856933594,42.51740991900762],[-71.29131317138672,42.51740991900762],[-71.29131317138672,42.62663343969058],[-71.44065856933594,42.62663343969058],[-71.44065856933594,42.51740991900762]]]}}` // features := expectJSON(t, featuresJSON, nil) // rect := expectJSON(t, rectJSON, nil) // clipped := features.Clipped(rect) // println(clipped.String()) // } ================================================ FILE: internal/clip/collection.go ================================================ package clip import ( "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) func clipCollection( collection geojson.Collection, clipper geojson.Object, opts *geometry.IndexOptions, ) geojson.Object { var features []geojson.Object for _, feature := range collection.Children() { feature = Clip(feature, clipper, opts) if feature.Empty() { continue } if _, ok := feature.(*geojson.Feature); !ok { feature = geojson.NewFeature(feature, "") } features = append(features, feature) } return geojson.NewFeatureCollection(features) } ================================================ FILE: internal/clip/feature.go ================================================ package clip import ( "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) func clipFeature( feature *geojson.Feature, clipper geojson.Object, opts *geometry.IndexOptions, ) geojson.Object { newFeature := Clip(feature.Base(), clipper, opts) if _, ok := newFeature.(*geojson.Feature); !ok { newFeature = geojson.NewFeature(newFeature, feature.Members()) } return newFeature } ================================================ FILE: internal/clip/linestring.go ================================================ package clip import ( "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) func clipLineString( lineString *geojson.LineString, clipper geojson.Object, opts *geometry.IndexOptions, ) geojson.Object { bbox := clipper.Rect() var newPoints [][]geometry.Point var clipped geometry.Segment var rejected bool var line []geometry.Point base := lineString.Base() nSegments := base.NumSegments() for i := 0; i < nSegments; i++ { clipped, rejected = clipSegment(base.SegmentAt(i), bbox) if rejected { continue } if len(line) > 0 && line[len(line)-1] != clipped.A { newPoints = append(newPoints, line) line = []geometry.Point{clipped.A} } else if len(line) == 0 { line = append(line, clipped.A) } line = append(line, clipped.B) } if len(line) > 0 { newPoints = append(newPoints, line) } var children []*geometry.Line for _, points := range newPoints { children = append(children, geometry.NewLine(points, opts)) } if len(children) == 1 { return geojson.NewLineString(children[0]) } return geojson.NewMultiLineString(children) } ================================================ FILE: internal/clip/point.go ================================================ package clip import ( "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) func clipPoint( point *geojson.Point, clipper geojson.Object, opts *geometry.IndexOptions, ) geojson.Object { if point.IntersectsRect(clipper.Rect()) { return point } return geojson.NewMultiPoint(nil) } ================================================ FILE: internal/clip/polygon.go ================================================ package clip import ( "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) func clipPolygon( polygon *geojson.Polygon, clipper geojson.Object, opts *geometry.IndexOptions, ) geojson.Object { rect := clipper.Rect() var newPoints [][]geometry.Point base := polygon.Base() rings := []geometry.Ring{base.Exterior} rings = append(rings, base.Holes...) for _, ring := range rings { ringPoints := make([]geometry.Point, ring.NumPoints()) for i := 0; i < len(ringPoints); i++ { ringPoints[i] = ring.PointAt(i) } if clippedRing := clipRing(ringPoints, rect); len(clippedRing) > 0 { newPoints = append(newPoints, clippedRing) } } var exterior []geometry.Point var holes [][]geometry.Point if len(newPoints) > 0 { exterior = newPoints[0] } if len(newPoints) > 1 { holes = newPoints[1:] } newPoly := geojson.NewPolygon( geometry.NewPoly(exterior, holes, opts), ) if newPoly.Empty() { return geojson.NewMultiPolygon(nil) } return newPoly } ================================================ FILE: internal/clip/rect.go ================================================ package clip import ( "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) func clipRect( rect *geojson.Rect, clipper geojson.Object, opts *geometry.IndexOptions, ) geojson.Object { base := rect.Base() points := make([]geometry.Point, base.NumPoints()) for i := 0; i < len(points); i++ { points[i] = base.PointAt(i) } poly := geometry.NewPoly(points, nil, opts) gPoly := geojson.NewPolygon(poly) return Clip(gPoly, clipper, opts) } ================================================ FILE: internal/collection/collection.go ================================================ package collection import ( "math" "runtime" "github.com/tidwall/btree" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/rtree" "github.com/tidwall/tile38/internal/deadline" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/object" ) // yieldStep forces the iterator to yield goroutine every 256 steps. const yieldStep = 256 // Cursor allows for quickly paging through Scan, Within, Intersects, and Nearby type Cursor interface { Offset() uint64 Step(count uint64) } func byID(a, b *object.Object) bool { return a.ID() < b.ID() } func byValue(a, b *object.Object) bool { value1 := a.String() value2 := b.String() if value1 < value2 { return true } if value1 > value2 { return false } // the values match so we'll compare IDs, which are always unique. return byID(a, b) } func byExpires(a, b *object.Object) bool { if a.Expires() < b.Expires() { return true } if a.Expires() > b.Expires() { return false } // the values match so we'll compare IDs, which are always unique. return byID(a, b) } // Collection represents a collection of geojson objects. type Collection struct { objs btree.Map[string, *object.Object] // sorted by id spatial rtree.RTreeGN[float32, *object.Object] // geospatially indexed values *btree.BTreeG[*object.Object] // sorted by value+id expires *btree.BTreeG[*object.Object] // sorted by ex+id weight int points int objects int // geometry count nobjects int // non-geometry count } var optsNoLock = btree.Options{NoLocks: true} // New creates an empty collection func New() *Collection { col := &Collection{ values: btree.NewBTreeGOptions(byValue, optsNoLock), expires: btree.NewBTreeGOptions(byExpires, optsNoLock), } return col } // Count returns the number of objects in collection. func (c *Collection) Count() int { return c.objects + c.nobjects } // StringCount returns the number of string values. func (c *Collection) StringCount() int { return c.nobjects } // PointCount returns the number of points (lat/lon coordinates) in collection. func (c *Collection) PointCount() int { return c.points } // TotalWeight calculates the in-memory cost of the collection in bytes. func (c *Collection) TotalWeight() int { return c.weight } // Bounds returns the bounds of all the items in the collection. func (c *Collection) Bounds() (minX, minY, maxX, maxY float64) { _, _, left := c.spatial.LeftMost() _, _, bottom := c.spatial.BottomMost() _, _, right := c.spatial.RightMost() _, _, top := c.spatial.TopMost() if left == nil { return } return left.Rect().Min.X, bottom.Rect().Min.Y, right.Rect().Max.X, top.Rect().Max.Y } func (c *Collection) indexDelete(item *object.Object) { if !item.Geo().Empty() { c.spatial.Delete(rtreeItem(item)) } } func (c *Collection) indexInsert(item *object.Object) { if !item.Geo().Empty() { c.spatial.Insert(rtreeItem(item)) } } const dRNDTOWARDS = (1.0 - 1.0/8388608.0) /* Round towards zero */ const dRNDAWAY = (1.0 + 1.0/8388608.0) /* Round away from zero */ func rtreeValueDown(d float64) float32 { f := float32(d) if float64(f) > d { if d < 0 { f = float32(d * dRNDAWAY) } else { f = float32(d * dRNDTOWARDS) } } return f } func rtreeValueUp(d float64) float32 { f := float32(d) if float64(f) < d { if d < 0 { f = float32(d * dRNDTOWARDS) } else { f = float32(d * dRNDAWAY) } } return f } func rtreeItem(item *object.Object) (min, max [2]float32, data *object.Object) { min, max = rtreeRect(item.Rect()) return min, max, item } func rtreeRect(rect geometry.Rect) (min, max [2]float32) { return [2]float32{ rtreeValueDown(rect.Min.X), rtreeValueDown(rect.Min.Y), }, [2]float32{ rtreeValueUp(rect.Max.X), rtreeValueUp(rect.Max.Y), } } // Set adds or replaces an object in the collection and returns the fields // array. func (c *Collection) Set(obj *object.Object) (prev *object.Object) { prev, _ = c.objs.Set(obj.ID(), obj) c.setFill(prev, obj) return prev } func (c *Collection) setFill(prev, obj *object.Object) { if prev != nil { if prev.IsSpatial() { c.indexDelete(prev) c.objects-- } else { c.values.Delete(prev) c.nobjects-- } if prev.Expires() != 0 { c.expires.Delete(prev) } c.points -= prev.Geo().NumPoints() c.weight -= prev.Weight() } if obj.IsSpatial() { c.indexInsert(obj) c.objects++ } else { c.values.Set(obj) c.nobjects++ } if obj.Expires() != 0 { c.expires.Set(obj) } c.points += obj.Geo().NumPoints() c.weight += obj.Weight() } // Delete removes an object and returns it. // If the object does not exist then the 'ok' return value will be false. func (c *Collection) Delete(id string) (prev *object.Object) { prev, _ = c.objs.Delete(id) if prev == nil { return nil } if prev.IsSpatial() { if !prev.Geo().Empty() { c.indexDelete(prev) } c.objects-- } else { c.values.Delete(prev) c.nobjects-- } if prev.Expires() != 0 { c.expires.Delete(prev) } c.points -= prev.Geo().NumPoints() c.weight -= prev.Weight() return prev } // Get returns an object. // If the object does not exist then the 'ok' return value will be false. func (c *Collection) Get(id string) *object.Object { obj, _ := c.objs.Get(id) return obj } // Scan iterates though the collection ids. func (c *Collection) Scan( desc bool, cursor Cursor, deadline *deadline.Deadline, iterator func(obj *object.Object) bool, ) bool { var keepon = true var count uint64 var offset uint64 if cursor != nil { offset = cursor.Offset() cursor.Step(offset) } iter := func(_ string, obj *object.Object) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) keepon = iterator(obj) return keepon } if desc { c.objs.Reverse(iter) } else { c.objs.Scan(iter) } return keepon } // ScanRange iterates though the collection starting with specified id. func (c *Collection) ScanRange( start, end string, desc bool, cursor Cursor, deadline *deadline.Deadline, iterator func(o *object.Object) bool, ) bool { var keepon = true var count uint64 var offset uint64 if cursor != nil { offset = cursor.Offset() cursor.Step(offset) } iter := func(_ string, o *object.Object) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) if !desc { if o.ID() >= end { return false } } else { if o.ID() <= end { return false } } keepon = iterator(o) return keepon } if desc { c.objs.Descend(start, iter) } else { c.objs.Ascend(start, iter) } return keepon } // SearchValues iterates though the collection values. func (c *Collection) SearchValues( desc bool, cursor Cursor, deadline *deadline.Deadline, iterator func(o *object.Object) bool, ) bool { var keepon = true var count uint64 var offset uint64 if cursor != nil { offset = cursor.Offset() cursor.Step(offset) } iter := func(o *object.Object) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) keepon = iterator(o) return keepon } if desc { c.values.Reverse(iter) } else { c.values.Scan(iter) } return keepon } // SearchValuesRange iterates though the collection values. func (c *Collection) SearchValuesRange(start, end string, desc bool, cursor Cursor, deadline *deadline.Deadline, iterator func(o *object.Object) bool, ) bool { var keepon = true var count uint64 var offset uint64 if cursor != nil { offset = cursor.Offset() cursor.Step(offset) } iter := func(o *object.Object) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) keepon = iterator(o) return keepon } pstart := object.New("", String(start), 0, field.List{}) pend := object.New("", String(end), 0, field.List{}) if desc { // descend range c.values.Descend(pstart, func(item *object.Object) bool { return bGT(c.values, item, pend) && iter(item) }) } else { c.values.Ascend(pstart, func(item *object.Object) bool { return bLT(c.values, item, pend) && iter(item) }) } return keepon } func bLT(tr *btree.BTreeG[*object.Object], a, b *object.Object) bool { return tr.Less(a, b) } func bGT(tr *btree.BTreeG[*object.Object], a, b *object.Object) bool { return tr.Less(b, a) } // ScanGreaterOrEqual iterates though the collection starting with specified id. func (c *Collection) ScanGreaterOrEqual(id string, desc bool, cursor Cursor, deadline *deadline.Deadline, iterator func(o *object.Object) bool, ) bool { var keepon = true var count uint64 var offset uint64 if cursor != nil { offset = cursor.Offset() cursor.Step(offset) } iter := func(_ string, o *object.Object) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) keepon = iterator(o) return keepon } if desc { c.objs.Descend(id, iter) } else { c.objs.Ascend(id, iter) } return keepon } func (c *Collection) geoSearch( rect geometry.Rect, iter func(o *object.Object) bool, ) bool { alive := true min, max := rtreeRect(rect) // avoid search if NaN present as it results in full search // https://github.com/tidwall/tile38/issues/793 if math.IsNaN(float64(min[0])) && math.IsNaN(float64(min[1])) && math.IsNaN(float64(max[0])) && math.IsNaN(float64(max[1])) { return alive } c.spatial.Search( min, max, func(_, _ [2]float32, o *object.Object) bool { alive = iter(o) return alive }, ) return alive } func (c *Collection) geoSparse( obj geojson.Object, sparse uint8, iter func(o *object.Object) (match, ok bool), ) bool { matches := make(map[string]bool) alive := true c.geoSparseInner(obj.Rect(), sparse, func(o *object.Object) (match, ok bool) { ok = true if !matches[o.ID()] { match, ok = iter(o) if match { matches[o.ID()] = true } } return match, ok }) return alive } func (c *Collection) geoSparseInner( rect geometry.Rect, sparse uint8, iter func(o *object.Object) (match, ok bool), ) bool { if sparse > 0 { w := rect.Max.X - rect.Min.X h := rect.Max.Y - rect.Min.Y quads := [4]geometry.Rect{ { Min: geometry.Point{X: rect.Min.X, Y: rect.Min.Y + h/2}, Max: geometry.Point{X: rect.Min.X + w/2, Y: rect.Max.Y}, }, { Min: geometry.Point{X: rect.Min.X + w/2, Y: rect.Min.Y + h/2}, Max: geometry.Point{X: rect.Max.X, Y: rect.Max.Y}, }, { Min: geometry.Point{X: rect.Min.X, Y: rect.Min.Y}, Max: geometry.Point{X: rect.Min.X + w/2, Y: rect.Min.Y + h/2}, }, { Min: geometry.Point{X: rect.Min.X + w/2, Y: rect.Min.Y}, Max: geometry.Point{X: rect.Max.X, Y: rect.Min.Y + h/2}, }, } for _, quad := range quads { if !c.geoSparseInner(quad, sparse-1, iter) { return false } } return true } alive := true c.geoSearch(rect, func(o *object.Object) bool { match, ok := iter(o) if !ok { alive = false return false } return !match }) return alive } // Within returns all object that are fully contained within an object or // bounding box. Set obj to nil in order to use the bounding box. func (c *Collection) Within( obj geojson.Object, sparse uint8, cursor Cursor, deadline *deadline.Deadline, iter func(o *object.Object) bool, ) bool { var count uint64 var offset uint64 if cursor != nil { offset = cursor.Offset() cursor.Step(offset) } if sparse > 0 { return c.geoSparse(obj, sparse, func(o *object.Object) (match, ok bool) { count++ if count <= offset { return false, true } nextStep(count, cursor, deadline) if match = o.Geo().Within(obj); match { ok = iter(o) } return match, ok }) } return c.geoSearch(obj.Rect(), func(o *object.Object) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) if o.Geo().Within(obj) { return iter(o) } return true }) } // Intersects returns all object that are intersect an object or bounding box. // Set obj to nil in order to use the bounding box. func (c *Collection) Intersects( gobj geojson.Object, sparse uint8, cursor Cursor, deadline *deadline.Deadline, iter func(o *object.Object) bool, ) bool { var count uint64 var offset uint64 if cursor != nil { offset = cursor.Offset() cursor.Step(offset) } if sparse > 0 { return c.geoSparse(gobj, sparse, func(o *object.Object) (match, ok bool) { count++ if count <= offset { return false, true } nextStep(count, cursor, deadline) if match = o.Geo().Intersects(gobj); match { ok = iter(o) } return match, ok }) } return c.geoSearch(gobj.Rect(), func(o *object.Object) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) if o.Geo().Intersects(gobj) { return iter(o) } return true }, ) } // Nearby returns the nearest neighbors func (c *Collection) Nearby( target geojson.Object, cursor Cursor, deadline *deadline.Deadline, iter func(o *object.Object, dist float64) bool, ) bool { alive := true center := target.Center() var count uint64 var offset uint64 if cursor != nil { offset = cursor.Offset() cursor.Step(offset) } distFn := geodeticDistAlgo([2]float64{center.X, center.Y}) c.spatial.Nearby( func(min, max [2]float32, data *object.Object, item bool) float64 { return distFn( [2]float64{float64(min[0]), float64(min[1])}, [2]float64{float64(max[0]), float64(max[1])}, data, item, ) }, func(_, _ [2]float32, o *object.Object, dist float64) bool { count++ if count <= offset { return true } nextStep(count, cursor, deadline) alive = iter(o, dist) return alive }, ) return alive } func nextStep(step uint64, cursor Cursor, deadline *deadline.Deadline) { if step&(yieldStep-1) == (yieldStep - 1) { runtime.Gosched() deadline.Check() } if cursor != nil { cursor.Step(1) } } // ScanExpires returns a list of all objects that have expired. func (c *Collection) ScanExpires(iter func(o *object.Object) bool) { c.expires.Scan(iter) } ================================================ FILE: internal/collection/collection_test.go ================================================ package collection import ( "fmt" "math/rand" "reflect" "strconv" "testing" "time" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/gjson" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/object" ) func PO(x, y float64) *geojson.Point { return geojson.NewPoint(geometry.Point{X: x, Y: y}) } func init() { seed := time.Now().UnixNano() println(seed) rand.Seed(seed) } func expect(t testing.TB, expect bool) { t.Helper() if !expect { t.Fatal("not what you expected") } } func bounds(c *Collection) geometry.Rect { minX, minY, maxX, maxY := c.Bounds() return geometry.Rect{ Min: geometry.Point{X: minX, Y: minY}, Max: geometry.Point{X: maxX, Y: maxY}, } } func TestCollectionNewCollection(t *testing.T) { const numItems = 10000 objs := make(map[string]geojson.Object) c := New() for i := 0; i < numItems; i++ { id := strconv.FormatInt(int64(i), 10) obj := PO(rand.Float64()*360-180, rand.Float64()*180-90) objs[id] = obj c.Set(object.New(id, obj, 0, field.List{})) } count := 0 bbox := geometry.Rect{ Min: geometry.Point{X: -180, Y: -90}, Max: geometry.Point{X: 180, Y: 90}, } c.geoSearch(bbox, func(o *object.Object) bool { count++ return true }) if count != len(objs) { t.Fatalf("count = %d, expect %d", count, len(objs)) } count = c.Count() if count != len(objs) { t.Fatalf("c.Count() = %d, expect %d", count, len(objs)) } testCollectionVerifyContents(t, c, objs) } func toFields(fNames, fValues []string) field.List { var fields field.List for i := 0; i < len(fNames); i++ { fields = fields.Set(field.Make(fNames[i], fValues[i])) } return fields } func TestCollectionSet(t *testing.T) { t.Run("AddString", func(t *testing.T) { c := New() str1 := String("hello") old := c.Set(object.New("str", str1, 0, field.List{})) expect(t, old == nil) }) t.Run("UpdateString", func(t *testing.T) { c := New() str1 := String("hello") str2 := String("world") old := c.Set(object.New("str", str1, 0, field.List{})) expect(t, old == nil) old = c.Set(object.New("str", str2, 0, field.List{})) expect(t, old.Geo() == str1) }) t.Run("AddPoint", func(t *testing.T) { c := New() point1 := PO(-112.1, 33.1) old := c.Set(object.New("point", point1, 0, field.List{})) expect(t, old == nil) }) t.Run("UpdatePoint", func(t *testing.T) { c := New() point1 := PO(-112.1, 33.1) point2 := PO(-112.2, 33.2) old := c.Set(object.New("point", point1, 0, field.List{})) expect(t, old == nil) old = c.Set(object.New("point", point2, 0, field.List{})) expect(t, old.Geo().Center() == point1.Base()) }) t.Run("Fields", func(t *testing.T) { c := New() str1 := String("hello") fNames := []string{"a", "b", "c"} fValues := []string{"1", "2", "3"} fields1 := toFields(fNames, fValues) old := c.Set(object.New("str", str1, 0, fields1)) expect(t, old == nil) str2 := String("hello") fNames = []string{"d", "e", "f"} fValues = []string{"4", "5", "6"} fields2 := toFields(fNames, fValues) old = c.Set(object.New("str", str2, 0, fields2)) expect(t, old.Geo() == str1) expect(t, reflect.DeepEqual(old.Fields(), fields1)) fNames = []string{"a", "b", "c", "d", "e", "f"} fValues = []string{"7", "8", "9", "10", "11", "12"} fields3 := toFields(fNames, fValues) old = c.Set(object.New("str", str1, 0, fields3)) expect(t, old.Geo() == str2) expect(t, reflect.DeepEqual(old.Fields(), fields2)) }) t.Run("Delete", func(t *testing.T) { c := New() c.Set(object.New("1", String("1"), 0, field.List{})) c.Set(object.New("2", String("2"), 0, field.List{})) c.Set(object.New("3", PO(1, 2), 0, field.List{})) expect(t, c.Count() == 3) expect(t, c.StringCount() == 2) expect(t, c.PointCount() == 1) expect(t, bounds(c) == geometry.Rect{ Min: geometry.Point{X: 1, Y: 2}, Max: geometry.Point{X: 1, Y: 2}}) var prev *object.Object prev = c.Delete("2") expect(t, prev.Geo().String() == "2") expect(t, c.Count() == 2) expect(t, c.StringCount() == 1) expect(t, c.PointCount() == 1) prev = c.Delete("1") expect(t, prev.Geo().String() == "1") expect(t, c.Count() == 1) expect(t, c.StringCount() == 0) expect(t, c.PointCount() == 1) prev = c.Delete("3") expect(t, prev.Geo().String() == `{"type":"Point","coordinates":[1,2]}`) expect(t, c.Count() == 0) expect(t, c.StringCount() == 0) expect(t, c.PointCount() == 0) prev = c.Delete("3") expect(t, prev == nil) expect(t, c.Count() == 0) expect(t, bounds(c) == geometry.Rect{}) expect(t, c.Get("3") == nil) }) } func fieldValueAt(fields field.List, index int) string { if index < 0 || index >= fields.Len() { panic("out of bounds") } var retval string var i int fields.Scan(func(f field.Field) bool { if i == index { retval = f.Value().Data() } i++ return true }) return retval } func TestCollectionScan(t *testing.T) { N := 256 c := New() for _, i := range rand.Perm(N) { id := fmt.Sprintf("%04d", i) c.Set(object.New(id, String(id), 0, makeFields( field.Make("ex", id), ))) } var n int var prevID string c.Scan(false, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.ID() > prevID) } expect(t, o.ID() == fieldValueAt(o.Fields(), 0)) n++ prevID = o.ID() return true }) expect(t, n == c.Count()) n = 0 c.Scan(true, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.ID() < prevID) } expect(t, o.ID() == fieldValueAt(o.Fields(), 0)) n++ prevID = o.ID() return true }) expect(t, n == c.Count()) n = 0 c.ScanRange("0060", "0070", false, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.ID() > prevID) } expect(t, o.ID() == fieldValueAt(o.Fields(), 0)) n++ prevID = o.ID() return true }) expect(t, n == 10) n = 0 c.ScanRange("0070", "0060", true, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.ID() < prevID) } expect(t, o.ID() == fieldValueAt(o.Fields(), 0)) n++ prevID = o.ID() return true }) expect(t, n == 10) n = 0 c.ScanGreaterOrEqual("0070", true, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.ID() < prevID) } expect(t, o.ID() == fieldValueAt(o.Fields(), 0)) n++ prevID = o.ID() return true }) expect(t, n == 71) n = 0 c.ScanGreaterOrEqual("0070", false, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.ID() > prevID) } expect(t, o.ID() == fieldValueAt(o.Fields(), 0)) n++ prevID = o.ID() return true }) expect(t, n == c.Count()-70) } func makeFields(entries ...field.Field) field.List { var fields field.List for _, f := range entries { fields = fields.Set(f) } return fields } func TestCollectionSearch(t *testing.T) { N := 256 c := New() for i, j := range rand.Perm(N) { id := fmt.Sprintf("%04d", j) ex := fmt.Sprintf("%04d", i) c.Set(object.New(id, String(ex), 0, makeFields( field.Make("i", ex), field.Make("j", id), ))) } var n int var prevValue string c.SearchValues(false, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.Geo().String() > prevValue) } expect(t, o.ID() == fieldValueAt(o.Fields(), 1)) n++ prevValue = o.Geo().String() return true }) expect(t, n == c.Count()) n = 0 c.SearchValues(true, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.Geo().String() < prevValue) } expect(t, o.ID() == fieldValueAt(o.Fields(), 1)) n++ prevValue = o.Geo().String() return true }) expect(t, n == c.Count()) n = 0 c.SearchValuesRange("0060", "0070", false, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.Geo().String() > prevValue) } expect(t, o.ID() == fieldValueAt(o.Fields(), 1)) n++ prevValue = o.Geo().String() return true }) expect(t, n == 10) n = 0 c.SearchValuesRange("0070", "0060", true, nil, nil, func(o *object.Object) bool { if n > 0 { expect(t, o.Geo().String() < prevValue) } expect(t, o.ID() == fieldValueAt(o.Fields(), 1)) n++ prevValue = o.Geo().String() return true }) expect(t, n == 10) } func TestCollectionWeight(t *testing.T) { c := New() c.Set(object.New("1", String("1"), 0, field.List{})) expect(t, c.TotalWeight() > 0) c.Delete("1") expect(t, c.TotalWeight() == 0) c.Set(object.New("1", String("1"), 0, toFields( []string{"a", "b", "c"}, []string{"1", "2", "3"}, ), )) expect(t, c.TotalWeight() > 0) c.Delete("1") expect(t, c.TotalWeight() == 0) c.Set(object.New("1", String("1"), 0, toFields( []string{"a", "b", "c"}, []string{"1", "2", "3"}, ), )) c.Set(object.New("2", String("2"), 0, toFields( []string{"d", "e", "f"}, []string{"4", "5", "6"}, ), )) c.Set(object.New("1", String("1"), 0, toFields( []string{"d", "e", "f"}, []string{"4", "5", "6"}, ), )) c.Delete("1") c.Delete("2") expect(t, c.TotalWeight() == 0) } func TestSpatialSearch(t *testing.T) { json := ` {"type":"FeatureCollection","features":[ {"type":"Feature","id":"p1","properties":{"marker-color":"#962d28","stroke":"#962d28","fill":"#962d28"},"geometry":{"type":"Point","coordinates":[-71.4743041992187,42.51867517417283]}}, {"type":"Feature","id":"p2","properties":{"marker-color":"#962d28","stroke":"#962d28","fill":"#962d28"},"geometry":{"type":"Point","coordinates":[-71.4056396484375,42.50197174319114]}}, {"type":"Feature","id":"p3","properties":{"marker-color":"#962d28","stroke":"#962d28","fill":"#962d28"},"geometry":{"type":"Point","coordinates":[-71.4619445800781,42.49437779897246]}}, {"type":"Feature","id":"p4","properties":{"marker-color":"#962d28","stroke":"#962d28","fill":"#962d28"},"geometry":{"type":"Point","coordinates":[-71.4337921142578,42.53891577257117]}}, {"type":"Feature","id":"r1","properties":{"marker-color":"#962d28","stroke":"#962d28","fill":"#962d28"},"geometry":{"type":"Polygon","coordinates":[[[-71.4279556274414,42.48804880765346],[-71.37439727783203,42.48804880765346],[-71.37439727783203,42.52322988064187],[-71.4279556274414,42.52322988064187],[-71.4279556274414,42.48804880765346]]]}}, {"type":"Feature","id":"r2","properties":{"marker-color":"#962d28","stroke":"#962d28","fill":"#962d28"},"geometry":{"type":"Polygon","coordinates":[[[-71.4825439453125,42.53588010092859],[-71.45027160644531,42.53588010092859],[-71.45027160644531,42.55839115400447],[-71.4825439453125,42.55839115400447],[-71.4825439453125,42.53588010092859]]]}}, {"type":"Feature","id":"r3","properties":{"marker-color":"#962d28","stroke":"#962d28","fill":"#962d28"},"geometry":{"type":"Polygon","coordinates": [[[-71.4111328125,42.53512115995963],[-71.3833236694336,42.53512115995963],[-71.3833236694336,42.54953946116446],[-71.4111328125,42.54953946116446],[-71.4111328125,42.53512115995963]]]}}, {"type":"Feature","id":"q1","properties":{},"geometry":{"type":"Polygon","coordinates":[[[-71.55258178710938,42.51361399979923],[-71.42074584960938,42.51361399979923],[-71.42074584960938,42.59100512331456],[-71.55258178710938,42.59100512331456],[-71.55258178710938,42.51361399979923]]]}}, {"type":"Feature","id":"q2","properties":{},"geometry":{"type":"Polygon","coordinates":[[[-71.52992248535156,42.48121277771616],[-71.36375427246092,42.48121277771616],[-71.36375427246092,42.57786045892046],[-71.52992248535156,42.57786045892046],[-71.52992248535156,42.48121277771616]]]}}, {"type":"Feature","id":"q3","properties":{},"geometry":{"type":"Polygon","coordinates":[[[-71.49490356445312,42.56673588590953],[-71.52236938476562,42.47462922809497],[-71.42898559570312,42.464499337722344],[-71.43241882324219,42.522217752342236],[-71.37954711914061,42.56420729713456],[-71.49490356445312,42.56673588590953]]]}}, {"type":"Feature","id":"q4","properties":{},"geometry":{"type":"Point","coordinates": [-71.46366119384766,42.54043355305221]}} ]} ` p1, _ := geojson.Parse(gjson.Get(json, `features.#[id=="p1"]`).Raw, nil) p2, _ := geojson.Parse(gjson.Get(json, `features.#[id=="p2"]`).Raw, nil) p3, _ := geojson.Parse(gjson.Get(json, `features.#[id=="p3"]`).Raw, nil) p4, _ := geojson.Parse(gjson.Get(json, `features.#[id=="p4"]`).Raw, nil) r1, _ := geojson.Parse(gjson.Get(json, `features.#[id=="r1"]`).Raw, nil) r2, _ := geojson.Parse(gjson.Get(json, `features.#[id=="r2"]`).Raw, nil) r3, _ := geojson.Parse(gjson.Get(json, `features.#[id=="r3"]`).Raw, nil) q1, _ := geojson.Parse(gjson.Get(json, `features.#[id=="q1"]`).Raw, nil) q2, _ := geojson.Parse(gjson.Get(json, `features.#[id=="q2"]`).Raw, nil) q3, _ := geojson.Parse(gjson.Get(json, `features.#[id=="q3"]`).Raw, nil) q4, _ := geojson.Parse(gjson.Get(json, `features.#[id=="q4"]`).Raw, nil) c := New() c.Set(object.New("p1", p1, 0, field.List{})) c.Set(object.New("p2", p2, 0, field.List{})) c.Set(object.New("p3", p3, 0, field.List{})) c.Set(object.New("p4", p4, 0, field.List{})) c.Set(object.New("r1", r1, 0, field.List{})) c.Set(object.New("r2", r2, 0, field.List{})) c.Set(object.New("r3", r3, 0, field.List{})) var n int n = 0 c.Within(q1, 0, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 3) n = 0 c.Within(q2, 0, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 7) n = 0 c.Within(q3, 0, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 4) n = 0 c.Intersects(q1, 0, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 4) n = 0 c.Intersects(q2, 0, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 7) n = 0 c.Intersects(q3, 0, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 5) n = 0 c.Intersects(q3, 0, nil, nil, func(o *object.Object) bool { n++ return n <= 1 }) expect(t, n == 2) var items []geojson.Object exitems := []geojson.Object{ r2, p4, p1, r1, r3, p3, p2, } lastDist := float64(-1) distsMonotonic := true c.Nearby(q4, nil, nil, func(o *object.Object, dist float64) bool { if dist < lastDist { distsMonotonic = false } items = append(items, o.Geo()) return true }) expect(t, len(items) == 7) expect(t, distsMonotonic) expect(t, reflect.DeepEqual(items, exitems)) } func TestCollectionSparse(t *testing.T) { rect := geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: -71.598930, Y: 42.4586739}, Max: geometry.Point{X: -71.37302, Y: 42.607937}, }) N := 10000 c := New() r := rect.Rect() for i := 0; i < N; i++ { x := (r.Max.X-r.Min.X)*rand.Float64() + r.Min.X y := (r.Max.Y-r.Min.Y)*rand.Float64() + r.Min.Y point := PO(x, y) c.Set(object.New(fmt.Sprintf("%d", i), point, 0, field.List{})) } var n int n = 0 c.Within(rect, 1, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 4) n = 0 c.Within(rect, 2, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 16) n = 0 c.Within(rect, 3, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 64) n = 0 c.Within(rect, 3, nil, nil, func(o *object.Object) bool { n++ return n <= 30 }) expect(t, n == 31) n = 0 c.Intersects(rect, 3, nil, nil, func(o *object.Object) bool { n++ return true }) expect(t, n == 64) n = 0 c.Intersects(rect, 3, nil, nil, func(o *object.Object) bool { n++ return n <= 30 }) expect(t, n == 31) } func testCollectionVerifyContents(t *testing.T, c *Collection, objs map[string]geojson.Object) { for id, o2 := range objs { o := c.Get(id) if o == nil { t.Fatalf("ok[%s] = false, expect true", id) } j1 := string(o.Geo().AppendJSON(nil)) j2 := string(o2.AppendJSON(nil)) if j1 != j2 { t.Fatalf("j1 == %s, expect %s", j1, j2) } } } func TestManyCollections(t *testing.T) { colsM := make(map[string]*Collection) cols := 100 objs := 1000 k := 0 for i := 0; i < cols; i++ { key := strconv.FormatInt(int64(i), 10) for j := 0; j < objs; j++ { id := strconv.FormatInt(int64(j), 10) p := geometry.Point{ X: rand.Float64()*360 - 180, Y: rand.Float64()*180 - 90, } obj := geojson.Object(PO(p.X, p.Y)) col, ok := colsM[key] if !ok { col = New() colsM[key] = col } col.Set(object.New(id, obj, 0, field.List{})) k++ } } col := colsM["13"] //println(col.Count()) bbox := geometry.Rect{ Min: geometry.Point{X: -180, Y: 30}, Max: geometry.Point{X: 34, Y: 100}, } col.geoSearch(bbox, func(o *object.Object) bool { //println(id) return true }) } type testPointItem struct { id string object geojson.Object fields field.List } func makeBenchFields(nFields int) field.List { var fields field.List for i := 0; i < nFields; i++ { key := fmt.Sprintf("%d", i) val := key fields = fields.Set(field.Make(key, val)) } return fields } func BenchmarkInsert_Fields(t *testing.B) { benchmarkInsert(t, 1) } func BenchmarkInsert_NoFields(t *testing.B) { benchmarkInsert(t, 0) } func benchmarkInsert(t *testing.B, nFields int) { rand.Seed(time.Now().UnixNano()) items := make([]testPointItem, t.N) for i := 0; i < t.N; i++ { items[i] = testPointItem{ fmt.Sprintf("%d", i), PO(rand.Float64()*360-180, rand.Float64()*180-90), makeBenchFields(nFields), } } col := New() t.ResetTimer() for i := 0; i < t.N; i++ { col.Set(object.New(items[i].id, items[i].object, 0, items[i].fields)) } } func BenchmarkReplace_Fields(t *testing.B) { benchmarkReplace(t, 1) } func BenchmarkReplace_NoFields(t *testing.B) { benchmarkReplace(t, 0) } func benchmarkReplace(t *testing.B, nFields int) { rand.Seed(time.Now().UnixNano()) items := make([]testPointItem, t.N) for i := 0; i < t.N; i++ { items[i] = testPointItem{ fmt.Sprintf("%d", i), PO(rand.Float64()*360-180, rand.Float64()*180-90), makeBenchFields(nFields), } } col := New() for i := 0; i < t.N; i++ { col.Set(object.New(items[i].id, items[i].object, 0, items[i].fields)) } t.ResetTimer() for _, i := range rand.Perm(t.N) { o := col.Set(object.New(items[i].id, items[i].object, 0, field.List{})) if o.Geo() != items[i].object { t.Fatal("shoot!") } } } func BenchmarkGet_Fields(t *testing.B) { benchmarkGet(t, 1) } func BenchmarkGet_NoFields(t *testing.B) { benchmarkGet(t, 0) } func benchmarkGet(t *testing.B, nFields int) { rand.Seed(time.Now().UnixNano()) items := make([]testPointItem, t.N) for i := 0; i < t.N; i++ { items[i] = testPointItem{ fmt.Sprintf("%d", i), PO(rand.Float64()*360-180, rand.Float64()*180-90), makeBenchFields(nFields), } } col := New() for i := 0; i < t.N; i++ { col.Set(object.New(items[i].id, items[i].object, 0, items[i].fields)) } t.ResetTimer() for _, i := range rand.Perm(t.N) { o := col.Get(items[i].id) if o.Geo() != items[i].object { t.Fatal("shoot!") } } } func BenchmarkRemove_Fields(t *testing.B) { benchmarkRemove(t, 1) } func BenchmarkRemove_NoFields(t *testing.B) { benchmarkRemove(t, 0) } func benchmarkRemove(t *testing.B, nFields int) { rand.Seed(time.Now().UnixNano()) items := make([]testPointItem, t.N) for i := 0; i < t.N; i++ { items[i] = testPointItem{ fmt.Sprintf("%d", i), PO(rand.Float64()*360-180, rand.Float64()*180-90), makeBenchFields(nFields), } } col := New() for i := 0; i < t.N; i++ { col.Set(object.New(items[i].id, items[i].object, 0, items[i].fields)) } t.ResetTimer() for _, i := range rand.Perm(t.N) { prev := col.Delete(items[i].id) if prev.Geo() != items[i].object { t.Fatal("shoot!") } } } func BenchmarkScan_Fields(t *testing.B) { benchmarkScan(t, 1) } func BenchmarkScan_NoFields(t *testing.B) { benchmarkScan(t, 0) } func benchmarkScan(t *testing.B, nFields int) { rand.Seed(time.Now().UnixNano()) items := make([]testPointItem, t.N) for i := 0; i < t.N; i++ { items[i] = testPointItem{ fmt.Sprintf("%d", i), PO(rand.Float64()*360-180, rand.Float64()*180-90), makeBenchFields(nFields), } } col := New() for i := 0; i < t.N; i++ { col.Set(object.New(items[i].id, items[i].object, 0, items[i].fields)) } t.ResetTimer() for i := 0; i < t.N; i++ { var scanIteration int col.Scan(true, nil, nil, func(o *object.Object) bool { scanIteration++ return scanIteration <= 500 }) } } ================================================ FILE: internal/collection/geodesic.go ================================================ package collection import ( "math" "github.com/tidwall/tile38/internal/object" ) func geodeticDistAlgo(center [2]float64) ( algo func(min, max [2]float64, obj *object.Object, item bool) (dist float64), ) { const earthRadius = 6371e3 return func(min, max [2]float64, obj *object.Object, item bool) (dist float64) { if item { r := obj.Rect() min[0] = r.Min.X min[1] = r.Min.Y max[0] = r.Max.X max[1] = r.Max.Y } return earthRadius * pointRectDistGeodeticDeg( center[1], center[0], min[1], min[0], max[1], max[0], ) } } func pointRectDistGeodeticDeg(pLat, pLng, minLat, minLng, maxLat, maxLng float64) float64 { result := pointRectDistGeodeticRad( pLat*math.Pi/180, pLng*math.Pi/180, minLat*math.Pi/180, minLng*math.Pi/180, maxLat*math.Pi/180, maxLng*math.Pi/180, ) return result } func pointRectDistGeodeticRad(φq, λq, φl, λl, φh, λh float64) float64 { // Algorithm from: // Schubert, E., Zimek, A., & Kriegel, H.-P. (2013). // Geodetic Distance Queries on R-Trees for Indexing Geographic Data. // Lecture Notes in Computer Science, 146–164. // doi:10.1007/978-3-642-40235-7_9 const ( twoΠ = 2 * math.Pi halfΠ = math.Pi / 2 ) // distance on the unit sphere computed using Haversine formula distRad := func(φa, λa, φb, λb float64) float64 { if φa == φb && λa == λb { return 0 } Δφ := φa - φb Δλ := λa - λb sinΔφ := math.Sin(Δφ / 2) sinΔλ := math.Sin(Δλ / 2) cosφa := math.Cos(φa) cosφb := math.Cos(φb) return 2 * math.Asin(math.Sqrt(sinΔφ*sinΔφ+sinΔλ*sinΔλ*cosφa*cosφb)) } // Simple case, point or invalid rect if φl >= φh && λl >= λh { return distRad(φl, λl, φq, λq) } if λl <= λq && λq <= λh { // q is between the bounding meridians of r // hence, q is north, south or within r if φl <= φq && φq <= φh { // Inside return 0 } if φq < φl { // South return φl - φq } return φq - φh // North } // determine if q is closer to the east or west edge of r to select edge for // tests below Δλe := λl - λq Δλw := λq - λh if Δλe < 0 { Δλe += twoΠ } if Δλw < 0 { Δλw += twoΠ } var Δλ float64 // distance to closest edge var λedge float64 // longitude of closest edge if Δλe <= Δλw { Δλ = Δλe λedge = λl } else { Δλ = Δλw λedge = λh } sinΔλ, cosΔλ := math.Sincos(Δλ) tanφq := math.Tan(φq) if Δλ >= halfΠ { // If Δλ > 90 degrees (1/2 pi in radians) we're in one of the corners // (NW/SW or NE/SE depending on the edge selected). Compare against the // center line to decide which case we fall into φmid := (φh + φl) / 2 if tanφq >= math.Tan(φmid)*cosΔλ { return distRad(φq, λq, φh, λedge) // North corner } return distRad(φq, λq, φl, λedge) // South corner } if tanφq >= math.Tan(φh)*cosΔλ { return distRad(φq, λq, φh, λedge) // North corner } if tanφq <= math.Tan(φl)*cosΔλ { return distRad(φq, λq, φl, λedge) // South corner } // We're to the East or West of the rect, compute distance using cross-track // Note that this is a simplification of the cross track distance formula // valid since the track in question is a meridian. return math.Asin(math.Cos(φq) * sinΔλ) } ================================================ FILE: internal/collection/string.go ================================================ package collection import ( "encoding/json" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" ) type String string var _ geojson.Object = String("") func (s String) Spatial() geojson.Spatial { return geojson.EmptySpatial{} } func (s String) ForEach(iter func(geom geojson.Object) bool) bool { return iter(s) } func (s String) Empty() bool { return true } func (s String) Valid() bool { return false } func (s String) Rect() geometry.Rect { return geometry.Rect{} } func (s String) Center() geometry.Point { return geometry.Point{} } func (s String) AppendJSON(dst []byte) []byte { data, _ := json.Marshal(string(s)) return append(dst, data...) } func (s String) String() string { return string(s) } func (s String) JSON() string { return string(s.AppendJSON(nil)) } func (s String) MarshalJSON() ([]byte, error) { return s.AppendJSON(nil), nil } func (s String) Within(obj geojson.Object) bool { return false } func (s String) Contains(obj geojson.Object) bool { return false } func (s String) Intersects(obj geojson.Object) bool { return false } func (s String) NumPoints() int { return 0 } func (s String) Distance(obj geojson.Object) float64 { return 0 } func (s String) Members() string { return "" } ================================================ FILE: internal/deadline/deadline.go ================================================ package deadline import "time" // Deadline allows for commands to expire when they run too long type Deadline struct { unixNano int64 hit bool } // New returns a new deadline object func New(dl time.Time) *Deadline { return &Deadline{unixNano: dl.UnixNano()} } // Check the deadline and panic when reached // //go:noinline func (dl *Deadline) Check() { if dl == nil || dl.unixNano == 0 { return } if !dl.hit && time.Now().UnixNano() > dl.unixNano { dl.hit = true panic("deadline") } } // Hit returns true if the deadline has been hit func (dl *Deadline) Hit() bool { return dl.hit } // GetDeadlineTime returns the time object for the deadline, and an // "empty" boolean func (dl *Deadline) GetDeadlineTime() time.Time { return time.Unix(0, dl.unixNano) } ================================================ FILE: internal/endpoint/amqp.go ================================================ package endpoint import ( "fmt" "net" "sync" "time" "github.com/streadway/amqp" ) const amqpExpiresAfter = time.Second * 30 // AMQPConn is an endpoint connection type AMQPConn struct { mu sync.Mutex ep Endpoint conn *amqp.Connection channel *amqp.Channel ex bool t time.Time } // Expired returns true if the connection has expired func (conn *AMQPConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > amqpExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *AMQPConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *AMQPConn) close() { if conn.conn != nil { conn.conn.Close() conn.conn = nil conn.channel = nil } } // Send sends a message func (conn *AMQPConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() if conn.conn == nil { prefix := "amqp://" if conn.ep.AMQP.SSL { prefix = "amqps://" } var cfg amqp.Config cfg.Dial = func(network, addr string) (net.Conn, error) { return net.DialTimeout(network, addr, time.Second) } c, err := amqp.DialConfig(fmt.Sprintf("%s%s", prefix, conn.ep.AMQP.URI), cfg) if err != nil { return err } channel, err := c.Channel() if err != nil { return err } // Declare new exchange if err := channel.ExchangeDeclare( conn.ep.AMQP.QueueName, conn.ep.AMQP.Type, conn.ep.AMQP.Durable, conn.ep.AMQP.AutoDelete, conn.ep.AMQP.Internal, conn.ep.AMQP.NoWait, nil, ); err != nil { return err } if conn.ep.AMQP.Type != "topic" { // Create queue if queue don't exists if _, err := channel.QueueDeclare( conn.ep.AMQP.QueueName, conn.ep.AMQP.Durable, conn.ep.AMQP.AutoDelete, false, conn.ep.AMQP.NoWait, nil, ); err != nil { return err } // Binding exchange to queue if err := channel.QueueBind( conn.ep.AMQP.QueueName, conn.ep.AMQP.RouteKey, conn.ep.AMQP.QueueName, conn.ep.AMQP.NoWait, nil, ); err != nil { return err } } conn.conn = c conn.channel = channel } return conn.channel.Publish( conn.ep.AMQP.QueueName, conn.ep.AMQP.RouteKey, conn.ep.AMQP.Mandatory, conn.ep.AMQP.Immediate, amqp.Publishing{ Headers: amqp.Table{}, ContentType: "application/json", ContentEncoding: "", Body: []byte(msg), DeliveryMode: conn.ep.AMQP.DeliveryMode, Priority: conn.ep.AMQP.Priority, }, ) } func newAMQPConn(ep Endpoint) *AMQPConn { return &AMQPConn{ ep: ep, t: time.Now(), } } ================================================ FILE: internal/endpoint/cfqueue.go ================================================ package endpoint import ( "context" "sync" "time" "github.com/cloudflare/cloudflare-go/v4" "github.com/cloudflare/cloudflare-go/v4/option" "github.com/cloudflare/cloudflare-go/v4/queues" ) const cfqueueExpiresAfter = time.Second * 30 // CFQueueConn is an endpoint connection type CFQueueConn struct { mu sync.Mutex ep Endpoint client *cloudflare.Client ex bool t time.Time } // Expired returns true if the connection has expired func (conn *CFQueueConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > cfqueueExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *CFQueueConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *CFQueueConn) close() { if conn.client != nil { conn.client = nil } } // Send sends a message func (conn *CFQueueConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() // Initialize client if not already done if conn.client == nil { conn.client = cloudflare.NewClient( option.WithAPIToken(conn.ep.CFQueue.APIToken), ) } // Push message to CF Queue _, err := conn.client.Queues.Messages.Push( context.TODO(), conn.ep.CFQueue.QueueID, queues.MessagePushParams{ AccountID: cloudflare.String(conn.ep.CFQueue.AccountID), Body: queues.MessagePushParamsBodyMqQueueMessageText{ Body: cloudflare.String(msg), ContentType: cloudflare.F(queues.MessagePushParamsBodyMqQueueMessageTextContentTypeText), }, }, ) if err != nil { return err } return nil } func newCFQueueConn(ep Endpoint) *CFQueueConn { return &CFQueueConn{ ep: ep, t: time.Now(), } } ================================================ FILE: internal/endpoint/disque.go ================================================ package endpoint import ( "fmt" "sync" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/tile38/internal/log" ) const disqueExpiresAfter = time.Second * 30 // DisqueConn is an endpoint connection type DisqueConn struct { mu sync.Mutex ep Endpoint ex bool t time.Time conn redis.Conn } func newDisqueConn(ep Endpoint) *DisqueConn { return &DisqueConn{ ep: ep, t: time.Now(), } } // Expired returns true if the connection has expired func (conn *DisqueConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > disqueExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *DisqueConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *DisqueConn) close() { if conn.conn != nil { conn.conn.Close() conn.conn = nil } } // Send sends a message func (conn *DisqueConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() if conn.conn == nil { addr := fmt.Sprintf("%s:%d", conn.ep.Disque.Host, conn.ep.Disque.Port) var err error conn.conn, err = redis.Dial("tcp", addr) if err != nil { return err } } var args []interface{} args = append(args, conn.ep.Disque.QueueName, msg, 0) if conn.ep.Disque.Options.Replicate > 0 { args = append(args, "REPLICATE", conn.ep.Disque.Options.Replicate) } reply, err := redis.String(conn.conn.Do("ADDJOB", args...)) if err != nil { conn.close() return err } log.Debugf("Disque: ADDJOB '%s'", reply) return nil } ================================================ FILE: internal/endpoint/endpoint.go ================================================ package endpoint import ( "errors" "net/url" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/streadway/amqp" ) var errExpired = errors.New("expired") // Protocol is the type of protocol that the endpoint represents. type Protocol string const ( // Local protocol Local = Protocol("local") // HTTP protocol HTTP = Protocol("http") // Disque protocol Disque = Protocol("disque") // GRPC protocol GRPC = Protocol("grpc") // Redis protocol Redis = Protocol("redis") // Kafka protocol Kafka = Protocol("kafka") // MQTT protocol MQTT = Protocol("mqtt") // AMQP protocol AMQP = Protocol("amqp") // SQS protocol SQS = Protocol("sqs") // Google Cloud Pubsub protocol PubSub = Protocol("pubsub") // NATS protocol NATS = Protocol("nats") // EventHub protocol EventHub = Protocol("sb") // CFQueue protocol CFQueue = Protocol("cf-queue") ) // Endpoint represents an endpoint. type Endpoint struct { Protocol Protocol Original string GRPC struct { Host string Port int } Disque struct { Host string Port int QueueName string Options struct { Replicate int } } Redis struct { Host string Port int Channel string } Kafka struct { Host string Port int TopicName string Auth string SSL bool SASLSHA256 bool SASLSHA512 bool CACertFile string CertFile string KeyFile string } AMQP struct { URI string SSL bool QueueName string RouteKey string Type string Durable bool AutoDelete bool Internal bool NoWait bool Mandatory bool Immediate bool DeliveryMode uint8 Priority uint8 } MQTT struct { Host string Port int QueueName string Qos byte Retained bool CACertFile string CertFile string KeyFile string } PubSub struct { Project string Topic string CredPath string } SQS struct { PlainURL string QueueID string Region string CredPath string CredProfile string QueueName string CreateQueue bool } NATS struct { Host string Port int User string Pass string Topic string Token string TLS bool TLSCert string TLSKey string Secure bool // Jetstream indicates publishing via jetstream acknowledgements. Jetstream bool UserCredentialPath string } EventHub struct { ConnectionString string } CFQueue struct { AccountID string QueueID string APIToken string } Local struct { Channel string } } // Conn is an endpoint connection type Conn interface { ExpireNow() Expired() bool Send(val string) error } // Manager manages all endpoints type Manager struct { mu sync.RWMutex conns map[string]Conn publisher LocalPublisher shutdown atomic.Bool // atomic bool wg sync.WaitGroup // run wait group } // NewManager returns a new manager func NewManager(publisher LocalPublisher) *Manager { epc := &Manager{ conns: make(map[string]Conn), publisher: publisher, } epc.wg.Add(1) go epc.run() return epc } func (epc *Manager) Shutdown() { defer epc.wg.Wait() epc.shutdown.Store(true) // expire the connections epc.mu.Lock() defer epc.mu.Unlock() for _, conn := range epc.conns { conn.ExpireNow() } } // Run starts the managing of endpoints func (epc *Manager) run() { defer epc.wg.Done() for { if epc.shutdown.Load() { return } time.Sleep(time.Second) func() { epc.mu.Lock() defer epc.mu.Unlock() for endpoint, conn := range epc.conns { if conn.Expired() { delete(epc.conns, endpoint) } } }() } } // Validate an endpoint url func (epc *Manager) Validate(url string) error { _, err := parseEndpoint(url) return err } // Send send a message to an endpoint func (epc *Manager) Send(endpoint, msg string) error { for { epc.mu.Lock() conn, exists := epc.conns[endpoint] if !exists || conn.Expired() { ep, err := parseEndpoint(endpoint) if err != nil { epc.mu.Unlock() return err } switch ep.Protocol { default: return errors.New("invalid protocol") case HTTP: conn = newHTTPConn(ep) case Disque: conn = newDisqueConn(ep) case GRPC: conn = newGRPCConn(ep) case Redis: conn = newRedisConn(ep) case Kafka: conn = newKafkaConn(ep) case MQTT: conn = newMQTTConn(ep) case AMQP: conn = newAMQPConn(ep) case PubSub: conn = newPubSubConn(ep) case SQS: conn = newSQSConn(ep) case NATS: conn = newNATSConn(ep) case Local: conn = newLocalConn(ep, epc.publisher) case EventHub: conn = newEventHubConn(ep) case CFQueue: conn = newCFQueueConn(ep) } epc.conns[endpoint] = conn } epc.mu.Unlock() err := conn.Send(msg) if err != nil { if err == errExpired { // it's possible that the connection has expired in-between // the last conn.Expired() check and now. If so, we should // just try the send again. continue } return err } return nil } } func parseEndpoint(s string) (Endpoint, error) { var endpoint Endpoint endpoint.Original = s switch { default: return endpoint, errors.New("unknown scheme") case strings.HasPrefix(s, "local:"): endpoint.Protocol = Local case strings.HasPrefix(s, "http:"): endpoint.Protocol = HTTP case strings.HasPrefix(s, "https:"): if probeSQS(s) { endpoint.SQS.PlainURL = s endpoint.Protocol = SQS } else { endpoint.Protocol = HTTP } case strings.HasPrefix(s, "disque:"): endpoint.Protocol = Disque case strings.HasPrefix(s, "grpc:"): endpoint.Protocol = GRPC case strings.HasPrefix(s, "redis:"): endpoint.Protocol = Redis case strings.HasPrefix(s, "kafka:"): endpoint.Protocol = Kafka case strings.HasPrefix(s, "amqp:"): endpoint.Protocol = AMQP case strings.HasPrefix(s, "amqps:"): endpoint.Protocol = AMQP case strings.HasPrefix(s, "mqtt:"): endpoint.Protocol = MQTT case strings.HasPrefix(s, "pubsub:"): endpoint.Protocol = PubSub case strings.HasPrefix(s, "sqs:"): endpoint.Protocol = SQS case strings.HasPrefix(s, "nats:"): endpoint.Protocol = NATS case strings.HasPrefix(s, "Endpoint="): endpoint.Protocol = EventHub case strings.HasPrefix(s, "cf-queue:"): endpoint.Protocol = CFQueue } s = s[strings.Index(s, ":")+1:] if !strings.HasPrefix(s, "//") { return endpoint, errors.New("missing the two slashes") } sqp := strings.Split(s[2:], "?") sp := strings.Split(sqp[0], "/") s = sp[0] if s == "" { if endpoint.Protocol == Local { return endpoint, errors.New("missing channel") } return endpoint, errors.New("missing host") } // Local PubSub channel // local:// if endpoint.Protocol == Local { endpoint.Local.Channel = s } if endpoint.Protocol == GRPC { dp := strings.Split(s, ":") switch len(dp) { default: return endpoint, errors.New("invalid grpc url") case 1: endpoint.GRPC.Host = dp[0] endpoint.GRPC.Port = 80 case 2: endpoint.GRPC.Host = dp[0] n, err := strconv.ParseUint(dp[1], 10, 16) if err != nil { return endpoint, errors.New("invalid grpc url") } endpoint.GRPC.Port = int(n) } } if endpoint.Protocol == Redis { dp := strings.Split(s, ":") switch len(dp) { default: return endpoint, errors.New("invalid redis url") case 1: endpoint.Redis.Host = dp[0] endpoint.Redis.Port = 6379 case 2: endpoint.Redis.Host = dp[0] n, err := strconv.ParseUint(dp[1], 10, 16) if err != nil { return endpoint, errors.New("invalid redis url port") } endpoint.Redis.Port = int(n) } if len(sp) > 1 { var err error endpoint.Redis.Channel, err = url.QueryUnescape(sp[1]) if err != nil { return endpoint, errors.New("invalid redis channel name") } } } if endpoint.Protocol == Disque { dp := strings.Split(s, ":") switch len(dp) { default: return endpoint, errors.New("invalid disque url") case 1: endpoint.Disque.Host = dp[0] endpoint.Disque.Port = 7711 case 2: endpoint.Disque.Host = dp[0] n, err := strconv.ParseUint(dp[1], 10, 16) if err != nil { return endpoint, errors.New("invalid disque url") } endpoint.Disque.Port = int(n) } if len(sp) > 1 { var err error endpoint.Disque.QueueName, err = url.QueryUnescape(sp[1]) if err != nil { return endpoint, errors.New("invalid disque queue name") } } if len(sqp) > 1 { m, err := url.ParseQuery(sqp[1]) if err != nil { return endpoint, errors.New("invalid disque url") } for key, val := range m { if len(val) == 0 { continue } switch key { case "replicate": n, err := strconv.ParseUint(val[0], 10, 8) if err != nil { return endpoint, errors.New("invalid disque replicate value") } endpoint.Disque.Options.Replicate = int(n) } } } if endpoint.Disque.QueueName == "" { return endpoint, errors.New("missing disque queue name") } } if endpoint.Protocol == Kafka { // Parsing connection from URL string hp := strings.Split(s, ":") switch len(hp) { default: return endpoint, errors.New("invalid kafka url") case 1: endpoint.Kafka.Host = hp[0] endpoint.Kafka.Port = 9092 case 2: n, err := strconv.ParseUint(hp[1], 10, 16) if err != nil { return endpoint, errors.New("invalid kafka url port") } endpoint.Kafka.Host = hp[0] endpoint.Kafka.Port = int(n) } // Parsing Kafka queue name if len(sp) > 1 { var err error endpoint.Kafka.TopicName, err = url.QueryUnescape(sp[1]) if err != nil { return endpoint, errors.New("invalid kafka topic name") } } // Throw error if we not provide any queue name if endpoint.Kafka.TopicName == "" { return endpoint, errors.New("missing kafka topic name") } // Parsing additional params if len(sqp) > 1 { m, err := url.ParseQuery(sqp[1]) if err != nil { return endpoint, errors.New("invalid kafka url") } for key, val := range m { if len(val) == 0 { continue } switch key { case "auth": endpoint.Kafka.Auth = val[0] case "ssl": endpoint.Kafka.SSL, _ = strconv.ParseBool(val[0]) case "cacert": endpoint.Kafka.CACertFile = val[0] case "cert": endpoint.Kafka.CertFile = val[0] case "key": endpoint.Kafka.KeyFile = val[0] case "sha256": endpoint.Kafka.SASLSHA256, _ = strconv.ParseBool(val[0]) case "sha512": endpoint.Kafka.SASLSHA512, _ = strconv.ParseBool(val[0]) } } } } if endpoint.Protocol == MQTT { // Parsing connection from URL string hp := strings.Split(s, ":") switch len(hp) { default: return endpoint, errors.New("invalid MQTT url") case 1: endpoint.MQTT.Host = hp[0] endpoint.MQTT.Port = 1883 case 2: n, err := strconv.ParseUint(hp[1], 10, 16) if err != nil { return endpoint, errors.New("invalid MQTT url port") } endpoint.MQTT.Host = hp[0] endpoint.MQTT.Port = int(n) } // Parsing MQTT queue name if len(sp) > 1 { var err error var parts []string for _, part := range sp[1:] { part, err = url.QueryUnescape(part) if err != nil { return endpoint, errors.New("invalid MQTT topic name") } parts = append(parts, part) } endpoint.MQTT.QueueName = strings.Join(parts, "/") } // Parsing additional params if len(sqp) > 1 { m, err := url.ParseQuery(sqp[1]) if err != nil { return endpoint, errors.New("invalid MQTT url") } for key, val := range m { if len(val) == 0 { continue } switch key { case "qos": n, err := strconv.ParseUint(val[0], 10, 8) if err != nil { return endpoint, errors.New("invalid MQTT qos value") } endpoint.MQTT.Qos = byte(n) case "retained": n, err := strconv.ParseUint(val[0], 10, 8) if err != nil { return endpoint, errors.New("invalid MQTT retained value") } if n != 1 && n != 0 { return endpoint, errors.New("invalid MQTT retained, should be [0, 1]") } if n == 1 { endpoint.MQTT.Retained = true } case "cacert": endpoint.MQTT.CACertFile = val[0] case "cert": endpoint.MQTT.CertFile = val[0] case "key": endpoint.MQTT.KeyFile = val[0] } } } // Throw error if we not provide any queue name if endpoint.MQTT.QueueName == "" { return endpoint, errors.New("missing MQTT topic name") } } // Basic SQS connection strings in HOOKS interface // sqs://://?params=value // // params are: // // credpath - path where aws credentials are located // credprofile - credential profile if endpoint.Protocol == SQS { if endpoint.SQS.PlainURL == "" { // Parsing connection from URL string hp := strings.Split(s, ":") switch len(hp) { default: return endpoint, errors.New("invalid SQS url") case 2: endpoint.SQS.Region = hp[0] endpoint.SQS.QueueID = hp[1] } // Parsing SQS queue name if len(sp) > 1 { var err error endpoint.SQS.QueueName, err = url.QueryUnescape(sp[1]) if err != nil { return endpoint, errors.New("invalid SQS queue name") } } // Throw error if we not provide any queue name if endpoint.SQS.QueueName == "" { return endpoint, errors.New("missing SQS queue name") } } // Parsing additional params if len(sqp) > 1 { m, err := url.ParseQuery(sqp[1]) if err != nil { return endpoint, errors.New("invalid SQS url") } for key, val := range m { if len(val) == 0 { continue } switch key { case "credpath": endpoint.SQS.CredPath = val[0] case "credprofile": endpoint.SQS.CredProfile = val[0] case "createqueue": switch strings.ToLower(val[0]) { case "0", "false": default: endpoint.SQS.CreateQueue = true } } } } } // Basic Pubsub connection strings in HOOKS interface // pubsub://:?params=value // // params are: // // credpath - path where gcp credentials are located if endpoint.Protocol == PubSub { split := strings.Split(s, ":") if len(split) != 2 { return endpoint, errors.New("invalid PubSub format should be project/topic") } endpoint.PubSub.Project = split[0] endpoint.PubSub.Topic = split[1] if len(sqp) > 1 { m, err := url.ParseQuery(sqp[1]) if err != nil { return endpoint, errors.New("invalid Pubsub url") } for key, val := range m { if len(val) == 0 { continue } switch key { case "credpath": endpoint.PubSub.CredPath = val[0] } } } } // Basic AMQP connection strings in HOOKS interface // amqp://guest:guest@localhost:5672//?params=value // or amqp://guest:guest@localhost:5672///?params=value // // Default params are: // // Mandatory - false // Immeditate - false // Durable - true // Routing-Key - tile38 // // - "route" - [string] routing key // if endpoint.Protocol == AMQP { // Bind connection information endpoint.AMQP.URI = s endpoint.AMQP.Type = "direct" endpoint.AMQP.Durable = true endpoint.AMQP.DeliveryMode = amqp.Transient // Fix incase of namespace, e.g. example.com/namespace/queue // but not example.com/queue/ - with an endslash. if len(sp) > 2 && len(sp[2]) > 0 { endpoint.AMQP.URI = endpoint.AMQP.URI + "/" + sp[1] sp = append([]string{endpoint.AMQP.URI}, sp[2:]...) } // Bind queue name with no namespace if len(sp) > 1 { var err error endpoint.AMQP.QueueName, err = url.QueryUnescape(sp[1]) if err != nil { return endpoint, errors.New("invalid AMQP queue name") } } // Parsing additional attributes if len(sqp) > 1 { m, err := url.ParseQuery(sqp[1]) if err != nil { return endpoint, errors.New("invalid AMQP url") } for key, val := range m { if len(val) == 0 { continue } switch key { case "route": endpoint.AMQP.RouteKey = val[0] case "type": endpoint.AMQP.Type = val[0] case "durable": endpoint.AMQP.Durable = queryBool(val[0]) case "internal": endpoint.AMQP.Internal = queryBool(val[0]) case "no_wait": endpoint.AMQP.NoWait = queryBool(val[0]) case "auto_delete": endpoint.AMQP.AutoDelete = queryBool(val[0]) case "immediate": endpoint.AMQP.Immediate = queryBool(val[0]) case "mandatory": endpoint.AMQP.Mandatory = queryBool(val[0]) case "delivery_mode": endpoint.AMQP.DeliveryMode = uint8(queryInt(val[0])) case "priority": endpoint.AMQP.Priority = uint8(queryInt(val[0])) } } } if strings.HasPrefix(endpoint.Original, "amqps:") { endpoint.AMQP.SSL = true } if endpoint.AMQP.QueueName == "" { return endpoint, errors.New("missing AMQP queue name") } if endpoint.AMQP.RouteKey == "" { endpoint.AMQP.RouteKey = "tile38" } } // Basic NATS connection strings in HOOKS interface // nats://://?params=value // // params are: // // user - username // pass - password // when user or pass is not set then login without password is used if endpoint.Protocol == NATS { // Parsing connection from URL string hp := strings.Split(s, ":") switch len(hp) { default: return endpoint, errors.New("invalid SQS url") case 2: endpoint.NATS.Host = hp[0] port, err := strconv.Atoi(hp[1]) if err != nil { endpoint.NATS.Port = 4222 // default nats port } else { endpoint.NATS.Port = port } } // Parsing NATS topic name if len(sp) > 1 { var err error endpoint.NATS.Topic, err = url.QueryUnescape(sp[1]) if err != nil { return endpoint, errors.New("invalid NATS topic name") } } // Parsing additional params if len(sqp) > 1 { m, err := url.ParseQuery(sqp[1]) if err != nil { return endpoint, errors.New("invalid NATS url") } for key, val := range m { if len(val) == 0 { continue } switch key { case "user": endpoint.NATS.User = val[0] case "pass": endpoint.NATS.Pass = val[0] case "token": endpoint.NATS.Token = val[0] case "secure": endpoint.NATS.Secure = queryBool(val[0]) case "credential": endpoint.NATS.UserCredentialPath = val[0] case "jetstream": endpoint.NATS.Jetstream = queryBool(val[0]) case "tls": endpoint.NATS.TLS = queryBool(val[0]) case "tlscert": endpoint.NATS.TLSCert = val[0] case "tlskey": endpoint.NATS.TLSKey = val[0] } } } } if endpoint.Protocol == EventHub { dp := strings.Split(endpoint.Original, ";") if len(dp) != 4 { return endpoint, errors.New("malformed EventHub connection string") } sakn := strings.Split(dp[1], "=") if sakn[0] != "SharedAccessKeyName" { return endpoint, errors.New("missing SharedAccessKeyName") } sak := strings.Split(dp[2], "=") if sak[0] != "SharedAccessKey" { return endpoint, errors.New("missing SharedAccessKey") } ep := strings.Split(dp[3], "=") if ep[0] != "EntityPath" { return endpoint, errors.New("missing EntityPath") } endpoint.EventHub.ConnectionString = endpoint.Original } // Basic CF Queue connection strings in HOOKS interface // cf-queue:///?token= // // params are: // // token - API token if endpoint.Protocol == CFQueue { // Parse account_id/queue_id from the path parts if len(sp) < 2 { return endpoint, errors.New("invalid CF Queue format, should be account_id/queue_id") } endpoint.CFQueue.AccountID = sp[0] endpoint.CFQueue.QueueID = sp[1] // Parse query parameters for API token if len(sqp) > 1 { m, err := url.ParseQuery(sqp[1]) if err != nil { return endpoint, errors.New("invalid CF Queue url") } for key, val := range m { if len(val) == 0 { continue } switch key { case "token": endpoint.CFQueue.APIToken = val[0] } } } if endpoint.CFQueue.AccountID == "" { return endpoint, errors.New("missing CF Queue account ID") } if endpoint.CFQueue.QueueID == "" { return endpoint, errors.New("missing CF Queue queue ID") } if endpoint.CFQueue.APIToken == "" { return endpoint, errors.New("missing CF Queue API token") } } return endpoint, nil } func queryInt(s string) int { x, _ := strconv.ParseInt(s, 10, 64) return int(x) } func queryBool(s string) bool { if len(s) > 0 { if s[0] >= '1' && s[0] <= '9' { return true } switch s[0] { case 'Y', 'y', 'T', 't': return true } } return false } ================================================ FILE: internal/endpoint/eventHub.go ================================================ package endpoint import ( "context" "fmt" "time" "github.com/tidwall/gjson" eventhub "github.com/Azure/azure-event-hubs-go/v3" ) const () // HTTPConn is an endpoint connection type EvenHubConn struct { ep Endpoint } func newEventHubConn(ep Endpoint) *EvenHubConn { return &EvenHubConn{ ep: ep, } } // Expired returns true if the connection has expired func (conn *EvenHubConn) Expired() bool { return false } // ExpireNow forces the connection to expire func (conn *EvenHubConn) ExpireNow() { } // Send sends a message func (conn *EvenHubConn) Send(msg string) error { hub, err := eventhub.NewHubFromConnectionString(conn.ep.EventHub.ConnectionString) if err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() // parse json again to get out info for our kafka key key := gjson.Get(msg, "key") id := gjson.Get(msg, "id") keyValue := fmt.Sprintf("%s-%s", key.String(), id.String()) evtHubMsg := eventhub.NewEventFromString(msg) evtHubMsg.PartitionKey = &keyValue err = hub.Send(ctx, evtHubMsg) if err != nil { return err } return nil } ================================================ FILE: internal/endpoint/grpc.go ================================================ package endpoint import ( "errors" "fmt" "sync" "time" "github.com/tidwall/tile38/internal/hservice" "golang.org/x/net/context" "google.golang.org/grpc" ) const grpcExpiresAfter = time.Second * 30 // GRPCConn is an endpoint connection type GRPCConn struct { mu sync.Mutex ep Endpoint ex bool t time.Time conn *grpc.ClientConn sconn hservice.HookServiceClient } func newGRPCConn(ep Endpoint) *GRPCConn { return &GRPCConn{ ep: ep, t: time.Now(), } } // Expired returns true if the connection has expired func (conn *GRPCConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > grpcExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *GRPCConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *GRPCConn) close() { if conn.conn != nil { conn.conn.Close() conn.conn = nil } } // Send sends a message func (conn *GRPCConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() if conn.conn == nil { addr := fmt.Sprintf("%s:%d", conn.ep.GRPC.Host, conn.ep.GRPC.Port) var err error conn.conn, err = grpc.Dial(addr, grpc.WithInsecure()) if err != nil { conn.close() return err } conn.sconn = hservice.NewHookServiceClient(conn.conn) } r, err := conn.sconn.Send(context.Background(), &hservice.MessageRequest{Value: msg}) if err != nil { conn.close() return err } if !r.Ok { conn.close() return errors.New("invalid grpc reply") } return nil } ================================================ FILE: internal/endpoint/http.go ================================================ package endpoint import ( "bytes" "fmt" "io" "net/http" "time" ) const ( httpExpiresAfter = time.Second * 30 httpRequestTimeout = time.Second * 5 httpMaxIdleConnections = 20 ) // HTTPConn is an endpoint connection type HTTPConn struct { ep Endpoint client *http.Client } func newHTTPConn(ep Endpoint) *HTTPConn { return &HTTPConn{ ep: ep, client: &http.Client{ Transport: &http.Transport{ MaxIdleConnsPerHost: httpMaxIdleConnections, IdleConnTimeout: httpExpiresAfter, }, Timeout: httpRequestTimeout, }, } } // Expired returns true if the connection has expired func (conn *HTTPConn) Expired() bool { return false } // ExpireNow forces the connection to expire func (conn *HTTPConn) ExpireNow() { } // Send sends a message func (conn *HTTPConn) Send(msg string) error { req, err := http.NewRequest("POST", conn.ep.Original, bytes.NewBufferString(msg)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") resp, err := conn.client.Do(req) if err != nil { return err } // close the connection to reuse it defer resp.Body.Close() // discard response if _, err := io.Copy(io.Discard, resp.Body); err != nil { return err } // Only allow responses with status code 200, 201, and 202 if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted { return fmt.Errorf("invalid status: %s", resp.Status) } return nil } ================================================ FILE: internal/endpoint/kafka.go ================================================ package endpoint import ( "crypto/tls" "crypto/x509" "errors" "fmt" "os" "sync" "time" lg "log" "github.com/IBM/sarama" "github.com/tidwall/gjson" "github.com/tidwall/tile38/internal/log" ) const kafkaExpiresAfter = time.Second * 30 // KafkaConn is an endpoint connection type KafkaConn struct { mu sync.Mutex ep Endpoint conn sarama.SyncProducer cfg *sarama.Config ex bool t time.Time } // Expired returns true if the connection has expired func (conn *KafkaConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > kafkaExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *KafkaConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *KafkaConn) close() { if conn.conn != nil { conn.conn.Close() conn.conn = nil conn.cfg.MetricRegistry.UnregisterAll() conn.cfg = nil } } // Send sends a message func (conn *KafkaConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() if log.Level() > 2 { sarama.Logger = lg.New(log.Output(), "[sarama] ", 0) } uri := fmt.Sprintf("%s:%d", conn.ep.Kafka.Host, conn.ep.Kafka.Port) if conn.conn == nil { cfg := sarama.NewConfig() cfg.Net.DialTimeout = time.Second cfg.Net.ReadTimeout = time.Second * 5 cfg.Net.WriteTimeout = time.Second * 5 // Fix #333 : fix backward incompatibility introduced by sarama library cfg.Producer.Return.Successes = true // Sarama now sets the version based on the broker version in the release 1.46.0 as of August 25 2025. // There is no need to force this anymore as it is breaking the version check in Kafka 4.0 and above. // cfg.Version = sarama.V0_10_0_0 switch conn.ep.Kafka.Auth { case "sasl": // This path allows to either provide a custom ca certificate // or, because RootCAs is nil, is using the hosts ca set // to verify the server certificate if conn.ep.Kafka.SSL { tlsConfig := tls.Config{} if conn.ep.Kafka.CACertFile != "" { caCertPool, err := loadRootTLSCert(conn.ep.Kafka.CACertFile) if err != nil { return err } tlsConfig.RootCAs = &caCertPool } cfg.Net.TLS.Enable = true cfg.Net.TLS.Config = &tlsConfig } cfg.Net.SASL.Enable = true cfg.Net.SASL.User = os.Getenv("KAFKA_USERNAME") cfg.Net.SASL.Password = os.Getenv("KAFKA_PASSWORD") cfg.Net.SASL.Handshake = true cfg.Net.SASL.Mechanism = sarama.SASLTypePlaintext if conn.ep.Kafka.SASLSHA256 { cfg.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA256} } cfg.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256 } if conn.ep.Kafka.SASLSHA512 { cfg.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA512} } cfg.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512 } case "tls": tlsConfig := tls.Config{} cfg.Net.TLS.Enable = true certificates, err := loadClientTLSCert(conn.ep.Kafka.KeyFile, conn.ep.Kafka.CertFile) if err != nil { cfg.MetricRegistry.UnregisterAll() return err } tlsConfig.Certificates = certificates // This path allows to either provide a custom ca certificate // or, because RootCAs is nil, is using the hosts ca set // to verify server certificate if conn.ep.Kafka.CACertFile != "" { caCertPool, err := loadRootTLSCert(conn.ep.Kafka.CACertFile) if err != nil { return err } tlsConfig.RootCAs = &caCertPool } cfg.Net.TLS.Config = &tlsConfig case "none": // This path allows to either provide a custom ca certificate // or, because RootCAs is nil, is using the hosts ca set // to verify the server certificate if conn.ep.Kafka.SSL { tlsConfig := tls.Config{} if conn.ep.Kafka.CACertFile != "" { caCertPool, err := loadRootTLSCert(conn.ep.Kafka.CACertFile) if err != nil { return err } tlsConfig.RootCAs = &caCertPool } cfg.Net.TLS.Enable = true cfg.Net.TLS.Config = &tlsConfig } } c, err := sarama.NewSyncProducer([]string{uri}, cfg) if err != nil { cfg.MetricRegistry.UnregisterAll() return err } conn.conn = c conn.cfg = cfg } // parse json again to get out info for our kafka key key := gjson.Get(msg, "key") id := gjson.Get(msg, "id") keyValue := fmt.Sprintf("%s-%s", key.String(), id.String()) message := &sarama.ProducerMessage{ Topic: conn.ep.Kafka.TopicName, Key: sarama.StringEncoder(keyValue), Value: sarama.StringEncoder(msg), } _, offset, err := conn.conn.SendMessage(message) if err != nil { conn.close() return err } if offset < 0 { conn.close() return errors.New("invalid kafka reply") } return nil } func newKafkaConn(ep Endpoint) *KafkaConn { return &KafkaConn{ ep: ep, t: time.Now(), } } func loadClientTLSCert(KeyFile, CertFile string) ([]tls.Certificate, error) { // load client cert cert, err := tls.LoadX509KeyPair(CertFile, KeyFile) if err != nil { return []tls.Certificate{cert}, err } return []tls.Certificate{cert}, err } func loadRootTLSCert(CACertFile string) (x509.CertPool, error) { // Load CA cert caCert, err := os.ReadFile(CACertFile) if err != nil { return x509.CertPool{}, err } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) return *caCertPool, err } ================================================ FILE: internal/endpoint/local.go ================================================ package endpoint // LocalPublisher is used to publish local notifications type LocalPublisher interface { Publish(channel string, message ...string) int } // LocalConn is an endpoint connection type LocalConn struct { ep Endpoint publisher LocalPublisher } func newLocalConn(ep Endpoint, publisher LocalPublisher) *LocalConn { return &LocalConn{ ep: ep, publisher: publisher, } } // Expired returns true if the connection has expired func (conn *LocalConn) Expired() bool { return false } // ExpireNow forces the connection to expire func (conn *LocalConn) ExpireNow() { } // Send sends a message func (conn *LocalConn) Send(msg string) error { conn.publisher.Publish(conn.ep.Local.Channel, msg) return nil } ================================================ FILE: internal/endpoint/mqtt.go ================================================ package endpoint import ( "crypto/tls" "crypto/x509" "fmt" "math/rand" "os" "sync" "time" paho "github.com/eclipse/paho.mqtt.golang" "github.com/tidwall/tile38/internal/log" ) const ( mqttExpiresAfter = time.Second * 30 mqttPublishTimeout = time.Second * 5 ) // MQTTConn is an endpoint connection type MQTTConn struct { mu sync.Mutex ep Endpoint conn paho.Client ex bool t time.Time } // Expired returns true if the connection has expired func (conn *MQTTConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > mqttExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *MQTTConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *MQTTConn) close() { if conn.conn != nil { if conn.conn.IsConnected() { conn.conn.Disconnect(250) } conn.conn = nil } } // Send sends a message func (conn *MQTTConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() if conn.conn == nil { uri := fmt.Sprintf("tcp://%s:%d", conn.ep.MQTT.Host, conn.ep.MQTT.Port) ops := paho.NewClientOptions() if conn.ep.MQTT.CertFile != "" || conn.ep.MQTT.KeyFile != "" || conn.ep.MQTT.CACertFile != "" { var config tls.Config if conn.ep.MQTT.CertFile != "" || conn.ep.MQTT.KeyFile != "" { cert, err := tls.LoadX509KeyPair(conn.ep.MQTT.CertFile, conn.ep.MQTT.KeyFile) if err != nil { return err } config.Certificates = append(config.Certificates, cert) } if conn.ep.MQTT.CACertFile != "" { // Load CA cert caCert, err := os.ReadFile(conn.ep.MQTT.CACertFile) if err != nil { return err } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) config.RootCAs = caCertPool } ops = ops.SetTLSConfig(&config) } //generate UUID for the client-id. b := make([]byte, 16) _, err := rand.Read(b) if err != nil { log.Debugf("Failed to generate guid for the mqtt client. The endpoint will not work") return err } uuid := fmt.Sprintf("tile38-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]) ops = ops.SetClientID(uuid).AddBroker(uri) c := paho.NewClient(ops) if token := c.Connect(); token.Wait() && token.Error() != nil { return token.Error() } conn.conn = c } t := conn.conn.Publish(conn.ep.MQTT.QueueName, conn.ep.MQTT.Qos, conn.ep.MQTT.Retained, msg) if !t.WaitTimeout(mqttPublishTimeout) || t.Error() != nil { conn.close() return t.Error() } return nil } func newMQTTConn(ep Endpoint) *MQTTConn { return &MQTTConn{ ep: ep, t: time.Now(), } } ================================================ FILE: internal/endpoint/nats.go ================================================ package endpoint import ( "context" "fmt" "sync" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) const natsExpiresAfter = time.Second * 30 // NATSConn is an endpoint connection type NATSConn struct { mu sync.Mutex ep Endpoint ex bool t time.Time conn *nats.Conn js jetstream.JetStream } func newNATSConn(ep Endpoint) *NATSConn { return &NATSConn{ ep: ep, t: time.Now(), } } // Expired returns true if the connection has expired func (conn *NATSConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > natsExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *NATSConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *NATSConn) close() { if conn.conn != nil { conn.conn.Close() conn.conn = nil } } // Send sends a message func (conn *NATSConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() if conn.conn == nil { addr := fmt.Sprintf("%s:%d", conn.ep.NATS.Host, conn.ep.NATS.Port) scheme := "nats" // 'nats://' by default var err error var opts []nats.Option if conn.ep.NATS.User != "" && conn.ep.NATS.Pass != "" { opts = append(opts, nats.UserInfo(conn.ep.NATS.User, conn.ep.NATS.Pass)) } if conn.ep.NATS.TLS { opts = append(opts, nats.ClientCert( conn.ep.NATS.TLSCert, conn.ep.NATS.TLSKey, )) } if conn.ep.NATS.Token != "" { opts = append(opts, nats.Token(conn.ep.NATS.Token)) } if conn.ep.NATS.UserCredentialPath != "" { opts = append(opts, nats.UserCredentials(conn.ep.NATS.UserCredentialPath)) } if conn.ep.NATS.Secure { scheme = "tls" } addr = fmt.Sprintf("%s://%s", scheme, addr) conn.conn, err = nats.Connect(addr, opts...) if err != nil { conn.close() return err } if conn.ep.NATS.Jetstream { conn.js, err = jetstream.New(conn.conn) if err != nil { conn.close() return err } } } if conn.js == nil { return conn.publish(msg) } return conn.publishJS(msg) } // publishJS will publish the message to the subject using core nats. func (conn *NATSConn) publish(msg string) error { err := conn.conn.Publish(conn.ep.NATS.Topic, []byte(msg)) if err != nil { conn.close() return err } return nil } // publishJS will publish the message expecting a jetstream acknowledgement. func (conn *NATSConn) publishJS(msg string) error { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() _, err := conn.js.Publish(ctx, conn.ep.NATS.Topic, []byte(msg)) if err != nil { conn.close() return err } return nil } ================================================ FILE: internal/endpoint/pubsub.go ================================================ package endpoint import ( "context" "fmt" "sync" "time" "cloud.google.com/go/pubsub" "google.golang.org/api/option" ) const pubsubExpiresAfter = time.Second * 30 // SQSConn is an endpoint connection type PubSubConn struct { mu sync.Mutex ep Endpoint svc *pubsub.Client topic *pubsub.Topic ex bool t time.Time } func (conn *PubSubConn) close() { if conn.svc != nil { conn.svc.Close() conn.svc = nil } } // Send sends a message func (conn *PubSubConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } ctx := context.Background() conn.t = time.Now() if conn.svc == nil { var creds option.ClientOption var svc *pubsub.Client var err error credPath := conn.ep.PubSub.CredPath if credPath != "" { creds = option.WithCredentialsFile(credPath) svc, err = pubsub.NewClient(ctx, conn.ep.PubSub.Project, creds) } else { svc, err = pubsub.NewClient(ctx, conn.ep.PubSub.Project) } if err != nil { fmt.Println(err) return err } topic := svc.Topic(conn.ep.PubSub.Topic) conn.svc = svc conn.topic = topic } // Send message res := conn.topic.Publish(ctx, &pubsub.Message{ Data: []byte(msg), }) _, err := res.Get(ctx) if err != nil { fmt.Println(err) return err } return nil } func (conn *PubSubConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > pubsubExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *PubSubConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func newPubSubConn(ep Endpoint) *PubSubConn { return &PubSubConn{ ep: ep, t: time.Now(), } } ================================================ FILE: internal/endpoint/redis.go ================================================ package endpoint import ( "fmt" "sync" "time" "github.com/gomodule/redigo/redis" ) const redisExpiresAfter = time.Second * 30 // RedisConn is an endpoint connection type RedisConn struct { mu sync.Mutex ep Endpoint ex bool t time.Time conn redis.Conn } func newRedisConn(ep Endpoint) *RedisConn { return &RedisConn{ ep: ep, t: time.Now(), } } // Expired returns true if the connection has expired func (conn *RedisConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > redisExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *RedisConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *RedisConn) close() { if conn.conn != nil { conn.conn.Close() conn.conn = nil } } // Send sends a message func (conn *RedisConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() if conn.conn == nil { addr := fmt.Sprintf("%s:%d", conn.ep.Redis.Host, conn.ep.Redis.Port) var err error conn.conn, err = redis.Dial("tcp", addr) if err != nil { conn.close() return err } } _, err := redis.Int(conn.conn.Do("PUBLISH", conn.ep.Redis.Channel, msg)) if err != nil { conn.close() return err } return nil } ================================================ FILE: internal/endpoint/scram_client.go ================================================ package endpoint import ( "crypto/sha256" "crypto/sha512" "github.com/xdg-go/scram" ) var ( SHA256 scram.HashGeneratorFcn = sha256.New SHA512 scram.HashGeneratorFcn = sha512.New ) type XDGSCRAMClient struct { *scram.Client *scram.ClientConversation scram.HashGeneratorFcn } func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) { x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID) if err != nil { return err } x.ClientConversation = x.Client.NewConversation() return nil } func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) { response, err = x.ClientConversation.Step(challenge) return } func (x *XDGSCRAMClient) Done() bool { return x.ClientConversation.Done() } ================================================ FILE: internal/endpoint/sqs.go ================================================ package endpoint import ( "fmt" "strings" "sync" "time" "github.com/tidwall/gjson" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/sqs" "github.com/tidwall/tile38/internal/log" ) const sqsExpiresAfter = time.Second * 30 // SQSConn is an endpoint connection type SQSConn struct { mu sync.Mutex ep Endpoint session *session.Session svc *sqs.SQS ex bool t time.Time } func (conn *SQSConn) generateSQSURL() string { if conn.ep.SQS.PlainURL != "" { return conn.ep.SQS.PlainURL } return "https://sqs." + conn.ep.SQS.Region + ".amazonaws.com/" + conn.ep.SQS.QueueID + "/" + conn.ep.SQS.QueueName } // Expired returns true if the connection has expired func (conn *SQSConn) Expired() bool { conn.mu.Lock() defer conn.mu.Unlock() if !conn.ex { if time.Since(conn.t) > sqsExpiresAfter { conn.close() conn.ex = true } } return conn.ex } // ExpireNow forces the connection to expire func (conn *SQSConn) ExpireNow() { conn.mu.Lock() defer conn.mu.Unlock() conn.close() conn.ex = true } func (conn *SQSConn) close() { if conn.svc != nil { conn.svc = nil conn.session = nil } } // Send sends a message func (conn *SQSConn) Send(msg string) error { conn.mu.Lock() defer conn.mu.Unlock() if conn.ex { return errExpired } conn.t = time.Now() if conn.svc == nil && conn.session == nil { var creds *credentials.Credentials credPath := conn.ep.SQS.CredPath if credPath != "" { credProfile := conn.ep.SQS.CredProfile if credProfile == "" { credProfile = "default" } creds = credentials.NewSharedCredentials(credPath, credProfile) } var region string if conn.ep.SQS.Region != "" { region = conn.ep.SQS.Region } else { region = sqsRegionFromPlainURL(conn.ep.SQS.PlainURL) } sess := session.Must(session.NewSession(&aws.Config{ Region: ®ion, Credentials: creds, CredentialsChainVerboseErrors: aws.Bool(log.Level() >= 3), MaxRetries: aws.Int(5), })) svc := sqs.New(sess) if conn.ep.SQS.CreateQueue { svc.CreateQueue(&sqs.CreateQueueInput{ QueueName: aws.String(conn.ep.SQS.QueueName), Attributes: map[string]*string{ "DelaySeconds": aws.String("60"), "MessageRetentionPeriod": aws.String("86400"), }, }) } conn.session = sess conn.svc = svc } queueURL := conn.generateSQSURL() // Create message sendParams := &sqs.SendMessageInput{ MessageBody: aws.String(msg), QueueUrl: aws.String(queueURL), } if isFifoQueue(queueURL) { key := gjson.Get(msg, "key") id := gjson.Get(msg, "id") keyValue := fmt.Sprintf("%s#%s", key.String(), id.String()) sendParams.MessageGroupId = aws.String(keyValue) } _, err := conn.svc.SendMessage(sendParams) if err != nil { fmt.Println(err) return err } return nil } func newSQSConn(ep Endpoint) *SQSConn { return &SQSConn{ ep: ep, t: time.Now(), } } func probeSQS(s string) bool { // https://sqs.eu-central-1.amazonaws.com/123456789/myqueue return strings.HasPrefix(s, "https://sqs.") && strings.Contains(s, ".amazonaws.com") } func sqsRegionFromPlainURL(s string) string { parts := strings.Split(s, "https://sqs.") if len(parts) > 1 { parts = strings.Split(parts[1], ".amazonaws.com") if len(parts) > 1 { return parts[0] } } return "" } func isFifoQueue(s string) bool { return strings.HasSuffix(s, ".fifo") } ================================================ FILE: internal/field/field.go ================================================ package field import ( "math" "strconv" "strings" "github.com/tidwall/gjson" "github.com/tidwall/pretty" ) var ZeroValue = Value{kind: Number, data: "0", num: 0} var ZeroField = Field{name: "", value: ZeroValue} type Kind byte const ( Null = Kind(gjson.Null) False = Kind(gjson.False) Number = Kind(gjson.Number) String = Kind(gjson.String) True = Kind(gjson.True) JSON = Kind(gjson.JSON) ) type Value struct { kind Kind data string num float64 } func (v Value) IsZero() bool { return (v.kind == Number && v.data == "0" && v.num == 0) || v == (Value{}) } func (v Value) Equals(b Value) bool { return !v.Less(b) && !b.Less(v) } func (v Value) Kind() Kind { return v.kind } func (v Value) Data() string { return v.data } func (v Value) Num() float64 { return v.num } func (v Value) JSON() string { switch v.Kind() { case Number: switch v.Data() { case "NaN": return `"NaN"` case "+Inf": return `"+Inf"` case "-Inf": return `"-Inf"` default: return v.Data() } case String: return string(gjson.AppendJSONString(nil, v.Data())) case True: return "true" case False: return "false" case Null: if v != (Value{}) { return "null" } case JSON: return v.Data() } return "0" } func stringLessInsensitive(a, b string) bool { for i := 0; i < len(a) && i < len(b); i++ { if a[i] >= 'A' && a[i] <= 'Z' { if b[i] >= 'A' && b[i] <= 'Z' { // both are uppercase, do nothing if a[i] < b[i] { return true } else if a[i] > b[i] { return false } } else { // a is uppercase, convert a to lowercase if a[i]+32 < b[i] { return true } else if a[i]+32 > b[i] { return false } } } else if b[i] >= 'A' && b[i] <= 'Z' { // b is uppercase, convert b to lowercase if a[i] < b[i]+32 { return true } else if a[i] > b[i]+32 { return false } } else { // neither are uppercase if a[i] < b[i] { return true } else if a[i] > b[i] { return false } } } return len(a) < len(b) } // Less return true if a value is less than another value. // The caseSensitive parameter is used when the value are Strings. // The order when comparing two different kinds is: // // Null < False < Number < String < True < JSON // // Pulled from github.com/tidwall/gjson func (v Value) LessCase(b Value, caseSensitive bool) bool { if v.kind < b.kind { return true } if v.kind > b.kind { return false } if v.kind == Number { return v.num < b.num } if v.kind == String { if caseSensitive { return v.data < b.data } return stringLessInsensitive(v.data, b.data) } return v.data < b.data } // Less return true if a value is less than another value. // // Null < False < Number < String < True < JSON // // Pulled from github.com/tidwall/gjson func (v Value) Less(b Value) bool { return v.LessCase(b, false) } type Field struct { name string value Value } func (f Field) Name() string { return f.name } func (f Field) Value() Value { return f.value } func (f Field) Weight() int { return len(f.name) + 8 + len(f.value.data) } var nan = math.NaN() var pinf = math.Inf(+1) var ninf = math.Inf(-1) func ValueOf(data string) Value { data = strings.TrimSpace(data) num, err := strconv.ParseFloat(data, 64) if err == nil { if math.IsInf(num, 0) { if math.IsInf(num, +1) { return Value{kind: Number, data: "+Inf", num: pinf} } else { return Value{kind: Number, data: "-Inf", num: ninf} } } else if math.IsNaN(num) { return Value{kind: Number, data: "NaN", num: nan} } // Make sure that this is a JSON compatible number. // For example, "000123" and "000_123" both parse as floats but aren't // really Numbers that can be represents in JSON. if gjson.Valid(data) { return Value{kind: Number, data: data, num: num} } } else if gjson.Valid(data) { data = strings.TrimSpace(data) r := gjson.Parse(data) switch r.Type { case gjson.Null: return Value{kind: Null, data: "null"} case gjson.JSON: return Value{kind: JSON, data: string(pretty.Ugly([]byte(data)))} case gjson.True: return Value{kind: True, data: "true"} case gjson.False: return Value{kind: False, data: "false"} case gjson.Number: // Ignore. Numbers will always be picked up by the ParseFloat above. case gjson.String: // Ignore. Strings fallthrough by default } // Extract String from JSON data = r.String() } // Check if string is NaN, Inf(inity), +Inf(inity), -Inf(inity) if len(data) >= 3 && len(data) <= 9 { switch data[0] { case '-', '+', 'I', 'i', 'N', 'n': switch strings.ToLower(data) { case "nan": return Value{kind: Number, data: "NaN", num: nan} case "inf", "+inf", "infinity", "+infinity": return Value{kind: Number, data: "+Inf", num: pinf} case "-inf", "-infinity": return Value{kind: Number, data: "-Inf", num: ninf} } } } return Value{kind: String, data: data} } func Make(name, data string) Field { return Field{ strings.TrimSpace(name), ValueOf(data), } } ================================================ FILE: internal/field/field_test.go ================================================ package field import ( "testing" "github.com/tidwall/assert" ) func mLT(a, b Value) bool { return a.Less(b) } func mLTE(a, b Value) bool { return !mLT(b, a) } func mGT(a, b Value) bool { return mLT(b, a) } func mGTE(a, b Value) bool { return !mLT(a, b) } func mEQ(a, b Value) bool { return !mLT(a, b) && !mLT(b, a) } func TestOrder(t *testing.T) { assert.Assert(mLT(ValueOf("hello"), ValueOf("jello"))) assert.Assert(mLT(ValueOf("hello"), ValueOf("JELLO"))) assert.Assert(mLT(ValueOf("HELLO"), ValueOf("JELLO"))) assert.Assert(mLT(ValueOf("HELLO"), ValueOf("jello"))) assert.Assert(!mLT(ValueOf("hello"), ValueOf("hello"))) assert.Assert(!mLT(ValueOf("jello"), ValueOf("hello"))) assert.Assert(!mLT(ValueOf("Jello"), ValueOf("Hello"))) assert.Assert(!mLT(ValueOf("Jello"), ValueOf("hello"))) assert.Assert(!mLT(ValueOf("jello"), ValueOf("Hello"))) assert.Assert(mGT(ValueOf("jello"), ValueOf("hello"))) assert.Assert(!mGT(ValueOf("jello"), ValueOf("jello"))) assert.Assert(!mGT(ValueOf("hello"), ValueOf("jello"))) assert.Assert(mLTE(ValueOf("hello"), ValueOf("jello"))) assert.Assert(mLTE(ValueOf("hello"), ValueOf("hello"))) assert.Assert(mLTE(ValueOf("hello"), ValueOf("HELLO"))) assert.Assert(!mLTE(ValueOf("jello"), ValueOf("hello"))) assert.Assert(mGTE(ValueOf("jello"), ValueOf("jello"))) assert.Assert(mGTE(ValueOf("jello"), ValueOf("hello"))) assert.Assert(mGTE(ValueOf("jello"), ValueOf("JELLO"))) assert.Assert(!mGTE(ValueOf("hello"), ValueOf("jello"))) assert.Assert(mEQ(ValueOf("jello"), ValueOf("jello"))) assert.Assert(mEQ(ValueOf("jello"), ValueOf("JELLO"))) assert.Assert(!mEQ(ValueOf("jello"), ValueOf("hello"))) } func TestLess(t *testing.T) { assert.Assert(mLT(ValueOf("null"), ValueOf("false"))) assert.Assert(mLT(ValueOf("null"), ValueOf("123"))) assert.Assert(mLT(ValueOf("null"), ValueOf("hello"))) assert.Assert(mLT(ValueOf("null"), ValueOf("true"))) assert.Assert(mLT(ValueOf("null"), ValueOf("[]"))) assert.Assert(mLT(ValueOf("false"), ValueOf("123"))) assert.Assert(mLT(ValueOf("false"), ValueOf("hello"))) assert.Assert(mLT(ValueOf("false"), ValueOf("true"))) assert.Assert(mLT(ValueOf("false"), ValueOf("[]"))) assert.Assert(mLT(ValueOf("123"), ValueOf("hello"))) assert.Assert(mLT(ValueOf("123"), ValueOf("true"))) assert.Assert(mLT(ValueOf("123"), ValueOf("[]"))) assert.Assert(mLT(ValueOf("hello"), ValueOf("true"))) assert.Assert(mLT(ValueOf("hello"), ValueOf("[]"))) assert.Assert(mLT(ValueOf("true"), ValueOf("[]"))) assert.Assert(!mLT(ValueOf("false"), ValueOf("null"))) assert.Assert(!mLT(ValueOf("123"), ValueOf("null"))) assert.Assert(!mLT(ValueOf("hello"), ValueOf("null"))) assert.Assert(!mLT(ValueOf("true"), ValueOf("null"))) assert.Assert(!mLT(ValueOf("[]"), ValueOf("null"))) assert.Assert(!mLT(ValueOf("123"), ValueOf("false"))) assert.Assert(!mLT(ValueOf("hello"), ValueOf("false"))) assert.Assert(!mLT(ValueOf("true"), ValueOf("false"))) assert.Assert(!mLT(ValueOf("[]"), ValueOf("false"))) assert.Assert(!mLT(ValueOf("hello"), ValueOf("123"))) assert.Assert(!mLT(ValueOf("true"), ValueOf("123"))) assert.Assert(!mLT(ValueOf("[]"), ValueOf("123"))) assert.Assert(!mLT(ValueOf("true"), ValueOf("hello"))) assert.Assert(!mLT(ValueOf("[]"), ValueOf("hello"))) assert.Assert(!mLT(ValueOf("[]"), ValueOf("true"))) assert.Assert(mLT(ValueOf("123"), ValueOf("456"))) assert.Assert(mLT(ValueOf("[1]"), ValueOf("[2]"))) } func TestLessCase(t *testing.T) { assert.Assert(ValueOf("A").LessCase(ValueOf("B"), true)) assert.Assert(!ValueOf("A").LessCase(ValueOf("A"), true)) assert.Assert(!ValueOf("B").LessCase(ValueOf("A"), true)) } func TestVarious(t *testing.T) { assert.Assert(!ValueOf("A").IsZero()) assert.Assert(ValueOf("0").IsZero()) assert.Assert(Value{}.IsZero()) assert.Assert(ZeroValue.IsZero()) assert.Assert(ZeroValue.Equals(ZeroValue)) assert.Assert(ZeroValue.Kind() == Number) assert.Assert(ValueOf("0").Kind() == Number) assert.Assert(ValueOf("hello").Kind() == String) assert.Assert(ValueOf(`"hello"`).Kind() == String) assert.Assert(ValueOf(`"123"`).Kind() == String) assert.Assert(ValueOf(`"123"`).Data() == `123`) assert.Assert(ValueOf(`"123"`).Num() == 0) } func TestJSON(t *testing.T) { assert.Assert(ValueOf(`A`).JSON() == `"A"`) assert.Assert(ValueOf(`"A"`).JSON() == `"A"`) assert.Assert(ValueOf(`123`).JSON() == `123`) assert.Assert(ValueOf(`{}`).JSON() == `{}`) assert.Assert(ValueOf(`{ }`).JSON() == `{}`) assert.Assert(ValueOf(` -Inf `).JSON() == `"-Inf"`) assert.Assert(ValueOf(` "-Inf" `).JSON() == `"-Inf"`) assert.Assert(ValueOf(`+Inf`).JSON() == `"+Inf"`) assert.Assert(ValueOf(`"+Inf"`).JSON() == `"+Inf"`) assert.Assert(ValueOf(`Inf`).JSON() == `"+Inf"`) assert.Assert(ValueOf(`"Inf"`).JSON() == `"+Inf"`) assert.Assert(ValueOf(`NaN`).JSON() == `"NaN"`) assert.Assert(ValueOf(`"NaN"`).JSON() == `"NaN"`) assert.Assert(ValueOf(`nan`).JSON() == `"NaN"`) assert.Assert(ValueOf(`infinity`).JSON() == `"+Inf"`) assert.Assert(ValueOf(` true `).JSON() == `true`) assert.Assert(ValueOf(` false `).JSON() == `false`) assert.Assert(ValueOf(` null `).JSON() == `null`) assert.Assert(Value{}.JSON() == `0`) assert.Assert(Value{}.JSON() == `0`) } func TestField(t *testing.T) { assert.Assert(Make("hello", "123").Name() == "hello") assert.Assert(Make("HELLO", "123").Name() == "HELLO") assert.Assert(Make("HELLO", "123").Value().Num() == 123) assert.Assert(Make("HELLO", "123").Value().JSON() == "123") assert.Assert(Make("HELLO", "123").Value().Num() == 123) } func TestWeight(t *testing.T) { assert.Assert(Make("hello", "123").Weight() == 16) } func TestNumber(t *testing.T) { assert.Assert(ValueOf("12").Num() == 12) assert.Assert(ValueOf("012").Num() == 0) } ================================================ FILE: internal/field/list_binary.go ================================================ package field import ( "encoding/binary" "strconv" "strings" "unsafe" "github.com/tidwall/gjson" "github.com/tidwall/pretty" "github.com/tidwall/tile38/internal/sstring" ) // binary format // (size,entry,[entry...]) // size: uvarint -- size of the full byte slice, excluding itself. // entry: (name,value) -- one field entry // name: shared string num -- field name, string data, uses the shared library // size: uvarint -- number of bytes in data // value: (kind,vdata) -- field value // kind: byte -- value kind // vdata: (size,data) -- value data, string data // useSharedNames will results in smaller memory usage by sharing the names // of fields using the sstring package. Otherwise the names are embedded with // the list. const useSharedNames = true // List of fields, ordered by Name. type List struct { p *byte } type bytes struct { p *byte l int c int } func ptob(p *byte) []byte { if p == nil { return nil } // Get the size of the bytes (excluding the header) x, n := uvarint(*(*[]byte)(unsafe.Pointer(&bytes{p, 10, 10}))) // Return the byte slice (excluding the header) return (*(*[]byte)(unsafe.Pointer(&bytes{p, n + x, n + x})))[n:] } func btoa(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } // uvarint is a slightly modified version of binary.Uvarint, and it's a little // faster. But it lacks overflow checks which are not needed for our use. func uvarint(buf []byte) (int, int) { var x uint64 for i := 0; i < len(buf); i++ { b := buf[i] if b < 0x80 { return int(x | uint64(b)<<(i*7)), i + 1 } x |= uint64(b&0x7f) << (i * 7) } return 0, 0 } func datakind(kind Kind) bool { switch kind { case Number, String, JSON: return true } return false } func bfield(name string, kind Kind, data string) Field { var num float64 switch kind { case Number: num, _ = strconv.ParseFloat(data, 64) case Null: data = "null" case False: data = "false" case True: data = "true" } return Field{ name: name, value: Value{ kind: Kind(kind), data: data, num: num, }, } } // Set a field in the list. // If the input field value is zero `f.Value().IsZero()` then the field is // deleted or removed from the list since lists cannot have Zero values. // Returns a newly allocated list the updated field. // The original (receiver) list is not modified. func (fields List) Set(field Field) List { b := ptob(fields.p) var i int for { s := i // read the name var name string x, n := uvarint(b[i:]) if n == 0 { break } if useSharedNames { name = sstring.Load(x) i += n } else { name = btoa(b[i+n : i+n+x]) i += n + x } kind := Kind(b[i]) i++ var data string if datakind(kind) { x, n = uvarint(b[i:]) data = btoa(b[i+n : i+n+x]) i += n + x } if field.name < name { // insert before i = s break } if name == field.name { if field.Value().IsZero() { // delete return List{delfield(b, s, i)} } prev := bfield(name, kind, data) if prev.Value().Equals(field.Value()) { // no change return fields } // replace return List{putfield(b, field, s, i)} } } if field.Value().IsZero() { return fields } // insert after return List{putfield(b, field, i, i)} } func delfield(b []byte, s, e int) *byte { totallen := s + (len(b) - e) if totallen == 0 { return nil } var psz [10]byte pn := binary.PutUvarint(psz[:], uint64(totallen)) plen := pn + totallen p := make([]byte, plen) // copy each component i := 0 // -- header size copy(p[i:], psz[:pn]) i += pn // -- head entries copy(p[i:], b[:s]) i += s // -- tail entries copy(p[i:], b[e:]) return &p[0] } func putfield(b []byte, f Field, s, e int) *byte { name := f.Name() var namesz [10]byte var namen int if useSharedNames { num := sstring.Store(name) namen = binary.PutUvarint(namesz[:], uint64(num)) } else { namen = binary.PutUvarint(namesz[:], uint64(len(name))) } value := f.Value() kind := value.Kind() isdatakind := datakind(kind) var data string var datasz [10]byte var datan int if isdatakind { data = value.Data() datan = binary.PutUvarint(datasz[:], uint64(len(data))) } var totallen int if useSharedNames { totallen = s + namen + 1 + (len(b) - e) } else { totallen = s + namen + len(name) + 1 + +(len(b) - e) } if isdatakind { totallen += datan + len(data) } var psz [10]byte pn := binary.PutUvarint(psz[:], uint64(totallen)) plen := pn + totallen p := make([]byte, plen) // copy each component i := 0 // -- header size copy(p[i:], psz[:pn]) i += pn // -- head entries copy(p[i:], b[:s]) i += s // -- name copy(p[i:], namesz[:namen]) i += namen if !useSharedNames { copy(p[i:], name) i += len(name) } // -- kind p[i] = byte(kind) i++ if isdatakind { // -- data copy(p[i:], datasz[:datan]) i += datan copy(p[i:], data) i += len(data) } // -- tail entries copy(p[i:], b[e:]) return &p[0] } // Get a field from the list. Or returns ZeroField if not found. func (fields List) Get(name string) Field { var isj bool var jname string var jpath string dot := strings.IndexByte(name, '.') if dot != -1 { isj = true jname = name[:dot] jpath = name[dot+1:] } b := ptob(fields.p) var i int for { // read the fname var fname string x, n := uvarint(b[i:]) if n == 0 { break } if useSharedNames { fname = sstring.Load(x) i += n } else { fname = btoa(b[i+n : i+n+x]) i += n + x } kind := Kind(b[i]) i++ var data string if datakind(kind) { x, n = uvarint(b[i:]) data = btoa(b[i+n : i+n+x]) i += n + x } if kind == JSON && isj { if jname < fname { break } if fname == jname { res := gjson.Get(data, jpath) if res.Exists() { return bfield(name, Kind(res.Type), res.String()) } } } else { if name < fname { break } if fname == name { return bfield(name, kind, data) } } } return ZeroField } // Scan each field in list func (fields List) Scan(iter func(field Field) bool) { b := ptob(fields.p) var i int for { // read the fname var fname string x, n := uvarint(b[i:]) if n == 0 { break } if useSharedNames { fname = sstring.Load(x) i += n } else { fname = btoa(b[i+n : i+n+x]) i += n + x } kind := Kind(b[i]) i++ var data string if datakind(kind) { x, n = uvarint(b[i:]) data = btoa(b[i+n : i+n+x]) i += n + x } if !iter(bfield(fname, kind, data)) { return } } } // Len return the number of fields in list. func (fields List) Len() int { var count int b := ptob(fields.p) var i int for { x, n := uvarint(b[i:]) if n == 0 { break } if useSharedNames { i += n } else { i += n + x } isdatakind := datakind(Kind(b[i])) i++ if isdatakind { x, n = uvarint(b[i:]) i += n + x } count++ } return count } // Weight is the number of bytes of the list. func (fields List) Weight() int { if fields.p == nil { return 0 } x, n := uvarint(*(*[]byte)(unsafe.Pointer(&bytes{fields.p, 10, 10}))) return x + n } // MakeList returns a field list from an array of fields. func MakeList(fields []Field) List { // TODO: optimize to reduce allocations. var list List for _, f := range fields { list = list.Set(f) } return list } func (fields List) String() string { var dst []byte dst = append(dst, '{') var i int fields.Scan(func(f Field) bool { if i > 0 { dst = append(dst, ',') } dst = gjson.AppendJSONString(dst, f.Name()) dst = append(dst, ':') dst = append(dst, f.Value().JSON()...) i++ return true }) dst = append(dst, '}') return string(pretty.UglyInPlace(dst)) } ================================================ FILE: internal/field/list_struct.go ================================================ //go:build exclude package field type List struct { entries []Field } // bsearch searches array for value. func (fields List) bsearch(name string) (index int, found bool) { i, j := 0, len(fields.entries) for i < j { h := i + (j-i)/2 if name >= fields.entries[h].name { i = h + 1 } else { j = h } } if i > 0 && fields.entries[i-1].name >= name { return i - 1, true } return i, false } func (fields List) Set(field Field) List { var updated List index, found := fields.bsearch(field.name) if found { if field.value.IsZero() { // delete if len(fields.entries) > 1 { updated.entries = make([]Field, len(fields.entries)-1) copy(updated.entries, fields.entries[:index]) copy(updated.entries[index:], fields.entries[index+1:]) } } else if !fields.entries[index].value.Equals(field.value) { // update updated.entries = make([]Field, len(fields.entries)) copy(updated.entries, fields.entries) updated.entries[index].value = field.value } else { // nothing changes updated = fields } return updated } if field.Value().IsZero() { return fields } updated.entries = make([]Field, len(fields.entries)+1) copy(updated.entries, fields.entries[:index]) copy(updated.entries[index+1:], fields.entries[index:]) updated.entries[index] = field return updated } func (fields List) Get(name string) Field { index, found := fields.bsearch(name) if !found { return ZeroField } return fields.entries[index] } func (fields List) Scan(iter func(field Field) bool) { for _, f := range fields.entries { if !iter(f) { return } } } func (fields List) Len() int { return len(fields.entries) } func (fields List) Weight() int { var weight int for _, f := range fields.entries { weight += f.Weight() } return weight } ================================================ FILE: internal/field/list_test.go ================================================ package field import ( "fmt" "math/rand" "testing" "time" "github.com/tidwall/assert" "github.com/tidwall/btree" ) func TestList(t *testing.T) { var fields List fields = fields.Set(Make("hello", "123")) assert.Assert(fields.Len() == 1) // println(fields.Weight()) // assert.Assert(fields.Weight() == 16) fields = fields.Set(Make("jello", "456")) assert.Assert(fields.Len() == 2) // assert.Assert(fields.Weight() == 32) value := fields.Get("jello") assert.Assert(value.Value().Data() == "456") assert.Assert(value.Value().JSON() == "456") assert.Assert(value.Value().Num() == 456) value = fields.Get("nello") assert.Assert(value.Name() == "") assert.Assert(value.Value().IsZero()) fields = fields.Set(Make("jello", "789")) assert.Assert(fields.Len() == 2) // assert.Assert(fields.Weight() == 32) fields = fields.Set(Make("nello", "0")) assert.Assert(fields.Len() == 2) // assert.Assert(fields.Weight() == 32) fields = fields.Set(Make("jello", "789")) assert.Assert(fields.Len() == 2) // assert.Assert(fields.Weight() == 32) fields = fields.Set(Make("jello", "0")) assert.Assert(fields.Len() == 1) // assert.Assert(fields.Weight() == 16) fields = fields.Set(Make("nello", "012")) fields = fields.Set(Make("hello", "456")) fields = fields.Set(Make("fello", "123")) fields = fields.Set(Make("jello", "789")) var names string var datas string var nums float64 fields.Scan(func(f Field) bool { names += f.Name() datas += f.Value().Data() nums += f.Value().Num() return true }) assert.Assert(names == "fellohellojellonello") assert.Assert(datas == "123456789012") assert.Assert(nums == 1368) names = "" datas = "" nums = 0 fields.Scan(func(f Field) bool { names += f.Name() datas += f.Value().Data() nums += f.Value().Num() return false }) assert.Assert(names == "fello") assert.Assert(datas == "123") assert.Assert(nums == 123) } func randStr(n int) string { b := make([]byte, n) rand.Read(b) for i := 0; i < n; i++ { b[i] = 'a' + b[i]%26 } return string(b) } func randVal(n int) string { switch rand.Intn(10) { case 0: return "null" case 1: return "true" case 2: return "false" case 3: return `{"a":"` + randStr(n) + `"}` case 4: return `["` + randStr(n) + `"]` case 5: return `"` + randStr(n) + `"` case 6: return randStr(n) default: return fmt.Sprintf("%f", rand.Float64()*360) } } func TestRandom(t *testing.T) { seed := time.Now().UnixNano() // seed = 1663607868546669000 rand.Seed(seed) start := time.Now() var total int for time.Since(start) < time.Second*2 { N := rand.Intn(500) var org []Field var tr btree.Map[string, Field] var fields List for i := 0; i < N; i++ { name := randStr(rand.Intn(10)) value := randVal(rand.Intn(10)) field := Make(name, value) org = append(org, field) fields = fields.Set(field) v := fields.Get(name) // println(name, v.Value().Data(), field.Value().Data()) if !v.Value().Equals(field.Value()) { t.Fatalf("seed: %d, expected true", seed) } tr.Set(name, field) if fields.Len() != tr.Len() { t.Fatalf("seed: %d, expected %d, got %d", seed, tr.Len(), fields.Len()) } } comp := func() { var all []Field fields.Scan(func(f Field) bool { all = append(all, f) return true }) if len(all) != fields.Len() { t.Fatalf("seed: %d, expected %d, got %d", seed, fields.Len(), len(all)) } if fields.Len() != tr.Len() { t.Fatalf("seed: %d, expected %d, got %d", seed, tr.Len(), fields.Len()) } var i int tr.Scan(func(name string, f Field) bool { if name != f.Name() || all[i].Name() != f.Name() { t.Fatalf("seed: %d, out of order", seed) } i++ return true }) } comp() rand.Shuffle(len(org), func(i, j int) { org[i], org[j] = org[j], org[i] }) for _, f := range org { comp() tr.Delete(f.Name()) fields = fields.Set(Make(f.Name(), "0")) if fields.Len() != tr.Len() { t.Fatalf("seed: %d, expected %d, got %d", seed, tr.Len(), fields.Len()) } comp() } total++ } } func TestJSONGet(t *testing.T) { var list List list = list.Set(Make("hello", "world")) list = list.Set(Make("hello", `"world"`)) list = list.Set(Make("jello", "planet")) list = list.Set(Make("telly", `{"a":[1,2,3],"b":null,"c":true,"d":false}`)) list = list.Set(Make("belly", `{"a":{"b":{"c":"fancy"}}}`)) json := list.String() exp := `{"belly":{"a":{"b":{"c":"fancy"}}},"hello":"world","jello":` + `"planet","telly":{"a":[1,2,3],"b":null,"c":true,"d":false}}` if json != exp { t.Fatalf("expected '%s', got '%s'", exp, json) } data := list.Get("hello").Value().Data() if data != "world" { t.Fatalf("expected '%s', got '%s'", "world", data) } data = list.Get("telly").Value().Data() if data != `{"a":[1,2,3],"b":null,"c":true,"d":false}` { t.Fatalf("expected '%s', got '%s'", `{"a":[1,2,3],"b":null,"c":true,"d":false}`, data) } data = list.Get("belly").Value().Data() if data != `{"a":{"b":{"c":"fancy"}}}` { t.Fatalf("expected '%s', got '%s'", `{"a":{"b":{"c":"fancy"}}}`, data) } data = list.Get("belly.a").Value().Data() if data != `{"b":{"c":"fancy"}}` { t.Fatalf("expected '%s', got '%s'", `{"b":{"c":"fancy"}}`, data) } data = list.Get("belly.a.b").Value().Data() if data != `{"c":"fancy"}` { t.Fatalf("expected '%s', got '%s'", `{"c":"fancy"}`, data) } data = list.Get("belly.a.b.c").Value().Data() if data != `fancy` { t.Fatalf("expected '%s', got '%s'", `fancy`, data) } // Tile38 defaults non-existent fields to zero. data = list.Get("belly.a.b.c.d").Value().Data() if data != `0` { t.Fatalf("expected '%s', got '%s'", `0`, data) } } ================================================ FILE: internal/glob/glob.go ================================================ package glob import "strings" // Glob structure for simple string matching type Glob struct { Pattern string Desc bool Limits []string IsGlob bool } // Match returns true when string matches pattern. Returns an error when the // pattern is invalid. func Match(pattern, str string) (matched bool, err error) { return wildcardMatch(pattern, str) } // IsGlob returns true when the pattern is a valid glob func IsGlob(pattern string) bool { for i := 0; i < len(pattern); i++ { switch pattern[i] { case '[', '*', '?': _, err := Match(pattern, "whatever") return err == nil } } return false } // Parse returns a glob structure from the pattern. func Parse(pattern string, desc bool) *Glob { g := &Glob{Pattern: pattern, Desc: desc, Limits: []string{"", ""}} if strings.HasPrefix(pattern, "*") { g.IsGlob = true return g } if pattern == "" { g.IsGlob = false return g } n := 0 isGlob := false outer: for i := 0; i < len(pattern); i++ { switch pattern[i] { case '[', '*', '?': _, err := Match(pattern, "whatever") if err == nil { isGlob = true } break outer } n++ } if n == 0 { g.Limits = []string{pattern, pattern} g.IsGlob = false return g } var a, b string if desc { a = pattern[:n] b = a if b[n-1] == 0x00 { for len(b) > 0 && b[len(b)-1] == 0x00 { if len(b) > 1 { if b[len(b)-2] == 0x00 { b = b[:len(b)-1] } else { b = string(append([]byte(b[:len(b)-2]), b[len(b)-2]-1, 0xFF)) } } else { b = "" } } } else { b = string(append([]byte(b[:n-1]), b[n-1]-1)) } if a[n-1] == 0xFF { a = string(append([]byte(a), 0x00)) } else { a = string(append([]byte(a[:n-1]), a[n-1]+1)) } } else { a = pattern[:n] if a[n-1] == 0xFF { b = string(append([]byte(a), 0x00)) } else { b = string(append([]byte(a[:n-1]), a[n-1]+1)) } } g.Limits = []string{a, b} g.IsGlob = isGlob return g } ================================================ FILE: internal/glob/glob_test.go ================================================ package glob import ( "math/rand" "testing" "time" ) func test(t *testing.T, pattern string, desc bool, limitsExpect []string, isGlobExpect bool) { g := Parse(pattern, desc) if g.IsGlob != isGlobExpect { t.Fatalf("pattern[%v] desc[%v] (isGlob=%v, expected=%v)", pattern, desc, g.IsGlob, isGlobExpect) } if g.Limits[0] != limitsExpect[0] || g.Limits[1] != limitsExpect[1] { t.Fatalf("pattern[%v] desc[%v] (limits=%v, expected=%v)", pattern, desc, g.Limits, limitsExpect) } if g.Pattern != pattern { t.Fatalf("pattern[%v] desc[%v] (pattern=%v, expected=%v)", pattern, desc, g.Pattern, pattern) } if g.Desc != desc { t.Fatalf("pattern[%v] desc[%v] (desc=%v, expected=%v)", pattern, desc, g.Desc, desc) } } func TestGlob(t *testing.T) { test(t, "*", false, []string{"", ""}, true) test(t, "", false, []string{"", ""}, false) test(t, "hello*", false, []string{"hello", "hellp"}, true) test(t, "hello", false, []string{"hello", "hellp"}, false) test(t, "\xff*", false, []string{"\xff", "\xff\x00"}, true) test(t, "\x00*", false, []string{"\x00", "\x01"}, true) test(t, "\xff", false, []string{"\xff", "\xff\x00"}, false) test(t, "*", true, []string{"", ""}, true) test(t, "", true, []string{"", ""}, false) test(t, "hello*", true, []string{"hellp", "helln"}, true) test(t, "hello", true, []string{"hellp", "helln"}, false) test(t, "a\xff*", true, []string{"a\xff\x00", "a\xfe"}, true) test(t, "\x00*", true, []string{"\x01", ""}, true) test(t, "\x01*", true, []string{"\x02", "\x00"}, true) test(t, "b\x00*", true, []string{"b\x01", "a\xff"}, true) test(t, "\x00\x00*", true, []string{"\x00\x01", ""}, true) test(t, "\x00\x01\x00*", true, []string{"\x00\x01\x01", "\x00\x00\xff"}, true) } func testMatch(s, pattern string) bool { ok, _ := Match(pattern, s) return ok } func TestMatch(t *testing.T) { if !testMatch("hello world", "hello world") { t.Fatal("fail") } if testMatch("hello world", "jello world") { t.Fatal("fail") } if !testMatch("hello world", "hello*") { t.Fatal("fail") } if testMatch("hello world", "jello*") { t.Fatal("fail") } if !testMatch("hello world", "hello?world") { t.Fatal("fail") } if testMatch("hello world", "jello?world") { t.Fatal("fail") } if !testMatch("hello world", "he*o?world") { t.Fatal("fail") } if !testMatch("hello world", "he*o?wor*") { t.Fatal("fail") } if !testMatch("hello world", "he*o?*r*") { t.Fatal("fail") } if !testMatch("的情况下解析一个", "*") { t.Fatal("fail") } if !testMatch("的情况下解析一个", "*况下*") { t.Fatal("fail") } if !testMatch("的情况下解析一个", "*况?*") { t.Fatal("fail") } if !testMatch("的情况下解析一个", "的情况?解析一个") { t.Fatal("fail") } } // TestWildcardMatch - Tests validate the logic of wild card matching. // `WildcardMatch` supports '*' and '?' wildcards. // Sample usage: In resource matching for folder policy validation. func TestWildcardMatch(t *testing.T) { testCases := []struct { pattern string text string matched bool }{ // Test case - 1. // Test case with pattern containing key name with a prefix. Should accept the same text without a "*". { pattern: "my-folder/oo*", text: "my-folder/oo", matched: true, }, // Test case - 2. // Test case with "*" at the end of the pattern. { pattern: "my-folder/In*", text: "my-folder/India/Karnataka/", matched: true, }, // Test case - 3. // Test case with prefixes shuffled. // This should fail. { pattern: "my-folder/In*", text: "my-folder/Karnataka/India/", matched: false, }, // Test case - 4. // Test case with text expanded to the wildcards in the pattern. { pattern: "my-folder/In*/Ka*/Ban", text: "my-folder/India/Karnataka/Ban", matched: true, }, // Test case - 5. // Test case with the keyname part is repeated as prefix several times. // This is valid. { pattern: "my-folder/In*/Ka*/Ban", text: "my-folder/India/Karnataka/Ban/Ban/Ban/Ban/Ban", matched: true, }, // Test case - 6. // Test case to validate that `*` can be expanded into multiple prefixes. { pattern: "my-folder/In*/Ka*/Ban", text: "my-folder/India/Karnataka/Area1/Area2/Area3/Ban", matched: true, }, // Test case - 7. // Test case to validate that `*` can be expanded into multiple prefixes. { pattern: "my-folder/In*/Ka*/Ban", text: "my-folder/India/State1/State2/Karnataka/Area1/Area2/Area3/Ban", matched: true, }, // Test case - 8. // Test case where the keyname part of the pattern is expanded in the text. { pattern: "my-folder/In*/Ka*/Ban", text: "my-folder/India/Karnataka/Bangalore", matched: false, }, // Test case - 9. // Test case with prefixes and wildcard expanded for all "*". { pattern: "my-folder/In*/Ka*/Ban*", text: "my-folder/India/Karnataka/Bangalore", matched: true, }, // Test case - 10. // Test case with keyname part being a wildcard in the pattern. {pattern: "my-folder/*", text: "my-folder/India", matched: true, }, // Test case - 11. { pattern: "my-folder/oo*", text: "my-folder/odo", matched: false, }, // Test case with pattern containing wildcard '?'. // Test case - 12. // "my-folder?/" matches "my-folder1/", "my-folder2/", "my-folder3" etc... // doesn't match "myfolder/". { pattern: "my-folder?/abc*", text: "myfolder/abc", matched: false, }, // Test case - 13. { pattern: "my-folder?/abc*", text: "my-folder1/abc", matched: true, }, // Test case - 14. { pattern: "my-?-folder/abc*", text: "my--folder/abc", matched: false, }, // Test case - 15. { pattern: "my-?-folder/abc*", text: "my-1-folder/abc", matched: true, }, // Test case - 16. { pattern: "my-?-folder/abc*", text: "my-k-folder/abc", matched: true, }, // Test case - 17. { pattern: "my??folder/abc*", text: "myfolder/abc", matched: false, }, // Test case - 18. { pattern: "my??folder/abc*", text: "my4afolder/abc", matched: true, }, // Test case - 19. { pattern: "my-folder?abc*", text: "my-folder/abc", matched: true, }, // Test case 20-21. // '?' matches '/' too. (works with s3). // This is because the namespace is considered flat. // "abc?efg" matches both "abcdefg" and "abc/efg". { pattern: "my-folder/abc?efg", text: "my-folder/abcdefg", matched: true, }, { pattern: "my-folder/abc?efg", text: "my-folder/abc/efg", matched: true, }, // Test case - 22. { pattern: "my-folder/abc????", text: "my-folder/abc", matched: false, }, // Test case - 23. { pattern: "my-folder/abc????", text: "my-folder/abcde", matched: false, }, // Test case - 24. { pattern: "my-folder/abc????", text: "my-folder/abcdefg", matched: true, }, // Test case 25-26. // test case with no '*'. { pattern: "my-folder/abc?", text: "my-folder/abc", matched: false, }, { pattern: "my-folder/abc?", text: "my-folder/abcd", matched: true, }, { pattern: "my-folder/abc?", text: "my-folder/abcde", matched: false, }, // Test case 27. { pattern: "my-folder/mnop*?", text: "my-folder/mnop", matched: false, }, // Test case 28. { pattern: "my-folder/mnop*?", text: "my-folder/mnopqrst/mnopqr", matched: true, }, // Test case 29. { pattern: "my-folder/mnop*?", text: "my-folder/mnopqrst/mnopqrs", matched: true, }, // Test case 30. { pattern: "my-folder/mnop*?", text: "my-folder/mnop", matched: false, }, // Test case 31. { pattern: "my-folder/mnop*?", text: "my-folder/mnopq", matched: true, }, // Test case 32. { pattern: "my-folder/mnop*?", text: "my-folder/mnopqr", matched: true, }, // Test case 33. { pattern: "my-folder/mnop*?and", text: "my-folder/mnopqand", matched: true, }, // Test case 34. { pattern: "my-folder/mnop*?and", text: "my-folder/mnopand", matched: false, }, // Test case 35. { pattern: "my-folder/mnop*?and", text: "my-folder/mnopqand", matched: true, }, // Test case 36. { pattern: "my-folder/mnop*?", text: "my-folder/mn", matched: false, }, // Test case 37. { pattern: "my-folder/mnop*?", text: "my-folder/mnopqrst/mnopqrs", matched: true, }, // Test case 38. { pattern: "my-folder/mnop*??", text: "my-folder/mnopqrst", matched: true, }, // Test case 39. { pattern: "my-folder/mnop*qrst", text: "my-folder/mnopabcdegqrst", matched: true, }, // Test case 40. { pattern: "my-folder/mnop*?and", text: "my-folder/mnopqand", matched: true, }, // Test case 41. { pattern: "my-folder/mnop*?and", text: "my-folder/mnopand", matched: false, }, // Test case 42. { pattern: "my-folder/mnop*?and?", text: "my-folder/mnopqanda", matched: true, }, // Test case 43. { pattern: "my-folder/mnop*?and", text: "my-folder/mnopqanda", matched: false, }, // Test case 44. { pattern: "my-?-folder/abc*", text: "my-folder/mnopqanda", matched: false, }, } // Iterating over the test cases, call the function under test and asert the output. for i, testCase := range testCases { actualResult := testMatch(testCase.text, testCase.pattern) if testCase.matched != actualResult { t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult) } } } func TestRandomInput(t *testing.T) { rand.Seed(time.Now().UnixNano()) b1 := make([]byte, 100) b2 := make([]byte, 100) for i := 0; i < 1000000; i++ { if _, err := rand.Read(b1); err != nil { t.Fatal(err) } if _, err := rand.Read(b2); err != nil { t.Fatal(err) } testMatch(string(b1), string(b2)) } } func BenchmarkAscii(t *testing.B) { for i := 0; i < t.N; i++ { if !testMatch("hello", "hello") { t.Fatal("fail") } } } func BenchmarkUnicode(t *testing.B) { for i := 0; i < t.N; i++ { if !testMatch("h情llo", "h情llo") { t.Fatal("fail") } } } ================================================ FILE: internal/glob/match.go ================================================ // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package glob import ( "errors" "unicode/utf8" ) // ErrBadPattern indicates a globbing pattern was malformed. var ErrBadPattern = errors.New("syntax error in pattern") // Match reports whether name matches the shell file name pattern. // The pattern syntax is: // // pattern: // { term } // term: // '*' matches any sequence of non-Separator characters // '?' matches any single non-Separator character // '[' [ '^' ] { character-range } ']' // character class (must be non-empty) // c matches character c (c != '*', '?', '\\', '[') // '\\' c matches character c // // character-range: // c matches character c (c != '\\', '-', ']') // '\\' c matches character c // lo '-' hi matches character c for lo <= c <= hi // // Match requires pattern to match all of name, not just a substring. // The only possible returned error is ErrBadPattern, when pattern // is malformed. // // On Windows, escaping is disabled. Instead, '\\' is treated as // path separator. func wildcardMatch(pattern, name string) (matched bool, err error) { Pattern: for len(pattern) > 0 { var star bool var chunk string star, chunk, pattern = scanChunk(pattern) if star && chunk == "" { // Trailing * matches rest of string unless it has a /. return true, nil } // Look for match at current position. t, ok, err := matchChunk(chunk, name) // if we're the last chunk, make sure we've exhausted the name // otherwise we'll give a false result even if we could still match // using the star if ok && (len(t) == 0 || len(pattern) > 0) { name = t continue } if err != nil { return false, err } if star { // Look for match skipping i+1 bytes. // Cannot skip /. for i := 0; i < len(name); i++ { t, ok, err := matchChunk(chunk, name[i+1:]) if ok { // if we're the last chunk, make sure we exhausted the name if len(pattern) == 0 && len(t) > 0 { continue } name = t continue Pattern } if err != nil { return false, err } } } return false, nil } return len(name) == 0, nil } // scanChunk gets the next segment of pattern, which is a non-star string // possibly preceded by a star. func scanChunk(pattern string) (star bool, chunk, rest string) { for len(pattern) > 0 && pattern[0] == '*' { pattern = pattern[1:] star = true } inrange := false var i int Scan: for i = 0; i < len(pattern); i++ { switch pattern[i] { case '\\': // error check handled in matchChunk: bad pattern. if i+1 < len(pattern) { i++ } case '[': inrange = true case ']': inrange = false case '*': if !inrange { break Scan } } } return star, pattern[0:i], pattern[i:] } // matchChunk checks whether chunk matches the beginning of s. // If so, it returns the remainder of s (after the match). // Chunk is all single-character operators: literals, char classes, and ?. func matchChunk(chunk, s string) (rest string, ok bool, err error) { for len(chunk) > 0 { if len(s) == 0 { return } switch chunk[0] { case '[': // character class r, n := utf8.DecodeRuneInString(s) s = s[n:] chunk = chunk[1:] // We can't end right after '[', we're expecting at least // a closing bracket and possibly a caret. if len(chunk) == 0 { err = ErrBadPattern return } // possibly negated negated := chunk[0] == '^' if negated { chunk = chunk[1:] } // parse all ranges match := false nrange := 0 for { if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 { chunk = chunk[1:] break } var lo, hi rune if lo, chunk, err = getEsc(chunk); err != nil { return } hi = lo if chunk[0] == '-' { if hi, chunk, err = getEsc(chunk[1:]); err != nil { return } } if lo <= r && r <= hi { match = true } nrange++ } if match == negated { return } case '?': _, n := utf8.DecodeRuneInString(s) s = s[n:] chunk = chunk[1:] case '\\': chunk = chunk[1:] if len(chunk) == 0 { err = ErrBadPattern return } fallthrough default: if chunk[0] != s[0] { return } s = s[1:] chunk = chunk[1:] } } return s, true, nil } // getEsc gets a possibly-escaped character from chunk, for a character class. func getEsc(chunk string) (r rune, nchunk string, err error) { if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' { err = ErrBadPattern return } if chunk[0] == '\\' { chunk = chunk[1:] if len(chunk) == 0 { err = ErrBadPattern return } } r, n := utf8.DecodeRuneInString(chunk) if r == utf8.RuneError && n == 1 { err = ErrBadPattern } nchunk = chunk[n:] if len(nchunk) == 0 { err = ErrBadPattern } return } ================================================ FILE: internal/hservice/gen.sh ================================================ #!/bin/bash cd $(dirname "${BASH_SOURCE[0]}") protoc --go_out=plugins=grpc,import_path=hservice:. *.proto ================================================ FILE: internal/hservice/hservice.pb.go ================================================ // Code generated by protoc-gen-go. // source: hservice.proto // DO NOT EDIT! /* Package hservice is a generated protocol buffer package. It is generated from these files: hservice.proto It has these top-level messages: MessageRequest MessageReply */ package hservice import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" import ( context "golang.org/x/net/context" grpc "google.golang.org/grpc" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package // The request message containing the message value type MessageRequest struct { Value string `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"` } func (m *MessageRequest) Reset() { *m = MessageRequest{} } func (m *MessageRequest) String() string { return proto.CompactTextString(m) } func (*MessageRequest) ProtoMessage() {} func (*MessageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } // The response message containing an ok (true or false) type MessageReply struct { Ok bool `protobuf:"varint,1,opt,name=ok" json:"ok,omitempty"` } func (m *MessageReply) Reset() { *m = MessageReply{} } func (m *MessageReply) String() string { return proto.CompactTextString(m) } func (*MessageReply) ProtoMessage() {} func (*MessageReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } func init() { proto.RegisterType((*MessageRequest)(nil), "hservice.MessageRequest") proto.RegisterType((*MessageReply)(nil), "hservice.MessageReply") } // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. const _ = grpc.SupportPackageIsVersion3 // Client API for HookService service type HookServiceClient interface { // Sends a greeting Send(ctx context.Context, in *MessageRequest, opts ...grpc.CallOption) (*MessageReply, error) } type hookServiceClient struct { cc *grpc.ClientConn } func NewHookServiceClient(cc *grpc.ClientConn) HookServiceClient { return &hookServiceClient{cc} } func (c *hookServiceClient) Send(ctx context.Context, in *MessageRequest, opts ...grpc.CallOption) (*MessageReply, error) { out := new(MessageReply) err := grpc.Invoke(ctx, "/hservice.HookService/Send", in, out, c.cc, opts...) if err != nil { return nil, err } return out, nil } // Server API for HookService service type HookServiceServer interface { // Sends a greeting Send(context.Context, *MessageRequest) (*MessageReply, error) } func RegisterHookServiceServer(s *grpc.Server, srv HookServiceServer) { s.RegisterService(&_HookService_serviceDesc, srv) } func _HookService_Send_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MessageRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(HookServiceServer).Send(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/hservice.HookService/Send", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(HookServiceServer).Send(ctx, req.(*MessageRequest)) } return interceptor(ctx, in, info, handler) } var _HookService_serviceDesc = grpc.ServiceDesc{ ServiceName: "hservice.HookService", HandlerType: (*HookServiceServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Send", Handler: _HookService_Send_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: fileDescriptor0, } func init() { proto.RegisterFile("hservice.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ // 168 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xcb, 0x28, 0x4e, 0x2d, 0x2a, 0xcb, 0x4c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x80, 0xf1, 0x95, 0xd4, 0xb8, 0xf8, 0x7c, 0x53, 0x8b, 0x8b, 0x13, 0xd3, 0x53, 0x83, 0x52, 0x0b, 0x4b, 0x53, 0x8b, 0x4b, 0x84, 0x44, 0xb8, 0x58, 0xcb, 0x12, 0x73, 0x4a, 0x53, 0x25, 0x18, 0x15, 0x18, 0x35, 0x38, 0x83, 0x20, 0x1c, 0x25, 0x39, 0x2e, 0x1e, 0xb8, 0xba, 0x82, 0x9c, 0x4a, 0x21, 0x3e, 0x2e, 0xa6, 0xfc, 0x6c, 0xb0, 0x12, 0x8e, 0x20, 0xa6, 0xfc, 0x6c, 0x23, 0x4f, 0x2e, 0x6e, 0x8f, 0xfc, 0xfc, 0xec, 0x60, 0x88, 0xb1, 0x42, 0x56, 0x5c, 0x2c, 0xc1, 0xa9, 0x79, 0x29, 0x42, 0x12, 0x7a, 0x70, 0x9b, 0x51, 0xad, 0x91, 0x12, 0xc3, 0x22, 0x53, 0x90, 0x53, 0xa9, 0xc4, 0xe0, 0xa4, 0xc9, 0x25, 0x9c, 0x9c, 0x9f, 0xab, 0x57, 0x92, 0x99, 0x93, 0x6a, 0x6c, 0x01, 0x57, 0xe5, 0x24, 0x80, 0x64, 0x7e, 0x00, 0xc8, 0x17, 0x01, 0x8c, 0x49, 0x6c, 0x60, 0xef, 0x18, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, 0x6d, 0xd0, 0x2b, 0x13, 0xe0, 0x00, 0x00, 0x00, } ================================================ FILE: internal/hservice/hservice.proto ================================================ syntax = "proto3"; option java_multiple_files = true; option java_package = "com.tile38.hservice"; option java_outer_classname = "HookServiceProto"; package hservice; // The greeting service definition. service HookService { // Sends a greeting rpc Send (MessageRequest) returns (MessageReply) {} } // The request message containing the message value message MessageRequest { string value = 1; } // The response message containing an ok (true or false) message MessageReply { bool ok = 1; } ================================================ FILE: internal/log/log.go ================================================ package log import ( "encoding/json" "fmt" "io" "os" "sync" "sync/atomic" "time" "go.uber.org/zap" "golang.org/x/term" ) var wmu sync.Mutex var wr io.Writer var zmu sync.Mutex var zlogger *zap.SugaredLogger var tty atomic.Bool var ljson atomic.Bool var llevel atomic.Int32 func init() { SetOutput(os.Stderr) SetLevel(1) } // Level is the log level // 0: silent - do not log // 1: normal - show everything except debug and warn // 2: verbose - show everything except debug // 3: very verbose - show everything func SetLevel(level int) { if level < 0 { level = 0 } else if level > 3 { level = 3 } llevel.Store(int32(level)) } // Level returns the log level func Level() int { return int(llevel.Load()) } func SetLogJSON(logJSON bool) { ljson.Store(logJSON) } func LogJSON() bool { return ljson.Load() } // SetOutput sets the output of the logger func SetOutput(w io.Writer) { f, ok := w.(*os.File) tty.Store(ok && term.IsTerminal(int(f.Fd()))) wmu.Lock() wr = w wmu.Unlock() } // Build a zap logger from default or custom config func Build(c string) error { var zcfg zap.Config if c == "" { zcfg = zap.NewProductionConfig() // to be able to filter with Tile38 levels zcfg.Level.SetLevel(zap.DebugLevel) // disable caller because caller is always log.go zcfg.DisableCaller = true } else { err := json.Unmarshal([]byte(c), &zcfg) if err != nil { return err } // to be able to filter with Tile38 levels zcfg.Level.SetLevel(zap.DebugLevel) // disable caller because caller is always log.go zcfg.DisableCaller = true } core, err := zcfg.Build() if err != nil { return err } defer core.Sync() zmu.Lock() zlogger = core.Sugar() zmu.Unlock() return nil } // Set a zap logger func Set(sl *zap.SugaredLogger) { zmu.Lock() zlogger = sl zmu.Unlock() } // Get a zap logger func Get() *zap.SugaredLogger { zmu.Lock() sl := zlogger zmu.Unlock() return sl } // Output returns the output writer func Output() io.Writer { wmu.Lock() defer wmu.Unlock() return wr } func log(level int, tag, color string, formatted bool, format string, args ...interface{}) { if llevel.Load() < int32(level) { return } var msg string if formatted { msg = fmt.Sprintf(format, args...) } else { msg = fmt.Sprint(args...) } if ljson.Load() { zmu.Lock() defer zmu.Unlock() switch tag { case "ERRO": zlogger.Error(msg) case "FATA": zlogger.Fatal(msg) case "WARN": zlogger.Warn(msg) case "DEBU": zlogger.Debug(msg) default: zlogger.Info(msg) } return } s := []byte(time.Now().Format("2006/01/02 15:04:05")) s = append(s, ' ') if tty.Load() { s = append(s, color...) } s = append(s, '[') s = append(s, tag...) s = append(s, ']') if tty.Load() { s = append(s, "\x1b[0m"...) } s = append(s, ' ') s = append(s, msg...) if s[len(s)-1] != '\n' { s = append(s, '\n') } wmu.Lock() wr.Write(s) wmu.Unlock() } var emptyFormat string // Infof ... func Infof(format string, args ...interface{}) { if llevel.Load() >= 1 { log(1, "INFO", "\x1b[36m", true, format, args...) } } // Info ... func Info(args ...interface{}) { if llevel.Load() >= 1 { log(1, "INFO", "\x1b[36m", false, emptyFormat, args...) } } // HTTPf ... func HTTPf(format string, args ...interface{}) { if llevel.Load() >= 1 { log(1, "HTTP", "\x1b[1m\x1b[30m", true, format, args...) } } // HTTP ... func HTTP(args ...interface{}) { if llevel.Load() >= 1 { log(1, "HTTP", "\x1b[1m\x1b[30m", false, emptyFormat, args...) } } // Errorf ... func Errorf(format string, args ...interface{}) { if llevel.Load() >= 1 { log(1, "ERRO", "\x1b[1m\x1b[31m", true, format, args...) } } // Error .. func Error(args ...interface{}) { if llevel.Load() >= 1 { log(1, "ERRO", "\x1b[1m\x1b[31m", false, emptyFormat, args...) } } // Warnf ... func Warnf(format string, args ...interface{}) { if llevel.Load() >= 1 { log(2, "WARN", "\x1b[33m", true, format, args...) } } // Warn ... func Warn(args ...interface{}) { if llevel.Load() >= 1 { log(2, "WARN", "\x1b[33m", false, emptyFormat, args...) } } // Debugf ... func Debugf(format string, args ...interface{}) { if llevel.Load() >= 3 { log(3, "DEBU", "\x1b[35m", true, format, args...) } } // Debug ... func Debug(args ...interface{}) { if llevel.Load() >= 3 { log(3, "DEBU", "\x1b[35m", false, emptyFormat, args...) } } // Printf ... func Printf(format string, args ...interface{}) { Infof(format, args...) } // Print ... func Print(args ...interface{}) { Info(args...) } // Fatalf ... func Fatalf(format string, args ...interface{}) { log(1, "FATA", "\x1b[31m", true, format, args...) os.Exit(1) } // Fatal ... func Fatal(args ...interface{}) { log(1, "FATA", "\x1b[31m", false, emptyFormat, args...) os.Exit(1) } ================================================ FILE: internal/log/log_test.go ================================================ package log import ( "bytes" "io" "strings" "testing" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest/observer" ) func TestLog(t *testing.T) { f := &bytes.Buffer{} SetLogJSON(false) SetOutput(f) Printf("hello %v", "everyone") if !strings.HasSuffix(f.String(), "hello everyone\n") { t.Fatal("fail") } } func TestLogJSON(t *testing.T) { SetLogJSON(true) Build("") type tcase struct { level int format string args string ops func(...interface{}) fops func(string, ...interface{}) expMsg string expLvl zapcore.Level } fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { observedZapCore, observedLogs := observer.New(zap.DebugLevel) Set(zap.New(observedZapCore).Sugar()) SetLevel(tc.level) if tc.format != "" { tc.fops(tc.format, tc.args) } else { tc.ops(tc.args) } if observedLogs.Len() < 1 { t.Fatal("fail") } allLogs := observedLogs.All() if allLogs[0].Message != tc.expMsg { t.Fatal("fail") } if allLogs[0].Level != tc.expLvl { t.Fatal("fail") } } } tests := map[string]tcase{ "Print": { level: 1, args: "Print json logger", ops: func(args ...interface{}) { Print(args...) }, expMsg: "Print json logger", expLvl: zapcore.InfoLevel, }, "Printf": { level: 1, format: "Printf json %v", args: "logger", fops: func(format string, args ...interface{}) { Printf(format, args...) }, expMsg: "Printf json logger", expLvl: zapcore.InfoLevel, }, "Info": { level: 1, args: "Info json logger", ops: func(args ...interface{}) { Info(args...) }, expMsg: "Info json logger", expLvl: zapcore.InfoLevel, }, "Infof": { level: 1, format: "Infof json %v", args: "logger", fops: func(format string, args ...interface{}) { Infof(format, args...) }, expMsg: "Infof json logger", expLvl: zapcore.InfoLevel, }, "Debug": { level: 3, args: "Debug json logger", ops: func(args ...interface{}) { Debug(args...) }, expMsg: "Debug json logger", expLvl: zapcore.DebugLevel, }, "Debugf": { level: 3, format: "Debugf json %v", args: "logger", fops: func(format string, args ...interface{}) { Debugf(format, args...) }, expMsg: "Debugf json logger", expLvl: zapcore.DebugLevel, }, "Warn": { level: 2, args: "Warn json logger", ops: func(args ...interface{}) { Warn(args...) }, expMsg: "Warn json logger", expLvl: zapcore.WarnLevel, }, "Warnf": { level: 2, format: "Warnf json %v", args: "logger", fops: func(format string, args ...interface{}) { Warnf(format, args...) }, expMsg: "Warnf json logger", expLvl: zapcore.WarnLevel, }, "Error": { level: 1, args: "Error json logger", ops: func(args ...interface{}) { Error(args...) }, expMsg: "Error json logger", expLvl: zapcore.ErrorLevel, }, "Errorf": { level: 1, format: "Errorf json %v", args: "logger", fops: func(format string, args ...interface{}) { Errorf(format, args...) }, expMsg: "Errorf json logger", expLvl: zapcore.ErrorLevel, }, "Http": { level: 1, args: "Http json logger", ops: func(args ...interface{}) { HTTP(args...) }, expMsg: "Http json logger", expLvl: zapcore.InfoLevel, }, "Httpf": { level: 1, format: "Httpf json %v", args: "logger", fops: func(format string, args ...interface{}) { HTTPf(format, args...) }, expMsg: "Httpf json logger", expLvl: zapcore.InfoLevel, }, } for name, tc := range tests { t.Run(name, fn(tc)) } } func BenchmarkLogPrintf(t *testing.B) { SetLogJSON(false) SetLevel(1) SetOutput(io.Discard) t.ResetTimer() for i := 0; i < t.N; i++ { Printf("X %s", "Y") } } func BenchmarkLogJSONPrintf(t *testing.B) { SetLogJSON(true) SetLevel(1) ec := zap.NewProductionEncoderConfig() ec.EncodeDuration = zapcore.NanosDurationEncoder ec.EncodeTime = zapcore.EpochNanosTimeEncoder enc := zapcore.NewJSONEncoder(ec) logger := zap.New( zapcore.NewCore( enc, zapcore.AddSync(io.Discard), zap.DebugLevel, )).Sugar() Set(logger) t.ResetTimer() for i := 0; i < t.N; i++ { Printf("X %s", "Y") } } ================================================ FILE: internal/object/object_binary.go ================================================ package object import ( "encoding/binary" "unsafe" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/tile38/internal/field" ) type pointObject struct { base Object pt geojson.SimplePoint } type geoObject struct { base Object geo geojson.Object } const opoint = 1 const ogeo = 2 type Object struct { head string // tuple (kind,expires,id) fields field.List } func (o *Object) geo() geojson.Object { if o != nil { switch o.head[0] { case opoint: return &(*pointObject)(unsafe.Pointer(o)).pt case ogeo: return (*geoObject)(unsafe.Pointer(o)).geo } } return nil } // uvarint is a slightly modified version of binary.Uvarint, and it's a little // faster. But it lacks overflow checks which are not needed for our use. func uvarint(s string) (uint64, int) { var x uint64 for i := 0; i < len(s); i++ { b := s[i] if b < 0x80 { return x | uint64(b)<<(i*7), i + 1 } x |= uint64(b&0x7f) << (i * 7) } return 0, 0 } func varint(s string) (int64, int) { ux, n := uvarint(s) x := int64(ux >> 1) if ux&1 != 0 { x = ^x } return x, n } func (o *Object) ID() string { if o.head[1] == 0 { return o.head[2:] } _, n := varint(o.head[1:]) return o.head[1+n:] } func (o *Object) Fields() field.List { return o.fields } func (o *Object) Expires() int64 { ex, _ := varint(o.head[1:]) return ex } func (o *Object) Rect() geometry.Rect { ogeo := o.geo() if ogeo == nil { return geometry.Rect{} } return ogeo.Rect() } func (o *Object) Geo() geojson.Object { return o.geo() } func (o *Object) String() string { ogeo := o.geo() if ogeo == nil { return "" } return ogeo.String() } func (o *Object) IsSpatial() bool { _, ok := o.geo().(geojson.Spatial) return ok } func (o *Object) Weight() int { var weight int weight += len(o.ID()) ogeo := o.geo() if ogeo != nil { if o.IsSpatial() { weight += ogeo.NumPoints() * 16 } else { weight += len(ogeo.String()) } } weight += o.Fields().Weight() return weight } func makeHead(kind byte, id string, expires int64) string { var exb [20]byte exn := 1 if expires != 0 { exn = binary.PutVarint(exb[:], expires) } n := 1 + exn + len(id) head := make([]byte, n) head[0] = kind copy(head[1:], exb[:exn]) copy(head[1+exn:], id) return *(*string)(unsafe.Pointer(&head)) } func newPoint(id string, pt geometry.Point, expires int64, fields field.List, ) *Object { return (*Object)(unsafe.Pointer(&pointObject{ Object{ head: makeHead(opoint, id, expires), fields: fields, }, geojson.SimplePoint{Point: pt}, })) } func newGeo(id string, geo geojson.Object, expires int64, fields field.List, ) *Object { return (*Object)(unsafe.Pointer(&geoObject{ Object{ head: makeHead(ogeo, id, expires), fields: fields, }, geo, })) } func New(id string, geo geojson.Object, expires int64, fields field.List, ) *Object { switch p := geo.(type) { case *geojson.SimplePoint: return newPoint(id, p.Base(), expires, fields) case *geojson.Point: if p.IsSimple() { return newPoint(id, p.Base(), expires, fields) } } return newGeo(id, geo, expires, fields) } ================================================ FILE: internal/object/object_struct.go ================================================ //go:build exclude package object import ( "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/tile38/internal/field" ) type Object struct { id string geo geojson.Object expires int64 // unix nano expiration fields field.List } func (o *Object) ID() string { if o == nil { return "" } return o.id } func (o *Object) Fields() field.List { if o == nil { return field.List{} } return o.fields } func (o *Object) Expires() int64 { if o == nil { return 0 } return o.expires } func (o *Object) Rect() geometry.Rect { if o == nil || o.geo == nil { return geometry.Rect{} } return o.geo.Rect() } func (o *Object) Geo() geojson.Object { if o == nil || o.geo == nil { return nil } return o.geo } func (o *Object) String() string { if o == nil || o.geo == nil { return "" } return o.geo.String() } func (o *Object) IsSpatial() bool { _, ok := o.geo.(geojson.Spatial) return ok } func (o *Object) Weight() int { if o == nil { return 0 } var weight int weight += len(o.ID()) if o.IsSpatial() { weight += o.Geo().NumPoints() * 16 } else { weight += len(o.Geo().String()) } weight += o.Fields().Weight() return weight } func New(id string, geo geojson.Object, expires int64, fields field.List, ) *Object { return &Object{ id: id, geo: geo, expires: expires, fields: fields, } } ================================================ FILE: internal/object/object_test.go ================================================ package object import ( "testing" "github.com/tidwall/assert" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/tile38/internal/field" ) func P(x, y float64) geojson.Object { return geojson.NewSimplePoint(geometry.Point{X: 10, Y: 20}) } func TestObject(t *testing.T) { o := New("hello", P(10, 20), 99, field.List{}) assert.Assert(o.ID() == "hello") } ================================================ FILE: internal/server/aof.go ================================================ package server import ( "errors" "fmt" "io" "math" "net" "os" "sort" "strconv" "strings" "sync" "time" "github.com/tidwall/buntdb" "github.com/tidwall/gjson" "github.com/tidwall/redcon" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) type errAOFHook struct { err error } func (err errAOFHook) Error() string { return fmt.Sprintf("hook: %v", err.err) } func (s *Server) loadAOF() (err error) { fi, err := s.aof.Stat() if err != nil { return err } start := time.Now() var count int defer func() { d := time.Since(start) ps := float64(count) / (float64(d) / float64(time.Second)) suf := []string{"bytes/s", "KB/s", "MB/s", "GB/s", "TB/s"} bps := float64(fi.Size()) / (float64(d) / float64(time.Second)) for i := 0; bps > 1024 && len(suf) > 1; i++ { bps /= 1024 suf = suf[1:] } byteSpeed := fmt.Sprintf("%.0f %s", bps, suf[0]) log.Infof("AOF loaded %d commands: %.2fs, %.0f/s, %s", count, float64(d)/float64(time.Second), ps, byteSpeed) }() var buf []byte var args [][]byte var packet [0xFFFF]byte for { n, err := s.aof.Read(packet[:]) if err != nil { if err != io.EOF { return err } if len(buf) > 0 { // There was an incomplete command or other data at the end of // the AOF file. Attempt to recover the file by truncating the // file at the end position of the last complete command. log.Warnf("Truncating %d bytes due to an incomplete command\n", len(buf)) s.aofsz -= len(buf) if err := s.aof.Truncate(int64(s.aofsz)); err != nil { return err } if _, err := s.aof.Seek(int64(s.aofsz), 0); err != nil { return err } } return nil } s.aofsz += n data := packet[:n] if len(buf) > 0 { data = append(buf, data...) } var complete bool for { if len(data) > 0 && data[0] == 0 { // Zeros found in AOF file (issue #230). // Just ignore it and move the next byte. data = data[1:] continue } complete, args, _, data, err = redcon.ReadNextCommand(data, args[:0]) if err != nil { return err } if !complete { break } if len(args) > 0 { var msg Message msg.Args = msg.Args[:0] for _, arg := range args { msg.Args = append(msg.Args, string(arg)) } if _, _, err := s.command(&msg, nil); err != nil { if commandErrIsFatal(err) { return err } } count++ } } if len(data) > 0 { buf = append(buf[:0], data...) } else if len(buf) > 0 { buf = buf[:0] } } } func commandErrIsFatal(err error) bool { // FSET (and other writable commands) may return errors that we need // to ignore during the loading process. These errors may occur (though unlikely) // due to the aof rewrite operation. return !(err == errKeyNotFound || err == errIDNotFound) } // flushAOF flushes all aof buffer data to disk. Set sync to true to sync the // fsync the file. func (s *Server) flushAOF(sync bool) { if len(s.aofbuf) > 0 { _, err := s.aof.Write(s.aofbuf) if err != nil { panic(err) } // send a broadcast to all sleeping followers s.fcond.Broadcast() if sync { if err := s.aof.Sync(); err != nil { panic(err) } } if cap(s.aofbuf) > 1024*1024*32 { s.aofbuf = make([]byte, 0, 1024*1024*32) } else { s.aofbuf = s.aofbuf[:0] } } } func (s *Server) writeAOF(args []string, d *commandDetails) error { if d != nil && !d.updated { // just ignore writes if the command did not update return nil } if s.shrinking { nargs := make([]string, len(args)) copy(nargs, args) s.shrinklog = append(s.shrinklog, nargs) } if s.aof != nil { s.aofdirty.Store(true) // prewrite optimization flag n := len(s.aofbuf) s.aofbuf = redcon.AppendArray(s.aofbuf, len(args)) for _, arg := range args { s.aofbuf = redcon.AppendBulkString(s.aofbuf, arg) } s.aofsz += len(s.aofbuf) - n } // process geofences if d != nil { // webhook geofences if s.config.followHost() == "" { // for leader only if d.parent { // queue children for _, d := range d.children { if err := s.queueHooks(d); err != nil { return err } } } else { // queue parent if err := s.queueHooks(d); err != nil { return err } } } // live geofences s.lcond.L.Lock() if len(s.lives) > 0 { if d.parent { // queue children s.lstack = append(s.lstack, d.children...) } else { // queue parent s.lstack = append(s.lstack, d) } s.lcond.Broadcast() } s.lcond.L.Unlock() } return nil } func (s *Server) getQueueCandidates(d *commandDetails) []*Hook { candidates := make(map[*Hook]bool) // add the hooks with "outside" detection s.hooksOut.Ascend(nil, func(v interface{}) bool { hook := v.(*Hook) if hook.Key == d.key { candidates[hook] = true } return true }) // look for candidates that might "cross" geofences if d.old != nil && d.obj != nil && s.hookCross.Len() > 0 { r1, r2 := d.old.Rect(), d.obj.Rect() s.hookCross.Search( [2]float64{ math.Min(r1.Min.X, r2.Min.X), math.Min(r1.Min.Y, r2.Min.Y), }, [2]float64{ math.Max(r1.Max.X, r2.Max.X), math.Max(r1.Max.Y, r2.Max.Y), }, func(min, max [2]float64, value interface{}) bool { hook := value.(*Hook) if hook.Key == d.key { candidates[hook] = true } return true }) } // look for candidates that overlap the old object if d.old != nil { r1 := d.old.Rect() s.hookTree.Search( [2]float64{r1.Min.X, r1.Min.Y}, [2]float64{r1.Max.X, r1.Max.Y}, func(min, max [2]float64, value interface{}) bool { hook := value.(*Hook) if hook.Key == d.key { candidates[hook] = true } return true }) } // look for candidates that overlap the new object if d.obj != nil { r1 := d.obj.Rect() s.hookTree.Search( [2]float64{r1.Min.X, r1.Min.Y}, [2]float64{r1.Max.X, r1.Max.Y}, func(min, max [2]float64, value interface{}) bool { hook := value.(*Hook) if hook.Key == d.key { candidates[hook] = true } return true }) } if len(candidates) == 0 { return nil } // return the candidates as a slice ret := make([]*Hook, 0, len(candidates)) for hook := range candidates { ret = append(ret, hook) } return ret } func (s *Server) queueHooks(d *commandDetails) error { // Create the slices that will store all messages and hooks var cmsgs, wmsgs []string var whooks []*Hook // Compile a slice of potential hook recipients candidates := s.getQueueCandidates(d) for _, hook := range candidates { // Calculate all matching fence messages for all candidates and append // them to the appropriate message slice msgs := FenceMatch(hook.Name, hook.ScanWriter, hook.Fence, hook.Metas, d) if len(msgs) > 0 { if hook.channel { cmsgs = append(cmsgs, msgs...) } else { wmsgs = append(wmsgs, msgs...) whooks = append(whooks, hook) } } } // Return nil if there are no messages to be sent if len(cmsgs)+len(wmsgs) == 0 { return nil } // Sort both message channel and webhook message slices if len(cmsgs) > 1 { sortMsgs(cmsgs) } if len(wmsgs) > 1 { sortMsgs(wmsgs) } // Publish all channel messages if any exist if len(cmsgs) > 0 { for _, m := range cmsgs { s.Publish(gjson.Get(m, "hook").String(), m) } } // Queue the webhook messages in the buntdb database err := s.qdb.Update(func(tx *buntdb.Tx) error { for _, msg := range wmsgs { s.qidx++ // increment the log id key := hookLogPrefix + uint64ToString(s.qidx) _, _, err := tx.Set(key, msg, hookLogSetDefaults) if err != nil { return err } log.Debugf("queued hook: %d", s.qidx) } _, _, err := tx.Set("hook:idx", uint64ToString(s.qidx), nil) if err != nil { return err } return nil }) if err != nil { return err } // all the messages have been queued. // notify the hooks for _, hook := range whooks { hook.Signal() } return nil } // sortMsgs sorts passed notification messages by their detect and hook fields func sortMsgs(msgs []string) { sort.SliceStable(msgs, func(i, j int) bool { detectI := msgDetectCode(gjson.Get(msgs[i], "detect").String()) detectJ := msgDetectCode(gjson.Get(msgs[j], "detect").String()) if detectI < detectJ { return true } if detectI > detectJ { return false } hookI := gjson.Get(msgs[i], "hook").String() hookJ := gjson.Get(msgs[j], "hook").String() return hookI < hookJ }) } // msgDetectCode returns a weight value for the passed detect value func msgDetectCode(detect string) int { switch detect { case "exit": return 1 case "outside": return 2 case "enter": return 3 case "inside": return 4 default: return 0 } } // Converts string to an integer func stringToUint64(s string) uint64 { n, _ := strconv.ParseUint(s, 10, 64) return n } // Converts a uint to a string func uint64ToString(u uint64) string { s := strings.Repeat("0", 20) + strconv.FormatUint(u, 10) return s[len(s)-20:] } type liveAOFSwitches struct { pos int64 } func (s liveAOFSwitches) Error() string { return goingLive } // AOFMD5 pos size func (s *Server) cmdAOFMD5(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 3 { return retrerr(errInvalidNumberOfArguments) } pos, err := strconv.ParseInt(args[1], 10, 64) if err != nil || pos < 0 { return retrerr(errInvalidArgument(args[1])) } size, err := strconv.ParseInt(args[2], 10, 64) if err != nil || size < 0 { return retrerr(errInvalidArgument(args[2])) } // >> Operation sum, err := s.checksum(pos, size) if err != nil { return retrerr(err) } // >> Response if msg.OutputType == JSON { return resp.StringValue(fmt.Sprintf( `{"ok":true,"md5":"%s","elapsed":"%s"}`, sum, time.Since(start))), nil } return resp.SimpleStringValue(sum), nil } // AOF pos func (s *Server) cmdAOF(msg *Message) (resp.Value, error) { if s.aof == nil { return retrerr(errors.New("aof disabled")) } // >> Args args := msg.Args if len(args) != 2 { return retrerr(errInvalidNumberOfArguments) } pos, err := strconv.ParseInt(args[1], 10, 64) if err != nil || pos < 0 { return retrerr(errInvalidArgument(args[1])) } // >> Operation f, err := os.Open(s.aof.Name()) if err != nil { return retrerr(err) } defer f.Close() n, err := f.Seek(0, 2) if err != nil { return retrerr(err) } if n < pos { return retrerr(errors.New( "pos is too big, must be less that the aof_size of leader")) } // >> Response var ls liveAOFSwitches ls.pos = pos return NOMessage, ls } func (s *Server) liveAOF(pos int64, conn net.Conn, rd *PipelineReader, msg *Message) error { s.mu.RLock() f, err := os.Open(s.aof.Name()) s.mu.RUnlock() if err != nil { return err } s.mu.Lock() s.aofconnM[conn] = f s.mu.Unlock() defer func() { s.mu.Lock() delete(s.aofconnM, conn) s.mu.Unlock() conn.Close() f.Close() }() if _, err := conn.Write([]byte("+OK\r\n")); err != nil { return err } if _, err := f.Seek(pos, 0); err != nil { return err } var wg sync.WaitGroup wg.Add(1) go func() { defer func() { f.Close() conn.Close() wg.Done() }() // Any incoming message should end the connection rd.ReadMessages() }() _, err = io.Copy(conn, f) if err != nil { return err } b := make([]byte, 4096*2) for { n, err := f.Read(b) if n > 0 { if _, err := conn.Write(b[:n]); err != nil { return err } } if err == io.EOF { s.fcond.L.Lock() s.fcond.Wait() s.fcond.L.Unlock() } else if err != nil { if errors.Is(err, os.ErrClosed) { // The live aof file can be closed when a client (follower) has // closed their connection or following an AOFSHRINK operation. err = nil } return err } } } ================================================ FILE: internal/server/aofmigrate.go ================================================ package server import ( "bufio" "encoding/binary" "errors" "io" "os" "path" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) var errCorruptedAOF = errors.New("corrupted aof file") // LegacyAOFReader represents the older AOF file reader. type LegacyAOFReader struct { r io.Reader // reader rerr error // read error chunk []byte // chunk buffer buf []byte // main buffer l int // length of valid data in buffer p int // pointer } // ReadCommand reads an old command. func (rd *LegacyAOFReader) ReadCommand() ([]byte, error) { if rd.l >= 4 { sz1 := int(binary.LittleEndian.Uint32(rd.buf[rd.p:])) if rd.l >= sz1+9 { // we have enough data for a record sz2 := int(binary.LittleEndian.Uint32(rd.buf[rd.p+4+sz1:])) if sz2 != sz1 || rd.buf[rd.p+4+sz1+4] != 0 { return nil, errCorruptedAOF } buf := rd.buf[rd.p+4 : rd.p+4+sz1] rd.p += sz1 + 9 rd.l -= sz1 + 9 return buf, nil } } // need more data if rd.rerr != nil { if rd.rerr == io.EOF { rd.rerr = nil // we want to return EOF, but we want to be able to try again if rd.l != 0 { return nil, io.ErrUnexpectedEOF } return nil, io.EOF } return nil, rd.rerr } if rd.p != 0 { // move p to the beginning copy(rd.buf, rd.buf[rd.p:rd.p+rd.l]) rd.p = 0 } var n int n, rd.rerr = rd.r.Read(rd.chunk) if n > 0 { cbuf := rd.chunk[:n] if len(rd.buf)-rd.l < n { if len(rd.buf) == 0 { rd.buf = make([]byte, len(cbuf)) copy(rd.buf, cbuf) } else { copy(rd.buf[rd.l:], cbuf[:len(rd.buf)-rd.l]) rd.buf = append(rd.buf, cbuf[len(rd.buf)-rd.l:]...) } } else { copy(rd.buf[rd.l:], cbuf) } rd.l += n } return rd.ReadCommand() } // NewLegacyAOFReader creates a new LegacyAOFReader. func NewLegacyAOFReader(r io.Reader) *LegacyAOFReader { rd := &LegacyAOFReader{r: r, chunk: make([]byte, 0xFFFF)} return rd } func (s *Server) migrateAOF() error { _, err := os.Stat(path.Join(s.dir, "appendonly.aof")) if err == nil { return nil } if !os.IsNotExist(err) { return err } _, err = os.Stat(path.Join(s.dir, "aof")) if err != nil { if os.IsNotExist(err) { return nil } return err } log.Warn("Migrating aof to new format") newf, err := os.Create(path.Join(s.dir, "migrate.aof")) if err != nil { return err } defer newf.Close() oldf, err := os.Open(path.Join(s.dir, "aof")) if err != nil { return err } defer oldf.Close() start := time.Now() count := 0 wr := bufio.NewWriter(newf) rd := NewLegacyAOFReader(oldf) for { cmdb, err := rd.ReadCommand() if err != nil { if err == io.EOF { break } return err } line := string(cmdb) var tok string values := make([]resp.Value, 0, 64) for line != "" { line, tok = token(line) if len(tok) > 0 && tok[0] == '{' { if line != "" { tok = tok + " " + line line = "" } } values = append(values, resp.StringValue(tok)) } data, err := resp.ArrayValue(values).MarshalRESP() if err != nil { return err } if _, err := wr.Write(data); err != nil { return err } if wr.Buffered() > 1024*1024 { if err := wr.Flush(); err != nil { return err } } count++ } if err := wr.Flush(); err != nil { return err } oldf.Close() newf.Close() log.Debugf("%d items: %.0f/sec", count, float64(count)/(float64(time.Since(start))/float64(time.Second))) return os.Rename(path.Join(s.dir, "migrate.aof"), path.Join(s.dir, "appendonly.aof")) } ================================================ FILE: internal/server/aofshrink.go ================================================ package server import ( "math" "os" "strconv" "strings" "time" "github.com/tidwall/btree" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/object" ) const maxkeys = 8 const maxids = 32 const maxchunk = 4 * 1024 * 1024 func (s *Server) aofshrink() { start := time.Now() s.mu.Lock() if s.aof == nil || s.shrinking { s.mu.Unlock() return } s.shrinking = true s.shrinklog = nil s.mu.Unlock() defer func() { s.mu.Lock() s.shrinking = false s.shrinklog = nil s.mu.Unlock() log.Infof("aof shrink ended %v", time.Since(start)) }() err := func() error { f, err := os.Create(s.opts.AppendFileName + "-shrink") if err != nil { return err } defer f.Close() var aofbuf []byte var values []string var keys []string var nextkey string var keysdone bool for { if len(keys) == 0 { // load more keys if keysdone { break } keysdone = true func() { s.mu.Lock() defer s.mu.Unlock() s.cols.Ascend(nextkey, func(key string, col *collection.Collection) bool { if len(keys) == maxkeys { keysdone = false nextkey = key return false } keys = append(keys, key) return true }, ) }() continue } var idsdone bool var nextid string for { if idsdone { keys = keys[1:] break } // load more objects func() { idsdone = true s.mu.Lock() defer s.mu.Unlock() col, ok := s.cols.Get(keys[0]) if !ok { return } var now = time.Now().UnixNano() // used for expiration var count = 0 // the object count col.ScanGreaterOrEqual(nextid, false, nil, nil, func(o *object.Object) bool { if count == maxids { // we reached the max number of ids for one batch nextid = o.ID() idsdone = false return false } // here we fill the values array with a new command values = values[:0] values = append(values, "set") values = append(values, keys[0]) values = append(values, o.ID()) o.Fields().Scan(func(f field.Field) bool { if !f.Value().IsZero() { values = append(values, "field") values = append(values, f.Name()) values = append(values, f.Value().JSON()) } return true }) if o.Expires() != 0 { ttl := math.Floor(float64(o.Expires()-now)/float64(time.Second)*10) / 10 if ttl < 0.1 { // always leave a little bit of ttl. ttl = 0.1 } values = append(values, "ex") values = append(values, strconv.FormatFloat(ttl, 'f', -1, 64)) } if objIsSpatial(o.Geo()) { values = append(values, "object") values = append(values, string(o.Geo().AppendJSON(nil))) } else { values = append(values, "string") values = append(values, o.Geo().String()) } // append the values to the aof buffer aofbuf = append(aofbuf, '*') aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...) aofbuf = append(aofbuf, '\r', '\n') for _, value := range values { aofbuf = append(aofbuf, '$') aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...) aofbuf = append(aofbuf, '\r', '\n') aofbuf = append(aofbuf, value...) aofbuf = append(aofbuf, '\r', '\n') } // increment the object count count++ return true }, ) }() if len(aofbuf) > maxchunk { if _, err := f.Write(aofbuf); err != nil { return err } aofbuf = aofbuf[:0] } } } // load hooks // first load the names of the hooks var hnames []string func() { s.mu.Lock() defer s.mu.Unlock() hnames = make([]string, 0, s.hooks.Len()) s.hooks.Walk(func(v []interface{}) { for _, v := range v { hnames = append(hnames, v.(*Hook).Name) } }) }() var hookHint btree.PathHint for _, name := range hnames { func() { s.mu.Lock() defer s.mu.Unlock() hook, _ := s.hooks.GetHint(&Hook{Name: name}, &hookHint).(*Hook) if hook == nil { return } hook.cond.L.Lock() defer hook.cond.L.Unlock() var values []string if hook.channel { values = append(values, "setchan", name) } else { values = append(values, "sethook", name, strings.Join(hook.Endpoints, ",")) } for _, meta := range hook.Metas { values = append(values, "meta", meta.Name, meta.Value) } if !hook.expires.IsZero() { ex := float64(time.Until(hook.expires)) / float64(time.Second) values = append(values, "ex", strconv.FormatFloat(ex, 'f', 1, 64)) } values = append(values, hook.Message.Args...) // append the values to the aof buffer aofbuf = append(aofbuf, '*') aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...) aofbuf = append(aofbuf, '\r', '\n') for _, value := range values { aofbuf = append(aofbuf, '$') aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...) aofbuf = append(aofbuf, '\r', '\n') aofbuf = append(aofbuf, value...) aofbuf = append(aofbuf, '\r', '\n') } }() } if len(aofbuf) > 0 { if _, err := f.Write(aofbuf); err != nil { return err } aofbuf = aofbuf[:0] } if err := f.Sync(); err != nil { return err } // finally grab any new data that may have been written since // the aofshrink has started and swap out the files. return func() error { s.mu.Lock() defer s.mu.Unlock() // kill all followers connections and close their files. This // ensures that there is only one opened AOF at a time which is // what Windows requires in order to perform the Rename function // below. for conn, f := range s.aofconnM { conn.Close() f.Close() } // send a broadcast to all sleeping followers s.fcond.Broadcast() // flush the aof buffer s.flushAOF(false) aofbuf = aofbuf[:0] for _, values := range s.shrinklog { // append the values to the aof buffer aofbuf = append(aofbuf, '*') aofbuf = append(aofbuf, strconv.FormatInt(int64(len(values)), 10)...) aofbuf = append(aofbuf, '\r', '\n') for _, value := range values { aofbuf = append(aofbuf, '$') aofbuf = append(aofbuf, strconv.FormatInt(int64(len(value)), 10)...) aofbuf = append(aofbuf, '\r', '\n') aofbuf = append(aofbuf, value...) aofbuf = append(aofbuf, '\r', '\n') } } if _, err := f.Write(aofbuf); err != nil { return err } if err := f.Sync(); err != nil { return err } // we now have a shrunken aof file that is fully in-sync with // the current dataset. let's swap out the on disk files and // point to the new file. // anything below this point is unrecoverable. just log and exit process // back up the live aof, just in case of fatal error if err := s.aof.Close(); err != nil { log.Fatalf("shrink live aof close fatal operation: %v", err) } if err := f.Close(); err != nil { log.Fatalf("shrink new aof close fatal operation: %v", err) } if err := os.Rename(s.opts.AppendFileName, s.opts.AppendFileName+"-bak"); err != nil { log.Fatalf("shrink backup fatal operation: %v", err) } if err := os.Rename(s.opts.AppendFileName+"-shrink", s.opts.AppendFileName); err != nil { log.Fatalf("shrink rename fatal operation: %v", err) } s.aof, err = os.OpenFile(s.opts.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) if err != nil { log.Fatalf("shrink openfile fatal operation: %v", err) } var n int64 n, err = s.aof.Seek(0, 2) if err != nil { log.Fatalf("shrink seek end fatal operation: %v", err) } s.aofsz = int(n) os.Remove(s.opts.AppendFileName + "-bak") // ignore error return nil }() }() if err != nil { log.Errorf("aof shrink failed: %v", err) return } } ================================================ FILE: internal/server/bson.go ================================================ package server import ( "crypto/md5" "crypto/rand" "encoding/binary" "encoding/hex" "os" "sync/atomic" "time" ) func bsonID() string { b := make([]byte, 12) binary.BigEndian.PutUint32(b, uint32(time.Now().Unix())) copy(b[4:], bsonMachine) binary.BigEndian.PutUint32(b[8:], atomic.AddUint32(&bsonCounter, 1)) binary.BigEndian.PutUint16(b[7:], bsonProcess) return hex.EncodeToString(b) } var ( bsonProcess = uint16(os.Getpid()) bsonMachine = func() []byte { host, _ := os.Hostname() b := make([]byte, 3) Must(rand.Read(b)) host = Default(host, string(b)) hw := md5.New() hw.Write([]byte(host)) return hw.Sum(nil)[:3] }() bsonCounter = func() uint32 { b := make([]byte, 4) Must(rand.Read(b)) return binary.BigEndian.Uint32(b) }() ) ================================================ FILE: internal/server/bson_test.go ================================================ package server import "testing" func TestBSON(t *testing.T) { id := bsonID() if len(id) != 24 { t.Fail() } } ================================================ FILE: internal/server/checksum.go ================================================ package server import ( "crypto/md5" "errors" "fmt" "io" "os" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) // checksum performs a simple md5 checksum on the aof file func (s *Server) checksum(pos, size int64) (sum string, err error) { if pos+size > int64(s.aofsz) { return "", io.EOF } var f *os.File f, err = os.Open(s.aof.Name()) if err != nil { return } defer f.Close() sumr := md5.New() err = func() error { if size == 0 { n, err := f.Seek(int64(s.aofsz), 0) if err != nil { return err } if pos >= n { return io.EOF } return nil } _, err = f.Seek(pos, 0) if err != nil { return err } _, err = io.CopyN(sumr, f, size) if err != nil { return err } return nil }() if err != nil { if err == io.ErrUnexpectedEOF { err = io.EOF } return "", err } return fmt.Sprintf("%x", sumr.Sum(nil)), nil } func connAOFMD5(conn *RESPConn, pos, size int64) (sum string, err error) { v, err := conn.Do("aofmd5", pos, size) if err != nil { return "", err } if v.Error() != nil { errmsg := v.Error().Error() if errmsg == "ERR EOF" || errmsg == "EOF" { return "", io.EOF } return "", v.Error() } sum = v.String() if len(sum) != 32 { return "", errors.New("checksum not ok") } return sum, nil } func (s *Server) matchChecksums(conn *RESPConn, pos, size int64) (match bool, err error) { sum, err := s.checksum(pos, size) if err != nil { if err == io.EOF { return false, nil } return false, err } csum, err := connAOFMD5(conn, pos, size) if err != nil { if err == io.EOF { return false, nil } return false, err } return csum == sum, nil } // getEndOfLastValuePositionInFile is a very slow operation because it reads the file // backwards on byte at a time. Eek. It seek+read, seek+read, etc. func getEndOfLastValuePositionInFile(fname string, startPos int64) (int64, error) { pos := startPos f, err := os.Open(fname) if err != nil { return 0, err } defer f.Close() readByte := func() (byte, error) { if pos <= 0 { return 0, io.EOF } pos-- if _, err := f.Seek(pos, 0); err != nil { return 0, err } b := make([]byte, 1) if n, err := f.Read(b); err != nil { return 0, err } else if n != 1 { return 0, errors.New("invalid read") } return b[0], nil } for { c, err := readByte() if err != nil { return 0, err } if c == '*' { if _, err := f.Seek(pos, 0); err != nil { return 0, err } rd := resp.NewReader(f) _, telnet, n, err := rd.ReadMultiBulk() if err != nil || telnet { continue // keep reading backwards } return pos + int64(n), nil } } } // followCheckSome is not a full checksum. It just "checks some" data. // We will do some various checksums on the leader until we find the correct position to start at. func (s *Server) followCheckSome(addr string, followc int, auth string, ) (pos int64, err error) { if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":check some") } s.mu.Lock() defer s.mu.Unlock() if int(s.followc.Load()) != followc { return 0, errNoLongerFollowing } if s.aofsz < checksumsz { return 0, nil } conn, err := DialTimeout(addr, time.Second*2) if err != nil { return 0, err } defer conn.Close() if auth != "" { if err := s.followDoLeaderAuth(conn, auth); err != nil { return 0, err } } min := int64(0) max := int64(s.aofsz) - checksumsz limit := int64(s.aofsz) match, err := s.matchChecksums(conn, min, checksumsz) if err != nil { return 0, err } if match { min += checksumsz // bump up the min for { if max < min || max+checksumsz > limit { pos = min break } else { match, err = s.matchChecksums(conn, max, checksumsz) if err != nil { return 0, err } if match { min = max + checksumsz } else { limit = max } max = (limit-min)/2 - checksumsz/2 + min // multiply } } } fullpos := pos fname := s.aof.Name() if pos == 0 { s.aof.Close() s.aof, err = os.Create(fname) if err != nil { log.Fatalf("could not recreate aof, possible data loss. %s", err.Error()) return 0, err } return 0, nil } // we want to truncate at a command location // search for nearest command pos, err = getEndOfLastValuePositionInFile(s.aof.Name(), fullpos) if err != nil { return 0, err } if pos == fullpos { if s.opts.ShowDebugMessages { log.Debug("follow: aof fully intact") } return pos, nil } log.Warnf("truncating aof to %d", pos) // any error below are fatal. s.aof.Close() if err := os.Truncate(fname, pos); err != nil { log.Fatalf("could not truncate aof, possible data loss. %s", err.Error()) return 0, err } s.aof, err = os.OpenFile(fname, os.O_CREATE|os.O_RDWR, 0600) if err != nil { log.Fatalf("could not create aof, possible data loss. %s", err.Error()) return 0, err } // reset the entire system. log.Infof("reloading aof commands") s.reset() if err := s.loadAOF(); err != nil { log.Fatalf("could not reload aof, possible data loss. %s", err.Error()) return 0, err } if int64(s.aofsz) != pos { log.Fatalf("aof size mismatch during reload, possible data loss.") return 0, errors.New("?") } return pos, nil } ================================================ FILE: internal/server/client.go ================================================ package server import ( "encoding/json" "fmt" "io" "sort" "strings" "sync" "time" "github.com/tidwall/resp" ) // Client is an remote connection into to Tile38 type Client struct { id int // unique id replPort int // the known replication port for follower connections replAddr string // the known replication addr for follower connections authd bool // client has been authenticated outputType Type // Null, JSON, or RESP strictRESP bool // client is in strict RESP mode remoteAddr string // original remote address in InputStream // input stream pr PipelineReader // command reader out []byte // output write buffer goLiveErr error // error type used for going line goLiveMsg *Message // last message for go live mu sync.Mutex // guard conn io.ReadWriteCloser // out-of-loop connection. name string // optional defined name opened time.Time // when the client was created/opened, unix nano last time.Time // last client request/response, unix nano closer io.Closer // used to close the connection } // Write ... func (client *Client) Write(b []byte) (n int, err error) { client.out = append(client.out, b...) return len(b), nil } // CLIENT (LIST | KILL | GETNAME | SETNAME) func (s *Server) cmdCLIENT(msg *Message, client *Client) (resp.Value, error) { start := time.Now() args := msg.Args if len(args) == 1 { return retrerr(errInvalidNumberOfArguments) } switch strings.ToLower(args[1]) { case "list": if len(args) != 2 { return retrerr(errInvalidNumberOfArguments) } var list []*Client s.connsmu.RLock() for _, cc := range s.conns { list = append(list, cc) } s.connsmu.RUnlock() sort.Slice(list, func(i, j int) bool { return list[i].id < list[j].id }) now := time.Now() var buf []byte for _, client := range list { client.mu.Lock() buf = append(buf, fmt.Sprintf("id=%d addr=%s name=%s age=%d idle=%d\n", client.id, client.remoteAddr, client.name, now.Sub(client.opened)/time.Second, now.Sub(client.last)/time.Second, )..., ) client.mu.Unlock() } if msg.OutputType == JSON { // Create a map of all key/value info fields var cmap []map[string]interface{} clients := strings.Split(string(buf), "\n") for _, client := range clients { client = strings.TrimSpace(client) m := make(map[string]interface{}) var hasFields bool for _, kv := range strings.Split(client, " ") { kv = strings.TrimSpace(kv) if split := strings.SplitN(kv, "=", 2); len(split) == 2 { hasFields = true m[split[0]] = tryParseType(split[1]) } } if hasFields { cmap = append(cmap, m) } } data, _ := json.Marshal(cmap) return resp.StringValue(`{"ok":true,"list":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.BytesValue(buf), nil case "getname": if len(args) != 2 { return retrerr(errInvalidNumberOfArguments) } client.mu.Lock() name := client.name client.mu.Unlock() if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"name":` + jsonString(name) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.StringValue(name), nil case "setname": if len(args) != 3 { return retrerr(errInvalidNumberOfArguments) } name := msg.Args[2] for i := 0; i < len(name); i++ { if name[i] < '!' || name[i] > '~' { return retrerr(clientErrorf( "Client names cannot contain spaces, newlines or special characters.", )) } } client.mu.Lock() client.name = name client.mu.Unlock() if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.SimpleStringValue("OK"), nil case "kill": if len(args) < 3 { return retrerr(errInvalidNumberOfArguments) } var useAddr bool var addr string var useID bool var id string for i := 2; i < len(args); i++ { if useAddr || useID { return retrerr(errInvalidNumberOfArguments) } arg := args[i] if strings.Contains(arg, ":") { addr = arg useAddr = true } else { switch strings.ToLower(arg) { case "addr": i++ if i == len(args) { return retrerr(errInvalidNumberOfArguments) } addr = args[i] useAddr = true case "id": i++ if i == len(args) { return retrerr(errInvalidNumberOfArguments) } id = args[i] useID = true default: return retrerr(clientErrorf("No such client")) } } } var closing []io.Closer s.connsmu.RLock() for _, cc := range s.conns { if useID && fmt.Sprintf("%d", cc.id) == id { if cc.closer != nil { closing = append(closing, cc.closer) } } else if useAddr { if cc.remoteAddr == addr { if cc.closer != nil { closing = append(closing, cc.closer) } } } } s.connsmu.RUnlock() if len(closing) == 0 { return retrerr(clientErrorf("No such client")) } // go func() { // close the connections behind the scene for _, closer := range closing { closer.Close() } // }() if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.SimpleStringValue("OK"), nil default: return retrerr(clientErrorf( "Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)", )) } } ================================================ FILE: internal/server/config.go ================================================ package server import ( "encoding/json" "os" "strconv" "strings" "sync" "time" "github.com/tidwall/gjson" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/glob" ) const ( defaultKeepAlive = 300 // seconds defaultProtectedMode = "yes" ) // Config keys const ( FollowHost = "follow_host" FollowPort = "follow_port" FollowID = "follow_id" FollowPos = "follow_pos" ReplicaPriority = "replica-priority" ServerID = "server_id" ReadOnly = "read_only" RequirePass = "requirepass" LeaderAuth = "leaderauth" ProtectedMode = "protected-mode" MaxMemory = "maxmemory" AutoGC = "autogc" KeepAlive = "keepalive" LogConfig = "logconfig" AnnounceIP = "replica_announce_ip" AnnouncePort = "replica_announce_port" ) var validProperties = []string{RequirePass, LeaderAuth, ProtectedMode, MaxMemory, AutoGC, KeepAlive, LogConfig, ReplicaPriority, AnnouncePort, AnnounceIP} // Config is a tile38 config type Config struct { path string mu sync.RWMutex _followHost string _followPort int64 _followID string _followPos int64 _replicaPriority int64 _serverID string _readOnly bool _requirePassP string _requirePass string _leaderAuthP string _leaderAuth string _protectedModeP string _protectedMode string _maxMemoryP string _maxMemory int64 _autoGCP string _autoGC uint64 _keepAliveP string _keepAlive int64 _logConfigP interface{} _logConfig string _announceIPP string _announceIP string _announcePortP string _announcePort int64 } func loadConfig(path string) (*Config, error) { var json string data, err := os.ReadFile(path) if err != nil { if !os.IsNotExist(err) { return nil, err } } else { json = string(data) } config := &Config{ path: path, _followHost: gjson.Get(json, FollowHost).String(), _followPort: gjson.Get(json, FollowPort).Int(), _followID: gjson.Get(json, FollowID).String(), _followPos: gjson.Get(json, FollowPos).Int(), _serverID: gjson.Get(json, ServerID).String(), _readOnly: gjson.Get(json, ReadOnly).Bool(), _requirePassP: gjson.Get(json, RequirePass).String(), _leaderAuthP: gjson.Get(json, LeaderAuth).String(), _protectedModeP: gjson.Get(json, ProtectedMode).String(), _maxMemoryP: gjson.Get(json, MaxMemory).String(), _autoGCP: gjson.Get(json, AutoGC).String(), _keepAliveP: gjson.Get(json, KeepAlive).String(), _logConfig: gjson.Get(json, LogConfig).String(), _announceIPP: gjson.Get(json, AnnounceIP).String(), _announcePortP: gjson.Get(json, AnnouncePort).String(), } if config._serverID == "" { config._serverID = randomKey(16) } // Need to be sure we look for existence vs not zero because zero is an intentional setting // anything less than zero will be considered default and will result in no slave_priority // being output when INFO is called. if gjson.Get(json, ReplicaPriority).Exists() { config._replicaPriority = gjson.Get(json, ReplicaPriority).Int() } else { config._replicaPriority = -1 } // load properties if err := config.setProperty(RequirePass, config._requirePassP, true); err != nil { return nil, err } if err := config.setProperty(LeaderAuth, config._leaderAuthP, true); err != nil { return nil, err } if err := config.setProperty(ProtectedMode, config._protectedModeP, true); err != nil { return nil, err } if err := config.setProperty(MaxMemory, config._maxMemoryP, true); err != nil { return nil, err } if err := config.setProperty(AutoGC, config._autoGCP, true); err != nil { return nil, err } if err := config.setProperty(KeepAlive, config._keepAliveP, true); err != nil { return nil, err } if err := config.setProperty(LogConfig, config._logConfig, true); err != nil { return nil, err } if err := config.setProperty(AnnounceIP, config._announceIPP, true); err != nil { return nil, err } if err := config.setProperty(AnnouncePort, config._announcePortP, true); err != nil { return nil, err } config.write(false) return config, nil } func (config *Config) write(writeProperties bool) { config.mu.Lock() defer config.mu.Unlock() if writeProperties { // save properties config._requirePassP = config._requirePass config._leaderAuthP = config._leaderAuth if config._protectedMode == defaultProtectedMode { config._protectedModeP = "" } else { config._protectedModeP = config._protectedMode } config._maxMemoryP = formatMemSize(config._maxMemory) if config._autoGC == 0 { config._autoGCP = "" } else { config._autoGCP = strconv.FormatUint(config._autoGC, 10) } if config._keepAlive == defaultKeepAlive { config._keepAliveP = "" } else { config._keepAliveP = strconv.FormatUint(uint64(config._keepAlive), 10) } if config._logConfig != "" { config._logConfigP = config._logConfig } if config._announceIP != "" { config._announceIPP = config._announceIP } if config._announcePort == 0 { config._announcePortP = "" } else { config._announcePortP = strconv.FormatUint(uint64(config._announcePort), 10) } } m := make(map[string]interface{}) if config._followHost != "" { m[FollowHost] = config._followHost } if config._followPort != 0 { m[FollowPort] = config._followPort } if config._followID != "" { m[FollowID] = config._followID } if config._followPos != 0 { m[FollowPos] = config._followPos } if config._replicaPriority >= 0 { m[ReplicaPriority] = config._replicaPriority } if config._serverID != "" { m[ServerID] = config._serverID } if config._readOnly { m[ReadOnly] = config._readOnly } if config._requirePassP != "" { m[RequirePass] = config._requirePassP } if config._leaderAuthP != "" { m[LeaderAuth] = config._leaderAuthP } if config._protectedModeP != "" { m[ProtectedMode] = config._protectedModeP } if config._maxMemoryP != "" { m[MaxMemory] = config._maxMemoryP } if config._autoGCP != "" { m[AutoGC] = config._autoGCP } if config._keepAliveP != "" { m[KeepAlive] = config._keepAliveP } if config._logConfigP != "" { var lcfg map[string]interface{} json.Unmarshal([]byte(config._logConfig), &lcfg) if len(lcfg) > 0 { m[LogConfig] = lcfg } } if config._announceIPP != "" { m[AnnounceIP] = config._announceIPP } if config._announcePortP != "" { m[AnnouncePort] = config._announcePortP } data, err := json.MarshalIndent(m, "", "\t") if err != nil { panic(err) } data = append(data, '\n') err = os.WriteFile(config.path, data, 0600) if err != nil { panic(err) } } func parseMemSize(s string) (bytes int64, ok bool) { if s == "" { return 0, true } s = strings.ToLower(s) var n uint64 var sz int64 var err error if strings.HasSuffix(s, "gb") { n, err = strconv.ParseUint(s[:len(s)-2], 10, 64) sz = int64(n * 1024 * 1024 * 1024) } else if strings.HasSuffix(s, "mb") { n, err = strconv.ParseUint(s[:len(s)-2], 10, 64) sz = int64(n * 1024 * 1024) } else if strings.HasSuffix(s, "kb") { n, err = strconv.ParseUint(s[:len(s)-2], 10, 64) sz = int64(n * 1024) } else { n, err = strconv.ParseUint(s, 10, 64) sz = int64(n) } if err != nil { return 0, false } return sz, true } func formatMemSize(sz int64) string { if sz <= 0 { return "" } if sz < 1024 { return strconv.FormatInt(sz, 10) } sz /= 1024 if sz < 1024 { return strconv.FormatInt(sz, 10) + "kb" } sz /= 1024 if sz < 1024 { return strconv.FormatInt(sz, 10) + "mb" } sz /= 1024 return strconv.FormatInt(sz, 10) + "gb" } func (config *Config) setProperty(name, value string, fromLoad bool) error { config.mu.Lock() defer config.mu.Unlock() var invalid bool switch name { default: return clientErrorf("Unsupported CONFIG parameter: %s", name) case RequirePass: config._requirePass = value case LeaderAuth: config._leaderAuth = value case AutoGC: if value == "" { config._autoGC = 0 } else { gc, err := strconv.ParseUint(value, 10, 64) if err != nil { return err } config._autoGC = gc } case MaxMemory: sz, ok := parseMemSize(value) if !ok { return clientErrorf("Invalid argument '%s' for CONFIG SET '%s'", value, name) } config._maxMemory = sz case ProtectedMode: switch strings.ToLower(value) { case "": if fromLoad { config._protectedMode = defaultProtectedMode } else { invalid = true } case "yes", "no": config._protectedMode = strings.ToLower(value) default: invalid = true } case KeepAlive: if value == "" { config._keepAlive = defaultKeepAlive } else { keepalive, err := strconv.ParseUint(value, 10, 64) if err != nil { invalid = true } else { config._keepAlive = int64(keepalive) } } case LogConfig: if value == "" { config._logConfig = "" } else { config._logConfig = value } case ReplicaPriority: replicaPriority, err := strconv.ParseInt(value, 10, 64) if err != nil || replicaPriority < 0 { invalid = true } else { config._replicaPriority = replicaPriority } case AnnounceIP: config._announceIP = value case AnnouncePort: if value == "" { config._announcePort = 0 } else { announcePort, err := strconv.ParseUint(value, 10, 64) if err != nil { invalid = true } else { config._announcePort = int64(announcePort) } } } if invalid { return clientErrorf("Invalid argument '%s' for CONFIG SET '%s'", value, name) } return nil } func (config *Config) getProperties(pattern string) map[string]interface{} { m := make(map[string]interface{}) for _, name := range validProperties { matched, _ := glob.Match(pattern, name) if matched { m[name] = config.getProperty(name) } } return m } func (config *Config) getProperty(name string) string { config.mu.RLock() defer config.mu.RUnlock() switch name { default: return "" case AutoGC: return strconv.FormatUint(config._autoGC, 10) case RequirePass: return config._requirePass case LeaderAuth: return config._leaderAuth case ProtectedMode: return config._protectedMode case MaxMemory: return formatMemSize(config._maxMemory) case KeepAlive: return strconv.FormatUint(uint64(config._keepAlive), 10) case LogConfig: return config._logConfig case ReplicaPriority: if config._replicaPriority < 0 { return "" } else { return strconv.FormatUint(uint64(config._replicaPriority), 10) } case AnnounceIP: return config._announceIP case AnnouncePort: return strconv.FormatUint(uint64(config._announcePort), 10) } } func (s *Server) cmdConfigGet(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] var ok bool var name string if vs, name, ok = tokenval(vs); !ok { return NOMessage, errInvalidNumberOfArguments } if len(vs) != 0 { return NOMessage, errInvalidNumberOfArguments } m := s.config.getProperties(name) switch msg.OutputType { case JSON: data, err := json.Marshal(m) if err != nil { return NOMessage, err } res = resp.StringValue(`{"ok":true,"properties":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: vals := respValuesSimpleMap(m) res = resp.ArrayValue(vals) } return } func (s *Server) cmdConfigSet(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] var ok bool var name string if vs, name, ok = tokenval(vs); !ok { return NOMessage, errInvalidNumberOfArguments } var value string if vs, value, ok = tokenval(vs); !ok { if strings.ToLower(name) != RequirePass { return NOMessage, errInvalidNumberOfArguments } } if len(vs) != 0 { return NOMessage, errInvalidNumberOfArguments } if err := s.config.setProperty(name, value, false); err != nil { return NOMessage, err } if name == MaxMemory { s.checkOutOfMemory() } return OKMessage(msg, start), nil } func (s *Server) cmdConfigRewrite(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] if len(vs) != 0 { return NOMessage, errInvalidNumberOfArguments } s.config.write(true) return OKMessage(msg, start), nil } func (config *Config) followHost() string { config.mu.RLock() v := config._followHost config.mu.RUnlock() return v } func (config *Config) followPort() int { config.mu.RLock() v := config._followPort config.mu.RUnlock() return int(v) } func (config *Config) replicaPriority() int { config.mu.RLock() v := config._replicaPriority config.mu.RUnlock() return int(v) } func (config *Config) serverID() string { config.mu.RLock() v := config._serverID config.mu.RUnlock() return v } func (config *Config) readOnly() bool { config.mu.RLock() v := config._readOnly config.mu.RUnlock() return v } func (config *Config) requirePass() string { config.mu.RLock() v := config._requirePass config.mu.RUnlock() return v } func (config *Config) leaderAuth() string { config.mu.RLock() v := config._leaderAuth config.mu.RUnlock() return v } func (config *Config) protectedMode() string { config.mu.RLock() v := config._protectedMode config.mu.RUnlock() return v } func (config *Config) maxMemory() int { config.mu.RLock() v := config._maxMemory config.mu.RUnlock() return int(v) } func (config *Config) autoGC() uint64 { config.mu.RLock() v := config._autoGC config.mu.RUnlock() return v } func (config *Config) keepAlive() int64 { config.mu.RLock() v := config._keepAlive config.mu.RUnlock() return v } func (config *Config) announceIP() string { config.mu.RLock() v := config._announceIP config.mu.RUnlock() return v } func (config *Config) announcePort() int { config.mu.RLock() v := config._announcePort config.mu.RUnlock() return int(v) } func (config *Config) setFollowHost(v string) { config.mu.Lock() config._followHost = v config.mu.Unlock() } func (config *Config) setFollowPort(v int) { config.mu.Lock() config._followPort = int64(v) config.mu.Unlock() } func (config *Config) setReadOnly(v bool) { config.mu.Lock() config._readOnly = v config.mu.Unlock() } ================================================ FILE: internal/server/crud.go ================================================ package server import ( "bytes" "math" "strconv" "strings" "time" "github.com/mmcloughlin/geohash" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/object" ) // BOUNDS key func (s *Server) cmdBOUNDS(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 2 { return retrerr(errInvalidNumberOfArguments) } key := args[1] // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } return retrerr(errKeyNotFound) } // >> Response vals := make([]resp.Value, 0, 2) var buf bytes.Buffer if msg.OutputType == JSON { buf.WriteString(`{"ok":true`) } minX, minY, maxX, maxY := col.Bounds() bbox := geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: minX, Y: minY}, Max: geometry.Point{X: maxX, Y: maxY}, }) if msg.OutputType == JSON { buf.WriteString(`,"bounds":`) buf.WriteString(string(bbox.AppendJSON(nil))) buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil } // RESP vals = append(vals, resp.ArrayValue([]resp.Value{ resp.ArrayValue([]resp.Value{ resp.FloatValue(minX), resp.FloatValue(minY), }), resp.ArrayValue([]resp.Value{ resp.FloatValue(maxX), resp.FloatValue(maxY), }), })) return vals[0], nil } // TYPE key // undocumented return "none" or "hash" func (s *Server) cmdTYPE(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 2 { return retrerr(errInvalidNumberOfArguments) } key := args[1] // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.SimpleStringValue("none"), nil } return retrerr(errKeyNotFound) } // >> Response typ := "hash" if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"type":` + jsonString(typ) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.SimpleStringValue(typ), nil } // GET key id [WITHFIELDS] [OBJECT|POINT|BOUNDS|(HASH geohash)] func (s *Server) cmdGET(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) < 3 { return retrerr(errInvalidNumberOfArguments) } key, id := args[1], args[2] withfields := false kind := "object" var precision int64 for i := 3; i < len(args); i++ { switch strings.ToLower(args[i]) { case "withfields": withfields = true case "object": kind = "object" case "point": kind = "point" case "bounds": kind = "bounds" case "hash": kind = "hash" i++ if i == len(args) { return retrerr(errInvalidNumberOfArguments) } var err error precision, err = strconv.ParseInt(args[i], 10, 64) if err != nil || precision < 1 || precision > 12 { return retrerr(errInvalidArgument(args[i])) } default: return retrerr(errInvalidNumberOfArguments) } } // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } return retrerr(errKeyNotFound) } o := col.Get(id) if o == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } return retrerr(errIDNotFound) } // >> Response oval := buildObjectResponse(msg, o, start, kind, precision, withfields, msg.OutputType == JSON) return oval, nil } func buildObjectResponse(msg *Message, o *object.Object, start time.Time, kind string, precision int64, withfields, json bool) resp.Value { vals := make([]resp.Value, 0, 2) var buf bytes.Buffer if msg.OutputType == JSON { buf.WriteString(`{"ok":true`) } switch kind { case "object": if msg.OutputType == JSON { buf.WriteString(`,"object":`) buf.WriteString(string(o.Geo().AppendJSON(nil))) } else { vals = append(vals, resp.StringValue(o.Geo().String())) } case "point": if msg.OutputType == JSON { buf.WriteString(`,"point":`) buf.Write(appendJSONSimplePoint(nil, o.Geo())) } else { point := o.Geo().Center() z := extractZCoordinate(o.Geo()) if z != 0 { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)), resp.StringValue(strconv.FormatFloat(z, 'f', -1, 64)), })) } else { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.StringValue(strconv.FormatFloat(point.Y, 'f', -1, 64)), resp.StringValue(strconv.FormatFloat(point.X, 'f', -1, 64)), })) } } case "hash": if msg.OutputType == JSON { buf.WriteString(`,"hash":`) } center := o.Geo().Center() p := geohash.EncodeWithPrecision(center.Y, center.X, uint(precision)) if msg.OutputType == JSON { buf.WriteString(`"` + p + `"`) } else { vals = append(vals, resp.StringValue(p)) } case "bounds": if msg.OutputType == JSON { buf.WriteString(`,"bounds":`) buf.Write(appendJSONSimpleBounds(nil, o.Geo())) } else { bbox := o.Rect() vals = append(vals, resp.ArrayValue([]resp.Value{ resp.ArrayValue([]resp.Value{ resp.FloatValue(bbox.Min.Y), resp.FloatValue(bbox.Min.X), }), resp.ArrayValue([]resp.Value{ resp.FloatValue(bbox.Max.Y), resp.FloatValue(bbox.Max.X), }), })) } } if withfields { nfields := o.Fields().Len() if nfields > 0 { fvals := make([]resp.Value, 0, nfields*2) if msg.OutputType == JSON { buf.WriteString(`,"fields":{`) } var i int o.Fields().Scan(func(f field.Field) bool { if json { if i > 0 { buf.WriteString(`,`) } buf.WriteString(jsonString(f.Name()) + ":" + f.Value().JSON()) } else { fvals = append(fvals, resp.StringValue(f.Name()), resp.StringValue(f.Value().Data())) } i++ return true }) if json { buf.WriteString(`}`) } else { vals = append(vals, resp.ArrayValue(fvals)) } } } if json { buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()) } var oval resp.Value if withfields { oval = resp.ArrayValue(vals) } else { oval = vals[0] } return oval } // DEL key id [ERRON404] func (s *Server) cmdDEL(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args args := msg.Args if len(args) < 3 { return retwerr(errInvalidNumberOfArguments) } key := args[1] id := args[2] erron404 := false for i := 3; i < len(args); i++ { switch strings.ToLower(args[i]) { case "erron404": erron404 = true default: return retwerr(errInvalidArgument(args[i])) } } // >> Operation updated := false var old *object.Object col, _ := s.cols.Get(key) if col != nil { old = col.Delete(id) if old != nil { if col.Count() == 0 { s.cols.Delete(key) } updated = true } else if erron404 { return retwerr(errIDNotFound) } } else if erron404 { return retwerr(errKeyNotFound) } s.groupDisconnectObject(key, id) // >> Response var d commandDetails d.command = "del" d.key = key d.obj = old d.updated = updated d.timestamp = time.Now() var res resp.Value switch msg.OutputType { case JSON: res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: if d.updated { res = resp.IntegerValue(1) } else { res = resp.IntegerValue(0) } } return res, d, nil } // PDEL key pattern func (s *Server) cmdPDEL(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 3 { return retwerr(errInvalidNumberOfArguments) } key := args[1] pattern := args[2] // >> Operation now := time.Now() var children []*commandDetails col, _ := s.cols.Get(key) if col != nil { g := glob.Parse(pattern, false) var ids []string iter := func(o *object.Object) bool { if match, _ := glob.Match(pattern, o.ID()); match { ids = append(ids, o.ID()) } return true } if g.Limits[0] == "" && g.Limits[1] == "" { col.Scan(false, nil, msg.Deadline, iter) } else { col.ScanRange(g.Limits[0], g.Limits[1], false, nil, msg.Deadline, iter) } for _, id := range ids { obj := col.Delete(id) children = append(children, &commandDetails{ command: "del", updated: true, timestamp: now, key: key, obj: obj, }) s.groupDisconnectObject(key, id) } if col.Count() == 0 { s.cols.Delete(key) } } // >> Response var d commandDetails var res resp.Value d.command = "pdel" d.children = children d.key = key d.pattern = pattern d.updated = len(d.children) > 0 d.timestamp = now d.parent = true switch msg.OutputType { case JSON: res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: res = resp.IntegerValue(len(d.children)) } return res, d, nil } func (s *Server) cmdDROPop(key string) *collection.Collection { col, _ := s.cols.Get(key) if col != nil { s.cols.Delete(key) } s.groupDisconnectCollection(key) return col } // DROP key func (s *Server) cmdDROP(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 2 { return retwerr(errInvalidNumberOfArguments) } key := args[1] // >> Operation col := s.cmdDROPop(key) // >> Response var res resp.Value var d commandDetails d.key = key d.updated = col != nil d.command = "drop" d.timestamp = time.Now() switch msg.OutputType { case JSON: res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: if d.updated { res = resp.IntegerValue(1) } else { res = resp.IntegerValue(0) } } return res, d, nil } // RENAME key newkey // RENAMENX key newkey func (s *Server) cmdRENAME(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 3 { return retwerr(errInvalidNumberOfArguments) } nx := strings.ToLower(args[0]) == "renamenx" key := args[1] newKey := args[2] // >> Operation col, _ := s.cols.Get(key) if col == nil { return retwerr(errKeyNotFound) } var hasHook, hasChannel bool s.hooks.Ascend(nil, func(v interface{}) bool { h := v.(*Hook) if h.Key == key || h.Key == newKey { if h.channel { hasChannel = true } else { hasHook = true } } return true }) if hasHook { return retwerr(errKeyHasHooksSet) } if hasChannel { return retwerr(errKeyHasChannelsSet) } var updated bool newCol, _ := s.cols.Get(newKey) if newCol == nil { updated = true } else if !nx { s.cols.Delete(newKey) updated = true } if updated { s.cols.Delete(key) s.cols.Set(newKey, col) } // >> Response var d commandDetails var res resp.Value d.command = "rename" d.key = key d.newKey = newKey d.updated = updated d.timestamp = time.Now() switch msg.OutputType { case JSON: res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: if !nx { res = resp.SimpleStringValue("OK") } else if d.updated { res = resp.IntegerValue(1) } else { res = resp.IntegerValue(0) } } return res, d, nil } // FLUSHDB func (s *Server) cmdFLUSHDB(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 1 { return retwerr(errInvalidNumberOfArguments) } // >> Operation // clear the entire database // drop each collection keys := s.cols.Keys() for _, key := range keys { s.cmdDROPop(key) } // delete all channels var names []string s.hooks.Ascend(nil, func(item any) bool { hook := item.(*Hook) if hook.channel { names = append(names, hook.Name) } return true }) for _, name := range names { s.cmdDELHOOKop(name, true) } // delete all hooks names = names[:0] s.hooks.Ascend(nil, func(item any) bool { hook := item.(*Hook) if !hook.channel { names = append(names, hook.Name) } return true }) for _, name := range names { s.cmdDELHOOKop(name, false) } s.cols.Clear() s.groupHooks.Clear() s.groupObjects.Clear() s.hookExpires.Clear() s.hooks.Clear() s.hooksOut.Clear() s.hookTree.Clear() s.hookCross.Clear() // >> Response var d commandDetails d.command = "flushdb" d.updated = true d.timestamp = time.Now() var res resp.Value if msg.OutputType == JSON { res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") } else { res = resp.SimpleStringValue("OK") } return res, d, nil } // SET key id [FIELD name value ...] [EX seconds] [NX|XX] // (OBJECT geojson)|(POINT lat lon z)|(BOUNDS minlat minlon maxlat maxlon)| // (HASH geohash)|(STRING value) func (s *Server) cmdSET(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() if s.config.maxMemory() > 0 && s.outOfMemory.Load() { return retwerr(errOOM) } // >> Args var key string var id string var fields []field.Field var ex int64 var xx bool var nx bool var ret bool var withfields bool kind := "object" var precision int64 var oobj geojson.Object args := msg.Args if len(args) < 3 { return retwerr(errInvalidNumberOfArguments) } key, id = args[1], args[2] for i := 3; i < len(args); i++ { switch strings.ToLower(args[i]) { case "field": if i+2 >= len(args) { return retwerr(errInvalidNumberOfArguments) } fkey := args[i+1] fval := args[i+2] i += 2 if isReservedFieldName(fkey) { return retwerr(errInvalidArgument(fkey)) } fields = append(fields, field.Make(fkey, fval)) case "ex": if i+1 >= len(args) { return retwerr(errInvalidNumberOfArguments) } exval := args[i+1] i += 1 x, err := strconv.ParseFloat(exval, 64) if err != nil { return retwerr(errInvalidArgument(exval)) } ex = time.Now().UnixNano() + int64(float64(time.Second)*x) case "nx": if xx { return retwerr(errInvalidArgument(args[i])) } nx = true case "xx": if nx { return retwerr(errInvalidArgument(args[i])) } xx = true case "return": if ret { return retwerr(errInvalidArgument(args[i])) } ret = true for j := i; j < i+3; j++ { if j >= len(args) { break } switch strings.ToLower(args[j]) { case "withfields": withfields = true i += 1 case "object": kind = "object" i += 1 case "point": kind = "point" i += 1 case "bounds": kind = "bounds" i += 1 case "hash": kind = "hash" j++ if j == len(args) { return retwerr(errInvalidNumberOfArguments) } var err error precision, err = strconv.ParseInt(args[j], 10, 64) if err != nil || precision < 1 || precision > 12 { return retwerr(errInvalidArgument(args[j])) } i += 2 } } case "string": if i+1 >= len(args) { return retwerr(errInvalidNumberOfArguments) } str := args[i+1] i += 1 oobj = collection.String(str) case "point": if i+2 >= len(args) { return retwerr(errInvalidNumberOfArguments) } slat := args[i+1] slon := args[i+2] i += 2 var z float64 var hasZ bool if i+1 < len(args) { // probe for possible z coordinate var err error z, err = strconv.ParseFloat(args[i+1], 64) if err == nil { hasZ = true i++ } } y, err := strconv.ParseFloat(slat, 64) if err != nil { return retwerr(errInvalidArgument(slat)) } x, err := strconv.ParseFloat(slon, 64) if err != nil { return retwerr(errInvalidArgument(slon)) } if !hasZ { oobj = geojson.NewPoint(geometry.Point{X: x, Y: y}) } else { oobj = geojson.NewPointZ(geometry.Point{X: x, Y: y}, z) } case "bounds": if i+4 >= len(args) { return retwerr(errInvalidNumberOfArguments) } var vals [4]float64 for j := 0; j < 4; j++ { var err error vals[j], err = strconv.ParseFloat(args[i+1+j], 64) if err != nil { return retwerr(errInvalidArgument(args[i+1+j])) } } i += 4 oobj = geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: vals[1], Y: vals[0]}, Max: geometry.Point{X: vals[3], Y: vals[2]}, }) case "hash": if i+1 >= len(args) { return retwerr(errInvalidNumberOfArguments) } shash := args[i+1] i += 1 lat, lon := geohash.Decode(shash) oobj = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) case "object": if i+1 >= len(args) { return retwerr(errInvalidNumberOfArguments) } json := args[i+1] i += 1 var err error oobj, err = geojson.Parse(json, &s.geomParseOpts) if err != nil { return retwerr(err) } default: return retwerr(errInvalidArgument(args[i])) } } if oobj == nil { return retwerr(errInvalidNumberOfArguments) } // >> Operation nada := func() (resp.Value, commandDetails, error) { // exclude operation due to 'xx' or 'nx' match if msg.OutputType == JSON { if nx { return retwerr(errIDAlreadyExists) } else { return retwerr(errIDNotFound) } } return resp.NullValue(), commandDetails{}, nil } col, ok := s.cols.Get(key) if !ok { if xx { return nada() } col = collection.New() s.cols.Set(key, col) } if xx || nx { if col.Get(id) == nil { if xx { return nada() } } else { if nx { return nada() } } } var flist field.List if old := col.Get(id); old != nil { flist = old.Fields() } for _, f := range fields { flist = flist.Set(f) } obj := object.New(id, oobj, ex, flist) old := col.Set(obj) // >> Response var d commandDetails d.command = "set" d.key = key d.obj = obj d.old = old d.updated = true // perhaps we should do a diff on the previous object? d.timestamp = time.Now() if ret { res := buildObjectResponse(msg, obj, start, kind, precision, withfields, msg.OutputType == JSON) return res, d, nil } var res resp.Value switch msg.OutputType { default: case JSON: res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: res = resp.SimpleStringValue("OK") } return res, d, nil } func retwerr(err error) (resp.Value, commandDetails, error) { return resp.Value{}, commandDetails{}, err } func retrerr(err error) (resp.Value, error) { return resp.Value{}, err } // FSET key id [XX] field value [field value...] func (s *Server) cmdFSET(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() if s.config.maxMemory() > 0 && s.outOfMemory.Load() { return retwerr(errOOM) } // >> Args var id string var key string var xx bool var ret bool var withfields bool kind := "object" var precision int64 var fields []field.Field // raw fields args := msg.Args if len(args) < 5 { return retwerr(errInvalidNumberOfArguments) } key, id = args[1], args[2] for i := 3; i < len(args); i++ { arg := args[i] switch strings.ToLower(arg) { case "xx": xx = true case "return": if ret { return retwerr(errInvalidArgument(args[i])) } ret = true for j := i; j < i+3; j++ { if j >= len(args) { break } switch strings.ToLower(args[j]) { case "withfields": withfields = true i += 1 case "object": kind = "object" i += 1 case "point": kind = "point" i += 1 case "bounds": kind = "bounds" i += 1 case "hash": kind = "hash" j++ if j == len(args) { return retwerr(errInvalidNumberOfArguments) } var err error precision, err = strconv.ParseInt(args[j], 10, 64) if err != nil || precision < 1 || precision > 12 { return retwerr(errInvalidArgument(args[j])) } i += 2 } } default: fkey := arg i++ if i == len(args) { return retwerr(errInvalidNumberOfArguments) } if isReservedFieldName(fkey) { return retwerr(errInvalidArgument(fkey)) } fval := args[i] fields = append(fields, field.Make(fkey, fval)) } } // >> Operation var d commandDetails var updateCount int col, ok := s.cols.Get(key) if !ok { return retwerr(errKeyNotFound) } o := col.Get(id) ok = o != nil if !(ok || xx) { return retwerr(errIDNotFound) } if ok { ofields := o.Fields() for _, f := range fields { prev := ofields.Get(f.Name()) if !prev.Value().Equals(f.Value()) { ofields = ofields.Set(f) updateCount++ } } obj := object.New(id, o.Geo(), o.Expires(), ofields) col.Set(obj) d.command = "fset" d.key = key d.obj = obj d.timestamp = time.Now() d.updated = updateCount > 0 } // >> Response var res resp.Value if ret { res := buildObjectResponse(msg, d.obj, start, kind, precision, withfields, msg.OutputType == JSON) return res, d, nil } switch msg.OutputType { case JSON: res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: res = resp.IntegerValue(updateCount) } return res, d, nil } // FGET key id field func (s *Server) cmdFGET(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) < 4 { return retrerr(errInvalidNumberOfArguments) } key, id, field := args[1], args[2], args[3] // >> Operation col, _ := s.cols.Get(key) if col == nil { return retrerr(errKeyNotFound) } o := col.Get(id) if o == nil { return retrerr(errIDNotFound) } f := o.Fields().Get(field) // >> Response var buf bytes.Buffer switch msg.OutputType { case JSON: buf.WriteString(`{"ok":true`) buf.WriteString(`,"value":` + f.Value().JSON()) buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil case RESP: return resp.StringValue(f.Value().Data()), nil } return NOMessage, nil } // EXPIRE key id seconds func (s *Server) cmdEXPIRE(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 4 { return retwerr(errInvalidNumberOfArguments) } key, id, svalue := args[1], args[2], args[3] value, err := strconv.ParseFloat(svalue, 64) if err != nil { return retwerr(errInvalidArgument(svalue)) } // >> Operation var ok bool var obj *object.Object col, _ := s.cols.Get(key) if col != nil { // replace the expiration by getting the old object ex := time.Now().Add( time.Duration(float64(time.Second) * value)).UnixNano() o := col.Get(id) ok = o != nil if ok { obj = object.New(id, o.Geo(), ex, o.Fields()) col.Set(obj) } } // >> Response var d commandDetails if ok { d.key = key d.obj = obj d.command = "expire" d.updated = true d.timestamp = time.Now() } var res resp.Value switch msg.OutputType { case JSON: if ok { res = resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") } else if col == nil { return retwerr(errKeyNotFound) } else { return retwerr(errIDNotFound) } case RESP: if ok { res = resp.IntegerValue(1) } else { res = resp.IntegerValue(0) } } return res, d, nil } // PERSIST key id func (s *Server) cmdPERSIST(msg *Message) (resp.Value, commandDetails, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 3 { return retwerr(errInvalidNumberOfArguments) } key, id := args[1], args[2] // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.IntegerValue(0), commandDetails{}, nil } return retwerr(errKeyNotFound) } o := col.Get(id) if o == nil { if msg.OutputType == RESP { return resp.IntegerValue(0), commandDetails{}, nil } return retwerr(errIDNotFound) } var obj *object.Object var cleared bool if o.Expires() != 0 { obj = object.New(id, o.Geo(), 0, o.Fields()) col.Set(obj) cleared = true } // >> Response var res resp.Value var d commandDetails d.command = "persist" d.key = key d.obj = obj d.updated = cleared d.timestamp = time.Now() switch msg.OutputType { case JSON: res = resp.SimpleStringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: if cleared { res = resp.IntegerValue(1) } else { res = resp.IntegerValue(0) } } return res, d, nil } // TTL key id func (s *Server) cmdTTL(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 3 { return retrerr(errInvalidNumberOfArguments) } key, id := args[1], args[2] // >> Operation col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == JSON { return retrerr(errKeyNotFound) } return resp.IntegerValue(-2), nil } o := col.Get(id) if o == nil { if msg.OutputType == JSON { return retrerr(errIDNotFound) } return resp.IntegerValue(-2), nil } var ttl float64 if o.Expires() == 0 { ttl = -1 } else { now := start.UnixNano() ttl = math.Max(float64(o.Expires()-now)/float64(time.Second), 0) } // >> Response if msg.OutputType == JSON { return resp.SimpleStringValue( `{"ok":true,"ttl":` + strconv.Itoa(int(ttl)) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.IntegerValue(int(ttl)), nil } // EXISTS key id func (s *Server) cmdEXISTS(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 3 { return retrerr(errInvalidNumberOfArguments) } key, id := args[1], args[2] // >> Operation col, _ := s.cols.Get(key) if col == nil { return retrerr(errKeyNotFound) } o := col.Get(id) exists := o != nil // >> Response if msg.OutputType == JSON { return resp.SimpleStringValue( `{"ok":true,"exists":` + strconv.FormatBool(exists) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.BoolValue(exists), nil } // FEXISTS key id field func (s *Server) cmdFEXISTS(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 4 { return retrerr(errInvalidNumberOfArguments) } key, id, field := args[1], args[2], args[3] // >> Operation col, _ := s.cols.Get(key) if col == nil { return retrerr(errKeyNotFound) } o := col.Get(id) if o == nil { return retrerr(errIDNotFound) } f := o.Fields().Get(field) exists := f.Name() != "" // >> Response if msg.OutputType == JSON { return resp.SimpleStringValue( `{"ok":true,"exists":` + strconv.FormatBool(exists) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.BoolValue(exists), nil } ================================================ FILE: internal/server/dev.go ================================================ package server import ( "errors" "fmt" "math/rand" "strconv" "sync/atomic" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) // MASSINSERT num_keys num_points [minx miny maxx maxy] func randMassInsertPosition(minLat, minLon, maxLat, maxLon float64) (float64, float64) { lat, lon := (rand.Float64()*(maxLat-minLat))+minLat, (rand.Float64()*(maxLon-minLon))+minLon return lat, lon } func (s *Server) cmdMassInsert(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] minLat, minLon, maxLat, maxLon := -90.0, -180.0, 90.0, 180.0 //37.10776, -122.67145, 38.19502, -121.62775 var snumCols, snumPoints string var cols, objs int var ok bool if vs, snumCols, ok = tokenval(vs); !ok || snumCols == "" { return NOMessage, errInvalidNumberOfArguments } if vs, snumPoints, ok = tokenval(vs); !ok || snumPoints == "" { return NOMessage, errInvalidNumberOfArguments } if len(vs) != 0 { var sminLat, sminLon, smaxLat, smaxLon string if vs, sminLat, ok = tokenval(vs); !ok || sminLat == "" { return NOMessage, errInvalidNumberOfArguments } if vs, sminLon, ok = tokenval(vs); !ok || sminLon == "" { return NOMessage, errInvalidNumberOfArguments } if vs, smaxLat, ok = tokenval(vs); !ok || smaxLat == "" { return NOMessage, errInvalidNumberOfArguments } if vs, smaxLon, ok = tokenval(vs); !ok || smaxLon == "" { return NOMessage, errInvalidNumberOfArguments } var err error if minLat, err = strconv.ParseFloat(sminLat, 64); err != nil { return NOMessage, err } if minLon, err = strconv.ParseFloat(sminLon, 64); err != nil { return NOMessage, err } if maxLat, err = strconv.ParseFloat(smaxLat, 64); err != nil { return NOMessage, err } if maxLon, err = strconv.ParseFloat(smaxLon, 64); err != nil { return NOMessage, err } if len(vs) != 0 { return NOMessage, errors.New("invalid number of arguments") } } n, err := strconv.ParseUint(snumCols, 10, 64) if err != nil { return NOMessage, errInvalidArgument(snumCols) } cols = int(n) n, err = strconv.ParseUint(snumPoints, 10, 64) if err != nil { return NOMessage, errInvalidArgument(snumPoints) } docmd := func(args []string) error { s.mu.Lock() defer s.mu.Unlock() nmsg := *msg nmsg._command = "" nmsg.Args = args _, d, err := s.command(&nmsg, nil) if err != nil { return err } return s.writeAOF(nmsg.Args, &d) } rand.Seed(time.Now().UnixNano()) objs = int(n) var k atomic.Uint64 for i := 0; i < cols; i++ { key := "mi:" + strconv.FormatInt(int64(i), 10) func(key string) { // lock cycle for j := 0; j < objs; j++ { id := strconv.FormatInt(int64(j), 10) var values []string values = append(values, "set", key, id) fvals := []float64{ 1, // one 0, // zero -1, // negOne 14, // nibble 20.5, // tinyDiv10 120, // int8 -120, // int8 20000, // int16 -20000, // int16 214748300, // int32 -214748300, // int32 2014748300, // float64 123.12312301, // float64 } for i, fval := range fvals { values = append(values, "FIELD", fmt.Sprintf("fname:%d", i), strconv.FormatFloat(fval, 'f', -1, 64)) } if rand.Int()%2 == 0 { values = append(values, "EX", fmt.Sprint(rand.Intn(25)+5)) } if j%8 == 0 { values = append(values, "STRING", fmt.Sprintf("str%v", j)) } else { lat, lon := randMassInsertPosition(minLat, minLon, maxLat, maxLon) values = append(values, "POINT", strconv.FormatFloat(lat, 'f', -1, 64), strconv.FormatFloat(lon, 'f', -1, 64), ) } err := docmd(values) if err != nil { log.Fatal(err) return } k.Add(1) if j%1000 == 1000-1 { log.Debugf("massinsert: %s %d/%d", key, k.Load(), cols*objs) } } }(key) } log.Infof("massinsert: done %d objects", k.Load()) return OKMessage(msg, start), nil } func (s *Server) cmdSleep(msg *Message) (res resp.Value, err error) { start := time.Now() if len(msg.Args) != 2 { return NOMessage, errInvalidNumberOfArguments } d, _ := strconv.ParseFloat(msg.Args[1], 64) time.Sleep(time.Duration(float64(time.Second) * d)) return OKMessage(msg, start), nil } ================================================ FILE: internal/server/expire.go ================================================ package server import ( "sync" "time" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/object" ) const bgExpireDelay = time.Second / 10 // backgroundExpiring deletes expired items from the database. // It's executes every 1/10 of a second. func (s *Server) backgroundExpiring(wg *sync.WaitGroup) { defer wg.Done() s.loopUntilServerStops(bgExpireDelay, func() { s.mu.LockLowPriority() defer s.mu.Unlock() now := time.Now() s.backgroundExpireObjects(now) s.backgroundExpireHooks(now) }) } func (s *Server) backgroundExpireObjects(now time.Time) { nano := now.UnixNano() var msgs []*Message s.cols.Scan(func(key string, col *collection.Collection) bool { col.ScanExpires(func(o *object.Object) bool { if nano < o.Expires() { return false } s.statsExpired.Add(1) msgs = append(msgs, &Message{Args: []string{"del", key, o.ID()}}) return true }) return true }) for _, msg := range msgs { _, d, err := s.cmdDEL(msg) if err != nil { log.Fatal(err) } if err := s.writeAOF(msg.Args, &d); err != nil { log.Fatal(err) } } if len(msgs) > 0 { log.Debugf("Expired %d objects\n", len(msgs)) } } func (s *Server) backgroundExpireHooks(now time.Time) { var msgs []*Message s.hookExpires.Ascend(nil, func(v interface{}) bool { h := v.(*Hook) if h.expires.After(now) { return false } msg := &Message{} if h.channel { msg.Args = []string{"delchan", h.Name} } else { msg.Args = []string{"delhook", h.Name} } msgs = append(msgs, msg) return true }) for _, msg := range msgs { _, d, err := s.cmdDelHook(msg) if err != nil { log.Fatal(err) } if err := s.writeAOF(msg.Args, &d); err != nil { log.Fatal(err) } } if len(msgs) > 0 { log.Debugf("Expired %d hooks\n", len(msgs)) } } ================================================ FILE: internal/server/expr.go ================================================ package server import ( "fmt" "regexp" "sync" "github.com/tidwall/expr" "github.com/tidwall/geojson" "github.com/tidwall/gjson" "github.com/tidwall/match" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/object" "github.com/tidwall/tinylru" ) type exprPool struct { pool *sync.Pool regexCache tinylru.LRUG[string, *regexp.Regexp] } func typeForObject(o *object.Object) expr.Value { switch o.Geo().(type) { case *geojson.Point, *geojson.SimplePoint: return expr.String("Point") case *geojson.LineString: return expr.String("LineString") case *geojson.Polygon, *geojson.Circle, *geojson.Rect: return expr.String("Polygon") case *geojson.MultiPoint: return expr.String("MultiPoint") case *geojson.MultiLineString: return expr.String("MultiLineString") case *geojson.MultiPolygon: return expr.String("MultiPolygon") case *geojson.GeometryCollection: return expr.String("GeometryCollection") case *geojson.Feature: return expr.String("Feature") case *geojson.FeatureCollection: return expr.String("FeatureCollection") default: return expr.Undefined } } func resultToValue(r gjson.Result) expr.Value { if !r.Exists() { return expr.Undefined } switch r.Type { case gjson.String: return expr.String(r.String()) case gjson.False: return expr.Bool(false) case gjson.True: return expr.Bool(true) case gjson.Number: return expr.Number(r.Float()) case gjson.JSON: return expr.Object(r) default: return expr.Null } } func objExpr(o *object.Object, info expr.RefInfo) (expr.Value, error) { if r := gjson.Get(o.Geo().Members(), info.Ident); r.Exists() { return resultToValue(r), nil } switch info.Ident { case "id": return expr.String(o.ID()), nil case "type": return typeForObject(o), nil default: var rf field.Field var ok bool o.Fields().Scan(func(f field.Field) bool { if f.Name() == info.Ident { rf = f ok = true return false } return true }) if ok { r := gjson.Parse(rf.Value().JSON()) return resultToValue(r), nil } } return expr.Number(0), nil } func newExprPool(s *Server) *exprPool { pool := &exprPool{} ext := expr.NewExtender( // ref func(info expr.RefInfo, ctx *expr.Context) (expr.Value, error) { o := ctx.UserData.(*object.Object) if !info.Chain { // root if info.Ident == "this" { return expr.Object(o), nil } return objExpr(o, info) } else { switch v := info.Value.Value().(type) { case *object.Object: return objExpr(o, info) case gjson.Result: return resultToValue(v.Get(info.Ident)), nil default: // object methods switch info.Ident { case "match": return expr.Function("match"), nil } } return expr.Undefined, nil } }, // call func(info expr.CallInfo, ctx *expr.Context) (expr.Value, error) { if info.Chain { switch info.Ident { case "match": if info.Args.Len() < 0 { return expr.Undefined, nil } t := match.MatchNoCase(info.Value.String(), info.Args.At(0).String()) return expr.Bool(t), nil } } return expr.Undefined, nil }, // op func(info expr.OpInfo, ctx *expr.Context) (expr.Value, error) { switch info.Op { case expr.OpRegex: field := info.Left.String() pattern := info.Right.String() re, ok := pool.regexCache.Get(pattern) if !ok { var err error re, err = regexp.Compile(pattern) if err != nil { return expr.Undefined, fmt.Errorf("invalid regex pattern: %v", err) } pool.regexCache.Set(pattern, re) } return expr.Bool(re.MatchString(field)), nil } return expr.Undefined, nil }, ) pool.pool = &sync.Pool{ New: func() any { ctx := &expr.Context{ Extender: ext, } return ctx }, } return pool } func (p *exprPool) Get(o *object.Object) *expr.Context { ctx := p.pool.Get().(*expr.Context) ctx.UserData = o ctx.NoCase = true return ctx } func (p *exprPool) Put(ctx *expr.Context) { p.pool.Put(ctx) } func (where whereT) matchExpr(s *Server, o *object.Object) bool { ctx := s.epool.Get(o) res, err := expr.Eval(where.name, ctx) if err != nil { log.Debugf("%v", err) } s.epool.Put(ctx) return res.Bool() } ================================================ FILE: internal/server/expression.go ================================================ package server import ( "strings" "github.com/tidwall/geojson" ) // BinaryOp represents various operators for expressions type BinaryOp byte // expression operator enum const ( NOOP BinaryOp = iota AND OR tokenAND = "and" tokenOR = "or" tokenNOT = "not" tokenLParen = "(" tokenRParen = ")" ) // areaExpression is (maybe negated) either an spatial object or operator + // children (other expressions). type areaExpression struct { negate bool obj geojson.Object op BinaryOp children children } type children []*areaExpression // String representation, helpful in logging. func (e *areaExpression) String() (res string) { if e.obj != nil { res = e.obj.String() } else { var chStrings []string for _, c := range e.children { chStrings = append(chStrings, c.String()) } switch e.op { case NOOP: res = "empty operator" case AND: res = "(" + strings.Join(chStrings, " "+tokenAND+" ") + ")" case OR: res = "(" + strings.Join(chStrings, " "+tokenOR+" ") + ")" default: res = "unknown operator" } } if e.negate { res = tokenNOT + " " + res } return } // Return boolean value modulo negate field of the expression. func (e *areaExpression) maybeNegate(val bool) bool { if e.negate { return !val } return val } // Methods for testing an areaExpression against the spatial object. func (e *areaExpression) testObject( o geojson.Object, objObjTest func(o1, o2 geojson.Object) bool, exprObjTest func(ae *areaExpression, ob geojson.Object) bool, ) bool { if e.obj != nil { return objObjTest(e.obj, o) } switch e.op { case AND: for _, c := range e.children { if !exprObjTest(c, o) { return false } } return true case OR: for _, c := range e.children { if exprObjTest(c, o) { return true } } return false } return false } func (e *areaExpression) rawIntersects(o geojson.Object) bool { return e.testObject(o, geojson.Object.Intersects, (*areaExpression).Intersects) } func (e *areaExpression) rawContains(o geojson.Object) bool { return e.testObject(o, geojson.Object.Contains, (*areaExpression).Contains) } func (e *areaExpression) rawWithin(o geojson.Object) bool { return e.testObject(o, geojson.Object.Within, (*areaExpression).Within) } func (e *areaExpression) Intersects(o geojson.Object) bool { return e.maybeNegate(e.rawIntersects(o)) } func (e *areaExpression) Contains(o geojson.Object) bool { return e.maybeNegate(e.rawContains(o)) } func (e *areaExpression) Within(o geojson.Object) bool { return e.maybeNegate(e.rawWithin(o)) } // Methods for testing an areaExpression against another areaExpression. func (e *areaExpression) testExpression( other *areaExpression, exprObjTest func(ae *areaExpression, ob geojson.Object) bool, rawExprExprTest func(ae1, ae2 *areaExpression) bool, exprExprTest func(ae1, ae2 *areaExpression) bool, ) bool { if other.negate { oppositeExp := &areaExpression{negate: !e.negate, obj: e.obj, op: e.op, children: e.children} nonNegateOther := &areaExpression{obj: other.obj, op: other.op, children: other.children} return exprExprTest(oppositeExp, nonNegateOther) } if other.obj != nil { return exprObjTest(e, other.obj) } switch other.op { case AND: for _, c := range other.children { if !rawExprExprTest(e, c) { return false } } return true case OR: for _, c := range other.children { if rawExprExprTest(e, c) { return true } } return false } return false } func (e *areaExpression) rawIntersectsExpr(other *areaExpression) bool { return e.testExpression( other, (*areaExpression).rawIntersects, (*areaExpression).rawIntersectsExpr, (*areaExpression).IntersectsExpr) } func (e *areaExpression) rawWithinExpr(other *areaExpression) bool { return e.testExpression( other, (*areaExpression).rawWithin, (*areaExpression).rawWithinExpr, (*areaExpression).WithinExpr) } func (e *areaExpression) rawContainsExpr(other *areaExpression) bool { return e.testExpression( other, (*areaExpression).rawContains, (*areaExpression).rawContainsExpr, (*areaExpression).ContainsExpr) } func (e *areaExpression) IntersectsExpr(other *areaExpression) bool { return e.maybeNegate(e.rawIntersectsExpr(other)) } func (e *areaExpression) WithinExpr(other *areaExpression) bool { return e.maybeNegate(e.rawWithinExpr(other)) } func (e *areaExpression) ContainsExpr(other *areaExpression) bool { return e.maybeNegate(e.rawContainsExpr(other)) } ================================================ FILE: internal/server/fence.go ================================================ package server import ( "math" "sort" "strconv" "time" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geo" "github.com/tidwall/geojson/geometry" "github.com/tidwall/gjson" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/object" ) // FenceMatch executes a fence match returns back json messages for fence detection. func FenceMatch(hookName string, sw *scanWriter, fence *liveFenceSwitches, metas []FenceMeta, details *commandDetails) []string { msgs := fenceMatch(hookName, sw, fence, metas, details) if len(fence.accept) == 0 { return msgs } nmsgs := make([]string, 0, len(msgs)) for _, msg := range msgs { if fence.accept[gjson.Get(msg, "command").String()] { nmsgs = append(nmsgs, msg) } } return nmsgs } func appendHookDetails(b []byte, hookName string, metas []FenceMeta) []byte { if len(hookName) > 0 { b = append(b, `,"hook":`...) b = appendJSONString(b, hookName) } if len(metas) > 0 { b = append(b, `,"meta":{`...) for i, meta := range metas { if i > 0 { b = append(b, ',') } b = appendJSONString(b, meta.Name) b = append(b, ':') b = appendJSONString(b, meta.Value) } b = append(b, '}') } return b } func objIsSpatial(obj geojson.Object) bool { _, ok := obj.(geojson.Spatial) return ok } func hookJSONString(hookName string, metas []FenceMeta) string { return string(appendHookDetails(nil, hookName, metas)) } func multiGlobMatch(globs []string, s string) bool { if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") { return true } for _, pattern := range globs { match, _ := glob.Match(pattern, s) if match { return true } } return false } func fenceMatch( hookName string, sw *scanWriter, fence *liveFenceSwitches, metas []FenceMeta, details *commandDetails, ) []string { if details.command == "drop" { return []string{ `{"command":"drop"` + hookJSONString(hookName, metas) + `,"key":` + jsonString(details.key) + `,"time":` + jsonTimeFormat(details.timestamp) + `}`, } } if details.obj == nil { return nil } if !multiGlobMatch(fence.globs, details.obj.ID()) { return nil } if !objIsSpatial(details.obj.Geo()) { return nil } if details.command == "fset" { nofields := sw.nofields if nofields { return nil } } if details.command == "del" { return []string{ `{"command":"del"` + hookJSONString(hookName, metas) + `,"key":` + jsonString(details.key) + `,"id":` + jsonString(details.obj.ID()) + `,"time":` + jsonTimeFormat(details.timestamp) + `}`, } } var roamNearbys, roamFaraways []roamMatch var detect = "outside" if fence != nil { if fence.roam.on { if details.command == "set" { roamNearbys, roamFaraways = fenceMatchRoam(sw.s, fence, details.obj, details.old) if len(roamNearbys) == 0 && len(roamFaraways) == 0 { return nil } } detect = "roam" } else { var nocross bool // not using roaming match1 := fenceMatchObject(fence, details.old) if match1 { match1, _, _ = sw.testObject(details.old) nocross = !match1 } match2 := fenceMatchObject(fence, details.obj) if match2 { match2, _, _ = sw.testObject(details.obj) nocross = !match2 } if match1 && match2 { detect = "inside" } else if match1 && !match2 { detect = "exit" } else if !match1 && match2 { detect = "enter" if details.command == "fset" { detect = "inside" } } else { if details.command != "fset" { // For cross detection, the object is outside the fence spatially, // so testObject wasn't called above. We need to check WHERE clause // before proceeding with cross detection. if match, _ := sw.fieldMatch(details.obj); !match { return nil } // Maybe the old object and new object create a line that crosses the fence. // Must detect for that possibility. if !nocross && details.old != nil { ls := geojson.NewLineString(geometry.NewLine( []geometry.Point{ details.old.Geo().Center(), details.obj.Geo().Center(), }, nil)) temp := false if fence.cmd == "within" { // because we are testing if the line croses the area we need to use // "intersects" instead of "within". fence.cmd = "intersects" temp = true } lso := object.New("", ls, 0, field.List{}) if fenceMatchObject(fence, lso) { detect = "cross" } if temp { fence.cmd = "within" } } } } } } // TODO: fields // if details.fmap == nil { // return nil // } for { if fence.detect != nil && !fence.detect[detect] { if detect == "enter" { detect = "inside" continue } if detect == "exit" || detect == "cross" { detect = "outside" continue } return nil } break } var distance float64 if fence.distance && fence.obj != nil { distance = details.obj.Geo().Distance(fence.obj) } sw.fullFields = true sw.msg.OutputType = JSON sw.writeObject(ScanWriterParams{ obj: details.obj, noTest: true, dist: distance, distOutput: fence.distance, }) if sw.wr.Len() == 0 { return nil } res := sw.wr.String() sw.wr.Reset() if len(res) > 0 && res[0] == ',' { res = res[1:] } if sw.output == outputIDs { res = `{"id":` + string(res) + `}` } var group string if detect == "enter" { group = sw.s.groupConnect(hookName, details.key, details.obj.ID()) } else if detect == "cross" { sw.s.groupDisconnect(hookName, details.key, details.obj.ID()) group = sw.s.groupConnect(hookName, details.key, details.obj.ID()) } else { group = sw.s.groupGet(hookName, details.key, details.obj.ID()) if group == "" { group = sw.s.groupConnect(hookName, details.key, details.obj.ID()) } } var msgs []string if fence.detect == nil || fence.detect[detect] { if len(res) > 0 && res[0] == '{' { msgs = append(msgs, makemsg(details.command, group, detect, hookName, metas, details.key, details.timestamp, res[1:])) } else { msgs = append(msgs, string(res)) } } switch detect { case "enter": if fence.detect == nil || fence.detect["inside"] { msgs = append(msgs, makemsg(details.command, group, "inside", hookName, metas, details.key, details.timestamp, res[1:])) } case "exit", "cross": if fence.detect == nil || fence.detect["outside"] { msgs = append(msgs, makemsg(details.command, group, "outside", hookName, metas, details.key, details.timestamp, res[1:])) } case "roam": if len(msgs) > 0 { var nmsgs []string for _, msg := range msgs { cmd := gjson.Get(msg, "command") if cmd.Exists() && cmd.String() != "set" { nmsgs = append(nmsgs, msg) } } for i := range roamNearbys { nmsg := extendRoamMessage(sw, fence, "nearby", msgs[0], roamNearbys[i]) nmsgs = append(nmsgs, string(nmsg)) } for i := range roamFaraways { nmsg := extendRoamMessage(sw, fence, "faraway", msgs[0], roamFaraways[i]) nmsgs = append(nmsgs, string(nmsg)) } msgs = nmsgs } } return msgs } func extendRoamMessage( sw *scanWriter, fence *liveFenceSwitches, kind string, baseMsg string, match roamMatch, ) string { // hack off the last '}' nmsg := []byte(baseMsg[:len(baseMsg)-1]) nmsg = append(nmsg, `,"`+kind+`":{"key":`...) nmsg = appendJSONString(nmsg, fence.roam.key) nmsg = append(nmsg, `,"id":`...) nmsg = appendJSONString(nmsg, match.id) nmsg = append(nmsg, `,"object":`...) nmsg = match.obj.AppendJSON(nmsg) nmsg = append(nmsg, `,"meters":`...) nmsg = strconv.AppendFloat(nmsg, math.Floor(match.meters*1000)/1000, 'f', -1, 64) if fence.roam.scan != "" { nmsg = append(nmsg, `,"scan":[`...) col, _ := sw.s.cols.Get(fence.roam.key) if col != nil { o := col.Get(match.id) if o != nil { nmsg = append(nmsg, `{"id":`...) nmsg = appendJSONString(nmsg, match.id) nmsg = append(nmsg, `,"self":true,"object":`...) nmsg = o.Geo().AppendJSON(nmsg) nmsg = append(nmsg, '}') } pattern := match.id + fence.roam.scan iterator := func(o *object.Object) bool { if o.ID() == match.id { return true } if matched, _ := glob.Match(pattern, o.ID()); matched { nmsg = append(nmsg, `,{"id":`...) nmsg = appendJSONString(nmsg, o.ID()) nmsg = append(nmsg, `,"object":`...) nmsg = o.Geo().AppendJSON(nmsg) nmsg = append(nmsg, '}') } return true } g := glob.Parse(pattern, false) if g.Limits[0] == "" && g.Limits[1] == "" { col.Scan(false, nil, nil, iterator) } else { col.ScanRange(g.Limits[0], g.Limits[1], false, nil, nil, iterator) } } nmsg = append(nmsg, ']') } nmsg = append(nmsg, '}') // re-add the last '}' nmsg = append(nmsg, '}') return string(nmsg) } func makemsg( command, group, detect, hookName string, metas []FenceMeta, key string, t time.Time, tail string, ) string { var buf []byte buf = append(append(buf, `{"command":"`...), command...) buf = append(append(buf, `","group":"`...), group...) buf = append(append(buf, `","detect":"`...), detect...) buf = append(buf, '"') buf = appendHookDetails(buf, hookName, metas) buf = appendJSONString(append(buf, `,"key":`...), key) buf = appendJSONTimeFormat(append(buf, `,"time":`...), t) buf = append(append(buf, ','), tail...) return string(buf) } func fenceMatchObject(fence *liveFenceSwitches, o *object.Object) bool { if o == nil { return false } if fence.roam.on { // we need to check this object against return false } switch fence.cmd { case "nearby": // nearby is an INTERSECT on a Circle return o.Geo().Intersects(fence.obj) case "within": return o.Geo().Within(fence.obj) case "intersects": return o.Geo().Intersects(fence.obj) } return false } func fenceMatchNearbys( s *Server, fence *liveFenceSwitches, obj *object.Object, ) (nearbys []roamMatch) { if obj == nil { return nil } col, _ := s.cols.Get(fence.roam.key) if col == nil { return nil } center := obj.Geo().Center() minLat, minLon, maxLat, maxLon := geo.RectFromCenter(center.Y, center.X, fence.roam.meters) rect := geometry.Rect{ Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, } col.Intersects(geojson.NewRect(rect), 0, nil, nil, func(o *object.Object) bool { var idMatch bool if o.ID() == obj.ID() { return true // skip self } meters := o.Geo().Distance(o.Geo()) if meters > fence.roam.meters { return true // skip outside radius } if fence.roam.pattern { idMatch, _ = glob.Match(fence.roam.id, o.ID()) } else { idMatch = fence.roam.id == o.ID() } if !idMatch { return true // skip non-id match } match := roamMatch{ id: o.ID(), obj: o.Geo(), meters: obj.Geo().Distance(o.Geo()), } nearbys = append(nearbys, match) return true }, ) return nearbys } func fenceMatchRoam( s *Server, fence *liveFenceSwitches, obj, old *object.Object, ) (nearbys, faraways []roamMatch) { oldNearbys := fenceMatchNearbys(s, fence, old) newNearbys := fenceMatchNearbys(s, fence, obj) // Go through all matching objects in new-nearbys and old-nearbys. for i := 0; i < len(oldNearbys); i++ { var match bool var j int for ; j < len(newNearbys); j++ { if newNearbys[j].id == oldNearbys[i].id { match = true break } } if match { // dwelling, more from old-nearbys oldNearbys[i] = oldNearbys[len(oldNearbys)-1] oldNearbys = oldNearbys[:len(oldNearbys)-1] i-- if fence.nodwell { // no dwelling allowed, remove from both lists newNearbys[j] = newNearbys[len(newNearbys)-1] newNearbys = newNearbys[:len(newNearbys)-1] } } } faraways, nearbys = oldNearbys, newNearbys // ensure the faraways distances are to the new object for i := 0; i < len(faraways); i++ { faraways[i].meters = faraways[i].obj.Distance(obj.Geo()) } sortRoamMatches(faraways) sortRoamMatches(nearbys) return nearbys, faraways } // sortRoamMatches stable sorts roam matches func sortRoamMatches(matches []roamMatch) { sort.Slice(matches, func(i, j int) bool { if matches[i].meters < matches[j].meters { return true } if matches[i].meters > matches[j].meters { return false } return matches[i].id < matches[j].id }) } ================================================ FILE: internal/server/follow.go ================================================ package server import ( "errors" "fmt" "io" "strconv" "strings" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) var errNoLongerFollowing = errors.New("no longer following") const checksumsz = 512 * 1024 func (s *Server) cmdFollow(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] var ok bool var host, sport string if vs, host, ok = tokenval(vs); !ok || host == "" { return NOMessage, errInvalidNumberOfArguments } if vs, sport, ok = tokenval(vs); !ok || sport == "" { return NOMessage, errInvalidNumberOfArguments } if len(vs) != 0 { return NOMessage, errInvalidNumberOfArguments } host = strings.ToLower(host) sport = strings.ToLower(sport) var update bool if host == "no" && sport == "one" { update = s.config.followHost() != "" || s.config.followPort() != 0 s.config.setFollowHost("") s.config.setFollowPort(0) } else { n, err := strconv.ParseUint(sport, 10, 64) if err != nil { return NOMessage, errInvalidArgument(sport) } port := int(n) update = s.config.followHost() != host || s.config.followPort() != port auth := s.config.leaderAuth() if update { s.mu.Unlock() conn, err := DialTimeout(fmt.Sprintf("%s:%d", host, port), time.Second*2) if err != nil { s.mu.Lock() return NOMessage, fmt.Errorf("cannot follow: %v", err) } defer conn.Close() if auth != "" { if err := s.followDoLeaderAuth(conn, auth); err != nil { return NOMessage, fmt.Errorf("cannot follow: %v", err) } } m, err := doServer(conn) if err != nil { s.mu.Lock() return NOMessage, fmt.Errorf("cannot follow: %v", err) } if m["id"] == "" { s.mu.Lock() return NOMessage, fmt.Errorf("cannot follow: invalid id") } if m["id"] == s.config.serverID() { s.mu.Lock() return NOMessage, fmt.Errorf("cannot follow self") } if m["following"] != "" { s.mu.Lock() return NOMessage, fmt.Errorf("cannot follow a follower") } s.mu.Lock() } s.config.setFollowHost(host) s.config.setFollowPort(port) } s.config.write(false) if update { s.followc.Add(1) if s.config.followHost() != "" { log.Infof("following new host '%s' '%s'.", host, sport) go s.follow(s.config.followHost(), s.config.followPort(), int(s.followc.Load())) } else { log.Infof("following no one") } } return OKMessage(msg, start), nil } // cmdReplConf is a command handler that sets replication configuration info func (s *Server) cmdReplConf(msg *Message, client *Client) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] var ok bool var cmd, val string // Parse the message if vs, cmd, ok = tokenval(vs); !ok || cmd == "" { return NOMessage, errInvalidNumberOfArguments } if _, val, ok = tokenval(vs); !ok || val == "" { return NOMessage, errInvalidNumberOfArguments } // Switch on the command received switch cmd { case "listening-port": // Parse the port as an integer port, err := strconv.Atoi(val) if err != nil { return NOMessage, errInvalidArgument(val) } // Apply the replication port to the client and return s.connsmu.RLock() defer s.connsmu.RUnlock() for _, c := range s.conns { if c.remoteAddr == client.remoteAddr { c.mu.Lock() c.replPort = port c.mu.Unlock() return OKMessage(msg, start), nil } } case "ip-address": // Apply the replication ip to the client and return s.connsmu.RLock() defer s.connsmu.RUnlock() for _, c := range s.conns { if c.remoteAddr == client.remoteAddr { c.mu.Lock() c.replAddr = val c.mu.Unlock() return OKMessage(msg, start), nil } } } return NOMessage, fmt.Errorf("cannot find follower") } func doServer(conn *RESPConn) (map[string]string, error) { v, err := conn.Do("server") if err != nil { return nil, err } if v.Error() != nil { return nil, v.Error() } arr := v.Array() m := make(map[string]string) for i := 0; i < len(arr)/2; i++ { m[arr[i*2+0].String()] = arr[i*2+1].String() } return m, err } func (s *Server) followHandleCommand(args []string, followc int, w io.Writer) (int, error) { s.mu.Lock() defer s.mu.Unlock() if int(s.followc.Load()) != followc { return s.aofsz, errNoLongerFollowing } msg := &Message{Args: args} _, d, err := s.command(msg, nil) if err != nil { if commandErrIsFatal(err) { return s.aofsz, err } } switch msg.Command() { case "publish": // Avoid writing these commands to the AOF default: if err := s.writeAOF(args, &d); err != nil { return s.aofsz, err } } if len(s.aofbuf) > 10240 { s.flushAOF(false) } return s.aofsz, nil } func (s *Server) followDoLeaderAuth(conn *RESPConn, auth string) error { v, err := conn.Do("auth", auth) if err != nil { return err } if v.Error() != nil { return v.Error() } if v.String() != "OK" { return errors.New("cannot follow: auth no ok") } return nil } // bit flags for the fcupflags server field const ( bitCaughtUpOnce int32 = 1 // follower caught up at least once in the past bitCaughtUp int32 = 2 // follower is fully caught up to leader ) func (s *Server) setCaughtUp(caughtUp bool) { var flags int32 if caughtUp { flags = bitCaughtUp | bitCaughtUpOnce } else { flags = s.fcupflags.Load() & bitCaughtUpOnce } s.fcupflags.Store(flags) } func (s *Server) caughtUp() bool { return (s.fcupflags.Load() & bitCaughtUp) == bitCaughtUp } func (s *Server) caughtUpOnce() bool { return (s.fcupflags.Load() & bitCaughtUpOnce) == bitCaughtUpOnce } func (s *Server) followStep(host string, port int, followc int) error { if int(s.followc.Load()) != followc { return errNoLongerFollowing } s.mu.Lock() s.faofsz = 0 s.setCaughtUp(false) auth := s.config.leaderAuth() s.mu.Unlock() addr := fmt.Sprintf("%s:%d", host, port) // check if we are following self conn, err := DialTimeout(addr, time.Second*2) if err != nil { return fmt.Errorf("cannot follow: %v", err) } defer conn.Close() if auth != "" { if err := s.followDoLeaderAuth(conn, auth); err != nil { return fmt.Errorf("cannot follow: %v", err) } } m, err := doServer(conn) if err != nil { return fmt.Errorf("cannot follow: %v", err) } if m["id"] == "" { return fmt.Errorf("cannot follow: invalid id") } if m["id"] == s.config.serverID() { return fmt.Errorf("cannot follow self") } if m["following"] != "" { return fmt.Errorf("cannot follow a follower") } // verify checksum pos, err := s.followCheckSome(addr, followc, auth) if err != nil { return err } // Send the replication port to the leader p := s.config.announcePort() if p == 0 { p = s.port } v, err := conn.Do("replconf", "listening-port", p) if err != nil { return err } if v.Error() != nil { return v.Error() } if v.String() != "OK" { return errors.New("invalid response to replconf request") } // Send the replication ip to the leader ip := s.config.announceIP() if ip != "" { v, err := conn.Do("replconf", "ip-address", ip) if err != nil { return err } if v.Error() != nil { return v.Error() } if v.String() != "OK" { return errors.New("invalid response to replconf request") } } if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":replconf") } v, err = conn.Do("aof", pos) if err != nil { return err } if v.Error() != nil { return v.Error() } if v.String() != "OK" { return errors.New("invalid response to aof live request") } if s.opts.ShowDebugMessages { log.Debug("follow:", addr, ":read aof") } aofSize, err := strconv.ParseInt(m["aof_size"], 10, 64) if err != nil { return err } s.mu.Lock() s.faofsz = int(aofSize) s.mu.Unlock() caughtUp := pos >= aofSize if caughtUp { s.setCaughtUp(true) log.Info("caught up") } nullw := io.Discard for { v, telnet, _, err := conn.rd.ReadMultiBulk() if err != nil { return err } vals := v.Array() if telnet || v.Type() != resp.Array { return errors.New("invalid multibulk") } svals := make([]string, len(vals)) for i := 0; i < len(vals); i++ { svals[i] = vals[i].String() } aofsz, err := s.followHandleCommand(svals, followc, nullw) if err != nil { return err } s.mu.Lock() s.faofsz = aofsz s.mu.Unlock() if !caughtUp { if aofsz >= int(aofSize) { caughtUp = true s.mu.Lock() s.flushAOF(false) s.setCaughtUp(true) s.mu.Unlock() log.Info("caught up") } } } } func (s *Server) follow(host string, port int, followc int) { for { err := s.followStep(host, port, followc) if err == errNoLongerFollowing { return } if err != nil && err != io.EOF { log.Error("follow: " + err.Error()) } time.Sleep(time.Second) } } ================================================ FILE: internal/server/group.go ================================================ package server import ( "github.com/tidwall/btree" ) func byGroupHook(va, vb interface{}) bool { a, b := va.(*groupItem), vb.(*groupItem) if a.hookName < b.hookName { return true } if a.hookName > b.hookName { return false } if a.colKey < b.colKey { return true } if a.colKey > b.colKey { return false } return a.objID < b.objID } func byGroupObject(va, vb interface{}) bool { a, b := va.(*groupItem), vb.(*groupItem) if a.colKey < b.colKey { return true } if a.colKey > b.colKey { return false } if a.objID < b.objID { return true } if a.objID > b.objID { return false } return a.hookName < b.hookName } type groupItem struct { hookName string colKey string objID string groupID string } func newGroupItem(hookName, colKey, objID string) *groupItem { groupID := bsonID() g := &groupItem{} // create a single string allocation ustr := hookName + colKey + objID + groupID var pos int g.hookName = ustr[pos : pos+len(hookName)] pos += len(hookName) g.colKey = ustr[pos : pos+len(colKey)] pos += len(colKey) g.objID = ustr[pos : pos+len(objID)] pos += len(objID) g.groupID = ustr[pos : pos+len(groupID)] pos += len(groupID) return g } func (s *Server) groupConnect(hookName, colKey, objID string) (groupID string) { g := newGroupItem(hookName, colKey, objID) s.groupHooks.Set(g) s.groupObjects.Set(g) return g.groupID } func (s *Server) groupDisconnect(hookName, colKey, objID string) { g := &groupItem{ hookName: hookName, colKey: colKey, objID: objID, } s.groupHooks.Delete(g) s.groupObjects.Delete(g) } func (s *Server) groupGet(hookName, colKey, objID string) (groupID string) { v := s.groupHooks.Get(&groupItem{ hookName: hookName, colKey: colKey, objID: objID, }) if v != nil { return v.(*groupItem).groupID } return "" } func deleteGroups(s *Server, groups []*groupItem) { var hhint btree.PathHint var ohint btree.PathHint for _, g := range groups { s.groupHooks.DeleteHint(g, &hhint) s.groupObjects.DeleteHint(g, &ohint) } } // groupDisconnectObject disconnects all hooks from provide object func (s *Server) groupDisconnectObject(colKey, objID string) { var groups []*groupItem s.groupObjects.Ascend(&groupItem{colKey: colKey, objID: objID}, func(v interface{}) bool { g := v.(*groupItem) if g.colKey != colKey || g.objID != objID { return false } groups = append(groups, g) return true }, ) deleteGroups(s, groups) } // groupDisconnectCollection disconnects all hooks from objects in provided // collection. func (s *Server) groupDisconnectCollection(colKey string) { var groups []*groupItem s.groupObjects.Ascend(&groupItem{colKey: colKey}, func(v interface{}) bool { g := v.(*groupItem) if g.colKey != colKey { return false } groups = append(groups, g) return true }, ) deleteGroups(s, groups) } // groupDisconnectHook disconnects all objects from provided hook. func (s *Server) groupDisconnectHook(hookName string) { var groups []*groupItem s.groupHooks.Ascend(&groupItem{hookName: hookName}, func(v interface{}) bool { g := v.(*groupItem) if g.hookName != hookName { return false } groups = append(groups, g) return true }, ) deleteGroups(s, groups) } ================================================ FILE: internal/server/hooks.go ================================================ package server import ( "bytes" "errors" "sort" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/tidwall/buntdb" "github.com/tidwall/gjson" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/endpoint" "github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/log" ) var hookLogSetDefaults = &buntdb.SetOptions{ Expires: true, // automatically delete after 30 seconds TTL: time.Second * 30, } func byHookName(a, b interface{}) bool { return a.(*Hook).Name < b.(*Hook).Name } func (s *Server) cmdSetHook(msg *Message) ( res resp.Value, d commandDetails, err error, ) { channel := msg.Command() == "setchan" start := time.Now() vs := msg.Args[1:] var name, urls, cmd string var ok bool if vs, name, ok = tokenval(vs); !ok || name == "" { return NOMessage, d, errInvalidNumberOfArguments } var endpoints []string if channel { endpoints = []string{"local://" + name} } else { if vs, urls, ok = tokenval(vs); !ok || urls == "" { return NOMessage, d, errInvalidNumberOfArguments } for _, url := range strings.Split(urls, ",") { url = strings.TrimSpace(url) err := s.epc.Validate(url) if err != nil { log.Errorf("sethook: %v", err) return resp.SimpleStringValue(""), d, errInvalidArgument(url) } endpoints = append(endpoints, url) } } var commandvs []string var cmdlc string var types map[string]bool var expires float64 var expiresSet bool metaMap := make(map[string]string) for { commandvs = vs if vs, cmd, ok = tokenval(vs); !ok || cmd == "" { return NOMessage, d, errInvalidNumberOfArguments } cmdlc = strings.ToLower(cmd) switch cmdlc { default: return NOMessage, d, errInvalidArgument(cmd) case "meta": var metakey string var metaval string if vs, metakey, ok = tokenval(vs); !ok || metakey == "" { return NOMessage, d, errInvalidNumberOfArguments } if vs, metaval, ok = tokenval(vs); !ok || metaval == "" { return NOMessage, d, errInvalidNumberOfArguments } metaMap[metakey] = metaval continue case "ex": var s string if vs, s, ok = tokenval(vs); !ok || s == "" { return NOMessage, d, errInvalidNumberOfArguments } v, err := strconv.ParseFloat(s, 64) if err != nil { return NOMessage, d, errInvalidArgument(s) } expires = v expiresSet = true continue case "nearby": types = nearbyTypes case "within", "intersects": types = withinOrIntersectsTypes } break } args, err := s.cmdSearchArgs(true, cmdlc, vs, types) if args.usingLua() { defer args.Close() } if err != nil { return NOMessage, d, err } if !args.fence { return NOMessage, d, errors.New("missing FENCE argument") } args.cmd = cmdlc cmsg := &Message{} *cmsg = *msg cmsg.Args = make([]string, len(commandvs)) copy(cmsg.Args, commandvs) metas := make([]FenceMeta, 0, len(metaMap)) for key, val := range metaMap { metas = append(metas, FenceMeta{key, val}) } sort.Sort(hookMetaByName(metas)) hook := &Hook{ Key: args.key, Name: name, Endpoints: endpoints, Fence: &args, Message: cmsg, epm: s.epc, Metas: metas, channel: channel, cond: sync.NewCond(&sync.Mutex{}), counter: &s.statsTotalMsgsSent, } if expiresSet { hook.expires = time.Now().Add(time.Duration(expires * float64(time.Second))) } if !channel { hook.db = s.qdb } var wr bytes.Buffer hook.ScanWriter, err = s.newScanWriter( &wr, cmsg, args.key, args.output, args.precision, args.globs, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, args.nofields, args.mvt, args.tileX, args.tileY, args.tileZ) if err != nil { return NOMessage, d, err } prevHook, _ := s.hooks.Get(&Hook{Name: name}).(*Hook) if prevHook != nil { if prevHook.channel != channel { return NOMessage, d, errors.New("hooks and channels cannot share the same name") } if prevHook.Equals(hook) { // it was a match so we do nothing. But let's signal just // for good measure. prevHook.Signal() if !hook.expires.IsZero() { s.hookExpires.Set(hook) } switch msg.OutputType { case JSON: return OKMessage(msg, start), d, nil case RESP: return resp.IntegerValue(0), d, nil } } prevHook.Close() s.hooks.Delete(prevHook) s.hooksOut.Delete(prevHook) if !prevHook.expires.IsZero() { s.hookExpires.Delete(prevHook) } s.groupDisconnectHook(name) } d.updated = true d.timestamp = time.Now() s.hooks.Set(hook) if hook.Fence.detect == nil || hook.Fence.detect["outside"] { s.hooksOut.Set(hook) } // remove previous hook from spatial index if prevHook != nil && prevHook.Fence != nil && prevHook.Fence.obj != nil { rect := prevHook.Fence.obj.Rect() s.hookTree.Delete( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, prevHook) if prevHook.Fence.detect["cross"] { s.hookCross.Delete( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, prevHook) } } // add hook to spatial index if hook != nil && hook.Fence != nil && hook.Fence.obj != nil { rect := hook.Fence.obj.Rect() s.hookTree.Insert( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, hook) if hook.Fence.detect["cross"] { s.hookCross.Insert( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, hook) } } hook.Open() // Opens a goroutine to notify the hook if !hook.expires.IsZero() { s.hookExpires.Set(hook) } switch msg.OutputType { case JSON: return OKMessage(msg, start), d, nil case RESP: return resp.IntegerValue(1), d, nil } return NOMessage, d, nil } func byHookExpires(a, b interface{}) bool { ha := a.(*Hook) hb := b.(*Hook) if ha.expires.Before(hb.expires) { return true } if ha.expires.After(hb.expires) { return false } return ha.Name < hb.Name } func (s *Server) cmdDELHOOKop(name string, channel bool) (updated bool) { hook, _ := s.hooks.Get(&Hook{Name: name}).(*Hook) if hook == nil || hook.channel != channel { return false } hook.Close() // remove hook from maps s.hooks.Delete(hook) s.hooksOut.Delete(hook) if !hook.expires.IsZero() { s.hookExpires.Delete(hook) } // remove any hook / object connections s.groupDisconnectHook(hook.Name) // remove hook from spatial index if hook.Fence != nil && hook.Fence.obj != nil { rect := hook.Fence.obj.Rect() s.hookTree.Delete( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, hook) if hook.Fence.detect["cross"] { s.hookCross.Delete( [2]float64{rect.Min.X, rect.Min.Y}, [2]float64{rect.Max.X, rect.Max.Y}, hook) } } return true } func (s *Server) cmdDelHook(msg *Message) ( res resp.Value, d commandDetails, err error, ) { channel := msg.Command() == "delchan" start := time.Now() vs := msg.Args[1:] var name string var ok bool if vs, name, ok = tokenval(vs); !ok || name == "" { return NOMessage, d, errInvalidNumberOfArguments } if len(vs) != 0 { return NOMessage, d, errInvalidNumberOfArguments } d.updated = s.cmdDELHOOKop(name, channel) d.timestamp = time.Now() switch msg.OutputType { case JSON: return OKMessage(msg, start), d, nil case RESP: if d.updated { return resp.IntegerValue(1), d, nil } return resp.IntegerValue(0), d, nil } return } func (s *Server) cmdPDelHook(msg *Message) ( res resp.Value, d commandDetails, err error, ) { channel := msg.Command() == "pdelchan" start := time.Now() vs := msg.Args[1:] var pattern string var ok bool if vs, pattern, ok = tokenval(vs); !ok || pattern == "" { return NOMessage, d, errInvalidNumberOfArguments } if len(vs) != 0 { return NOMessage, d, errInvalidNumberOfArguments } count := 0 var hooks []*Hook s.forEachHookByPattern(pattern, channel, func(hook *Hook) bool { hooks = append(hooks, hook) return true }) for _, hook := range hooks { if hook.channel != channel { continue } s.cmdDELHOOKop(hook.Name, channel) d.updated = true count++ } d.timestamp = time.Now() switch msg.OutputType { case JSON: return OKMessage(msg, start), d, nil case RESP: return resp.IntegerValue(count), d, nil } return } func (s *Server) forEachHookByPattern( pattern string, channel bool, iter func(hook *Hook) bool, ) { g := glob.Parse(pattern, false) hasUpperLimit := g.Limits[1] != "" s.hooks.Ascend(&Hook{Name: g.Limits[0]}, func(v interface{}) bool { hook := v.(*Hook) if hasUpperLimit && hook.Name > g.Limits[1] { return false } if hook.channel == channel { match, _ := glob.Match(pattern, hook.Name) if match { return iter(hook) } } return true }) } func (s *Server) cmdHooks(msg *Message) ( res resp.Value, err error, ) { channel := msg.Command() == "chans" start := time.Now() vs := msg.Args[1:] var pattern string var ok bool if vs, pattern, ok = tokenval(vs); !ok || pattern == "" { return NOMessage, errInvalidNumberOfArguments } if len(vs) != 0 { return NOMessage, errInvalidNumberOfArguments } switch msg.OutputType { case JSON: buf := &bytes.Buffer{} buf.WriteString(`{"ok":true,`) if channel { buf.WriteString(`"chans":[`) } else { buf.WriteString(`"hooks":[`) } var i int s.forEachHookByPattern(pattern, channel, func(hook *Hook) bool { var ttl = -1 if !hook.expires.IsZero() { ttl = int(hook.expires.Sub(start).Seconds()) if ttl < 0 { ttl = 0 } } if i > 0 { buf.WriteByte(',') } buf.WriteString(`{`) buf.WriteString(`"name":` + jsonString(hook.Name)) buf.WriteString(`,"key":` + jsonString(hook.Key)) buf.WriteString(`,"ttl":` + strconv.Itoa(ttl)) if !channel { buf.WriteString(`,"endpoints":[`) for i, endpoint := range hook.Endpoints { if i > 0 { buf.WriteByte(',') } buf.WriteString(jsonString(endpoint)) } buf.WriteString(`]`) } buf.WriteString(`,"command":[`) for i, v := range hook.Message.Args { if i > 0 { buf.WriteString(`,`) } buf.WriteString(jsonString(v)) } buf.WriteString(`],"meta":{`) for i, meta := range hook.Metas { if i > 0 { buf.WriteString(`,`) } buf.WriteString(jsonString(meta.Name)) buf.WriteString(`:`) buf.WriteString(jsonString(meta.Value)) } buf.WriteString(`}}`) i++ return true }) buf.WriteString(`],"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil case RESP: var vals []resp.Value s.forEachHookByPattern(pattern, channel, func(hook *Hook) bool { var hvals []resp.Value hvals = append(hvals, resp.StringValue(hook.Name)) hvals = append(hvals, resp.StringValue(hook.Key)) var evals []resp.Value for _, endpoint := range hook.Endpoints { evals = append(evals, resp.StringValue(endpoint)) } hvals = append(hvals, resp.ArrayValue(evals)) avals := make([]resp.Value, len(hook.Message.Args)) for i := 0; i < len(hook.Message.Args); i++ { avals[i] = resp.StringValue(hook.Message.Args[i]) } hvals = append(hvals, resp.ArrayValue(avals)) var metas []resp.Value for _, meta := range hook.Metas { metas = append(metas, resp.StringValue(meta.Name)) metas = append(metas, resp.StringValue(meta.Value)) } hvals = append(hvals, resp.ArrayValue(metas)) vals = append(vals, resp.ArrayValue(hvals)) return true }) return resp.ArrayValue(vals), nil } return resp.SimpleStringValue(""), nil } // Hook represents a hook. type Hook struct { cond *sync.Cond Key string Name string Endpoints []string Message *Message Fence *liveFenceSwitches ScanWriter *scanWriter Metas []FenceMeta db *buntdb.DB channel bool closed bool opened bool query string epm *endpoint.Manager expires time.Time counter *atomic.Int64 // counter that grows when a message was sent sig int } // Expires returns when the hook expires. Required by the expire.Item interface. func (h *Hook) Expires() time.Time { return h.expires } // Equals returns true if two hooks are equal func (h *Hook) Equals(hook *Hook) bool { if h.Key != hook.Key || h.Name != hook.Name || len(h.Endpoints) != len(hook.Endpoints) || len(h.Metas) != len(hook.Metas) { return false } if !h.expires.Equal(hook.expires) { return false } for i, endpoint := range h.Endpoints { if endpoint != hook.Endpoints[i] { return false } } for i, meta := range h.Metas { if meta.Name != hook.Metas[i].Name || meta.Value != hook.Metas[i].Value { return false } } if len(h.Message.Args) != len(hook.Message.Args) { return false } for i := 0; i < len(h.Message.Args); i++ { if h.Message.Args[i] != hook.Message.Args[i] { return false } } return true } // FenceMeta is a meta key/value pair for fences type FenceMeta struct { Name, Value string } type hookMetaByName []FenceMeta func (arr hookMetaByName) Len() int { return len(arr) } func (arr hookMetaByName) Less(a, b int) bool { return arr[a].Name < arr[b].Name } func (arr hookMetaByName) Swap(a, b int) { arr[a], arr[b] = arr[b], arr[a] } // Open is called when a hook is first created. It calls the manager // function in a goroutine func (h *Hook) Open() { if h.channel { // nothing to open for channels return } h.cond.L.Lock() defer h.cond.L.Unlock() if h.opened { return } h.opened = true h.query = `{"hook":` + jsonString(h.Name) + `}` go h.manager() } // Close closed the hook and stop the manager function func (h *Hook) Close() { if h.channel { // nothing to close for channels return } h.cond.L.Lock() defer h.cond.L.Unlock() if h.closed { return } h.closed = true h.cond.Broadcast() } // Signal can be called at any point to wake up the hook and // notify the manager that there may be something new in the queue. func (h *Hook) Signal() { if h.channel { // nothing to signal for channels return } h.cond.L.Lock() h.sig++ h.cond.Broadcast() h.cond.L.Unlock() } // the manager is a forever loop that calls proc whenever there's a signal. // it ends when the "closed" flag is set. func (h *Hook) manager() { // lock the hook to waiting on signals h.cond.L.Lock() defer h.cond.L.Unlock() var sig int for { if h.closed { // the hook has closed, end manager return } sig = h.sig // unlock/logk the hook and send outgoing messages if !func() bool { h.cond.L.Unlock() defer h.cond.L.Lock() return h.proc() }() { // a send failed, try again in a moment time.Sleep(time.Second / 2) continue } if sig != h.sig { // there was another incoming signal continue } // wait on signal h.cond.Wait() } } // proc processes queued hook logs. // returning true will indicate that all log entries have been // successfully handled. func (h *Hook) proc() (ok bool) { var keys, vals []string var ttls []time.Duration start := time.Now() err := h.db.Update(func(tx *buntdb.Tx) error { // get keys and vals err := tx.AscendGreaterOrEqual("hooks", h.query, func(key, val string) bool { if strings.HasPrefix(key, hookLogPrefix) { // Verify this hooks name matches the one in the notif if h.Name == gjson.Get(val, "hook").String() { keys = append(keys, key) vals = append(vals, val) } } return true }, ) if err != nil { return err } // delete the keys for _, key := range keys { ttl, err := tx.TTL(key) if err != nil { if err != buntdb.ErrNotFound { return err } } ttls = append(ttls, ttl) _, err = tx.Delete(key) if err != nil { if err != buntdb.ErrNotFound { return err } } } return nil }) if err != nil { log.Error(err) return false } // send each val. on failure reinsert that one and all of the following for i, key := range keys { val := vals[i] idx := stringToUint64(key[len(hookLogPrefix):]) var sent bool for _, endpoint := range h.Endpoints { err := h.epm.Send(endpoint, val) if err != nil { log.Debugf("Endpoint connect/send error: %v: %v: %v", idx, endpoint, err) continue } log.Debugf("Endpoint send ok: %v: %v: %v", idx, endpoint, err) sent = true h.counter.Add(1) break } if !sent { // failed to send. try to reinsert the remaining. // if this fails we lose log entries. keys = keys[i:] vals = vals[i:] ttls = ttls[i:] h.db.Update(func(tx *buntdb.Tx) error { for i, key := range keys { val := vals[i] ttl := ttls[i] - time.Since(start) if ttl > 0 { opts := &buntdb.SetOptions{ Expires: true, TTL: ttl, } _, _, err := tx.Set(key, val, opts) if err != nil { return err } } } return nil }) return false } } return true } ================================================ FILE: internal/server/json.go ================================================ package server import ( "bytes" "encoding/json" "strconv" "strings" "time" "github.com/tidwall/geojson" "github.com/tidwall/gjson" "github.com/tidwall/resp" "github.com/tidwall/sjson" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/object" ) func appendJSONString(b []byte, s string) []byte { for i := 0; i < len(s); i++ { if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 { d, _ := json.Marshal(s) return append(b, string(d)...) } } b = append(b, '"') b = append(b, s...) b = append(b, '"') return b } func jsonString(s string) string { for i := 0; i < len(s); i++ { if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 { d, _ := json.Marshal(s) return string(d) } } b := make([]byte, len(s)+2) b[0] = '"' copy(b[1:], s) b[len(b)-1] = '"' return string(b) } func isJSONNumber(data string) bool { // Returns true if the given string can be encoded as a JSON number value. // See: // https://json.org // http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf if data == "" { return false } i := 0 // sign if data[i] == '-' { i++ } if i == len(data) { return false } // int if data[i] == '0' { i++ } else { for ; i < len(data); i++ { if data[i] >= '0' && data[i] <= '9' { continue } break } } // frac if i == len(data) { return true } if data[i] == '.' { i++ if i == len(data) { return false } if data[i] < '0' || data[i] > '9' { return false } i++ for ; i < len(data); i++ { if data[i] >= '0' && data[i] <= '9' { continue } break } } // exp if i == len(data) { return true } if data[i] == 'e' || data[i] == 'E' { i++ if i == len(data) { return false } if data[i] == '+' || data[i] == '-' { i++ } if i == len(data) { return false } if data[i] < '0' || data[i] > '9' { return false } i++ for ; i < len(data); i++ { if data[i] >= '0' && data[i] <= '9' { continue } break } } return i == len(data) } func appendJSONSimpleBounds(dst []byte, o geojson.Object) []byte { bbox := o.Rect() dst = append(dst, `{"sw":{"lat":`...) dst = strconv.AppendFloat(dst, bbox.Min.Y, 'f', -1, 64) dst = append(dst, `,"lon":`...) dst = strconv.AppendFloat(dst, bbox.Min.X, 'f', -1, 64) dst = append(dst, `},"ne":{"lat":`...) dst = strconv.AppendFloat(dst, bbox.Max.Y, 'f', -1, 64) dst = append(dst, `,"lon":`...) dst = strconv.AppendFloat(dst, bbox.Max.X, 'f', -1, 64) dst = append(dst, `}}`...) return dst } func appendJSONSimplePoint(dst []byte, o geojson.Object) []byte { point := o.Center() z := extractZCoordinate(o) dst = append(dst, `{"lat":`...) dst = strconv.AppendFloat(dst, point.Y, 'f', -1, 64) dst = append(dst, `,"lon":`...) dst = strconv.AppendFloat(dst, point.X, 'f', -1, 64) if z != 0 { dst = append(dst, `,"z":`...) dst = strconv.AppendFloat(dst, z, 'f', -1, 64) } dst = append(dst, '}') return dst } func appendJSONTimeFormat(b []byte, t time.Time) []byte { b = append(b, '"') b = t.AppendFormat(b, "2006-01-02T15:04:05.999999999Z07:00") b = append(b, '"') return b } func jsonTimeFormat(t time.Time) string { var b []byte b = appendJSONTimeFormat(b, t) return string(b) } func (s *Server) cmdJget(msg *Message) (resp.Value, error) { start := time.Now() if len(msg.Args) < 3 { return NOMessage, errInvalidNumberOfArguments } if len(msg.Args) > 5 { return NOMessage, errInvalidNumberOfArguments } key := msg.Args[1] id := msg.Args[2] var doget bool var path string var raw bool if len(msg.Args) > 3 { doget = true path = msg.Args[3] if len(msg.Args) == 5 { if strings.ToLower(msg.Args[4]) == "raw" { raw = true } else { return NOMessage, errInvalidArgument(msg.Args[4]) } } } col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } return NOMessage, errKeyNotFound } o := col.Get(id) if o == nil { if msg.OutputType == RESP { return resp.NullValue(), nil } return NOMessage, errIDNotFound } var res gjson.Result if doget { res = gjson.Get(o.Geo().String(), path) } else { res = gjson.Parse(o.Geo().String()) } var val string if raw { val = res.Raw } else { val = res.String() } var buf bytes.Buffer if msg.OutputType == JSON { buf.WriteString(`{"ok":true`) } switch msg.OutputType { case JSON: if res.Exists() { buf.WriteString(`,"value":` + jsonString(val)) } buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil case RESP: if !res.Exists() { return resp.NullValue(), nil } return resp.StringValue(val), nil } return NOMessage, nil } func (s *Server) cmdJset(msg *Message) (res resp.Value, d commandDetails, err error) { // JSET key path value [RAW] start := time.Now() var raw, str bool switch len(msg.Args) { default: return NOMessage, d, errInvalidNumberOfArguments case 5: case 6: switch strings.ToLower(msg.Args[5]) { default: return NOMessage, d, errInvalidArgument(msg.Args[5]) case "raw": raw = true case "str": str = true } } key := msg.Args[1] id := msg.Args[2] path := msg.Args[3] val := msg.Args[4] if !str && !raw { switch val { default: raw = isJSONNumber(val) case "true", "false", "null": raw = true } } col, _ := s.cols.Get(key) var createcol bool if col == nil { col = collection.New() createcol = true } var json string var geoobj bool var fields field.List o := col.Get(id) if o != nil { geoobj = objIsSpatial(o.Geo()) json = o.Geo().String() fields = o.Fields() } if raw { // set as raw block json, err = sjson.SetRaw(json, path, val) } else { // set as a string json, err = sjson.Set(json, path, val) } if err != nil { return NOMessage, d, err } if geoobj { nmsg := *msg nmsg.Args = []string{"SET", key, id, "OBJECT", json} // SET key id OBJECT json return s.cmdSET(&nmsg) } if createcol { s.cols.Set(key, col) } var oobj geojson.Object = collection.String(json) obj := object.New(id, oobj, 0, fields) col.Set(obj) d.key = key d.obj = obj d.timestamp = time.Now() d.updated = true switch msg.OutputType { case JSON: var buf bytes.Buffer buf.WriteString(`{"ok":true`) buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), d, nil case RESP: return resp.SimpleStringValue("OK"), d, nil } return NOMessage, d, nil } func (s *Server) cmdJdel(msg *Message) (res resp.Value, d commandDetails, err error) { start := time.Now() if len(msg.Args) != 4 { return NOMessage, d, errInvalidNumberOfArguments } key := msg.Args[1] id := msg.Args[2] path := msg.Args[3] col, _ := s.cols.Get(key) if col == nil { if msg.OutputType == RESP { return resp.IntegerValue(0), d, nil } return NOMessage, d, errKeyNotFound } var json string var geoobj bool var fields field.List o := col.Get(id) if o != nil { geoobj = objIsSpatial(o.Geo()) json = o.Geo().String() fields = o.Fields() } njson, err := sjson.Delete(json, path) if err != nil { return NOMessage, d, err } if njson == json { switch msg.OutputType { case JSON: return NOMessage, d, errPathNotFound case RESP: return resp.IntegerValue(0), d, nil } return NOMessage, d, nil } json = njson if geoobj { nmsg := *msg nmsg.Args = []string{"SET", key, id, "OBJECT", json} // SET key id OBJECT json return s.cmdSET(&nmsg) } var oobj geojson.Object = collection.String(json) obj := object.New(id, oobj, 0, fields) col.Set(obj) d.key = key d.obj = obj d.timestamp = time.Now() d.updated = true switch msg.OutputType { case JSON: var buf bytes.Buffer buf.WriteString(`{"ok":true`) buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), d, nil case RESP: return resp.IntegerValue(1), d, nil } return NOMessage, d, nil } ================================================ FILE: internal/server/json_test.go ================================================ package server import ( "encoding/json" "testing" ) func BenchmarkJSONString(t *testing.B) { var s = "the need for mead" for i := 0; i < t.N; i++ { jsonString(s) } } func BenchmarkJSONMarshal(t *testing.B) { var s = "the need for mead" for i := 0; i < t.N; i++ { json.Marshal(s) } } func TestIsJsonNumber(t *testing.T) { test := func(expected bool, val string) { actual := isJSONNumber(val) if expected != actual { t.Fatalf("Expected %t == isJsonNumber(\"%s\") but was %t", expected, val, actual) } } test(false, "") test(false, "-") test(false, "foo") test(false, "0123") test(false, "1.") test(false, "1.0e") test(false, "1.0e-") test(false, "1.0E10NaN") test(false, "1.0ENaN") test(true, "-1") test(true, "0") test(true, "0.0") test(true, "42") test(true, "1.0E10") test(true, "1.0e10") test(true, "1E+5") test(true, "1E-10") } ================================================ FILE: internal/server/keys.go ================================================ package server import ( "encoding/json" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/glob" ) // KEYS pattern func (s *Server) cmdKEYS(msg *Message) (resp.Value, error) { var start = time.Now() // >> Args args := msg.Args if len(args) != 2 { return retrerr(errInvalidNumberOfArguments) } pattern := args[1] // >> Operation keys := []string{} g := glob.Parse(pattern, false) everything := g.Limits[0] == "" && g.Limits[1] == "" if everything { s.cols.Scan( func(key string, _ *collection.Collection) bool { match, _ := glob.Match(pattern, key) if match { keys = append(keys, key) } return true }, ) } else { s.cols.Ascend(g.Limits[0], func(key string, _ *collection.Collection) bool { if key > g.Limits[1] { return false } match, _ := glob.Match(pattern, key) if match { keys = append(keys, key) } return true }, ) } // >> Response if msg.OutputType == JSON { data, _ := json.Marshal(keys) return resp.StringValue(`{"ok":true,"keys":` + string(data) + `,"elapsed":"` + time.Since(start).String() + `"}`), nil } var vals []resp.Value for _, key := range keys { vals = append(vals, resp.StringValue(key)) } return resp.ArrayValue(vals), nil } ================================================ FILE: internal/server/live.go ================================================ package server import ( "bytes" "errors" "fmt" "io" "net" "sync" "time" "github.com/tidwall/redcon" "github.com/tidwall/tile38/internal/log" "go.uber.org/atomic" ) type liveBuffer struct { key string globs []string fence *liveFenceSwitches details []*commandDetails cond *sync.Cond } func (s *Server) processLives(wg *sync.WaitGroup) { defer wg.Done() var done atomic.Bool wg.Add(1) go func() { defer wg.Done() for { if done.Load() { break } s.lcond.Broadcast() time.Sleep(time.Second / 4) } }() s.lcond.L.Lock() defer s.lcond.L.Unlock() for { if s.stopServer.Load() { done.Store(true) return } for len(s.lstack) > 0 { item := s.lstack[0] s.lstack = s.lstack[1:] if len(s.lstack) == 0 { s.lstack = nil } for lb := range s.lives { lb.cond.L.Lock() if lb.key != "" && lb.key == item.key { lb.details = append(lb.details, item) lb.cond.Broadcast() } lb.cond.L.Unlock() } } s.lcond.Wait() } } func writeLiveMessage( conn net.Conn, message []byte, wrapRESP bool, connType Type, websocket bool, ) error { if len(message) == 0 { return nil } if websocket { return WriteWebSocketMessage(conn, message) } var err error switch connType { case RESP: if wrapRESP { _, err = fmt.Fprintf(conn, "$%d\r\n%s\r\n", len(message), string(message)) } else { _, err = conn.Write(message) } case Native: _, err = fmt.Fprintf(conn, "$%d %s\r\n", len(message), string(message)) } return err } func (s *Server) goLive( inerr error, conn net.Conn, rd *PipelineReader, msg *Message, websocket bool, ) error { addr := conn.RemoteAddr().String() log.Info("live " + addr) defer func() { log.Info("not live " + addr) }() switch lfs := inerr.(type) { default: return errors.New("invalid live type switches") case liveAOFSwitches: return s.liveAOF(lfs.pos, conn, rd, msg) case liveSubscriptionSwitches: return s.liveSubscription(conn, rd, msg, websocket) case liveMonitorSwitches: return s.liveMonitor(conn, rd, msg) case liveFenceSwitches: // fallthrough } // everything below is for live geofences lb := &liveBuffer{ cond: sync.NewCond(&sync.Mutex{}), } var err error var sw *scanWriter var wr bytes.Buffer lfs := inerr.(liveFenceSwitches) lb.globs = lfs.globs lb.key = lfs.key lb.fence = &lfs s.mu.RLock() sw, err = s.newScanWriter( &wr, msg, lfs.key, lfs.output, lfs.precision, lfs.globs, false, lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, lfs.nofields, lfs.mvt, lfs.tileX, lfs.tileY, lfs.tileZ) s.mu.RUnlock() // everything below if for live SCAN, NEARBY, WITHIN, INTERSECTS if err != nil { return err } s.lcond.L.Lock() s.lives[lb] = true s.lcond.L.Unlock() defer func() { s.lcond.L.Lock() delete(s.lives, lb) s.lcond.L.Unlock() conn.Close() }() var mustQuit bool go func() { defer func() { lb.cond.L.Lock() mustQuit = true lb.cond.Broadcast() lb.cond.L.Unlock() conn.Close() }() for { vs, err := rd.ReadMessages() if err != nil { if err != io.EOF && !(websocket && err == io.ErrUnexpectedEOF) { log.Error(err) } return } for _, v := range vs { if v == nil { continue } switch v.Command() { default: log.Error("received a live command that was not QUIT") return case "quit", "": return } } } }() outputType := msg.OutputType connType := msg.ConnType if websocket { outputType = JSON } var livemsg []byte switch outputType { case JSON: livemsg = redcon.AppendBulkString(nil, `{"ok":true,"live":true}`) case RESP: livemsg = redcon.AppendOK(nil) } if err := writeLiveMessage(conn, livemsg, false, connType, websocket); err != nil { return nil // nil return is fine here } for { lb.cond.L.Lock() if mustQuit { lb.cond.L.Unlock() return nil } for len(lb.details) > 0 { details := lb.details[0] lb.details = lb.details[1:] if len(lb.details) == 0 { lb.details = nil } fence := lb.fence lb.cond.L.Unlock() var msgs []string func() { // safely lock the fence because we are outside the main loop s.mu.RLock() defer s.mu.RUnlock() msgs = FenceMatch("", sw, fence, nil, details) }() for _, msg := range msgs { if err := writeLiveMessage(conn, []byte(msg), true, connType, websocket); err != nil { return nil // nil return is fine here } } s.statsTotalMsgsSent.Add(int64(len(msgs))) lb.cond.L.Lock() } lb.cond.Wait() lb.cond.L.Unlock() } } ================================================ FILE: internal/server/metrics.go ================================================ package server import ( "fmt" "net/http" "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/collection" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( metricDescriptions = map[string]*prometheus.Desc{ /* these metrics are taken from basicStats() / extStats() by accessing the map and directly exporting the value found */ "num_collections": prometheus.NewDesc("tile38_collections", "Total number of collections", nil, nil), "pid": prometheus.NewDesc("tile38_pid", "", nil, nil), "aof_size": prometheus.NewDesc("tile38_aof_size_bytes", "", nil, nil), "num_hooks": prometheus.NewDesc("tile38_hooks", "", nil, nil), "in_memory_size": prometheus.NewDesc("tile38_in_memory_size_bytes", "", nil, nil), "heap_size": prometheus.NewDesc("tile38_heap_size_bytes", "", nil, nil), "heap_released": prometheus.NewDesc("tile38_memory_reap_released_bytes", "", nil, nil), "max_heap_size": prometheus.NewDesc("tile38_memory_max_heap_size_bytes", "", nil, nil), "avg_item_size": prometheus.NewDesc("tile38_avg_item_size_bytes", "", nil, nil), "pointer_size": prometheus.NewDesc("tile38_pointer_size_bytes", "", nil, nil), "cpus": prometheus.NewDesc("tile38_num_cpus", "", nil, nil), "tile38_connected_clients": prometheus.NewDesc("tile38_connected_clients", "", nil, nil), "tile38_total_connections_received": prometheus.NewDesc("tile38_connections_received_total", "", nil, nil), "tile38_total_messages_sent": prometheus.NewDesc("tile38_messages_sent_total", "", nil, nil), "tile38_expired_keys": prometheus.NewDesc("tile38_expired_keys_total", "", nil, nil), /* these metrics are NOT taken from basicStats() / extStats() but are calculated independently */ "collection_objects": prometheus.NewDesc("tile38_collection_objects", "Total number of objects per collection", []string{"col"}, nil), "collection_points": prometheus.NewDesc("tile38_collection_points", "Total number of points per collection", []string{"col"}, nil), "collection_strings": prometheus.NewDesc("tile38_collection_strings", "Total number of strings per collection", []string{"col"}, nil), "collection_weight": prometheus.NewDesc("tile38_collection_weight_bytes", "Total weight of collection in bytes", []string{"col"}, nil), "server_info": prometheus.NewDesc("tile38_server_info", "Server info", []string{"id", "version"}, nil), "replication": prometheus.NewDesc("tile38_replication_info", "Replication info", []string{"role", "following", "caught_up", "caught_up_once"}, nil), "start_time": prometheus.NewDesc("tile38_start_time_seconds", "", nil, nil), } cmdDurations = prometheus.NewSummaryVec(prometheus.SummaryOpts{ Name: "tile38_cmd_duration_seconds", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001}, }, []string{"cmd"}, ) ) func (s *Server) MetricsIndexHandler(w http.ResponseWriter, r *http.Request) { w.Write([]byte(` Tile38 ` + core.Version + `

Tile38 ` + core.Version + `

Metrics

`)) } func (s *Server) MetricsHandler(w http.ResponseWriter, r *http.Request) { reg := prometheus.NewRegistry() reg.MustRegister( collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), collectors.NewGoCollector(), collectors.NewBuildInfoCollector(), cmdDurations, s, ) promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(w, r) } func (s *Server) Describe(ch chan<- *prometheus.Desc) { for _, desc := range metricDescriptions { ch <- desc } } func (s *Server) Collect(ch chan<- prometheus.Metric) { s.mu.RLock() defer s.mu.RUnlock() m := make(map[string]interface{}) s.basicStats(m) s.extStats(m) for metric, descr := range metricDescriptions { val, ok := toFloat(m[metric]) if ok { ch <- prometheus.MustNewConstMetric(descr, prometheus.GaugeValue, val) } } ch <- prometheus.MustNewConstMetric( metricDescriptions["server_info"], prometheus.GaugeValue, 1.0, s.config.serverID(), core.Version) ch <- prometheus.MustNewConstMetric( metricDescriptions["start_time"], prometheus.GaugeValue, float64(s.started.Unix())) replLbls := []string{"leader", "", "", ""} if s.config.followHost() != "" { replLbls = []string{"follower", fmt.Sprintf("%s:%d", s.config.followHost(), s.config.followPort()), fmt.Sprintf("%t", s.caughtUp()), fmt.Sprintf("%t", s.caughtUpOnce())} } ch <- prometheus.MustNewConstMetric( metricDescriptions["replication"], prometheus.GaugeValue, 1.0, replLbls...) /* add objects/points/strings stats for each collection */ s.cols.Scan(func(key string, col *collection.Collection) bool { ch <- prometheus.MustNewConstMetric( metricDescriptions["collection_objects"], prometheus.GaugeValue, float64(col.Count()), key, ) ch <- prometheus.MustNewConstMetric( metricDescriptions["collection_points"], prometheus.GaugeValue, float64(col.PointCount()), key, ) ch <- prometheus.MustNewConstMetric( metricDescriptions["collection_strings"], prometheus.GaugeValue, float64(col.StringCount()), key, ) ch <- prometheus.MustNewConstMetric( metricDescriptions["collection_weight"], prometheus.GaugeValue, float64(col.TotalWeight()), key, ) return true }) } func toFloat(val interface{}) (float64, bool) { switch v := val.(type) { case float64: return v, true case int64: return float64(v), true case uint64: return float64(v), true case float32: return float64(v), true case int: return float64(v), true case int32: return float64(v), true case uint32: return float64(v), true case int16: return float64(v), true case uint16: return float64(v), true case int8: return float64(v), true case uint8: return float64(v), true } return 0, false } ================================================ FILE: internal/server/monitor.go ================================================ package server import ( "fmt" "io" "net" "strconv" "strings" "time" "github.com/tidwall/resp" ) type liveMonitorSwitches struct { // no fields. everything is managed through the Message } func (sub liveMonitorSwitches) Error() string { return goingLive } func (s *Server) cmdMonitor(msg *Message) (resp.Value, error) { if len(msg.Args) != 1 { return resp.Value{}, errInvalidNumberOfArguments } return NOMessage, liveMonitorSwitches{} } func (s *Server) liveMonitor(conn net.Conn, rd *PipelineReader, msg *Message) error { s.monconnsMu.Lock() s.monconns[conn] = true s.monconnsMu.Unlock() defer func() { s.monconnsMu.Lock() delete(s.monconns, conn) s.monconnsMu.Unlock() conn.Close() }() s.monconnsMu.Lock() conn.Write([]byte("+OK\r\n")) s.monconnsMu.Unlock() msgs, err := rd.ReadMessages() if err != nil { if err == io.EOF { return nil } return err } for _, msg := range msgs { if len(msg.Args) == 1 && strings.ToLower(msg.Args[0]) == "quit" { s.monconnsMu.Lock() conn.Write([]byte("+OK\r\n")) s.monconnsMu.Unlock() return nil } } return nil } // send messages to live MONITOR clients func (s *Server) sendMonitor(err error, msg *Message, c *Client, lua bool) { s.monconnsMu.RLock() n := len(s.monconns) s.monconnsMu.RUnlock() if n == 0 { return } if (c == nil && !lua) || (err != nil && (err == errInvalidNumberOfArguments || strings.HasPrefix(err.Error(), "unknown command "))) { return } // accept all commands except for these: switch strings.ToLower(msg.Command()) { case "config", "config set", "config get", "config rewrite", "auth", "follow", "slaveof", "replconf", "aof", "aofmd5", "client", "monitor": return } var line []byte for i, arg := range msg.Args { if i > 0 { line = append(line, ' ') } line = append(line, strconv.Quote(arg)...) } tstr := fmt.Sprintf("%.6f", float64(time.Now().UnixNano())/1e9) var addr string if lua { addr = "lua" } else { addr = c.remoteAddr } s.monconnsMu.Lock() for conn := range s.monconns { fmt.Fprintf(conn, "+%s [0 %s] %s\r\n", tstr, addr, line) } s.monconnsMu.Unlock() } ================================================ FILE: internal/server/must.go ================================================ package server func Must[T any](a T, err error) T { if err != nil { panic(err) } return a } func Default[T comparable](a, b T) T { var c T if a == c { return b } return a } ================================================ FILE: internal/server/must_test.go ================================================ package server import ( "errors" "testing" ) func TestMust(t *testing.T) { if Must(1, nil) != 1 { t.Fail() } func() { var ended bool defer func() { if ended { t.Fail() } err, ok := recover().(error) if !ok { t.Fail() } if err.Error() != "ok" { t.Fail() } }() Must(1, errors.New("ok")) ended = true }() } func TestDefault(t *testing.T) { if Default("", "2") != "2" { t.Fail() } if Default("1", "2") != "1" { t.Fail() } } ================================================ FILE: internal/server/mvt.go ================================================ package server import ( "net/url" "strings" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/mvt" ) type mvtObj struct { id string obj geojson.Object } func mvtDrawRing(f *mvt.Feature, tileX, tileY, tileZ int, ring geometry.Series, hole bool, ) { npoints := ring.NumPoints() if npoints < 3 { return } cw := ring.Clockwise() reverse := (cw && hole) || (!cw && !hole) if reverse { p := ring.PointAt(npoints - 1) f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) for i := npoints - 2; i >= 0; i-- { p := ring.PointAt(i) f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) } } else { p := ring.PointAt(0) f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) for i := 1; i < npoints; i++ { p := ring.PointAt(i) f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) } } f.ClosePath() } func mvtAddFeature(l *mvt.Layer, tileX, tileY, tileZ int, o mvtObj) { var f *mvt.Feature switch g := o.obj.(type) { case *geojson.Point: f = l.AddFeature(mvt.Point) p := g.Base() f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) f.AddTag("type", "point") case *geojson.SimplePoint: f = l.AddFeature(mvt.Point) p := g f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) f.AddTag("type", "point") case *geojson.LineString: f = l.AddFeature(mvt.LineString) line := g.Base() npoints := line.NumPoints() if npoints < 2 { return } p := line.PointAt(0) f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) for i := 1; i < npoints; i++ { p := line.PointAt(i) f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) } f.AddTag("type", "linestring") case *geojson.Rect: f = l.AddFeature(mvt.Polygon) mvtDrawRing(f, tileX, tileY, tileZ, g.Base(), false) f.AddTag("type", "polygon") case *geojson.Polygon: f = l.AddFeature(mvt.Polygon) poly := g.Base() mvtDrawRing(f, tileX, tileY, tileZ, poly.Exterior, false) for _, hole := range poly.Holes { mvtDrawRing(f, tileX, tileY, tileZ, hole, true) } f.AddTag("type", "polygon") case *geojson.Feature: mvtAddFeature(l, tileX, tileY, tileZ, mvtObj{o.id, g.Base()}) return default: if g, ok := g.(geojson.Collection); ok { for _, g := range g.Children() { mvtAddFeature(l, tileX, tileY, tileZ, mvtObj{o.id, g}) } } return } f.AddTag("id", o.id) } func mvtRender(tileX, tileY, tileZ int, objs []mvtObj) []byte { var tile mvt.Tile l := tile.AddLayer("tile38") l.SetExtent(4096) for _, obj := range objs { mvtAddFeature(l, tileX, tileY, tileZ, obj) } return tile.Render() } func mvtFilterHTTPArgs(msg *Message, query string) (modified bool) { path := msg.Args[0] parts := strings.Split(path, "/") if len(parts) != 4 { return false } parts[3] = parts[3][:len(parts[3])-4] for i := 0; i < len(parts); i++ { var err error parts[i], err = url.PathUnescape(parts[i]) if err != nil { return false } } var limit string var sparse string if query != "" { q, _ := url.ParseQuery(query) sparse = q.Get("sparse") limit = q.Get("limit") } msg._command = "" msg.Args = []string{"INTERSECTS", parts[0]} if sparse != "" { msg.Args = append(msg.Args, "SPARSE", sparse) } else if limit != "" { msg.Args = append(msg.Args, "LIMIT", limit) } else { msg.Args = append(msg.Args, "LIMIT", "100000000") } msg.Args = append(msg.Args, "MVT", parts[2], parts[3], parts[1]) return true } ================================================ FILE: internal/server/mvt_test.go ================================================ package server import ( "bytes" "testing" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/mvt" ) func ls(points []geometry.Point) *geojson.LineString { return geojson.NewLineString(geometry.NewLine(points, nil)) } // Ensure that LineString features are encoded using // all points in the geometry, not just the first one. func TestMVTAddFeatureLineStringUsesAllPoints(t *testing.T) { tileX, tileY, tileZ := 0, 0, 0 line := ls([]geometry.Point{ {X: 1, Y: 1}, {X: 2, Y: 2}, {X: 3, Y: 3}, }) id := "line" actual := mvtRender(tileX, tileY, tileZ, []mvtObj{{id: id, obj: line}}) var tile mvt.Tile layer := tile.AddLayer("tile38") layer.SetExtent(4096) f := layer.AddFeature(mvt.LineString) series := line.Base() npoints := series.NumPoints() if npoints < 2 { t.Fatalf("expected at least two points, got %d", npoints) } p := series.PointAt(0) x, y := mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ) f.MoveTo(x, y) for i := 1; i < npoints; i++ { p = series.PointAt(i) x, y = mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ) f.LineTo(x, y) } f.AddTag("type", "linestring") f.AddTag("id", id) expected := tile.Render() if !bytes.Equal(actual, expected) { t.Fatalf("mvtAddFeature LineString encoding mismatch") } } // LineStrings with fewer than two points should not // produce any geometry commands. func TestMVTAddFeatureLineStringTooShort(t *testing.T) { tileX, tileY, tileZ := 0, 0, 0 line := ls([]geometry.Point{ {X: 1, Y: 1}, }) actual := mvtRender(tileX, tileY, tileZ, []mvtObj{{id: "short", obj: line}}) var tile mvt.Tile layer := tile.AddLayer("tile38") layer.SetExtent(4096) _ = layer.AddFeature(mvt.LineString) expected := tile.Render() if !bytes.Equal(actual, expected) { t.Fatalf("mvtAddFeature LineString with <2 points should not encode geometry") } } ================================================ FILE: internal/server/output.go ================================================ package server import ( "strings" "time" "github.com/tidwall/resp" ) // OUTPUT [resp|json] func (s *Server) cmdOUTPUT(msg *Message) (resp.Value, error) { start := time.Now() args := msg.Args switch len(args) { case 1: if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"output":"json","elapsed":` + time.Since(start).String() + `}`), nil } return resp.StringValue("resp"), nil case 2: // Setting the original message output type will be picked up by the // server prior to the next command being executed. switch strings.ToLower(args[1]) { default: return retrerr(errInvalidArgument(args[1])) case "json": msg.OutputType = JSON case "resp": msg.OutputType = RESP } return OKMessage(msg, start), nil default: return retrerr(errInvalidNumberOfArguments) } } ================================================ FILE: internal/server/pubqueue.go ================================================ package server import ( "net" "sync" "github.com/tidwall/redcon" ) type pubQueue struct { cond *sync.Cond entries []pubQueueEntry // follower publish queue closed bool } type pubQueueEntry struct { channel string messages []string } func (s *Server) startPublishQueue(wg *sync.WaitGroup) { defer wg.Done() var buf []byte var conns []net.Conn s.pubq.cond.L.Lock() for { for len(s.pubq.entries) > 0 { entries := s.pubq.entries s.pubq.entries = nil s.pubq.cond.L.Unlock() // Get follower connections s.mu.RLock() for conn := range s.aofconnM { conns = append(conns, conn) } s.mu.RUnlock() // Buffer the PUBLISH command pipeline buf = buf[:0] for _, entry := range entries { for _, message := range entry.messages { buf = redcon.AppendArray(buf, 3) buf = redcon.AppendBulkString(buf, "PUBLISH") buf = redcon.AppendBulkString(buf, entry.channel) buf = redcon.AppendBulkString(buf, message) } } // Publish to followers for i, conn := range conns { conn.Write(buf) conns[i] = nil } conns = conns[:0] s.pubq.cond.L.Lock() } if s.pubq.closed { break } s.pubq.cond.Wait() } s.pubq.cond.L.Unlock() } func (s *Server) stopPublishQueue() { s.pubq.cond.L.Lock() s.pubq.closed = true s.pubq.cond.Broadcast() s.pubq.cond.L.Unlock() } func (s *Server) sendPublishQueue(channel string, message ...string) { s.pubq.cond.L.Lock() if !s.pubq.closed { s.pubq.entries = append(s.pubq.entries, pubQueueEntry{ channel: channel, messages: message, }) } s.pubq.cond.Broadcast() s.pubq.cond.L.Unlock() } ================================================ FILE: internal/server/pubsub.go ================================================ package server import ( "io" "net" "strconv" "sync" "time" "github.com/tidwall/gjson" "github.com/tidwall/match" "github.com/tidwall/redcon" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) const ( pubsubChannel = iota pubsubPattern ) type pubsub struct { mu sync.RWMutex hubs [2]map[string]*subhub } func newPubsub() *pubsub { return &pubsub{ hubs: [2]map[string]*subhub{ make(map[string]*subhub), make(map[string]*subhub), }, } } // Publish a message to subscribers func (s *Server) Publish(channel string, message ...string) int { var msgs []submsg s.pubsub.mu.RLock() if hub := s.pubsub.hubs[pubsubChannel][channel]; hub != nil { for target := range hub.targets { for _, message := range message { msgs = append(msgs, submsg{ kind: pubsubChannel, target: target, channel: channel, message: message, }) } } } for pattern, hub := range s.pubsub.hubs[pubsubPattern] { if match.Match(channel, pattern) { for target := range hub.targets { for _, message := range message { msgs = append(msgs, submsg{ kind: pubsubPattern, target: target, channel: channel, pattern: pattern, message: message, }) } } } } s.pubsub.mu.RUnlock() // broadcast to clients for _, msg := range msgs { msg.target.cond.L.Lock() msg.target.msgs = append(msg.target.msgs, msg) msg.target.cond.Broadcast() msg.target.cond.L.Unlock() } // Broadcast to followers s.sendPublishQueue(channel, message...) return len(msgs) } func (ps *pubsub) register(kind int, channel string, target *subtarget) { ps.mu.Lock() hub, ok := ps.hubs[kind][channel] if !ok { hub = newSubhub() ps.hubs[kind][channel] = hub } hub.targets[target] = true ps.mu.Unlock() } func (ps *pubsub) unregister(kind int, channel string, target *subtarget) { ps.mu.Lock() hub, ok := ps.hubs[kind][channel] if ok { delete(hub.targets, target) if len(hub.targets) == 0 { delete(ps.hubs[kind], channel) } } ps.mu.Unlock() } type submsg struct { kind byte target *subtarget pattern string channel string message string } type subtarget struct { cond *sync.Cond msgs []submsg closed bool } func newSubtarget() *subtarget { target := new(subtarget) target.cond = sync.NewCond(&sync.Mutex{}) return target } type subhub struct { targets map[*subtarget]bool } func newSubhub() *subhub { hub := new(subhub) hub.targets = make(map[*subtarget]bool) return hub } type liveSubscriptionSwitches struct { // no fields. everything is managed through the Message } func (sub liveSubscriptionSwitches) Error() string { return goingLive } func (s *Server) cmdSubscribe(msg *Message) (resp.Value, error) { if len(msg.Args) < 2 { return resp.Value{}, errInvalidNumberOfArguments } return NOMessage, liveSubscriptionSwitches{} } func (s *Server) cmdPsubscribe(msg *Message) (resp.Value, error) { if len(msg.Args) < 2 { return resp.Value{}, errInvalidNumberOfArguments } return NOMessage, liveSubscriptionSwitches{} } func (s *Server) cmdPublish(msg *Message) (resp.Value, error) { start := time.Now() if len(msg.Args) != 3 { return resp.Value{}, errInvalidNumberOfArguments } channel := msg.Args[1] message := msg.Args[2] //geofence := gjson.Valid(message) && gjson.Get(message, "fence").Bool() n := s.Publish(channel, message) //, geofence) var res resp.Value switch msg.OutputType { case JSON: res = resp.StringValue(`{"ok":true` + `,"published":` + strconv.FormatInt(int64(n), 10) + `,"elapsed":"` + time.Since(start).String() + `"}`) case RESP: res = resp.IntegerValue(n) } return res, nil } func (s *Server) liveSubscription( conn net.Conn, rd *PipelineReader, msg *Message, websocket bool, ) error { defer conn.Close() // close connection when we are done outputType := msg.OutputType connType := msg.ConnType if websocket { outputType = JSON } else if msg.StrictRESP { outputType = RESP } var start time.Time // write helpers var writeLock sync.Mutex write := func(data []byte) { writeLock.Lock() defer writeLock.Unlock() writeLiveMessage(conn, data, outputType == JSON, connType, websocket) } writeOK := func() { switch outputType { case JSON: write([]byte(`{"ok":true` + `,"elapsed":"` + time.Since(start).String() + `"}`)) case RESP: write([]byte("+OK\r\n")) } } writePing := func(m *Message) { switch outputType { case JSON: if len(m.Args) > 1 { write([]byte(`{"ok":true,"ping":` + jsonString(m.Args[1]) + `,"elapsed":"` + time.Since(start).String() + `"}`)) } else { write([]byte(`{"ok":true,"ping":"pong","elapsed":"` + time.Since(start).String() + `"}`)) } case RESP: data := redcon.AppendArray(nil, 2) data = redcon.AppendBulkString(data, "PONG") if len(m.Args) > 1 { data = redcon.AppendBulkString(data, m.Args[1]) } else { data = redcon.AppendBulkString(data, "") } write(data) } } writeWrongNumberOfArgsErr := func(command string) { switch outputType { case JSON: write([]byte(`{"ok":false,"err":"invalid number of arguments"` + `,"elapsed":"` + time.Since(start).String() + `"}`)) case RESP: write([]byte("-ERR wrong number of arguments " + "for '" + command + "' command\r\n")) } } writeOnlyPubsubErr := func() { switch outputType { case JSON: write([]byte(`{"ok":false` + `,"err":"only (P)SUBSCRIBE / (P)UNSUBSCRIBE / ` + `PING / QUIT allowed in this context"` + `,"elapsed":"` + time.Since(start).String() + `"}`)) case RESP: write([]byte("-ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / " + "PING / QUIT allowed in this context\r\n")) } } writeSubscribe := func(command, channel string, num int) { switch outputType { case JSON: write([]byte(`{"ok":true` + `,"command":` + jsonString(command) + `,"channel":` + jsonString(channel) + `,"num":` + strconv.FormatInt(int64(num), 10) + `,"elapsed":"` + time.Since(start).String() + `"}`)) case RESP: b := redcon.AppendArray(nil, 3) b = redcon.AppendBulkString(b, command) b = redcon.AppendBulkString(b, channel) b = redcon.AppendInt(b, int64(num)) write(b) } } writeMessage := func(msg submsg) { if msg.kind == pubsubChannel { switch outputType { case JSON: var data []byte if !gjson.Valid(msg.message) { data = appendJSONString(nil, msg.message) } else { data = []byte(msg.message) } write(data) case RESP: b := redcon.AppendArray(nil, 3) b = redcon.AppendBulkString(b, "message") b = redcon.AppendBulkString(b, msg.channel) b = redcon.AppendBulkString(b, msg.message) write(b) } } else { switch outputType { case JSON: var data []byte if !gjson.Valid(msg.message) { data = appendJSONString(nil, msg.message) } else { data = []byte(msg.message) } write(data) case RESP: b := redcon.AppendArray(nil, 4) b = redcon.AppendBulkString(b, "pmessage") b = redcon.AppendBulkString(b, msg.pattern) b = redcon.AppendBulkString(b, msg.channel) b = redcon.AppendBulkString(b, msg.message) write(b) } } s.statsTotalMsgsSent.Add(1) } m := [2]map[string]bool{ make(map[string]bool), // pubsubChannel make(map[string]bool), // pubsubPattern } target := newSubtarget() defer func() { for i := 0; i < 2; i++ { for channel := range m[i] { s.pubsub.unregister(i, channel, target) } } target.cond.L.Lock() target.closed = true target.cond.Broadcast() target.cond.L.Unlock() }() go func() { log.Debugf("pubsub open") defer log.Debugf("pubsub closed") target.cond.L.Lock() defer target.cond.L.Unlock() for { for len(target.msgs) > 0 { msgs := target.msgs target.msgs = nil target.cond.L.Unlock() for _, msg := range msgs { writeMessage(msg) } target.cond.L.Lock() } if target.closed { break } target.cond.Wait() } }() msgs := []*Message{msg} for { for _, msg := range msgs { start = time.Now() var kind int var un bool switch msg.Command() { case "quit": writeOK() return nil case "ping": writePing(msg) continue case "psubscribe": kind, un = pubsubPattern, false case "punsubscribe": kind, un = pubsubPattern, true case "subscribe": kind, un = pubsubChannel, false case "unsubscribe": kind, un = pubsubChannel, true default: writeOnlyPubsubErr() continue } if len(msg.Args) < 2 { writeWrongNumberOfArgsErr(msg.Command()) } for i := 1; i < len(msg.Args); i++ { channel := msg.Args[i] if un { delete(m[kind], channel) s.pubsub.unregister(kind, channel, target) } else { m[kind][channel] = true s.pubsub.register(kind, channel, target) } writeSubscribe(msg.Command(), channel, len(m[0])+len(m[1])) } } var err error msgs, err = rd.ReadMessages() if err != nil { if err == io.EOF { return nil } return err } } } ================================================ FILE: internal/server/readonly.go ================================================ package server import ( "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" ) // READONLY yes|no func (s *Server) cmdREADONLY(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 2 { return retrerr(errInvalidNumberOfArguments) } switch args[1] { case "yes", "no": default: return retrerr(errInvalidArgument(args[1])) } // >> Operation var updated bool if args[1] == "yes" { if !s.config.readOnly() { updated = true s.config.setReadOnly(true) log.Info("read only") } } else { if s.config.readOnly() { updated = true s.config.setReadOnly(false) log.Info("read write") } } if updated { s.config.write(false) } // >> Response return OKMessage(msg, start), nil } ================================================ FILE: internal/server/respconn.go ================================================ package server import ( "net" "time" "github.com/tidwall/resp" ) // RESPConn represents a simple resp connection. type RESPConn struct { conn net.Conn rd *resp.Reader wr *resp.Writer } // DialTimeout dials a resp func DialTimeout(address string, timeout time.Duration) (*RESPConn, error) { tcpconn, err := net.DialTimeout("tcp", address, timeout) if err != nil { return nil, err } conn := &RESPConn{ conn: tcpconn, rd: resp.NewReader(tcpconn), wr: resp.NewWriter(tcpconn), } return conn, nil } // Close closes the connection. func (conn *RESPConn) Close() error { conn.wr.WriteMultiBulk("quit") return conn.conn.Close() } // Do performs a command and returns a resp value. func (conn *RESPConn) Do(commandName string, args ...interface{}) ( val resp.Value, err error, ) { if err := conn.wr.WriteMultiBulk(commandName, args...); err != nil { return val, err } val, _, err = conn.rd.ReadValue() return val, err } ================================================ FILE: internal/server/scan.go ================================================ package server import ( "bytes" "time" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/object" ) func (s *Server) cmdScanArgs(vs []string) ( ls liveFenceSwitches, err error, ) { var t searchScanBaseTokens vs, t, err = s.parseSearchScanBaseTokens("scan", t, vs) if err != nil { return } ls.searchScanBaseTokens = t if len(vs) != 0 { err = errInvalidNumberOfArguments return } return } func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] args, err := s.cmdScanArgs(vs) if args.usingLua() { defer args.Close() defer func() { // if r := recover(); r != nil { // res = NOMessage // err = fmt.Errorf("%v", r) // return // } }() } if err != nil { return NOMessage, err } wr := &bytes.Buffer{} sw, err := s.newScanWriter( wr, msg, args.key, args.output, args.precision, args.globs, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, args.nofields, args.mvt, args.tileX, args.tileY, args.tileZ) if err != nil { return NOMessage, err } if msg.OutputType == JSON { wr.WriteString(`{"ok":true`) } var ierr error if sw.col != nil { if sw.output == outputCount && len(sw.wheres) == 0 && len(sw.whereins) == 0 && len(sw.whereevals) == 0 && sw.globEverything { count := sw.col.Count() - int(args.cursor) if count < 0 { count = 0 } sw.count = uint64(count) } else { limits := multiGlobParse(sw.globs, args.desc) if limits[0] == "" && limits[1] == "" { sw.col.Scan(args.desc, sw, msg.Deadline, func(o *object.Object) bool { keepGoing, err := sw.pushObject(ScanWriterParams{ obj: o, }) if err != nil { ierr = err return false } return keepGoing }, ) } else { sw.col.ScanRange(limits[0], limits[1], args.desc, sw, msg.Deadline, func(o *object.Object) bool { keepGoing, err := sw.pushObject(ScanWriterParams{ obj: o, }) if err != nil { ierr = err return false } return keepGoing }, ) } } } if ierr != nil { return retrerr(ierr) } sw.writeFoot() if msg.OutputType == JSON { wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.BytesValue(wr.Bytes()), nil } return sw.respOut, nil } ================================================ FILE: internal/server/scanner.go ================================================ package server import ( "bytes" "encoding/base64" "errors" "math" "strconv" "strings" "github.com/mmcloughlin/geohash" "github.com/tidwall/btree" "github.com/tidwall/geojson" "github.com/tidwall/gjson" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/clip" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/object" ) const limitItems = 100 type outputT int const ( outputUnknown outputT = iota outputIDs outputObjects outputCount outputPoints outputHashes outputBounds ) type scanWriter struct { s *Server wr *bytes.Buffer name string msg *Message col *collection.Collection fkeys btree.Set[string] output outputT wheres []whereT whereins []whereinT whereevals []whereevalT numberIters uint64 numberItems uint64 nofields bool cursor uint64 limit uint64 hitLimit bool once bool count uint64 precision uint64 globs []string globEverything bool fullFields bool values []resp.Value matchValues bool respOut resp.Value filled []ScanWriterParams mvtObjs []mvtObj mvt bool tileX int tileY int tileZ int } type ScanWriterParams struct { obj *object.Object dist float64 distOutput bool // query or fence requested distance output noTest bool ignoreGlobMatch bool clip geojson.Object skipTesting bool } func (s *Server) newScanWriter( wr *bytes.Buffer, msg *Message, name string, output outputT, precision uint64, globs []string, matchValues bool, cursor, limit uint64, wheres []whereT, whereins []whereinT, whereevals []whereevalT, nofields, mvt bool, tileX, tileY, tileZ int, ) ( *scanWriter, error, ) { switch output { default: return nil, errors.New("invalid output type") case outputIDs, outputObjects, outputCount, outputBounds, outputPoints, outputHashes: } if limit == 0 { if output == outputCount { limit = math.MaxUint64 } else { limit = limitItems } } sw := &scanWriter{ s: s, wr: wr, name: name, msg: msg, globs: globs, limit: limit, cursor: cursor, output: output, nofields: nofields, precision: precision, whereevals: whereevals, matchValues: matchValues, } sw.mvt = mvt sw.tileX = tileX sw.tileY = tileY sw.tileZ = tileZ if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") { sw.globEverything = true } sw.wheres = wheres sw.whereins = whereins sw.col, _ = sw.s.cols.Get(sw.name) return sw, nil } func (sw *scanWriter) hasFieldsOutput() bool { switch sw.output { default: return false case outputObjects, outputPoints, outputHashes, outputBounds: return !sw.nofields } } func (sw *scanWriter) writeFoot() { if sw.mvt { sw.wr.WriteString(`,"mvt":"`) } else { switch sw.msg.OutputType { case JSON: if sw.fkeys.Len() > 0 && sw.hasFieldsOutput() { sw.wr.WriteString(`,"fields":[`) var i int sw.fkeys.Scan(func(name string) bool { if i > 0 { sw.wr.WriteByte(',') } sw.wr.WriteString(jsonString(name)) i++ return true }) sw.wr.WriteByte(']') } switch sw.output { case outputIDs: sw.wr.WriteString(`,"ids":[`) case outputObjects: sw.wr.WriteString(`,"objects":[`) case outputPoints: sw.wr.WriteString(`,"points":[`) case outputBounds: sw.wr.WriteString(`,"bounds":[`) case outputHashes: sw.wr.WriteString(`,"hashes":[`) case outputCount: } case RESP: } } var mvtTile []byte if sw.mvt { mvtTile = mvtRender(sw.tileX, sw.tileY, sw.tileZ, sw.mvtObjs) } else { for _, opts := range sw.filled { sw.writeFilled(opts) } } cursor := sw.numberIters if !sw.hitLimit { cursor = 0 } switch sw.msg.OutputType { case JSON: if sw.mvt { sw.wr.WriteString(base64.RawStdEncoding.EncodeToString(mvtTile)) sw.wr.WriteByte('"') } else { switch sw.output { default: sw.wr.WriteByte(']') case outputCount: } } sw.wr.WriteString(`,"count":` + strconv.FormatUint(sw.count, 10)) sw.wr.WriteString(`,"cursor":` + strconv.FormatUint(cursor, 10)) case RESP: if sw.output == outputCount { sw.respOut = resp.IntegerValue(int(sw.count)) } else { values := []resp.Value{resp.IntegerValue(int(cursor))} if sw.mvt { values = append(values, resp.BytesValue(mvtTile)) } else { values = append(values, resp.ArrayValue(sw.values)) } sw.respOut = resp.ArrayValue(values) } } } func extractZCoordinate(o geojson.Object) float64 { for { switch g := o.(type) { case *geojson.Point: return g.Z() case *geojson.Feature: o = g.Base() default: return 0 } } } func isPathKey(s string, key string) bool { return strings.HasPrefix(s, key) && (s == key || s[len(key)] == '.') } func getFieldValue(o *object.Object, name string) field.Value { if isPathKey(name, "properties") { if g := o.Geo(); g != nil { if res := gjson.Get(g.Members(), "properties"); res.Exists() { if name != "properties" { // We have a dot path suffix. res = res.Get(name[len("properties")+1:]) } if res.Exists() { return field.ValueOf(res.Raw) } } } } if name == "z" { z := extractZCoordinate(o.Geo()) return field.ValueOf(strconv.FormatFloat(z, 'f', -1, 64)) } return o.Fields().Get(name).Value() } func (sw *scanWriter) fieldMatch(o *object.Object) (bool, error) { for _, where := range sw.wheres { if where.expr { if !where.matchExpr(sw.s, o) { return false, nil } } else { if !where.matchField(getFieldValue(o, where.name)) { return false, nil } } } for _, wherein := range sw.whereins { if !wherein.match(getFieldValue(o, wherein.name)) { return false, nil } } if len(sw.whereevals) > 0 { fieldNames := make(map[string]field.Value) var props string if objIsSpatial(o.Geo()) { z := extractZCoordinate(o.Geo()) fieldNames["z"] = field.ValueOf(strconv.FormatFloat(z, 'f', -1, 64)) props = gjson.Get(o.Geo().Members(), "properties").Raw } o.Fields().Scan(func(f field.Field) bool { fieldNames[f.Name()] = f.Value() return true }) for _, whereval := range sw.whereevals { match, err := whereval.match(fieldNames, o.ID(), props) if err != nil { return false, err } if !match { return false, nil } } } return true, nil } func (sw *scanWriter) globMatch(o *object.Object) (ok, keepGoing bool) { if sw.globEverything { return true, true } var val string if sw.matchValues { val = o.String() } else { val = o.ID() } for _, pattern := range sw.globs { ok, _ := glob.Match(pattern, val) if ok { return true, true } } return false, true } // Increment cursor func (sw *scanWriter) Offset() uint64 { return sw.cursor } func (sw *scanWriter) Step(n uint64) { sw.numberIters += n } // ok is whether the object passes the test and should be written // keepGoing is whether there could be more objects to test func (sw *scanWriter) testObject(o *object.Object, ) (ok, keepGoing bool, err error) { match, kg := sw.globMatch(o) if !match { return false, kg, nil } ok, err = sw.fieldMatch(o) if err != nil { return false, false, err } return ok, true, nil } func (sw *scanWriter) pushObject(opts ScanWriterParams) (keepGoing bool, err error, ) { keepGoing = true if !opts.noTest { var ok bool var err error ok, keepGoing, err = sw.testObject(opts.obj) if err != nil { return false, err } if !ok { return keepGoing, nil } } sw.count++ if sw.output == outputCount { return sw.count < sw.limit, nil } if opts.clip != nil { // create a newly clipped object opts.obj = object.New( opts.obj.ID(), clip.Clip(opts.obj.Geo(), opts.clip, &sw.s.geomIndexOpts), opts.obj.Expires(), opts.obj.Fields(), ) } if sw.mvt { sw.mvtObjs = append(sw.mvtObjs, mvtObj{opts.obj.ID(), opts.obj.Geo()}) } if !sw.fullFields { opts.obj.Fields().Scan(func(f field.Field) bool { sw.fkeys.Insert(f.Name()) return true }) } sw.filled = append(sw.filled, opts) sw.numberItems++ if sw.numberItems == sw.limit { sw.hitLimit = true return false, nil } return keepGoing, nil } func (sw *scanWriter) writeObject(opts ScanWriterParams) { n := len(sw.filled) sw.pushObject(opts) if len(sw.filled) > n { sw.writeFilled(sw.filled[len(sw.filled)-1]) sw.filled = sw.filled[:n] } } func (sw *scanWriter) writeFilled(opts ScanWriterParams) { switch sw.msg.OutputType { case JSON: var wr bytes.Buffer var jsfields string if sw.once { wr.WriteByte(',') } else { sw.once = true } fieldsOutput := sw.hasFieldsOutput() if fieldsOutput && sw.fullFields { if opts.obj.Fields().Len() > 0 { jsfields = `,"fields":{` var i int opts.obj.Fields().Scan(func(f field.Field) bool { if !f.Value().IsZero() { if i > 0 { jsfields += `,` } jsfields += jsonString(f.Name()) + ":" + f.Value().JSON() i++ } return true }) jsfields += `}` } } else if fieldsOutput && sw.fkeys.Len() > 0 && !sw.fullFields { jsfields = `,"fields":[` var i int sw.fkeys.Scan(func(name string) bool { if i > 0 { jsfields += `,` } f := opts.obj.Fields().Get(name) jsfields += f.Value().JSON() i++ return true }) jsfields += `]` } if sw.output == outputIDs { if opts.distOutput || opts.dist > 0 { wr.WriteString(`{"id":` + jsonString(opts.obj.ID()) + `,"distance":` + strconv.FormatFloat(opts.dist, 'f', -1, 64) + "}") } else { wr.WriteString(jsonString(opts.obj.ID())) } } else { wr.WriteString(`{"id":` + jsonString(opts.obj.ID())) switch sw.output { case outputObjects: wr.WriteString(`,"object":` + string(opts.obj.Geo().AppendJSON(nil))) case outputPoints: wr.WriteString(`,"point":` + string(appendJSONSimplePoint(nil, opts.obj.Geo()))) case outputHashes: center := opts.obj.Geo().Center() p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision)) wr.WriteString(`,"hash":"` + p + `"`) case outputBounds: wr.WriteString(`,"bounds":` + string(appendJSONSimpleBounds(nil, opts.obj.Geo()))) } wr.WriteString(jsfields) if opts.distOutput || opts.dist > 0 { wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.dist, 'f', -1, 64)) } wr.WriteString(`}`) } sw.wr.Write(wr.Bytes()) case RESP: vals := make([]resp.Value, 1, 3) vals[0] = resp.StringValue(opts.obj.ID()) if sw.output == outputIDs { if opts.distOutput || opts.dist > 0 { vals = append(vals, resp.FloatValue(opts.dist)) sw.values = append(sw.values, resp.ArrayValue(vals)) } else { sw.values = append(sw.values, vals[0]) } } else { switch sw.output { case outputObjects: vals = append(vals, resp.StringValue(opts.obj.String())) case outputPoints: point := opts.obj.Geo().Center() z := extractZCoordinate(opts.obj.Geo()) if z != 0 { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.FloatValue(point.Y), resp.FloatValue(point.X), resp.FloatValue(z), })) } else { vals = append(vals, resp.ArrayValue([]resp.Value{ resp.FloatValue(point.Y), resp.FloatValue(point.X), })) } case outputHashes: center := opts.obj.Geo().Center() p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision)) vals = append(vals, resp.StringValue(p)) case outputBounds: bbox := opts.obj.Rect() vals = append(vals, resp.ArrayValue([]resp.Value{ resp.ArrayValue([]resp.Value{ resp.FloatValue(bbox.Min.Y), resp.FloatValue(bbox.Min.X), }), resp.ArrayValue([]resp.Value{ resp.FloatValue(bbox.Max.Y), resp.FloatValue(bbox.Max.X), }), })) } if sw.hasFieldsOutput() { var fvals []resp.Value var i int opts.obj.Fields().Scan(func(f field.Field) bool { if !f.Value().IsZero() { fvals = append(fvals, resp.StringValue(f.Name()), resp.StringValue(f.Value().Data())) i++ } return true }) if len(fvals) > 0 { vals = append(vals, resp.ArrayValue(fvals)) } } if opts.distOutput || opts.dist > 0 { vals = append(vals, resp.FloatValue(opts.dist)) } sw.values = append(sw.values, resp.ArrayValue(vals)) } } } ================================================ FILE: internal/server/scanner_test.go ================================================ package server import ( "fmt" "math" "math/rand" "testing" "time" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/object" ) type testPointItem struct { object geojson.Object fields field.List } func PO(x, y float64) *geojson.Point { return geojson.NewPoint(geometry.Point{X: x, Y: y}) } func BenchmarkFieldMatch(t *testing.B) { rand.Seed(time.Now().UnixNano()) items := make([]testPointItem, t.N) for i := 0; i < t.N; i++ { var fields field.List fields = fields.Set(field.Make("foo", fmt.Sprintf("%f", rand.Float64()*9+1))) fields = fields.Set(field.Make("bar", fmt.Sprintf("%f", math.Round(rand.Float64()*30)+1))) items[i] = testPointItem{ PO(rand.Float64()*360-180, rand.Float64()*180-90), fields, } } sw := &scanWriter{ wheres: []whereT{ {false, "foo", false, field.ValueOf("1"), false, field.ValueOf("3")}, {false, "bar", false, field.ValueOf("10"), false, field.ValueOf("30")}, }, whereins: []whereinT{ {"foo", []field.Value{field.ValueOf("1"), field.ValueOf("2")}}, {"bar", []field.Value{field.ValueOf("11"), field.ValueOf("25")}}, }, } t.ResetTimer() for i := 0; i < t.N; i++ { // one call is super fast, measurements are not reliable, let's do 100 for ix := 0; ix < 100; ix++ { sw.fieldMatch(object.New("", items[i].object, 0, items[i].fields)) } } } ================================================ FILE: internal/server/scripts.go ================================================ package server import ( "bytes" "context" "crypto/sha1" "encoding/hex" "encoding/json" "errors" "fmt" "math" "strconv" "strings" "sync" "time" "github.com/tidwall/geojson/geo" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/log" "github.com/tidwall/tinylru" lua "github.com/yuin/gopher-lua" luajson "layeh.com/gopher-json" ) const ( iniLuaPoolSize = 5 maxLuaPoolSize = 1000 ) // For Lua os.clock() impl var startedAt = time.Now() var errShaNotFound = errors.New("sha not found") var errCmdNotSupported = errors.New("command not supported in scripts") var errNotLeader = errors.New("not the leader") var errReadOnly = errors.New("read only") var errCatchingUp = errors.New("catching up to leader") var errNoLuasAvailable = errors.New("no interpreters available") var errTimeout = errors.New("timeout") // Go-routine-safe pool of read-to-go lua states type lStatePool struct { m sync.Mutex s *Server saved []*lua.LState total int } // newPool returns a new pool of lua states func (s *Server) newPool() *lStatePool { pl := &lStatePool{ saved: make([]*lua.LState, iniLuaPoolSize), s: s, } // Fill the pool with some ready handlers for i := 0; i < iniLuaPoolSize; i++ { pl.saved[i] = pl.New() pl.total++ } return pl } func (pl *lStatePool) Get() (*lua.LState, error) { pl.m.Lock() defer pl.m.Unlock() n := len(pl.saved) if n == 0 { if pl.total >= maxLuaPoolSize { return nil, errNoLuasAvailable } pl.total++ return pl.New(), nil } x := pl.saved[n-1] pl.saved = pl.saved[0 : n-1] return x, nil } // Prune removes some of the idle lua states from the pool func (pl *lStatePool) Prune() { pl.m.Lock() n := len(pl.saved) if n > iniLuaPoolSize { // drop half of the idle states that is above the minimum dropNum := (n - iniLuaPoolSize) / 2 if dropNum < 1 { dropNum = 1 } newSaved := make([]*lua.LState, n-dropNum) copy(newSaved, pl.saved[dropNum:]) pl.saved = newSaved pl.total -= dropNum } pl.m.Unlock() } func (pl *lStatePool) New() *lua.LState { // Prevent opening all Lua modules L := lua.NewState(lua.Options{SkipOpenLibs: true}) allowedModules := []struct { moduleName string moduleFn lua.LGFunction }{ {lua.BaseLibName, openBaseSubset}, {lua.TabLibName, lua.OpenTable}, {lua.MathLibName, lua.OpenMath}, {lua.StringLibName, lua.OpenString}, {lua.OsLibName, openOsSubset}, // See below for impl, only opens clock/difftime } // Open non-vulnerable modules (i.e. NOT io/os) for _, pair := range allowedModules { if err := L.CallByParam(lua.P{ Fn: L.NewFunction(pair.moduleFn), NRet: 0, Protect: true, }, lua.LString(pair.moduleName)); err != nil { panic(err) } } getArgs := func(ls *lua.LState) (evalCmd string, args []string) { evalCmd = ls.GetGlobal("EVAL_CMD").String() // Trying to work with unknown number of args. // When we see empty arg we call it enough. for i := 1; ; i++ { if arg := ls.ToString(i); arg == "" { break } else { args = append(args, arg) } } return } call := func(ls *lua.LState) int { evalCmd, args := getArgs(ls) var numRet int if res, err := pl.s.luaTile38Call(evalCmd, args[0], args[1:]...); err != nil { ls.RaiseError("ERR %s", err.Error()) numRet = 0 } else { ls.Push(ConvertToLua(ls, res)) numRet = 1 } return numRet } pcall := func(ls *lua.LState) int { evalCmd, args := getArgs(ls) if res, err := pl.s.luaTile38Call(evalCmd, args[0], args[1:]...); err != nil { ls.Push(ConvertToLua(ls, resp.ErrorValue(err))) } else { ls.Push(ConvertToLua(ls, res)) } return 1 } errorReply := func(ls *lua.LState) int { tbl := L.CreateTable(0, 1) tbl.RawSetString("err", lua.LString(ls.ToString(1))) ls.Push(tbl) return 1 } statusReply := func(ls *lua.LState) int { tbl := L.CreateTable(0, 1) tbl.RawSetString("ok", lua.LString(ls.ToString(1))) ls.Push(tbl) return 1 } sha1hex := func(ls *lua.LState) int { shaSum := Sha1Sum(ls.ToString(1)) ls.Push(lua.LString(shaSum)) return 1 } distanceTo := func(ls *lua.LState) int { dt := geo.DistanceTo( float64(ls.ToNumber(1)), float64(ls.ToNumber(2)), float64(ls.ToNumber(3)), float64(ls.ToNumber(4))) ls.Push(lua.LNumber(dt)) return 1 } var exports = map[string]lua.LGFunction{ "call": call, "pcall": pcall, "error_reply": errorReply, "status_reply": statusReply, "sha1hex": sha1hex, "distance_to": distanceTo, } L.SetGlobal("tile38", L.SetFuncs(L.NewTable(), exports)) // Load json L.SetGlobal("json", L.Get(luajson.Loader(L))) // Prohibit creating new globals in this state lockNewGlobals := func(ls *lua.LState) int { ls.RaiseError("attempt to create global variable '%s'", ls.ToString(2)) return 0 } mt := L.CreateTable(0, 1) mt.RawSetString("__newindex", L.NewFunction(lockNewGlobals)) L.SetMetatable(L.Get(lua.GlobalsIndex), mt) return L } func (pl *lStatePool) Put(L *lua.LState) { pl.m.Lock() pl.saved = append(pl.saved, L) pl.m.Unlock() } func (pl *lStatePool) Shutdown() { pl.m.Lock() for _, L := range pl.saved { L.Close() } pl.m.Unlock() } // Go-routine-safe map of compiled scripts type lScriptMap struct { m sync.Mutex scripts map[string]*lua.FunctionProto lru tinylru.LRUG[string, *lua.FunctionProto] } func (sm *lScriptMap) Get(key string) (script *lua.FunctionProto, ok bool) { sm.m.Lock() script, ok = sm.scripts[key] if !ok { script, ok = sm.lru.Get(key) } sm.m.Unlock() return script, ok } func (sm *lScriptMap) Put(key string, script *lua.FunctionProto) { sm.m.Lock() sm.scripts[key] = script sm.m.Unlock() } func (sm *lScriptMap) PutLRU(key string, script *lua.FunctionProto) { sm.m.Lock() sm.lru.Set(key, script) sm.m.Unlock() } func (sm *lScriptMap) Flush() { sm.m.Lock() sm.scripts = make(map[string]*lua.FunctionProto) sm.lru.Clear() sm.m.Unlock() } // NewScriptMap returns a new map with lua scripts func (s *Server) newScriptMap() *lScriptMap { return &lScriptMap{ scripts: make(map[string]*lua.FunctionProto), } } // ConvertToLua converts RESP value to lua LValue func ConvertToLua(L *lua.LState, val resp.Value) lua.LValue { if val.IsNull() { return lua.LFalse } switch val.Type() { case resp.Integer: return lua.LNumber(val.Integer()) case resp.BulkString: return lua.LString(val.String()) case resp.Error: tbl := L.CreateTable(0, 1) tbl.RawSetString("err", lua.LString(val.String())) return tbl case resp.SimpleString: tbl := L.CreateTable(0, 1) tbl.RawSetString("ok", lua.LString(val.String())) return tbl case resp.Array: tbl := L.CreateTable(len(val.Array()), 0) for _, item := range val.Array() { tbl.Append(ConvertToLua(L, item)) } return tbl } return lua.LString("ERR: unknown RESP type: " + val.Type().String()) } // ConvertToRESP convert lua LValue to RESP value func ConvertToRESP(val lua.LValue) resp.Value { switch val.Type() { case lua.LTNil: return resp.NullValue() case lua.LTBool: if val == lua.LTrue { return resp.IntegerValue(1) } return resp.NullValue() case lua.LTNumber: float := float64(val.(lua.LNumber)) if math.IsNaN(float) || math.IsInf(float, 0) { return resp.FloatValue(float) } return resp.IntegerValue(int(math.Floor(float))) case lua.LTString: return resp.StringValue(val.String()) case lua.LTTable: var values []resp.Value var specialValues []resp.Value var cb func(lk lua.LValue, lv lua.LValue) tbl := val.(*lua.LTable) if tbl.Len() != 0 { // list cb = func(lk lua.LValue, lv lua.LValue) { values = append(values, ConvertToRESP(lv)) } } else { // map cb = func(lk lua.LValue, lv lua.LValue) { if lk.Type() == lua.LTString { lks := lk.String() switch lks { case "ok": specialValues = append(specialValues, resp.SimpleStringValue(lv.String())) case "err": specialValues = append(specialValues, resp.ErrorValue(errors.New(lv.String()))) } } values = append(values, resp.ArrayValue( []resp.Value{ConvertToRESP(lk), ConvertToRESP(lv)})) } } tbl.ForEach(cb) if len(values) == 1 && len(specialValues) == 1 { return specialValues[0] } return resp.ArrayValue(values) } return resp.ErrorValue(errors.New("Unsupported lua type: " + val.Type().String())) } // ConvertToJSON converts lua LValue to JSON string func ConvertToJSON(val lua.LValue) string { switch val.Type() { case lua.LTNil: return "null" case lua.LTBool: if val == lua.LTrue { return "true" } return "false" case lua.LTNumber: return val.String() case lua.LTString: if b, err := json.Marshal(val.String()); err != nil { panic(err) } else { return string(b) } case lua.LTTable: var values []string var cb func(lk lua.LValue, lv lua.LValue) var start, end string tbl := val.(*lua.LTable) if tbl.Len() != 0 { // list start = `[` end = `]` cb = func(lk lua.LValue, lv lua.LValue) { values = append(values, ConvertToJSON(lv)) } } else { // map start = `{` end = `}` cb = func(lk lua.LValue, lv lua.LValue) { values = append( values, ConvertToJSON(lk)+`:`+ConvertToJSON(lv)) } } tbl.ForEach(cb) return start + strings.Join(values, `,`) + end } return "Unsupported lua type: " + val.Type().String() } func luaSetRawGlobals(ls *lua.LState, tbl map[string]lua.LValue) { gt := ls.Get(lua.GlobalsIndex).(*lua.LTable) for key, val := range tbl { gt.RawSetString(key, val) } } // Sha1Sum returns a string with hex representation of sha1 sum of a given string func Sha1Sum(s string) string { h := sha1.New() h.Write([]byte(s)) return hex.EncodeToString(h.Sum(nil)) } // Replace newlines with literal \n since RESP errors cannot have newlines func makeSafeErr(err error) error { return errors.New(strings.Replace(err.Error(), "\n", `\n`, -1)) } // Run eval/evalro/evalna command or it's -sha variant func (s *Server) cmdEvalUnified(scriptIsSha bool, msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] var ok bool var script, numkeysStr, key, arg string if vs, script, ok = tokenval(vs); !ok || script == "" { return NOMessage, errInvalidNumberOfArguments } if vs, numkeysStr, ok = tokenval(vs); !ok || numkeysStr == "" { return NOMessage, errInvalidNumberOfArguments } var i, numkeys uint64 if numkeys, err = strconv.ParseUint(numkeysStr, 10, 64); err != nil { err = errInvalidArgument(numkeysStr) return } luaState, err := s.luapool.Get() if err != nil { return } luaDeadline := lua.LNil if msg.Deadline != nil { dlTime := msg.Deadline.GetDeadlineTime() ctx, cancel := context.WithDeadline(context.Background(), dlTime) defer cancel() luaState.SetContext(ctx) defer luaState.RemoveContext() luaDeadline = lua.LNumber(float64(dlTime.UnixNano()) / 1e9) } defer s.luapool.Put(luaState) keysTbl := luaState.CreateTable(int(numkeys), 0) for i = 0; i < numkeys; i++ { if vs, key, ok = tokenval(vs); !ok || key == "" { err = errInvalidNumberOfArguments return } keysTbl.Append(lua.LString(key)) } argsTbl := luaState.CreateTable(len(vs), 0) for len(vs) > 0 { if vs, arg, ok = tokenval(vs); !ok || arg == "" { err = errInvalidNumberOfArguments return } argsTbl.Append(lua.LString(arg)) } var shaSum string if scriptIsSha { shaSum = script } else { shaSum = Sha1Sum(script) } luaSetRawGlobals( luaState, map[string]lua.LValue{ "KEYS": keysTbl, "ARGV": argsTbl, "DEADLINE": luaDeadline, "EVAL_CMD": lua.LString(msg.Command()), }) compiled, ok := s.luascripts.Get(shaSum) var fn *lua.LFunction if ok { fn = &lua.LFunction{ IsG: false, Env: luaState.Env, Proto: compiled, GFunction: nil, Upvalues: make([]*lua.Upvalue, 0), } } else if scriptIsSha { err = errShaNotFound return } else { fn, err = luaState.Load(strings.NewReader(script), "f_"+shaSum) if err != nil { return NOMessage, makeSafeErr(err) } s.luascripts.Put(shaSum, fn.Proto) } luaState.Push(fn) defer luaSetRawGlobals( luaState, map[string]lua.LValue{ "KEYS": lua.LNil, "ARGV": lua.LNil, "DEADLINE": lua.LNil, "EVAL_CMD": lua.LNil, }) if err := luaState.PCall(0, 1, nil); err != nil { if strings.Contains(err.Error(), "context deadline exceeded") { msg.Deadline.Check() } log.Debugf("%v", err.Error()) return NOMessage, makeSafeErr(err) } ret := luaState.Get(-1) // returned value luaState.Pop(1) switch msg.OutputType { case JSON: var buf bytes.Buffer buf.WriteString(`{"ok":true`) buf.WriteString(`,"result":` + ConvertToJSON(ret)) buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil case RESP: return ConvertToRESP(ret), nil } return NOMessage, nil } func (s *Server) cmdScriptLoad(msg *Message) (resp.Value, error) { start := time.Now() vs := msg.Args[1:] var ok bool var script string if _, script, ok = tokenval(vs); !ok || script == "" { return NOMessage, errInvalidNumberOfArguments } shaSum := Sha1Sum(script) luaState, err := s.luapool.Get() if err != nil { return NOMessage, err } defer s.luapool.Put(luaState) fn, err := luaState.Load(strings.NewReader(script), "f_"+shaSum) if err != nil { return NOMessage, makeSafeErr(err) } s.luascripts.Put(shaSum, fn.Proto) switch msg.OutputType { case JSON: var buf bytes.Buffer buf.WriteString(`{"ok":true`) buf.WriteString(`,"result":"` + shaSum + `"`) buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil case RESP: return resp.StringValue(shaSum), nil } return NOMessage, nil } func (s *Server) cmdScriptExists(msg *Message) (resp.Value, error) { start := time.Now() vs := msg.Args[1:] var ok bool var shaSum string var results []int var ires int for len(vs) > 0 { if vs, shaSum, ok = tokenval(vs); !ok || shaSum == "" { return NOMessage, errInvalidNumberOfArguments } _, ok = s.luascripts.Get(shaSum) if ok { ires = 1 } else { ires = 0 } results = append(results, ires) } switch msg.OutputType { case JSON: var buf bytes.Buffer buf.WriteString(`{"ok":true`) var resArray []string for _, ires := range results { resArray = append(resArray, fmt.Sprintf("%d", ires)) } buf.WriteString(`,"result":[` + strings.Join(resArray, ",") + `]`) buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil case RESP: var resArray []resp.Value for _, ires := range results { resArray = append(resArray, resp.IntegerValue(ires)) } return resp.ArrayValue(resArray), nil } return resp.SimpleStringValue(""), nil } func (s *Server) cmdScriptFlush(msg *Message) (resp.Value, error) { start := time.Now() s.luascripts.Flush() switch msg.OutputType { case JSON: var buf bytes.Buffer buf.WriteString(`{"ok":true`) buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil case RESP: return resp.StringValue("OK"), nil } return resp.SimpleStringValue(""), nil } func (s *Server) commandInScript(msg *Message) ( res resp.Value, d commandDetails, err error, ) { switch msg.Command() { default: err = fmt.Errorf("unknown command '%s'", msg.Args[0]) case "set": res, d, err = s.cmdSET(msg) case "fset": res, d, err = s.cmdFSET(msg) case "del": res, d, err = s.cmdDEL(msg) case "pdel": res, d, err = s.cmdPDEL(msg) case "drop": res, d, err = s.cmdDROP(msg) case "expire": res, d, err = s.cmdEXPIRE(msg) case "rename": res, d, err = s.cmdRENAME(msg) case "renamenx": res, d, err = s.cmdRENAME(msg) case "persist": res, d, err = s.cmdPERSIST(msg) case "ttl": res, err = s.cmdTTL(msg) case "stats": res, err = s.cmdSTATS(msg) case "scan": res, err = s.cmdScan(msg) case "nearby": res, err = s.cmdNearby(msg) case "within": res, err = s.cmdWITHIN(msg) case "intersects": res, err = s.cmdINTERSECTS(msg) case "search": res, err = s.cmdSearch(msg) case "bounds": res, err = s.cmdBOUNDS(msg) case "get": res, err = s.cmdGET(msg) case "fget": res, err = s.cmdFGET(msg) case "jget": res, err = s.cmdJget(msg) case "jset": res, d, err = s.cmdJset(msg) case "jdel": res, d, err = s.cmdJdel(msg) case "type": res, err = s.cmdTYPE(msg) case "keys": res, err = s.cmdKEYS(msg) case "exists": res, err = s.cmdEXISTS(msg) case "fexists": res, err = s.cmdFEXISTS(msg) case "test": res, err = s.cmdTEST(msg) case "server": res, err = s.cmdSERVER(msg) } s.sendMonitor(err, msg, nil, true) return } func (s *Server) luaTile38Call(evalcmd string, cmd string, args ...string) (resp.Value, error) { msg := &Message{} msg.OutputType = RESP msg.Args = append([]string{cmd}, args...) if msg.Command() == "timeout" { if err := rewriteTimeoutMsg(msg); err != nil { return resp.NullValue(), err } } switch msg.Command() { case "ping", "echo", "auth", "massinsert", "shutdown", "gc", "sethook", "pdelhook", "delhook", "follow", "readonly", "config", "output", "client", "aofshrink", "script load", "script exists", "script flush", "eval", "evalsha", "evalro", "evalrosha", "evalna", "evalnasha": return resp.NullValue(), errCmdNotSupported } switch evalcmd { case "eval", "evalsha": return s.luaTile38AtomicRW(msg) case "evalro", "evalrosha": return s.luaTile38AtomicRO(msg) case "evalna", "evalnasha": return s.luaTile38NonAtomic(msg) } return resp.NullValue(), errCmdNotSupported } // The eval command has already got the lock. No locking on the call from within the script. func (s *Server) luaTile38AtomicRW(msg *Message) (resp.Value, error) { var write bool switch msg.Command() { default: return resp.NullValue(), errCmdNotSupported case "set", "del", "drop", "fset", "flushdb", "expire", "persist", "jset", "pdel", "rename", "renamenx": // write operations write = true if s.config.followHost() != "" { return resp.NullValue(), errNotLeader } if s.config.readOnly() { return resp.NullValue(), errReadOnly } case "get", "keys", "scan", "nearby", "within", "intersects", "hooks", "search", "ttl", "bounds", "server", "info", "type", "jget", "fget", "exists", "fexists", "test": // read operations if s.config.followHost() != "" && !s.caughtUpOnce() { return resp.NullValue(), errCatchingUp } } res, d, err := func() (res resp.Value, d commandDetails, err error) { if msg.Deadline != nil { if write { res = NOMessage err = errTimeoutOnCmd(msg.Command()) return } defer func() { if msg.Deadline.Hit() { v := recover() if v != nil { if s, ok := v.(string); !ok || s != "deadline" { panic(v) } } res = NOMessage err = errTimeout } }() } return s.commandInScript(msg) }() if err != nil { return resp.NullValue(), err } if write { if err := s.writeAOF(msg.Args, &d); err != nil { return resp.NullValue(), err } } return res, nil } func (s *Server) luaTile38AtomicRO(msg *Message) (resp.Value, error) { switch msg.Command() { default: return resp.NullValue(), errCmdNotSupported case "set", "del", "drop", "fset", "flushdb", "expire", "persist", "jset", "pdel", "rename", "renamenx": // write operations return resp.NullValue(), errReadOnly case "get", "keys", "scan", "nearby", "within", "intersects", "hooks", "search", "ttl", "bounds", "server", "info", "type", "jget", "fget", "exists", "fexists", "test": // read operations if s.config.followHost() != "" && !s.caughtUpOnce() { return resp.NullValue(), errCatchingUp } } res, _, err := func() (res resp.Value, d commandDetails, err error) { if msg.Deadline != nil { defer func() { if msg.Deadline.Hit() { v := recover() if v != nil { if s, ok := v.(string); !ok || s != "deadline" { panic(v) } } res = NOMessage err = errTimeout } }() } return s.commandInScript(msg) }() if err != nil { return resp.NullValue(), err } return res, nil } func (s *Server) luaTile38NonAtomic(msg *Message) (resp.Value, error) { var write bool // choose the locking strategy switch msg.Command() { default: return resp.NullValue(), errCmdNotSupported case "set", "del", "drop", "fset", "flushdb", "expire", "persist", "jset", "pdel", "rename", "renamenx": // write operations write = true s.mu.Lock() defer s.mu.Unlock() if s.config.followHost() != "" { return resp.NullValue(), errNotLeader } if s.config.readOnly() { return resp.NullValue(), errReadOnly } case "get", "keys", "scan", "nearby", "within", "intersects", "hooks", "search", "ttl", "bounds", "server", "info", "type", "jget", "fget", "exists", "fexists", "test": // read operations s.mu.RLock() defer s.mu.RUnlock() if s.config.followHost() != "" && !s.caughtUpOnce() { return resp.NullValue(), errCatchingUp } } res, d, err := func() (res resp.Value, d commandDetails, err error) { if msg.Deadline != nil { if write { res = NOMessage err = errTimeoutOnCmd(msg.Command()) return } defer func() { if msg.Deadline.Hit() { v := recover() if v != nil { if s, ok := v.(string); !ok || s != "deadline" { panic(v) } } res = NOMessage err = errTimeout } }() } return s.commandInScript(msg) }() if err != nil { return resp.NullValue(), err } if write { if err := s.writeAOF(msg.Args, &d); err != nil { return resp.NullValue(), err } } return res, nil } // Opens a subset of the Lua 5.1 base module (tonumber, tostring) func openBaseSubset(L *lua.LState) int { basefns := map[string]lua.LGFunction{ "tonumber": baseToNumber, "tostring": baseToString, } global := L.Get(lua.GlobalsIndex).(*lua.LTable) L.SetGlobal("_G", global) L.SetGlobal("_VERSION", lua.LString(lua.LuaVersion)) L.SetGlobal("_GOPHER_LUA_VERSION", lua.LString(lua.PackageName+" "+lua.PackageVersion)) basemod := L.RegisterModule("_G", basefns) L.Push(basemod) return 1 } // Opens a subset of the Lua 5.1 os module (clock, difftime) func openOsSubset(L *lua.LState) int { osfns := map[string]lua.LGFunction{ "clock": osClock, "difftime": osDiffTime, } osmod := L.RegisterModule(lua.OsLibName, osfns) L.Push(osmod) return 1 } // Lua tonumber() func baseToNumber(L *lua.LState) int { base := L.OptInt(2, 10) noBase := L.Get(2) == lua.LNil switch lv := L.CheckAny(1).(type) { case lua.LNumber: L.Push(lv) case lua.LString: str := strings.Trim(string(lv), " \n\t") if strings.Contains(str, ".") { if v, err := strconv.ParseFloat(str, lua.LNumberBit); err != nil { L.Push(lua.LNil) } else { L.Push(lua.LNumber(v)) } } else { if noBase && strings.HasPrefix(strings.ToLower(str), "0x") { base, str = 16, str[2:] // Hex number } if v, err := strconv.ParseInt(str, base, lua.LNumberBit); err != nil { L.Push(lua.LNil) } else { L.Push(lua.LNumber(v)) } } default: L.Push(lua.LNil) } return 1 } // Lua tostring() func baseToString(L *lua.LState) int { v1 := L.CheckAny(1) L.Push(L.ToStringMeta(v1)) return 1 } // Lua os.clock() func osClock(L *lua.LState) int { L.Push(lua.LNumber(time.Since(startedAt).Seconds())) return 1 } // Lua os.difftime() func osDiffTime(L *lua.LState) int { L.Push(lua.LNumber(L.CheckInt64(1) - L.CheckInt64(2))) return 1 } ================================================ FILE: internal/server/search.go ================================================ package server import ( "bytes" "errors" "fmt" "strconv" "strings" "time" "github.com/iwpnd/sectr" "github.com/mmcloughlin/geohash" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/bing" "github.com/tidwall/tile38/internal/buffer" "github.com/tidwall/tile38/internal/clip" "github.com/tidwall/tile38/internal/glob" "github.com/tidwall/tile38/internal/object" ) const defaultCircleSteps = 64 type liveFenceSwitches struct { searchScanBaseTokens obj geojson.Object cmd string roam roamSwitches } type roamSwitches struct { on bool key string id string pattern bool meters float64 scan string } type roamMatch struct { id string obj geojson.Object meters float64 } func (lfs liveFenceSwitches) Error() string { return goingLive } func (lfs liveFenceSwitches) Close() { for _, whereeval := range lfs.whereevals { whereeval.Close() } } func (lfs liveFenceSwitches) usingLua() bool { return len(lfs.whereevals) > 0 } func parseRectArea(ltyp string, vs []string) (nvs []string, grect geojson.Object, tileX, tileY, tileZ int, err error, ) { var rect geometry.Rect var ok bool switch ltyp { default: err = errNotRectangle return case "bounds": var sminLat, sminLon, smaxlat, smaxlon string if vs, sminLat, ok = tokenval(vs); !ok || sminLat == "" { err = errInvalidNumberOfArguments return } if vs, sminLon, ok = tokenval(vs); !ok || sminLon == "" { err = errInvalidNumberOfArguments return } if vs, smaxlat, ok = tokenval(vs); !ok || smaxlat == "" { err = errInvalidNumberOfArguments return } if vs, smaxlon, ok = tokenval(vs); !ok || smaxlon == "" { err = errInvalidNumberOfArguments return } var minLat, minLon, maxLat, maxLon float64 if minLat, err = strconv.ParseFloat(sminLat, 64); err != nil { err = errInvalidArgument(sminLat) return } if minLon, err = strconv.ParseFloat(sminLon, 64); err != nil { err = errInvalidArgument(sminLon) return } if maxLat, err = strconv.ParseFloat(smaxlat, 64); err != nil { err = errInvalidArgument(smaxlat) return } if maxLon, err = strconv.ParseFloat(smaxlon, 64); err != nil { err = errInvalidArgument(smaxlon) return } rect = geometry.Rect{ Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, } case "hash": var hash string if vs, hash, ok = tokenval(vs); !ok || hash == "" { err = errInvalidNumberOfArguments return } box := geohash.BoundingBox(hash) rect = geometry.Rect{ Min: geometry.Point{X: box.MinLng, Y: box.MinLat}, Max: geometry.Point{X: box.MaxLng, Y: box.MaxLat}, } case "quadkey": var key string if vs, key, ok = tokenval(vs); !ok || key == "" { err = errInvalidNumberOfArguments return } var minLat, minLon, maxLat, maxLon float64 minLat, minLon, maxLat, maxLon, err = bing.QuadKeyToBounds(key) if err != nil { err = errInvalidArgument(key) return } rect = geometry.Rect{ Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, } case "tile", "mvt": var sx, sy, sz string if vs, sx, ok = tokenval(vs); !ok || sx == "" { err = errInvalidNumberOfArguments return } if vs, sy, ok = tokenval(vs); !ok || sy == "" { err = errInvalidNumberOfArguments return } if vs, sz, ok = tokenval(vs); !ok || sz == "" { err = errInvalidNumberOfArguments return } var x, y, z int if x, err = strconv.Atoi(sx); err != nil || x < 0 { err = errInvalidArgument(sx) return } if y, err = strconv.Atoi(sy); err != nil || y < 0 { err = errInvalidArgument(sy) return } if z, err = strconv.Atoi(sz); err != nil || z < 0 || z > 23 { err = errInvalidArgument(sz) return } var minLat, minLon, maxLat, maxLon float64 minLat, minLon, maxLat, maxLon = bing.TileXYToBounds(int64(x), int64(y), uint64(z)) rect = geometry.Rect{ Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, } tileX = x tileY = y tileZ = z } nvs = vs if ltyp == "mvt" { // Expand rectangle by 0.0625% rect.Min.Y -= (rect.Max.Y - rect.Min.Y) * 0.0625 rect.Max.Y += (rect.Max.Y - rect.Min.Y) * 0.0625 rect.Min.X -= (rect.Max.X - rect.Min.X) * 0.0625 rect.Max.X += (rect.Max.X - rect.Min.X) * 0.0625 if rect.Min.Y < bing.MinLatitude { rect.Min.Y = bing.MinLatitude } if rect.Max.Y > bing.MaxLatitude { rect.Max.Y = bing.MaxLatitude } if rect.Min.X < bing.MinLongitude { rect.Min.X = bing.MinLongitude } if rect.Max.X > bing.MaxLongitude { rect.Max.X = bing.MaxLongitude } } grect = geojson.NewRect(rect) return } func (s *Server) cmdSearchArgs( fromFenceCmd bool, cmd string, vs []string, types map[string]bool, ) (lfs liveFenceSwitches, err error) { var t searchScanBaseTokens if fromFenceCmd { t.fence = true } vs, t, err = s.parseSearchScanBaseTokens(cmd, t, vs) if err != nil { return } lfs.searchScanBaseTokens = t var typ string var ok bool if vs, typ, ok = tokenval(vs); !ok || typ == "" { err = errInvalidNumberOfArguments return } if lfs.searchScanBaseTokens.output == outputBounds { if cmd == "within" || cmd == "intersects" { if _, err := strconv.ParseFloat(typ, 64); err == nil { // It's likely that the output was not specified, but rather the search bounds. lfs.searchScanBaseTokens.output = defaultSearchOutput vs = append([]string{typ}, vs...) typ = "BOUNDS" } } } ltyp := strings.ToLower(typ) found := types[ltyp] if !found && lfs.searchScanBaseTokens.fence && ltyp == "roam" && cmd == "nearby" { // allow roaming for nearby fence searches. found = true } if !found { err = errInvalidArgument(typ) return } switch ltyp { case "point": var slat, slon, smeters string if vs, slat, ok = tokenval(vs); !ok || slat == "" { err = errInvalidNumberOfArguments return } if vs, slon, ok = tokenval(vs); !ok || slon == "" { err = errInvalidNumberOfArguments return } var lat, lon, meters float64 if lat, err = strconv.ParseFloat(slat, 64); err != nil { err = errInvalidArgument(slat) return } if lon, err = strconv.ParseFloat(slon, 64); err != nil { err = errInvalidArgument(slon) return } // radius is optional for nearby, but mandatory for others if cmd == "nearby" { if vs, smeters, ok = tokenval(vs); ok && smeters != "" { meters, err = strconv.ParseFloat(smeters, 64) if err != nil || meters < 0 { err = errInvalidArgument(smeters) return } } else { meters = -1 } // Nearby used the Circle type lfs.obj = geojson.NewCircle(geometry.Point{X: lon, Y: lat}, meters, defaultCircleSteps) } else { // Intersects and Within use the Point type lfs.obj = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) } case "circle": if lfs.clip { err = errInvalidArgument("cannot clip with " + ltyp) return } var slat, slon, smeters string if vs, slat, ok = tokenval(vs); !ok || slat == "" { err = errInvalidNumberOfArguments return } if vs, slon, ok = tokenval(vs); !ok || slon == "" { err = errInvalidNumberOfArguments return } var lat, lon, meters float64 if lat, err = strconv.ParseFloat(slat, 64); err != nil { err = errInvalidArgument(slat) return } if lon, err = strconv.ParseFloat(slon, 64); err != nil { err = errInvalidArgument(slon) return } if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { err = errInvalidNumberOfArguments return } meters, err = strconv.ParseFloat(smeters, 64) if err != nil || meters < 0 { err = errInvalidArgument(smeters) return } lfs.obj = geojson.NewCircle(geometry.Point{X: lon, Y: lat}, meters, defaultCircleSteps) case "object": if lfs.clip { err = errInvalidArgument("cannot clip with object") return } var obj string if vs, obj, ok = tokenval(vs); !ok || obj == "" { err = errInvalidNumberOfArguments return } lfs.obj, err = geojson.Parse(obj, &s.geomParseOpts) if err != nil { return } case "sector": if lfs.clip { err = errInvalidArgument("cannot clip with " + ltyp) return } var slat, slon, smeters, sb1, sb2 string if vs, slat, ok = tokenval(vs); !ok || slat == "" { err = errInvalidNumberOfArguments return } if vs, slon, ok = tokenval(vs); !ok || slon == "" { err = errInvalidNumberOfArguments return } if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { err = errInvalidNumberOfArguments return } if vs, sb1, ok = tokenval(vs); !ok || sb1 == "" { err = errInvalidNumberOfArguments return } if vs, sb2, ok = tokenval(vs); !ok || sb2 == "" { err = errInvalidNumberOfArguments return } var lat, lon, meters, b1, b2 float64 if lat, err = strconv.ParseFloat(slat, 64); err != nil { err = errInvalidArgument(slat) return } if lon, err = strconv.ParseFloat(slon, 64); err != nil { err = errInvalidArgument(slon) return } if meters, err = strconv.ParseFloat(smeters, 64); err != nil { err = errInvalidArgument(smeters) return } if b1, err = strconv.ParseFloat(sb1, 64); err != nil { err = errInvalidArgument(sb1) return } if b2, err = strconv.ParseFloat(sb2, 64); err != nil { err = errInvalidArgument(sb2) return } if b1 == b2 { err = fmt.Errorf("equal bearings (%s == %s), use CIRCLE instead", sb1, sb2) return } origin := sectr.Point{Lng: lon, Lat: lat} sector := sectr.NewSector(origin, meters, b1, b2) lfs.obj, err = geojson.Parse(string(sector.JSON()), &s.geomParseOpts) if err != nil { return } case "bounds", "hash", "tile", "mvt", "quadkey": vs, lfs.obj, lfs.tileX, lfs.tileY, lfs.tileZ, err = parseRectArea(ltyp, vs) if err != nil { return } if ltyp == "mvt" { lfs.mvt = true lfs.clip = true } case "get": if lfs.clip { err = errInvalidArgument("cannot clip with get") } var key, id string if vs, key, ok = tokenval(vs); !ok || key == "" { err = errInvalidNumberOfArguments return } if vs, id, ok = tokenval(vs); !ok || id == "" { err = errInvalidNumberOfArguments return } col, _ := s.cols.Get(key) if col == nil { err = errKeyNotFound return } o := col.Get(id) if o == nil { err = errIDNotFound return } lfs.obj = o.Geo() case "roam": lfs.roam.on = true if vs, lfs.roam.key, ok = tokenval(vs); !ok || lfs.roam.key == "" { err = errInvalidNumberOfArguments return } if vs, lfs.roam.id, ok = tokenval(vs); !ok || lfs.roam.id == "" { err = errInvalidNumberOfArguments return } lfs.roam.pattern = glob.IsGlob(lfs.roam.id) var smeters string if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { err = errInvalidNumberOfArguments return } if lfs.roam.meters, err = strconv.ParseFloat(smeters, 64); err != nil { err = errInvalidArgument(smeters) return } var scan string if vs, scan, ok = tokenval(vs); ok { if strings.ToLower(scan) != "scan" { err = errInvalidArgument(scan) return } if vs, scan, ok = tokenval(vs); !ok || scan == "" { err = errInvalidNumberOfArguments return } lfs.roam.scan = scan } } var clipRect geojson.Object var tok, ltok string for len(vs) > 0 { if vs, tok, ok = tokenval(vs); !ok || tok == "" { err = errInvalidNumberOfArguments return } if strings.ToLower(tok) != "clipby" { err = errInvalidNumberOfArguments return } if vs, tok, ok = tokenval(vs); !ok || tok == "" { err = errInvalidNumberOfArguments return } ltok = strings.ToLower(tok) switch ltok { case "bounds", "hash", "tile", "quadkey": vs, clipRect, lfs.tileX, lfs.tileY, lfs.tileZ, err = parseRectArea(ltok, vs) if err == errNotRectangle { err = errInvalidArgument("cannot clipby " + ltok) return } if err != nil { return } lfs.obj = clip.Clip(lfs.obj, clipRect, &s.geomIndexOpts) default: err = errInvalidArgument("cannot clipby " + ltok) return } } if lfs.hasbuffer { lfs.obj, err = buffer.Simple(lfs.obj, lfs.buffer) if err != nil { return } } return } var nearbyTypes = map[string]bool{ "point": true, } var withinOrIntersectsTypes = map[string]bool{ "geo": true, "bounds": true, "hash": true, "tile": true, "quadkey": true, "get": true, "object": true, "circle": true, "point": true, "sector": true, "mvt": true, } func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] wr := &bytes.Buffer{} sargs, err := s.cmdSearchArgs(false, "nearby", vs, nearbyTypes) if sargs.usingLua() { defer sargs.Close() defer func() { if r := recover(); r != nil { res = NOMessage err = errors.New(r.(string)) return } }() } if err != nil { return NOMessage, err } sargs.cmd = "nearby" if sargs.fence { return NOMessage, sargs } sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields, sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ) if err != nil { return NOMessage, err } if msg.OutputType == JSON { wr.WriteString(`{"ok":true`) } var ierr error if sw.col != nil { iterStep := func(o *object.Object, dist float64) bool { keepGoing, err := sw.pushObject(ScanWriterParams{ obj: o, dist: dist, distOutput: sargs.distance, ignoreGlobMatch: true, skipTesting: true, }) if err != nil { ierr = err return false } return keepGoing } maxDist := sargs.obj.(*geojson.Circle).Meters() if sargs.sparse > 0 { if maxDist < 0 { // error cannot use SPARSE and KNN together return NOMessage, errors.New("cannot use SPARSE without a point distance") } // An intersects operation is required for SPARSE iter := func(o *object.Object) bool { var dist float64 if sargs.distance { dist = o.Geo().Distance(sargs.obj) } return iterStep(o, dist) } sw.col.Intersects(sargs.obj, sargs.sparse, sw, msg.Deadline, iter) } else { iter := func(o *object.Object, dist float64) bool { if maxDist > 0 && dist > maxDist { return false } var meters float64 if sargs.distance { meters = dist } return iterStep(o, meters) } sw.col.Nearby(sargs.obj, sw, msg.Deadline, iter) } } if ierr != nil { return retrerr(ierr) } sw.writeFoot() if msg.OutputType == JSON { wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.BytesValue(wr.Bytes()), nil } return sw.respOut, nil } func (s *Server) cmdWITHIN(msg *Message) (res resp.Value, err error) { return s.cmdWITHINorINTERSECTS("within", msg) } func (s *Server) cmdINTERSECTS(msg *Message) (res resp.Value, err error) { return s.cmdWITHINorINTERSECTS("intersects", msg) } func (s *Server) cmdWITHINorINTERSECTS(cmd string, msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] wr := &bytes.Buffer{} sargs, err := s.cmdSearchArgs(false, cmd, vs, withinOrIntersectsTypes) if sargs.usingLua() { defer sargs.Close() defer func() { if r := recover(); r != nil { res = NOMessage err = errors.New(r.(string)) return } }() } if err != nil { return NOMessage, err } sargs.cmd = cmd if sargs.fence { return NOMessage, sargs } sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields, sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ) if err != nil { return NOMessage, err } if msg.OutputType == JSON { wr.WriteString(`{"ok":true`) } var ierr error if sw.col != nil { switch cmd { case "within": sw.col.Within(sargs.obj, sargs.sparse, sw, msg.Deadline, func(o *object.Object) bool { keepGoing, err := sw.pushObject(ScanWriterParams{obj: o}) if err != nil { ierr = err return false } return keepGoing }, ) case "intersects": sw.col.Intersects(sargs.obj, sargs.sparse, sw, msg.Deadline, func(o *object.Object) bool { params := ScanWriterParams{obj: o} if sargs.clip { params.clip = sargs.obj } keepGoing, err := sw.pushObject(params) if err != nil { ierr = err return false } return keepGoing }, ) } } if ierr != nil { return retrerr(ierr) } sw.writeFoot() if msg.OutputType == JSON { wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.BytesValue(wr.Bytes()), nil } return sw.respOut, nil } func (s *Server) cmdSeachValuesArgs(vs []string) ( lfs liveFenceSwitches, err error, ) { var t searchScanBaseTokens vs, t, err = s.parseSearchScanBaseTokens("search", t, vs) if err != nil { return } lfs.searchScanBaseTokens = t if len(vs) != 0 { err = errInvalidNumberOfArguments return } return } func multiGlobParse(globs []string, desc bool) [2]string { var limits [2]string for i, pattern := range globs { g := glob.Parse(pattern, desc) if g.Limits[0] == "" && g.Limits[1] == "" { limits[0], limits[1] = "", "" break } if i == 0 { limits[0], limits[1] = g.Limits[0], g.Limits[1] } else if desc { if g.Limits[0] > limits[0] { limits[0] = g.Limits[0] } if g.Limits[1] < limits[1] { limits[1] = g.Limits[1] } } else { if g.Limits[0] < limits[0] { limits[0] = g.Limits[0] } if g.Limits[1] > limits[1] { limits[1] = g.Limits[1] } } } return limits } func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] wr := &bytes.Buffer{} sargs, err := s.cmdSeachValuesArgs(vs) if sargs.usingLua() { defer sargs.Close() defer func() { if r := recover(); r != nil { res = NOMessage err = errors.New(r.(string)) return } }() } if err != nil { return NOMessage, err } sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, true, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields, sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ) if err != nil { return NOMessage, err } if msg.OutputType == JSON { wr.WriteString(`{"ok":true`) } var ierr error if sw.col != nil { if sw.output == outputCount && len(sw.wheres) == 0 && sw.globEverything { count := sw.col.Count() - int(sargs.cursor) if count < 0 { count = 0 } sw.count = uint64(count) } else { limits := multiGlobParse(sw.globs, sargs.desc) if limits[0] == "" && limits[1] == "" { sw.col.SearchValues(sargs.desc, sw, msg.Deadline, func(o *object.Object) bool { keepGoing, err := sw.pushObject(ScanWriterParams{ obj: o, }) if err != nil { ierr = err return false } return keepGoing }, ) } else { // must disable globSingle for string value type matching because // globSingle is only for ID matches, not values. sw.col.SearchValuesRange(limits[0], limits[1], sargs.desc, sw, msg.Deadline, func(o *object.Object) bool { keepGoing, err := sw.pushObject(ScanWriterParams{ obj: o, }) if err != nil { ierr = err return false } return keepGoing }, ) } } } if ierr != nil { return retrerr(ierr) } sw.writeFoot() if msg.OutputType == JSON { wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.BytesValue(wr.Bytes()), nil } return sw.respOut, nil } ================================================ FILE: internal/server/server.go ================================================ package server import ( "bytes" "crypto/rand" "crypto/sha1" "encoding/base64" "encoding/binary" "errors" "fmt" "io" "net" "net/http" "net/url" "os" "path" "path/filepath" "runtime" "runtime/debug" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/prometheus/client_golang/prometheus" "github.com/tidwall/btree" "github.com/tidwall/buntdb" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/gjson" "github.com/tidwall/redcon" "github.com/tidwall/resp" "github.com/tidwall/rtree" "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/collection" "github.com/tidwall/tile38/internal/deadline" "github.com/tidwall/tile38/internal/endpoint" "github.com/tidwall/tile38/internal/log" "github.com/tidwall/tile38/internal/object" "github.com/tidwall/tile38/internal/viewer" ) var errOOM = errors.New("OOM command not allowed when used memory > 'maxmemory'") func errTimeoutOnCmd(cmd string) error { return fmt.Errorf("timeout not supported for '%s'", cmd) } const ( goingLive = "going live" hookLogPrefix = "hook:log:" ) // commandDetails is detailed information about a mutable command. It's used // for geofence formulas. type commandDetails struct { command string // client command, like "SET" or "DEL" key string // collection key newKey string // new key, for RENAME command obj *object.Object // target object old *object.Object // previous object, if any updated bool // object was updated timestamp time.Time // timestamp when the update occurred parent bool // when true, only children are forwarded pattern string // PDEL key pattern children []*commandDetails // for multi actions such as "PDEL" } type rwlocker interface { LockLowPriority() Lock() Unlock() RLock() RUnlock() } // rwspinlock is the same as a RWLock, but uses spinlocks instead of mutexes. type rwspinlock struct { state atomic.Int32 } func (l *rwspinlock) Lock() { for { state := l.state.Load() if state == 0 && l.state.CompareAndSwap(state, -1) { return } runtime.Gosched() } } func (l *rwspinlock) LockLowPriority() { // All write locks are low priority and unfair. First come first serve. l.Lock() } func (l *rwspinlock) Unlock() { if l.state.Add(1) > 0 { panic("Unlock of unlocked rwspinlock") } } func (l *rwspinlock) RLock() { for { state := l.state.Load() if state >= 0 && l.state.CompareAndSwap(state, state+1) { return } runtime.Gosched() } } func (l *rwspinlock) RUnlock() { if l.state.Add(-1) < 0 { panic("RUnlock of unlocked rwspinlock") } } // rwmutex is the same as a RWLock but includes a LockLowPriority for // performing write locks that do not block the queue for readers. type rwmutex struct { mu sync.RWMutex } func (l *rwmutex) Lock() { start := time.Now() for { for range 100 { if l.mu.TryLock() { return } runtime.Gosched() } if time.Since(start) > time.Millisecond*50 { break } } l.mu.Lock() } func (l *rwmutex) LockLowPriority() { for !l.mu.TryLock() { runtime.Gosched() } } func (l *rwmutex) Unlock() { l.mu.Unlock() } func (l *rwmutex) RLock() { l.mu.RLock() } func (l *rwmutex) RUnlock() { l.mu.RUnlock() } // Server is a tile38 controller type Server struct { // user defined options opts Options // static values unix string host string port int http bool dir string started time.Time config *Config epc *endpoint.Manager epool *exprPool lnmu sync.Mutex ln net.Listener // server listener // env opts geomParseOpts geojson.ParseOptions geomIndexOpts geometry.IndexOptions http500Errors bool // atomics followc atomic.Int64 // counter when follow property changes statsTotalConns atomic.Int64 // counter for total connections statsTotalCommands atomic.Int64 // counter for total commands statsTotalMsgsSent atomic.Int64 // counter for total sent webhook messages statsExpired atomic.Int64 // item expiration counter lastShrinkDuration atomic.Int64 stopServer atomic.Bool outOfMemory atomic.Bool loadedAndReady atomic.Bool // server is loaded and ready for commands connsmu sync.RWMutex conns map[int]*Client mu rwlocker // sync.RWMutex // aof aof *os.File // active aof file aofdirty atomic.Bool // mark the aofbuf as having data aofbuf []byte // prewrite buffer aofsz int // active size of the aof file shrinking bool // aof shrinking flag shrinklog [][]string // aof shrinking log // database qdb *buntdb.DB // hook queue log qidx uint64 // hook queue log last idx cols *btree.Map[string, *collection.Collection] // data collections hooks *btree.BTree // hook name -- [string]*Hook hookCross *rtree.RTree // hook spatial tree for "cross" geofences hookTree *rtree.RTree // hook spatial tree for all hooksOut *btree.BTree // hooks with "outside" detection -- [string]*Hook groupHooks *btree.BTree // hooks that are connected to objects groupObjects *btree.BTree // objects that are connected to hooks hookExpires *btree.BTree // queue of all hooks marked for expiration // followers (external aof readers) follows map[*bytes.Buffer]bool fcond *sync.Cond lstack []*commandDetails lives map[*liveBuffer]bool lcond *sync.Cond // live geofence signal faofsz int // last reported aofsize fcupflags atomic.Int32 // follow caught up (caughtUp and caughtUpOnce) aofconnM map[net.Conn]io.Closer pubq pubQueue // lua scripts luascripts *lScriptMap luapool *lStatePool // pubsub system (SUBSCRIBE, PUBLISH, and SETCHAN) pubsub *pubsub // monitor connections (using the MONITOR command) monconnsMu sync.RWMutex monconns map[net.Conn]bool } // Options for Serve() type Options struct { Host string Port int Dir string UseHTTP bool MetricsAddr string UnixSocketPath string // path for unix socket ClientOutput string // "" or "resp" or "json" // DevMode puts application in to dev mode DevMode bool // ShowDebugMessages allows for log.Debug to print to console. ShowDebugMessages bool // ProtectedMode forces Tile38 to default in protected mode. ProtectedMode string // AppendOnly allows for disabling the appendonly file. AppendOnly bool // AppendFileName allows for custom appendonly file path AppendFileName string // QueueFileName allows for custom queue.db file path QueueFileName string // Shutdown allows for shutting down the server. Shutdown <-chan bool // Spinlock uses a spinlock instead of a mutex Spinlock bool } // Serve starts a new tile38 server func Serve(opts Options) error { if opts.AppendFileName == "" { opts.AppendFileName = path.Join(opts.Dir, "appendonly.aof") } if opts.QueueFileName == "" { opts.QueueFileName = path.Join(opts.Dir, "queue.db") } if opts.ProtectedMode == "" { opts.ProtectedMode = "no" } log.Infof("Server started, Tile38 version %s, git %s", core.Version, core.GitSHA) defer func() { log.Warn("Server has shutdown, bye now") if false { // prints the stack, looking for running goroutines. buf := make([]byte, 10000) n := runtime.Stack(buf, true) println(string(buf[:n])) } }() var lock rwlocker if opts.Spinlock { lock = new(rwspinlock) } else { lock = new(rwmutex) } // Initialize the s s := &Server{ mu: lock, unix: opts.UnixSocketPath, host: opts.Host, port: opts.Port, dir: opts.Dir, follows: make(map[*bytes.Buffer]bool), fcond: sync.NewCond(&sync.Mutex{}), lives: make(map[*liveBuffer]bool), lcond: sync.NewCond(&sync.Mutex{}), hooks: btree.NewNonConcurrent(byHookName), hooksOut: btree.NewNonConcurrent(byHookName), hookCross: &rtree.RTree{}, hookTree: &rtree.RTree{}, aofconnM: make(map[net.Conn]io.Closer), started: time.Now(), conns: make(map[int]*Client), http: opts.UseHTTP, pubsub: newPubsub(), pubq: pubQueue{cond: sync.NewCond(&sync.Mutex{})}, monconns: make(map[net.Conn]bool), cols: &btree.Map[string, *collection.Collection]{}, groupHooks: btree.NewNonConcurrent(byGroupHook), groupObjects: btree.NewNonConcurrent(byGroupObject), hookExpires: btree.NewNonConcurrent(byHookExpires), opts: opts, } s.epool = newExprPool(s) s.epc = endpoint.NewManager(s) defer s.epc.Shutdown() s.luascripts = s.newScriptMap() s.luapool = s.newPool() defer s.luapool.Shutdown() if err := os.MkdirAll(opts.Dir, 0700); err != nil { return err } var err error s.config, err = loadConfig(filepath.Join(opts.Dir, "config")) if err != nil { return err } // Send "500 Internal Server" error instead of "200 OK" for json responses // with `"ok":false`. T38HTTP500ERRORS=1 s.http500Errors, _ = strconv.ParseBool(os.Getenv("T38HTTP500ERRORS")) // Allow for geometry indexing options through environment variables: // T38IDXGEOMKIND -- None, RTree, QuadTree // T38IDXGEOM -- Min number of points in a geometry for indexing. // T38IDXMULTI -- Min number of object in a Multi/Collection for indexing. s.geomParseOpts = *geojson.DefaultParseOptions s.geomIndexOpts = *geometry.DefaultIndexOptions n, err := strconv.ParseUint(os.Getenv("T38IDXGEOM"), 10, 32) if err == nil { s.geomParseOpts.IndexGeometry = int(n) s.geomIndexOpts.MinPoints = int(n) } n, err = strconv.ParseUint(os.Getenv("T38IDXMULTI"), 10, 32) if err == nil { s.geomParseOpts.IndexChildren = int(n) } requireValid := os.Getenv("REQUIREVALID") if requireValid != "" { s.geomParseOpts.RequireValid = true } indexKind := os.Getenv("T38IDXGEOMKIND") switch indexKind { default: log.Errorf("Unknown index kind: %s", indexKind) case "": case "None": s.geomParseOpts.IndexGeometryKind = geometry.None s.geomIndexOpts.Kind = geometry.None case "RTree": s.geomParseOpts.IndexGeometryKind = geometry.RTree s.geomIndexOpts.Kind = geometry.RTree case "QuadTree": s.geomParseOpts.IndexGeometryKind = geometry.QuadTree s.geomIndexOpts.Kind = geometry.QuadTree } if s.geomParseOpts.IndexGeometryKind == geometry.None { log.Debugf("Geom indexing: %s", s.geomParseOpts.IndexGeometryKind, ) } else { log.Debugf("Geom indexing: %s (%d points)", s.geomParseOpts.IndexGeometryKind, s.geomParseOpts.IndexGeometry, ) } log.Debugf("Multi indexing: RTree (%d points)", s.geomParseOpts.IndexChildren) nerr := make(chan error) go func() { // Start the server in the background nerr <- s.netServe() }() var fstop atomic.Bool go func() { for !fstop.Load() { s.fcond.Broadcast() time.Sleep(time.Second / 4) } }() go func() { <-opts.Shutdown s.stopServer.Store(true) log.Warnf("Shutting down...") fstop.Store(true) s.lnmu.Lock() ln := s.ln s.ln = nil s.lnmu.Unlock() if ln != nil { ln.Close() } for conn, f := range s.aofconnM { conn.Close() f.Close() } }() // Load the queue before the aof qdb, err := buntdb.Open(opts.QueueFileName) if err != nil { return err } var qidx uint64 if err := qdb.View(func(tx *buntdb.Tx) error { val, err := tx.Get("hook:idx") if err != nil { if err == buntdb.ErrNotFound { return nil } return err } qidx = stringToUint64(val) return nil }); err != nil { return err } err = qdb.CreateIndex("hooks", hookLogPrefix+"*", buntdb.IndexJSONCaseSensitive("hook")) if err != nil { return err } s.qdb = qdb s.qidx = qidx if err := s.migrateAOF(); err != nil { return err } if opts.AppendOnly { f, err := os.OpenFile(opts.AppendFileName, os.O_CREATE|os.O_RDWR, 0600) if err != nil { return err } s.aof = f if err := s.loadAOF(); err != nil { return err } defer func() { s.flushAOF(false) s.aof.Sync() }() } // Start background routines var bgwg sync.WaitGroup if s.config.followHost() != "" { bgwg.Add(1) go func() { defer bgwg.Done() s.follow(s.config.followHost(), s.config.followPort(), int(s.followc.Load())) }() } var mln net.Listener if opts.MetricsAddr != "" { log.Infof("Listening for metrics at: %s", opts.MetricsAddr) mln, err = net.Listen("tcp", opts.MetricsAddr) if err != nil { return err } bgwg.Add(1) go func() { defer bgwg.Done() smux := http.NewServeMux() smux.HandleFunc("/", s.MetricsIndexHandler) smux.HandleFunc("/metrics", s.MetricsHandler) err := http.Serve(mln, smux) if err != nil { if !s.stopServer.Load() { log.Fatalf("metrics server: %s", err) } } }() } bgwg.Add(1) go s.processLives(&bgwg) bgwg.Add(1) go s.watchOutOfMemory(&bgwg) bgwg.Add(1) go s.watchLuaStatePool(&bgwg) bgwg.Add(1) go s.watchAutoGC(&bgwg) bgwg.Add(1) go s.backgroundExpiring(&bgwg) bgwg.Add(1) go s.backgroundSyncAOF(&bgwg) bgwg.Add(1) go s.startPublishQueue(&bgwg) defer func() { log.Debug("Stopping background routines") // Stop background routines s.stopPublishQueue() s.followc.Add(1) // this will force any follow communication to die s.stopServer.Store(true) if mln != nil { mln.Close() // Stop the metrics server } bgwg.Wait() }() // Server is now loaded and ready. Wait for network error messages. s.loadedAndReady.Store(true) return <-nerr } func (s *Server) isProtected() bool { if s.opts.ProtectedMode == "no" { // --protected-mode no return false } if s.host != "" && s.host != "127.0.0.1" && s.host != "::1" && s.host != "localhost" { // -h address return false } is := s.config.protectedMode() != "no" && s.config.requirePass() == "" return is } func (s *Server) netServe() error { var ln net.Listener var err error if s.unix != "" { os.RemoveAll(s.unix) ln, err = net.Listen("unix", s.unix) } else { tcpAddr := fmt.Sprintf("%s:%d", s.host, s.port) ln, err = net.Listen("tcp", tcpAddr) } if err != nil { return err } s.lnmu.Lock() s.ln = ln s.lnmu.Unlock() var wg sync.WaitGroup defer func() { log.Debug("Closing client connections...") s.connsmu.RLock() for _, c := range s.conns { if c.closer != nil { c.closer.Close() } } s.connsmu.RUnlock() wg.Wait() ln.Close() log.Debug("Client connection closed") }() var defaultOutputType Type switch s.opts.ClientOutput { case "resp": defaultOutputType = RESP case "json": defaultOutputType = JSON } log.Infof("Ready to accept connections at %s", ln.Addr()) var clientID int64 for { conn, err := ln.Accept() if err != nil { if s.stopServer.Load() { return nil } log.Warn(err) time.Sleep(time.Second / 5) continue } wg.Add(1) go func(conn net.Conn) { detached := false defer func() { if !detached { wg.Done() } }() // open connection // create the client client := new(Client) client.id = int(atomic.AddInt64(&clientID, 1)) client.opened = time.Now() client.remoteAddr = conn.RemoteAddr().String() client.closer = conn // add client to server map s.connsmu.Lock() s.conns[client.id] = client s.connsmu.Unlock() s.statsTotalConns.Add(1) // set the client keep-alive, if needed if s.config.keepAlive() > 0 { if conn, ok := conn.(*net.TCPConn); ok { conn.SetKeepAlive(true) conn.SetKeepAlivePeriod( time.Duration(s.config.keepAlive()) * time.Second, ) } } log.Debugf("Opened connection: %s", client.remoteAddr) defer func() { // close connection // delete from server map s.connsmu.Lock() delete(s.conns, client.id) s.connsmu.Unlock() log.Debugf("Closed connection: %s", client.remoteAddr) conn.Close() }() var lastConnType Type var lastOutputType Type // check if the connection is protected if !strings.HasPrefix(client.remoteAddr, "127.0.0.1:") && !strings.HasPrefix(client.remoteAddr, "[::1]:") { if s.isProtected() { // This is a protected server. Only loopback is allowed. conn.Write(deniedMessage) return // close connection } } packet := make([]byte, 0xFFFF) for { var close bool n, err := conn.Read(packet) if err != nil { return } in := packet[:n] // read the payload packet from the client input stream. packet := client.in.Begin(in) // load the pipeline reader pr := &client.pr rdbuf := bytes.NewBuffer(packet) pr.rd = rdbuf pr.wr = client msgs, err := pr.ReadMessages() for _, msg := range msgs { // Just closing connection if we have deprecated HTTP or WS connection, // And --http-transport = false if !s.http && (msg.ConnType == WebSocket || msg.ConnType == HTTP) { close = true // close connection break } if msg != nil && msg.Command() != "" { if client.outputType != Null { msg.OutputType = client.outputType } else if defaultOutputType != Null { msg.OutputType = defaultOutputType } msg.StrictRESP = client.strictRESP if msg.Command() == "quit" { if msg.OutputType == RESP { io.WriteString(client, "+OK\r\n") } close = true // close connection break } // increment last used client.mu.Lock() client.last = time.Now() client.mu.Unlock() // update total command count s.statsTotalCommands.Add(1) // handle the command err := s.handleInputCommand(client, msg) if err != nil { if err.Error() == goingLive { client.goLiveErr = err client.goLiveMsg = msg // detach var rwc io.ReadWriteCloser = conn client.conn = rwc if len(client.out) > 0 { client.conn.Write(client.out) client.out = nil } client.in = InputStream{} client.pr.rd = rwc client.pr.wr = rwc client.closer = nil wg.Done() detached = true log.Debugf("Detached connection: %s", client.remoteAddr) var wg2 sync.WaitGroup wg2.Add(1) go func() { defer wg2.Done() err := s.goLive( client.goLiveErr, &liveConn{conn.RemoteAddr(), rwc}, &client.pr, client.goLiveMsg, client.goLiveMsg.ConnType == WebSocket, ) if err != nil { log.Error(err) } }() wg2.Wait() return // close connection } log.Error(err) return // close connection, NOW } client.outputType = msg.OutputType client.strictRESP = msg.StrictRESP } else { client.Write([]byte("HTTP/1.1 500 Bad Request\r\nConnection: close\r\n\r\n")) break } if msg.ConnType == HTTP || msg.ConnType == WebSocket { close = true // close connection break } lastOutputType = msg.OutputType lastConnType = msg.ConnType } packet = packet[len(packet)-rdbuf.Len():] client.in.End(packet) // write to client if len(client.out) > 0 { if s.aofdirty.Load() { func() { // prewrite s.mu.Lock() defer s.mu.Unlock() s.flushAOF(false) }() s.aofdirty.Store(false) } conn.Write(client.out) client.out = nil } if close { break } if err != nil { log.Error(err) if lastConnType == RESP { var value resp.Value switch lastOutputType { case JSON: value = resp.StringValue(`{"ok":false,"err":` + jsonString(err.Error()) + "}") case RESP: value = resp.ErrorValue(err) } bytes, _ := value.MarshalRESP() conn.Write(bytes) } break // close connection } } }(conn) } } type liveConn struct { remoteAddr net.Addr rwc io.ReadWriteCloser } func (conn *liveConn) Close() error { return conn.rwc.Close() } func (conn *liveConn) LocalAddr() net.Addr { panic("not supported") } func (conn *liveConn) RemoteAddr() net.Addr { return conn.remoteAddr } func (conn *liveConn) Read(b []byte) (n int, err error) { return conn.rwc.Read(b) } func (conn *liveConn) Write(b []byte) (n int, err error) { return conn.rwc.Write(b) } func (conn *liveConn) SetDeadline(deadline time.Time) error { panic("not supported") } func (conn *liveConn) SetReadDeadline(deadline time.Time) error { panic("not supported") } func (conn *liveConn) SetWriteDeadline(deadline time.Time) error { panic("not supported") } func (s *Server) watchAutoGC(wg *sync.WaitGroup) { defer wg.Done() start := time.Now() s.loopUntilServerStops(time.Second, func() { autoGC := s.config.autoGC() if autoGC == 0 { return } if time.Since(start) < time.Second*time.Duration(autoGC) { return } var mem1, mem2 runtime.MemStats runtime.ReadMemStats(&mem1) log.Debugf("autogc(before): "+ "alloc: %v, heap_alloc: %v, heap_released: %v", mem1.Alloc, mem1.HeapAlloc, mem1.HeapReleased) runtime.GC() debug.FreeOSMemory() runtime.ReadMemStats(&mem2) log.Debugf("autogc(after): "+ "alloc: %v, heap_alloc: %v, heap_released: %v", mem2.Alloc, mem2.HeapAlloc, mem2.HeapReleased) start = time.Now() }) } func (s *Server) checkOutOfMemory() { if s.stopServer.Load() { return } oom := s.outOfMemory.Load() var mem runtime.MemStats if s.config.maxMemory() == 0 { if oom { s.outOfMemory.Store(false) } return } if oom { runtime.GC() } runtime.ReadMemStats(&mem) s.outOfMemory.Store(int(mem.HeapAlloc) > s.config.maxMemory()) } func (s *Server) loopUntilServerStops(dur time.Duration, op func()) { var last time.Time for { if s.stopServer.Load() { return } now := time.Now() if now.Sub(last) > dur { op() last = now } time.Sleep(time.Second / 5) } } func (s *Server) watchOutOfMemory(wg *sync.WaitGroup) { defer wg.Done() s.loopUntilServerStops(time.Second*4, func() { s.checkOutOfMemory() }) } func (s *Server) watchLuaStatePool(wg *sync.WaitGroup) { defer wg.Done() s.loopUntilServerStops(time.Second*10, func() { s.luapool.Prune() }) } // backgroundSyncAOF ensures that the aof buffer is does not grow too big. func (s *Server) backgroundSyncAOF(wg *sync.WaitGroup) { defer wg.Done() s.loopUntilServerStops(time.Second, func() { s.mu.LockLowPriority() defer s.mu.Unlock() s.flushAOF(true) }) } func isReservedFieldName(field string) bool { switch field { case "z", "lat", "lon": return true } return false } func rewriteTimeoutMsg(msg *Message) (err error) { vs := msg.Args[1:] var valStr string var ok bool if vs, valStr, ok = tokenval(vs); !ok || valStr == "" || len(vs) == 0 { err = errInvalidNumberOfArguments return } timeoutSec, _err := strconv.ParseFloat(valStr, 64) if _err != nil || timeoutSec < 0 { err = errInvalidArgument(valStr) return } msg.Args = vs[:] msg._command = "" msg.Deadline = deadline.New( time.Now().Add(time.Duration(timeoutSec * float64(time.Second)))) return } func (s *Server) handleInputCommand(client *Client, msg *Message) error { start := time.Now() var mvt bool if msg.ConnType == HTTP && len(msg.Args) == 1 { var query string if i := strings.IndexByte(msg.Args[0], '?'); i != -1 { query = msg.Args[0][i+1:] msg.Args[0] = msg.Args[0][:i] } if strings.HasSuffix(msg.Args[0], ".mvt") || strings.HasSuffix(msg.Args[0], ".pbf") { mvt = mvtFilterHTTPArgs(msg, query) } else if strings.HasPrefix(msg.Args[0], "viewer/") || msg.Args[0] == "viewer" { return viewer.HandleHTTP(client, "/"+strings.Join(msg.Args, "/"), s.opts.DevMode) } } serializeOutput := func(res resp.Value) (string, error) { var resStr string var err error switch msg.OutputType { case JSON: resStr = res.String() case RESP: var resBytes []byte resBytes, err = res.MarshalRESP() resStr = string(resBytes) } return resStr, err } writeOutput := func(res string) error { switch msg.ConnType { default: err := fmt.Errorf("unsupported conn type: %v", msg.ConnType) log.Error(err) return err case WebSocket: return WriteWebSocketMessage(client, []byte(res)) case HTTP: extraNL := 2 contentType := "application/json; charset=utf-8" status := "200 OK" if (s.http500Errors || msg._command == "healthz") && !gjson.Get(res, "ok").Bool() { status = "500 Internal Server Error" } else if mvt { v := gjson.Get(res, "mvt") if !v.Exists() { status = "500 Internal Server Error" } else { res = v.String() out, err := base64.RawStdEncoding.DecodeString(res) if err != nil { status = "500 Internal Server Error" } else { res = string(out) contentType = "application/vnd.mapbox-vector-tile" } } } _, err := fmt.Fprintf(client, ""+ "HTTP/1.1 %s\r\n"+ "Connection: close\r\n"+ "Content-Type: %s\r\n"+ "Content-Length: %d\r\n"+ "Access-Control-Allow-Origin: *\r\n"+ "\r\n", status, contentType, len(res)+extraNL) if err != nil { return err } _, err = io.WriteString(client, res) if err != nil { return err } if extraNL == 2 { _, err = io.WriteString(client, "\r\n") if err != nil { return err } } return nil case RESP: var err error if msg.OutputType == JSON { _, err = fmt.Fprintf(client, "$%d\r\n%s\r\n", len(res), res) } else { _, err = io.WriteString(client, res) } return err case Native: _, err := fmt.Fprintf(client, "$%d %s\r\n", len(res), res) return err } } cmd := msg.Command() defer func() { took := time.Since(start).Seconds() cmdDurations.With(prometheus.Labels{"cmd": cmd}).Observe(took) }() // Ping. Just send back the response. No need to put through the pipeline. if cmd == "ping" || cmd == "echo" { switch msg.OutputType { case JSON: if len(msg.Args) > 1 { return writeOutput(`{"ok":true,"` + cmd + `":` + jsonString(msg.Args[1]) + `,"elapsed":"` + time.Since(start).String() + `"}`) } return writeOutput(`{"ok":true,"` + cmd + `":"pong","elapsed":"` + time.Since(start).String() + `"}`) case RESP: if len(msg.Args) > 1 { data := redcon.AppendBulkString(nil, msg.Args[1]) return writeOutput(string(data)) } return writeOutput("+PONG\r\n") } s.sendMonitor(nil, msg, client, false) return nil } writeErr := func(errMsg string) error { switch msg.OutputType { case JSON: return writeOutput(`{"ok":false,"err":` + jsonString(errMsg) + `,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: if errMsg == errInvalidNumberOfArguments.Error() { return writeOutput("-ERR wrong number of arguments for '" + cmd + "' command\r\n") } var ucprefix bool word := strings.Split(errMsg, " ")[0] if len(word) > 0 { ucprefix = true for i := 0; i < len(word); i++ { if word[i] < 'A' || word[i] > 'Z' { ucprefix = false break } } } if !ucprefix { errMsg = "ERR " + errMsg } v, _ := resp.ErrorValue(errors.New(errMsg)).MarshalRESP() return writeOutput(string(v)) } return nil } if !s.loadedAndReady.Load() { switch msg.Command() { case "output", "ping", "echo", "auth": default: return writeErr("LOADING Tile38 is loading the dataset in memory") } } if cmd == "hello" { // Not Supporting RESP3+, returns an ERR instead. ot := msg.OutputType if len(msg.Args) > 1 && (msg.Args[1] >= "0" && msg.Args[1] <= "9") && s.opts.ClientOutput == "json" && ot == JSON && msg.ConnType == RESP { // Here we are making sure that we ignoring the '-o json' flag, if // used, otherwise the connection will fail for some redis clients // like "github.com/redis/go-redis/v9" are overly strict and expect // a map type or an error as the result of the HELLO command. msg.OutputType = RESP msg.StrictRESP = true } err := writeErr("unknown command '" + msg.Args[0] + "'") msg.OutputType = ot return err } if cmd == "command" && len(msg.Args) > 1 && msg.Args[1] == "DOCS" && s.opts.ClientOutput == "json" { // The standard Redis client typically sends a "COMMAND DOCS" to start // client connection. msg.StrictRESP = true } if cmd == "timeout" { if err := rewriteTimeoutMsg(msg); err != nil { return writeErr(err.Error()) } } var write bool if (!client.authd || cmd == "auth") && cmd != "output" && cmd != "healthz" { if s.config.requirePass() != "" { password := "" // This better be an AUTH command or the Message should contain an Auth if cmd != "auth" && msg.Auth == "" { // Just shut down the pipeline now. The less the client connection knows the better. return writeErr("authentication required") } if msg.Auth != "" { password = msg.Auth } else { if len(msg.Args) > 1 { password = msg.Args[1] } } if s.config.requirePass() != strings.TrimSpace(password) { return writeErr("invalid password") } client.authd = true if msg.ConnType != HTTP { resStr, _ := serializeOutput(OKMessage(msg, start)) return writeOutput(resStr) } } else if msg.Command() == "auth" { return writeErr("invalid password") } } // choose the locking strategy switch msg.Command() { default: s.mu.RLock() defer s.mu.RUnlock() case "set", "del", "drop", "fset", "flushdb", "setchan", "pdelchan", "delchan", "sethook", "pdelhook", "delhook", "expire", "persist", "jset", "pdel", "rename", "renamenx": // write operations write = true s.mu.Lock() defer s.mu.Unlock() if s.config.followHost() != "" { return writeErr("not the leader") } if s.config.readOnly() { return writeErr("read only") } case "eval", "evalsha": // write operations (potentially) but no AOF for the script command itself s.mu.Lock() defer s.mu.Unlock() if s.config.followHost() != "" { return writeErr("not the leader") } if s.config.readOnly() { return writeErr("read only") } case "get", "keys", "scan", "nearby", "within", "intersects", "hooks", "chans", "search", "ttl", "bounds", "server", "info", "type", "jget", "evalro", "evalrosha", "role", "fget", "exists", "fexists": // read operations s.mu.RLock() defer s.mu.RUnlock() // fallthrough to perform a "catching up to leader" check fallthrough case "healthz": // healthz operation does not require a read lock. It only needs // a caughtup once check to leader, which is atomic. if s.config.followHost() != "" && !s.caughtUpOnce() { return writeErr("catching up to leader") } case "follow", "slaveof", "replconf", "readonly", "config": // system operations // does not write to aof, but requires a write lock. s.mu.Lock() defer s.mu.Unlock() case "output": // this is local connection operation. Locks not needed. case "echo": case "massinsert": // dev operation case "sleep": // dev operation s.mu.RLock() defer s.mu.RUnlock() case "shutdown": // dev operation s.mu.Lock() defer s.mu.Unlock() case "aofshrink": s.mu.RLock() defer s.mu.RUnlock() case "client": s.mu.Lock() defer s.mu.Unlock() case "evalna", "evalnasha": // No locking for scripts, otherwise writes cannot happen within scripts case "subscribe", "psubscribe", "publish": // No locking for pubsub case "monitor": // No locking for monitor } res, d, err := func() (res resp.Value, d commandDetails, err error) { if msg.Deadline != nil { if write { res = NOMessage err = errTimeoutOnCmd(msg.Command()) return } defer func() { if msg.Deadline.Hit() { v := recover() if v != nil { if s, ok := v.(string); !ok || s != "deadline" { panic(v) } } res = NOMessage err = errTimeout } }() } res, d, err = s.command(msg, client) if msg.Deadline != nil { msg.Deadline.Check() } return res, d, err }() if res.Type() == resp.Error { return writeErr(res.String()) } if err != nil { if err.Error() == goingLive { return err } return writeErr(err.Error()) } if write { if err := s.writeAOF(msg.Args, &d); err != nil { if _, ok := err.(errAOFHook); ok { return writeErr(err.Error()) } log.Fatal(err) return err } } var resStr string resStr, err = serializeOutput(res) if err != nil { return err } if err := writeOutput(resStr); err != nil { return err } return nil } func randomKey(n int) string { b := make([]byte, n) nn, err := rand.Read(b) if err != nil { panic(err) } if nn != n { panic("random failed") } return fmt.Sprintf("%x", b) } func (s *Server) reset() { s.aofsz = 0 s.cols.Clear() } func (s *Server) command(msg *Message, client *Client) ( res resp.Value, d commandDetails, err error, ) { switch msg.Command() { default: err = fmt.Errorf("unknown command '%s'", msg.Args[0]) case "set": res, d, err = s.cmdSET(msg) case "fset": res, d, err = s.cmdFSET(msg) case "del": res, d, err = s.cmdDEL(msg) case "pdel": res, d, err = s.cmdPDEL(msg) case "drop": res, d, err = s.cmdDROP(msg) case "flushdb": res, d, err = s.cmdFLUSHDB(msg) case "rename": res, d, err = s.cmdRENAME(msg) case "renamenx": res, d, err = s.cmdRENAME(msg) case "sethook": res, d, err = s.cmdSetHook(msg) case "delhook": res, d, err = s.cmdDelHook(msg) case "pdelhook": res, d, err = s.cmdPDelHook(msg) case "hooks": res, err = s.cmdHooks(msg) case "setchan": res, d, err = s.cmdSetHook(msg) case "delchan": res, d, err = s.cmdDelHook(msg) case "pdelchan": res, d, err = s.cmdPDelHook(msg) case "chans": res, err = s.cmdHooks(msg) case "expire": res, d, err = s.cmdEXPIRE(msg) case "persist": res, d, err = s.cmdPERSIST(msg) case "ttl": res, err = s.cmdTTL(msg) case "shutdown": if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } log.Fatal("shutdown requested by developer") case "massinsert": if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } res, err = s.cmdMassInsert(msg) case "sleep": if !s.opts.DevMode { err = fmt.Errorf("unknown command '%s'", msg.Args[0]) return } res, err = s.cmdSleep(msg) case "follow", "slaveof": res, err = s.cmdFollow(msg) case "replconf": res, err = s.cmdReplConf(msg, client) case "readonly": res, err = s.cmdREADONLY(msg) case "stats": res, err = s.cmdSTATS(msg) case "server": res, err = s.cmdSERVER(msg) case "healthz": res, err = s.cmdHEALTHZ(msg) case "info": res, err = s.cmdINFO(msg) case "role": res, err = s.cmdROLE(msg) case "scan": res, err = s.cmdScan(msg) case "nearby": res, err = s.cmdNearby(msg) case "within": res, err = s.cmdWITHIN(msg) case "intersects": res, err = s.cmdINTERSECTS(msg) case "search": res, err = s.cmdSearch(msg) case "bounds": res, err = s.cmdBOUNDS(msg) case "get": res, err = s.cmdGET(msg) case "fget": res, err = s.cmdFGET(msg) case "jget": res, err = s.cmdJget(msg) case "jset": res, d, err = s.cmdJset(msg) case "jdel": res, d, err = s.cmdJdel(msg) case "type": res, err = s.cmdTYPE(msg) case "keys": res, err = s.cmdKEYS(msg) case "exists": res, err = s.cmdEXISTS(msg) case "fexists": res, err = s.cmdFEXISTS(msg) case "output": res, err = s.cmdOUTPUT(msg) case "aof": res, err = s.cmdAOF(msg) case "aofmd5": res, err = s.cmdAOFMD5(msg) case "gc": runtime.GC() debug.FreeOSMemory() res = OKMessage(msg, time.Now()) case "aofshrink": go s.aofshrink() res = OKMessage(msg, time.Now()) case "config get": res, err = s.cmdConfigGet(msg) case "config set": res, err = s.cmdConfigSet(msg) case "config rewrite": res, err = s.cmdConfigRewrite(msg) case "config", "script": // These get rewritten into "config foo" and "script bar" err = fmt.Errorf("unknown command '%s'", msg.Args[0]) if len(msg.Args) > 1 { msg.Args[1] = msg.Args[0] + " " + msg.Args[1] msg.Args = msg.Args[1:] msg._command = "" return s.command(msg, client) } case "client": res, err = s.cmdCLIENT(msg, client) case "eval", "evalro", "evalna": res, err = s.cmdEvalUnified(false, msg) case "evalsha", "evalrosha", "evalnasha": res, err = s.cmdEvalUnified(true, msg) case "script load": res, err = s.cmdScriptLoad(msg) case "script exists": res, err = s.cmdScriptExists(msg) case "script flush": res, err = s.cmdScriptFlush(msg) case "subscribe": res, err = s.cmdSubscribe(msg) case "psubscribe": res, err = s.cmdPsubscribe(msg) case "publish": res, err = s.cmdPublish(msg) case "test": res, err = s.cmdTEST(msg) case "monitor": res, err = s.cmdMonitor(msg) } s.sendMonitor(err, msg, client, false) return } // This phrase is copied nearly verbatim from Redis. var deniedMessage = []byte(strings.Replace(strings.TrimSpace(` -DENIED Tile38 is running in protected mode because protected mode is enabled, no bind address was specified, no authentication password is requested to clients. In this mode connections are only accepted from the loopback interface. If you want to connect from external computers to Tile38 you may adopt one of the following solutions: 1) Just disable protected mode sending the command 'CONFIG SET protected-mode no' from the loopback interface by connecting to Tile38 from the same host the server is running, however MAKE SURE Tile38 is not publicly accessible from internet if you do so. Use CONFIG REWRITE to make this change permanent. 2) Alternatively you can just disable the protected mode by editing the Tile38 configuration file, and setting the protected mode option to 'no', and then restarting the server. 3) If you started the server manually just for testing, restart it with the '--protected-mode no' option. 4) Setup a bind address or an authentication password. NOTE: You only need to do one of the above things in order for the server to start accepting connections from the outside. `), "\n", " ", -1) + "\r\n") // WriteWebSocketMessage write a websocket message to an io.Writer. func WriteWebSocketMessage(w io.Writer, data []byte) error { var msg []byte buf := make([]byte, 10+len(data)) buf[0] = 129 // FIN + TEXT if len(data) <= 125 { buf[1] = byte(len(data)) copy(buf[2:], data) msg = buf[:2+len(data)] } else if len(data) <= 0xFFFF { buf[1] = 126 binary.BigEndian.PutUint16(buf[2:], uint16(len(data))) copy(buf[4:], data) msg = buf[:4+len(data)] } else { buf[1] = 127 binary.BigEndian.PutUint64(buf[2:], uint64(len(data))) copy(buf[10:], data) msg = buf[:10+len(data)] } _, err := w.Write(msg) return err } // OKMessage returns a default OK message in JSON or RESP. func OKMessage(msg *Message, start time.Time) resp.Value { switch msg.OutputType { case JSON: return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}") case RESP: return resp.SimpleStringValue("OK") } return resp.SimpleStringValue("") } // NOMessage is no message var NOMessage = resp.SimpleStringValue("") var errInvalidHTTP = errors.New("invalid HTTP request") // Type is resp type type Type byte // Protocol Types const ( Null Type = iota RESP Telnet Native HTTP WebSocket JSON ) // Message is a resp message type Message struct { _command string Args []string StrictRESP bool ConnType Type OutputType Type Auth string AcceptEncoding string Deadline *deadline.Deadline } // Command returns the first argument as a lowercase string func (msg *Message) Command() string { if msg._command == "" { msg._command = strings.ToLower(msg.Args[0]) } return msg._command } // PipelineReader ... type PipelineReader struct { rd io.Reader wr io.Writer packet [0xFFFF]byte buf []byte } const kindHTTP redcon.Kind = 9999 // NewPipelineReader ... func NewPipelineReader(rd io.ReadWriter) *PipelineReader { return &PipelineReader{rd: rd, wr: rd} } func readcrlfline(packet []byte) (line string, leftover []byte, ok bool) { for i := 1; i < len(packet); i++ { if packet[i] == '\n' && packet[i-1] == '\r' { return string(packet[:i-1]), packet[i+1:], true } } return "", packet, false } func headerValue(header, name string) int { i := 0 for ; i < len(name) && i < len(header); i++ { a := header[i] if a >= 'A' && a <= 'Z' { a += 32 } b := name[i] if b >= 'A' && b <= 'Z' { b += 32 } if a != b { return -1 } } if i != len(name) || i == len(header) || header[i] != ':' { return -1 } i++ for ; i < len(header); i++ { if header[i] != ' ' && header[i] != '\t' { break } } return i } func readNextHTTPCommand(packet []byte, argsIn [][]byte, msg *Message, wr io.Writer) ( complete bool, args [][]byte, kind redcon.Kind, leftover []byte, err error, ) { args = argsIn[:0] msg.ConnType = HTTP msg.OutputType = JSON msg.StrictRESP = false opacket := packet ready, err := func() (bool, error) { var line string var ok bool // read header var headers []string for { line, packet, ok = readcrlfline(packet) if !ok { return false, nil } if line == "" { break } headers = append(headers, line) } parts := strings.Split(headers[0], " ") if len(parts) != 3 { return false, errInvalidHTTP } method := parts[0] path := parts[1] // Handle CORS request for allowed origins if method == "OPTIONS" { if wr == nil { return false, errors.New("connection is nil") } corshead := "HTTP/1.1 204 No Content\r\n" + "Connection: close\r\n" + "Access-Control-Allow-Origin: *\r\n" + "Access-Control-Allow-Headers: *, Authorization\r\n" + "Access-Control-Allow-Methods: POST, GET, OPTIONS\r\n\r\n" if _, err = wr.Write([]byte(corshead)); err != nil { return false, err } return false, nil } if len(path) == 0 || path[0] != '/' { return false, errInvalidHTTP } path, err = url.QueryUnescape(path[1:]) if err != nil { return false, errInvalidHTTP } if method != "GET" && method != "POST" { return false, errInvalidHTTP } contentLength := 0 websocket := false websocketVersion := 0 websocketKey := "" acceptEncoding := "" for _, hdr := range headers[1:] { var i int if i = headerValue(hdr, "Accept-Encoding"); i != -1 { acceptEncoding = strings.TrimSpace(hdr[i:]) } else if i = headerValue(hdr, "Authorization"); i != -1 { msg.Auth = strings.TrimSpace(hdr[i:]) } else if i = headerValue(hdr, "Upgrade"); i != -1 { val := strings.TrimSpace(hdr[i:]) if strings.ToLower(val) == "websocket" { websocket = true } } else if i = headerValue(hdr, "Sec-Websocket-Version"); i != -1 { val := strings.TrimSpace(hdr[i:]) n, err := strconv.ParseUint(strings.TrimSpace(val), 10, 64) if err != nil { return false, err } websocketVersion = int(n) } else if i = headerValue(hdr, "Sec-Websocket-Key"); i != -1 { websocketKey = strings.TrimSpace(hdr[i:]) } else if i = headerValue(hdr, "Content-Length"); i != -1 { val := strings.TrimSpace(hdr[i:]) n, err := strconv.ParseUint(strings.TrimSpace(val), 10, 64) if err != nil { return false, err } contentLength = int(n) } } if websocket && websocketVersion >= 13 && websocketKey != "" { msg.ConnType = WebSocket if wr == nil { return false, errors.New("connection is nil") } sum := sha1.Sum([]byte(websocketKey + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11")) accept := base64.StdEncoding.EncodeToString(sum[:]) wshead := "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " + accept + "\r\n\r\n" if _, err = wr.Write([]byte(wshead)); err != nil { return false, err } } else if contentLength > 0 { msg.ConnType = HTTP if len(packet) < contentLength { return false, nil } path += string(packet[:contentLength]) packet = packet[contentLength:] } if path == "" { return true, nil } nmsg, err := readNativeMessageLine([]byte(path)) if err != nil { return false, err } msg.AcceptEncoding = acceptEncoding msg.OutputType = JSON msg.StrictRESP = false msg.Args = nmsg.Args return true, nil }() if err != nil || !ready { return false, args[:0], kindHTTP, opacket, err } return true, args[:0], kindHTTP, packet, nil } func readNextCommand(packet []byte, argsIn [][]byte, msg *Message, wr io.Writer) ( complete bool, args [][]byte, kind redcon.Kind, leftover []byte, err error, ) { if packet[0] == 'G' || packet[0] == 'P' || packet[0] == 'O' { // could be an HTTP request var line []byte for i := 1; i < len(packet); i++ { if packet[i] == '\n' { if packet[i-1] == '\r' { line = packet[:i+1] break } } } if len(line) == 0 { return false, argsIn[:0], redcon.Redis, packet, nil } if len(line) > 11 && string(line[len(line)-11:len(line)-5]) == " HTTP/" { return readNextHTTPCommand(packet, argsIn, msg, wr) } } return redcon.ReadNextCommand(packet, args) } // ReadMessages ... func (rd *PipelineReader) ReadMessages() ([]*Message, error) { var msgs []*Message moreData: n, err := rd.rd.Read(rd.packet[:]) if err != nil { return nil, err } if n == 0 { // need more data goto moreData } data := rd.packet[:n] if len(rd.buf) > 0 { data = append(rd.buf, data...) } for len(data) > 0 { msg := &Message{} complete, args, kind, leftover, err2 := readNextCommand(data, nil, msg, rd.wr) if err2 != nil { err = err2 break } if !complete { break } if kind == kindHTTP { if len(msg.Args) == 0 { return nil, errInvalidHTTP } msgs = append(msgs, msg) } else if len(args) > 0 { for i := 0; i < len(args); i++ { msg.Args = append(msg.Args, string(args[i])) } switch kind { case redcon.Redis: msg.ConnType = RESP msg.OutputType = RESP msg.StrictRESP = false case redcon.Tile38: msg.ConnType = Native msg.OutputType = JSON msg.StrictRESP = false case redcon.Telnet: msg.ConnType = RESP msg.OutputType = RESP msg.StrictRESP = false } msgs = append(msgs, msg) } data = leftover } if len(data) > 0 { rd.buf = append(rd.buf[:0], data...) } else if len(rd.buf) > 0 { rd.buf = rd.buf[:0] } return msgs, err } func readNativeMessageLine(line []byte) (*Message, error) { var args []string reading: for len(line) != 0 { if line[0] == '{' { // The native protocol cannot understand json boundaries so it assumes that // a json element must be at the end of the line. args = append(args, string(line)) break } if line[0] == '"' && line[len(line)-1] == '"' { if len(args) > 0 && strings.ToLower(args[0]) == "set" && strings.ToLower(args[len(args)-1]) == "string" { // Setting a string value that is contained inside double quotes. // This is only because of the boundary issues of the native protocol. args = append(args, string(line[1:len(line)-1])) break } } i := 0 for ; i < len(line); i++ { if line[i] == ' ' { arg := string(line[:i]) if arg != "" { args = append(args, arg) } line = line[i+1:] continue reading } } args = append(args, string(line)) break } return &Message{Args: args, ConnType: Native, OutputType: JSON}, nil } // InputStream is a helper type for managing input streams from inside // the Data event. type InputStream struct{ b []byte } // Begin accepts a new packet and returns a working sequence of // unprocessed bytes. func (is *InputStream) Begin(packet []byte) (data []byte) { data = packet if len(is.b) > 0 { is.b = append(is.b, data...) data = is.b } return data } // End shifts the stream to match the unprocessed data. func (is *InputStream) End(data []byte) { if len(data) > 0 { if len(data) != len(is.b) { is.b = append(is.b[:0], data...) } } else if len(is.b) > 0 { is.b = is.b[:0] } } // clientErrorf is the same as fmt.Errorf, but is intented for errors that are // sent back to the client. This allows for the Go static checker to ignore // throwing warning for certain error strings. // https://staticcheck.io/docs/checks#ST1005 func clientErrorf(format string, args ...interface{}) error { return fmt.Errorf(format, args...) } ================================================ FILE: internal/server/stats.go ================================================ package server import ( "bytes" "encoding/json" "errors" "fmt" "os" "runtime" "sort" "strconv" "strings" "sync" "time" "github.com/tidwall/buntdb" "github.com/tidwall/resp" "github.com/tidwall/tile38/core" "github.com/tidwall/tile38/internal/collection" ) var memStats runtime.MemStats var memStatsMu sync.Mutex var memStatsBG bool // ReadMemStats returns the latest memstats. It provides an instant response. func readMemStats() runtime.MemStats { memStatsMu.Lock() if !memStatsBG { runtime.ReadMemStats(&memStats) go func() { var ms runtime.MemStats for { runtime.ReadMemStats(&ms) memStatsMu.Lock() memStats = ms memStatsMu.Unlock() time.Sleep(time.Second / 5) } }() memStatsBG = true } ms := memStats memStatsMu.Unlock() return ms } // STATS key [key...] func (s *Server) cmdSTATS(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) < 2 { return retrerr(errInvalidNumberOfArguments) } // >> Operation var vals []resp.Value var ms = []map[string]interface{}{} for i := 1; i < len(args); i++ { key := args[i] col, _ := s.cols.Get(key) if col != nil { m := make(map[string]interface{}) m["num_points"] = col.PointCount() m["in_memory_size"] = col.TotalWeight() m["num_objects"] = col.Count() m["num_strings"] = col.StringCount() switch msg.OutputType { case JSON: ms = append(ms, m) case RESP: vals = append(vals, resp.ArrayValue(respValuesSimpleMap(m))) } } else { switch msg.OutputType { case JSON: ms = append(ms, nil) case RESP: vals = append(vals, resp.NullValue()) } } } // >> Response if msg.OutputType == JSON { data, _ := json.Marshal(ms) return resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.ArrayValue(vals), nil } // HEALTHZ func (s *Server) cmdHEALTHZ(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args if len(args) != 1 { return retrerr(errInvalidNumberOfArguments) } // >> Operation if s.config.followHost() != "" { if !s.caughtUp() { return retrerr(errors.New("not caught up")) } } // >> Response if msg.OutputType == JSON { return resp.StringValue(`{"ok":true,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.SimpleStringValue("OK"), nil } // SERVER [ext] func (s *Server) cmdSERVER(msg *Message) (resp.Value, error) { start := time.Now() // >> Args args := msg.Args var ext bool for i := 1; i < len(args); i++ { switch strings.ToLower(args[i]) { case "ext": ext = true default: return retrerr(errInvalidArgument(args[i])) } } // >> Operation m := make(map[string]interface{}) if ext { s.extStats(m) } else { s.basicStats(m) } // >> Response if msg.OutputType == JSON { data, _ := json.Marshal(m) return resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.ArrayValue(respValuesSimpleMap(m)), nil } // basicStats populates the passed map with basic system/go/tile38 statistics func (s *Server) basicStats(m map[string]interface{}) { m["id"] = s.config.serverID() if s.config.followHost() != "" { m["following"] = fmt.Sprintf("%s:%d", s.config.followHost(), s.config.followPort()) m["caught_up"] = s.caughtUp() m["caught_up_once"] = s.caughtUpOnce() } m["http_transport"] = s.http m["pid"] = os.Getpid() m["aof_size"] = s.aofsz m["num_collections"] = s.cols.Len() m["num_hooks"] = s.hooks.Len() sz := 0 s.cols.Scan(func(key string, col *collection.Collection) bool { sz += col.TotalWeight() return true }) m["in_memory_size"] = sz points := 0 objects := 0 nstrings := 0 s.cols.Scan(func(key string, col *collection.Collection) bool { points += col.PointCount() objects += col.Count() nstrings += col.StringCount() return true }) m["num_points"] = points m["num_objects"] = objects m["num_strings"] = nstrings mem := readMemStats() avgsz := 0 if points != 0 { avgsz = int(mem.HeapAlloc) / points } m["mem_alloc"] = mem.Alloc m["heap_size"] = mem.HeapAlloc m["heap_released"] = mem.HeapReleased m["max_heap_size"] = s.config.maxMemory() m["avg_item_size"] = avgsz m["version"] = core.Version m["pointer_size"] = (32 << uintptr(uint64(^uintptr(0))>>63)) / 8 m["read_only"] = s.config.readOnly() m["cpus"] = runtime.NumCPU() n, _ := runtime.ThreadCreateProfile(nil) m["threads"] = float64(n) var nevents int s.qdb.View(func(tx *buntdb.Tx) error { // All entries in the buntdb log are events, except for one, which // is "hook:idx". nevents, _ = tx.Len() nevents -= 1 // Ignore the "hook:idx" if nevents < 0 { nevents = 0 } return nil }) m["pending_events"] = nevents } // extStats populates the passed map with extended system/go/tile38 statistics func (s *Server) extStats(m map[string]interface{}) { n, _ := runtime.ThreadCreateProfile(nil) mem := readMemStats() // Go/Memory Stats // Number of goroutines that currently exist m["go_goroutines"] = runtime.NumGoroutine() // Number of OS threads created m["go_threads"] = float64(n) // A summary of the GC invocation durations m["go_version"] = runtime.Version() // Number of bytes allocated and still in use m["alloc_bytes"] = mem.Alloc // Total number of bytes allocated, even if freed m["alloc_bytes_total"] = mem.TotalAlloc // Number of CPUS available on the system m["sys_cpus"] = runtime.NumCPU() // Number of bytes obtained from system m["sys_bytes"] = mem.Sys // Total number of pointer lookups m["lookups_total"] = mem.Lookups // Total number of mallocs m["mallocs_total"] = mem.Mallocs // Total number of frees m["frees_total"] = mem.Frees // Number of heap bytes allocated and still in use m["heap_alloc_bytes"] = mem.HeapAlloc // Number of heap bytes obtained from system m["heap_sys_bytes"] = mem.HeapSys // Number of heap bytes waiting to be used m["heap_idle_bytes"] = mem.HeapIdle // Number of heap bytes that are in use m["heap_inuse_bytes"] = mem.HeapInuse // Number of heap bytes released to OS m["heap_released_bytes"] = mem.HeapReleased // Number of allocated objects m["heap_objects"] = mem.HeapObjects // Number of bytes in use by the stack allocator m["stack_inuse_bytes"] = mem.StackInuse // Number of bytes obtained from system for stack allocator m["stack_sys_bytes"] = mem.StackSys // Number of bytes in use by mspan structures m["mspan_inuse_bytes"] = mem.MSpanInuse // Number of bytes used for mspan structures obtained from system m["mspan_sys_bytes"] = mem.MSpanSys // Number of bytes in use by mcache structures m["mcache_inuse_bytes"] = mem.MCacheInuse // Number of bytes used for mcache structures obtained from system m["mcache_sys_bytes"] = mem.MCacheSys // Number of bytes used by the profiling bucket hash table m["buck_hash_sys_bytes"] = mem.BuckHashSys // Number of bytes used for garbage collection system metadata m["gc_sys_bytes"] = mem.GCSys // Number of bytes used for other system allocations m["other_sys_bytes"] = mem.OtherSys // Number of heap bytes when next garbage collection will take place m["next_gc_bytes"] = mem.NextGC // Number of seconds since 1970 of last garbage collection m["last_gc_time_seconds"] = float64(mem.LastGC) / 1e9 // The fraction of this program's available CPU time used by the GC since // the program started m["gc_cpu_fraction"] = mem.GCCPUFraction // Tile38 Stats // ID of the server m["tile38_id"] = s.config.serverID() // The process ID of the server m["tile38_pid"] = os.Getpid() // Version of Tile38 running m["tile38_version"] = core.Version // Maximum heap size allowed m["tile38_max_heap_size"] = s.config.maxMemory() // Type of instance running if s.config.followHost() == "" { m["tile38_type"] = "leader" } else { m["tile38_type"] = "follower" } // Whether or not the server is read-only m["tile38_read_only"] = s.config.readOnly() // Size of pointer m["tile38_pointer_size"] = (32 << uintptr(uint64(^uintptr(0))>>63)) / 8 // Uptime of the Tile38 server in seconds m["tile38_uptime_in_seconds"] = time.Since(s.started).Seconds() // Number of currently connected Tile38 clients s.connsmu.RLock() m["tile38_connected_clients"] = len(s.conns) s.connsmu.RUnlock() // Whether or not a cluster is enabled m["tile38_cluster_enabled"] = false // Whether or not the Tile38 AOF is enabled m["tile38_aof_enabled"] = s.opts.AppendOnly // Whether or not an AOF shrink is currently in progress m["tile38_aof_rewrite_in_progress"] = s.shrinking // Length of time the last AOF shrink took m["tile38_aof_last_rewrite_time_sec"] = s.lastShrinkDuration.Load() / int64(time.Second) // Duration of the on-going AOF rewrite operation if any var currentShrinkStart time.Time if currentShrinkStart.IsZero() { m["tile38_aof_current_rewrite_time_sec"] = 0 } else { m["tile38_aof_current_rewrite_time_sec"] = time.Since(currentShrinkStart).Seconds() } // Total size of the AOF in bytes m["tile38_aof_size"] = s.aofsz // Whether or no the HTTP transport is being served m["tile38_http_transport"] = s.http // Number of connections accepted by the server m["tile38_total_connections_received"] = s.statsTotalConns.Load() // Number of commands processed by the server m["tile38_total_commands_processed"] = s.statsTotalCommands.Load() // Number of webhook messages sent by server m["tile38_total_messages_sent"] = s.statsTotalMsgsSent.Load() // Number of key expiration events m["tile38_expired_keys"] = s.statsExpired.Load() // Number of connected slaves m["tile38_connected_slaves"] = len(s.aofconnM) points := 0 objects := 0 strings := 0 s.cols.Scan(func(key string, col *collection.Collection) bool { points += col.PointCount() objects += col.Count() strings += col.StringCount() return true }) // Number of points in the database m["tile38_num_points"] = points // Number of objects in the database m["tile38_num_objects"] = objects // Number of string in the database m["tile38_num_strings"] = strings // Number of collections in the database m["tile38_num_collections"] = s.cols.Len() // Number of hooks in the database m["tile38_num_hooks"] = s.hooks.Len() // Number of hook groups in the database m["tile38_num_hook_groups"] = s.groupHooks.Len() // Number of object groups in the database m["tile38_num_object_groups"] = s.groupObjects.Len() avgsz := 0 if points != 0 { avgsz = int(mem.HeapAlloc) / points } // Average point size in bytes m["tile38_avg_point_size"] = avgsz sz := 0 s.cols.Scan(func(key string, col *collection.Collection) bool { sz += col.TotalWeight() return true }) // Total in memory size of all collections m["tile38_in_memory_size"] = sz } func (s *Server) writeInfoServer(w *bytes.Buffer) { fmt.Fprintf(w, "tile38_version:%s\r\n", core.Version) fmt.Fprintf(w, "redis_version:%s\r\n", core.Version) // Version of the Redis server fmt.Fprintf(w, "uptime_in_seconds:%d\r\n", int(time.Since(s.started).Seconds())) // Number of seconds since Redis server start } func (s *Server) writeInfoClients(w *bytes.Buffer) { s.connsmu.RLock() fmt.Fprintf(w, "connected_clients:%d\r\n", len(s.conns)) // Number of client connections (excluding connections from slaves) s.connsmu.RUnlock() } func (s *Server) writeInfoMemory(w *bytes.Buffer) { mem := readMemStats() fmt.Fprintf(w, "used_memory:%d\r\n", mem.Alloc) // total number of bytes allocated by Redis using its allocator (either standard libc, jemalloc, or an alternative allocator such as tcmalloc } func boolInt(t bool) int { if t { return 1 } return 0 } func (s *Server) writeInfoPersistence(w *bytes.Buffer) { fmt.Fprintf(w, "aof_enabled:%d\r\n", boolInt(s.opts.AppendOnly)) fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(s.shrinking)) // Flag indicating a AOF rewrite operation is on-going fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", s.lastShrinkDuration.Load()/int64(time.Second)) // Duration of the last AOF rewrite operation in seconds var currentShrinkStart time.Time // c.currentShrinkStart.get() if currentShrinkStart.IsZero() { fmt.Fprintf(w, "aof_current_rewrite_time_sec:0\r\n") // Duration of the on-going AOF rewrite operation if any } else { fmt.Fprintf(w, "aof_current_rewrite_time_sec:%d\r\n", time.Since(currentShrinkStart)/time.Second) // Duration of the on-going AOF rewrite operation if any } } func (s *Server) writeInfoStats(w *bytes.Buffer) { fmt.Fprintf(w, "total_connections_received:%d\r\n", s.statsTotalConns.Load()) // Total number of connections accepted by the server fmt.Fprintf(w, "total_commands_processed:%d\r\n", s.statsTotalCommands.Load()) // Total number of commands processed by the server fmt.Fprintf(w, "total_messages_sent:%d\r\n", s.statsTotalMsgsSent.Load()) // Total number of commands processed by the server fmt.Fprintf(w, "expired_keys:%d\r\n", s.statsExpired.Load()) // Total number of key expiration events } func replicaIPAndPort(cc *Client) (ip string, port int) { ip = cc.remoteAddr if cc.replAddr != "" { ip = cc.replAddr } i := strings.LastIndex(ip, ":") if i != -1 { ip = ip[:i] if ip == "[::1]" { ip = "localhost" } } port = cc.replPort return ip, port } // writeInfoReplication writes all replication data to the 'info' response func (s *Server) writeInfoReplication(w *bytes.Buffer) { if s.config.followHost() != "" { fmt.Fprintf(w, "role:slave\r\n") fmt.Fprintf(w, "master_host:%s\r\n", s.config.followHost()) fmt.Fprintf(w, "master_port:%v\r\n", s.config.followPort()) fmt.Fprintf(w, "slave_repl_offset:%v\r\n", int(s.faofsz)) if s.config.replicaPriority() >= 0 { fmt.Fprintf(w, "slave_priority:%v\r\n", s.config.replicaPriority()) } } else { fmt.Fprintf(w, "role:master\r\n") var i int s.connsmu.RLock() for _, cc := range s.conns { if cc.replPort != 0 { ip, port := replicaIPAndPort(cc) fmt.Fprintf(w, "slave%v:ip=%s,port=%v,state=online\r\n", i, ip, port) i++ } } s.connsmu.RUnlock() } fmt.Fprintf(w, "connected_slaves:%d\r\n", len(s.aofconnM)) // Number of connected slaves } func (s *Server) writeInfoCluster(w *bytes.Buffer) { fmt.Fprintf(w, "cluster_enabled:0\r\n") } // INFO [section ...] func (s *Server) cmdINFO(msg *Message) (res resp.Value, err error) { start := time.Now() // >> Args args := msg.Args msects := make(map[string]bool) allsects := []string{ "server", "clients", "memory", "persistence", "stats", "replication", "cpu", "cluster", "keyspace", } if len(args) == 1 { for _, s := range allsects { msects[s] = true } } for i := 1; i < len(args); i++ { section := strings.ToLower(args[i]) switch section { case "all", "default": for _, s := range allsects { msects[s] = true } default: for _, s := range allsects { if s == section { msects[section] = true } } } } // >> Operation var sects []string for _, s := range allsects { if msects[s] { sects = append(sects, s) } } w := &bytes.Buffer{} for i, section := range sects { if i > 0 { w.WriteString("\r\n") } switch strings.ToLower(section) { default: continue case "server": w.WriteString("# Server\r\n") s.writeInfoServer(w) case "clients": w.WriteString("# Clients\r\n") s.writeInfoClients(w) case "memory": w.WriteString("# Memory\r\n") s.writeInfoMemory(w) case "persistence": w.WriteString("# Persistence\r\n") s.writeInfoPersistence(w) case "stats": w.WriteString("# Stats\r\n") s.writeInfoStats(w) case "replication": w.WriteString("# Replication\r\n") s.writeInfoReplication(w) case "cpu": w.WriteString("# CPU\r\n") s.writeInfoCPU(w) case "cluster": w.WriteString("# Cluster\r\n") s.writeInfoCluster(w) } } // >> Response if msg.OutputType == JSON { // Create a map of all key/value info fields m := make(map[string]interface{}) for _, kv := range strings.Split(w.String(), "\r\n") { kv = strings.TrimSpace(kv) if !strings.HasPrefix(kv, "#") { if split := strings.SplitN(kv, ":", 2); len(split) == 2 { m[split[0]] = tryParseType(split[1]) } } } // Marshal the map and use the output in the JSON response data, _ := json.Marshal(m) return resp.StringValue(`{"ok":true,"info":` + string(data) + `,"elapsed":"` + time.Since(start).String() + "\"}"), nil } return resp.BytesValue(w.Bytes()), nil } // tryParseType attempts to parse the passed string as an integer, float64 and // a bool returning any successful parsed values. It returns the passed string // if all tries fail func tryParseType(str string) interface{} { if v, err := strconv.ParseInt(str, 10, 64); err == nil { return v } if v, err := strconv.ParseFloat(str, 64); err == nil { return v } if v, err := strconv.ParseBool(str); err == nil { return v } return str } func respValuesSimpleMap(m map[string]interface{}) []resp.Value { var keys []string for key := range m { keys = append(keys, key) } sort.Strings(keys) var vals []resp.Value for _, key := range keys { val := m[key] vals = append(vals, resp.StringValue(key)) vals = append(vals, resp.StringValue(fmt.Sprintf("%v", val))) } return vals } // ROLE func (s *Server) cmdROLE(msg *Message) (res resp.Value, err error) { start := time.Now() var role string var offset int var ips []string var ports []int var offsets []int var host string var port int var state string if s.config.followHost() == "" { role = "master" offset = s.aofsz s.connsmu.RLock() for _, cc := range s.conns { if cc.replPort != 0 { ip, port := replicaIPAndPort(cc) ips = append(ips, ip) ports = append(ports, port) offsets = append(offsets, s.aofsz) } } s.connsmu.RUnlock() } else { role = "slave" host = s.config.followHost() port = s.config.followPort() offset = int(s.faofsz) state = "connected" } if msg.OutputType == JSON { var json []byte json = append(json, `{"ok":true,"role":{`...) json = append(json, `"role":`...) json = appendJSONString(json, role) if role == "master" { json = append(json, `,"offset":`...) json = strconv.AppendInt(json, int64(offset), 10) json = append(json, `,"slaves":[`...) for i := range ips { if i > 0 { json = append(json, ',') } json = append(json, '{') json = append(json, `"ip":`...) json = appendJSONString(json, ips[i]) json = append(json, `,"port":`...) json = appendJSONString(json, fmt.Sprint(ports[i])) json = append(json, `,"offset":`...) json = appendJSONString(json, fmt.Sprint(offsets[i])) json = append(json, '}') } json = append(json, `]`...) } else if role == "slave" { json = append(json, `,"host":`...) json = appendJSONString(json, host) json = append(json, `,"port":`...) json = strconv.AppendInt(json, int64(port), 10) json = append(json, `,"state":`...) json = appendJSONString(json, state) json = append(json, `,"offset":`...) json = strconv.AppendInt(json, int64(offset), 10) } json = append(json, `},"elapsed":`...) json = appendJSONString(json, time.Since(start).String()) json = append(json, '}') return resp.StringValue(string(json)), nil } else { var vals []resp.Value vals = append(vals, resp.StringValue(role)) if role == "master" { vals = append(vals, resp.IntegerValue(offset)) var replicaVals []resp.Value for i := range ips { var vals []resp.Value vals = append(vals, resp.StringValue(ips[i])) vals = append(vals, resp.StringValue(fmt.Sprint(ports[i]))) vals = append(vals, resp.StringValue(fmt.Sprint(offsets[i]))) replicaVals = append(replicaVals, resp.ArrayValue(vals)) } vals = append(vals, resp.ArrayValue(replicaVals)) } else if role == "slave" { vals = append(vals, resp.StringValue(host)) vals = append(vals, resp.IntegerValue(port)) vals = append(vals, resp.StringValue(state)) vals = append(vals, resp.IntegerValue(offset)) } return resp.ArrayValue(vals), nil } } ================================================ FILE: internal/server/stats_cpu.go ================================================ //go:build !linux && !darwin package server import ( "bytes" "fmt" ) func (s *Server) writeInfoCPU(w *bytes.Buffer) { fmt.Fprintf(w, "used_cpu_sys:%.2f\r\n"+ "used_cpu_user:%.2f\r\n"+ "used_cpu_sys_children:%.2f\r\n"+ "used_cpu_user_children:%.2f\r\n", 0.0, 0.0, 0.0, 0.0, ) } ================================================ FILE: internal/server/stats_cpu_darlin.go ================================================ //go:build linux || darwin package server import ( "bytes" "fmt" "syscall" ) func (s *Server) writeInfoCPU(w *bytes.Buffer) { var selfRu syscall.Rusage var cRu syscall.Rusage syscall.Getrusage(syscall.RUSAGE_SELF, &selfRu) syscall.Getrusage(syscall.RUSAGE_CHILDREN, &cRu) fmt.Fprintf(w, "used_cpu_sys:%.2f\r\n"+ "used_cpu_user:%.2f\r\n"+ "used_cpu_sys_children:%.2f\r\n"+ "used_cpu_user_children:%.2f\r\n", float64(selfRu.Stime.Sec)+float64(selfRu.Stime.Usec/1000000), float64(selfRu.Utime.Sec)+float64(selfRu.Utime.Usec/1000000), float64(cRu.Stime.Sec)+float64(cRu.Stime.Usec/1000000), float64(cRu.Utime.Sec)+float64(cRu.Utime.Usec/1000000), ) } ================================================ FILE: internal/server/test.go ================================================ package server // TEST command: spatial tests without walking the tree. import ( "bytes" "fmt" "strconv" "strings" "time" "github.com/iwpnd/sectr" "github.com/mmcloughlin/geohash" "github.com/tidwall/geojson" "github.com/tidwall/geojson/geometry" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/bing" "github.com/tidwall/tile38/internal/clip" ) func (s *Server) parseArea(ovs []string, doClip bool) (vs []string, o geojson.Object, err error) { var ok bool var typ string vs = ovs[:] if vs, typ, ok = tokenval(vs); !ok || typ == "" { err = errInvalidNumberOfArguments return } ltyp := strings.ToLower(typ) switch ltyp { case "point": var slat, slon string if vs, slat, ok = tokenval(vs); !ok || slat == "" { err = errInvalidNumberOfArguments return } if vs, slon, ok = tokenval(vs); !ok || slon == "" { err = errInvalidNumberOfArguments return } var lat, lon float64 if lat, err = strconv.ParseFloat(slat, 64); err != nil { err = errInvalidArgument(slat) return } if lon, err = strconv.ParseFloat(slon, 64); err != nil { err = errInvalidArgument(slon) return } o = geojson.NewPoint(geometry.Point{X: lon, Y: lat}) case "sector": if doClip { err = fmt.Errorf("invalid clip type '%s'", typ) return } var slat, slon, smeters, sb1, sb2 string if vs, slat, ok = tokenval(vs); !ok || slat == "" { err = errInvalidNumberOfArguments return } if vs, slon, ok = tokenval(vs); !ok || slon == "" { err = errInvalidNumberOfArguments return } if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { err = errInvalidNumberOfArguments return } if vs, sb1, ok = tokenval(vs); !ok || sb1 == "" { err = errInvalidNumberOfArguments return } if vs, sb2, ok = tokenval(vs); !ok || sb2 == "" { err = errInvalidNumberOfArguments return } var lat, lon, meters, b1, b2 float64 if lat, err = strconv.ParseFloat(slat, 64); err != nil { err = errInvalidArgument(slat) return } if lon, err = strconv.ParseFloat(slon, 64); err != nil { err = errInvalidArgument(slon) return } if meters, err = strconv.ParseFloat(smeters, 64); err != nil { err = errInvalidArgument(smeters) return } if b1, err = strconv.ParseFloat(sb1, 64); err != nil { err = errInvalidArgument(sb1) return } if b2, err = strconv.ParseFloat(sb2, 64); err != nil { err = errInvalidArgument(sb2) return } if b1 == b2 { err = fmt.Errorf("equal bearings (%s == %s), use CIRCLE instead", sb1, sb2) return } origin := sectr.Point{Lng: lon, Lat: lat} sector := sectr.NewSector(origin, meters, b1, b2) o, err = geojson.Parse(string(sector.JSON()), &s.geomParseOpts) if err != nil { return } case "circle": if doClip { err = fmt.Errorf("invalid clip type '%s'", typ) return } var slat, slon, smeters string if vs, slat, ok = tokenval(vs); !ok || slat == "" { err = errInvalidNumberOfArguments return } if vs, slon, ok = tokenval(vs); !ok || slon == "" { err = errInvalidNumberOfArguments return } var lat, lon, meters float64 if lat, err = strconv.ParseFloat(slat, 64); err != nil { err = errInvalidArgument(slat) return } if lon, err = strconv.ParseFloat(slon, 64); err != nil { err = errInvalidArgument(slon) return } if vs, smeters, ok = tokenval(vs); !ok || smeters == "" { err = errInvalidNumberOfArguments return } if meters, err = strconv.ParseFloat(smeters, 64); err != nil { err = errInvalidArgument(smeters) return } if meters < 0 { err = errInvalidArgument(smeters) return } o = geojson.NewCircle(geometry.Point{X: lon, Y: lat}, meters, defaultCircleSteps) case "object": if doClip { err = fmt.Errorf("invalid clip type '%s'", typ) return } var obj string if vs, obj, ok = tokenval(vs); !ok || obj == "" { err = errInvalidNumberOfArguments return } o, err = geojson.Parse(obj, &s.geomParseOpts) if err != nil { return } case "bounds": var sminLat, sminLon, smaxlat, smaxlon string if vs, sminLat, ok = tokenval(vs); !ok || sminLat == "" { err = errInvalidNumberOfArguments return } if vs, sminLon, ok = tokenval(vs); !ok || sminLon == "" { err = errInvalidNumberOfArguments return } if vs, smaxlat, ok = tokenval(vs); !ok || smaxlat == "" { err = errInvalidNumberOfArguments return } if vs, smaxlon, ok = tokenval(vs); !ok || smaxlon == "" { err = errInvalidNumberOfArguments return } var minLat, minLon, maxLat, maxLon float64 if minLat, err = strconv.ParseFloat(sminLat, 64); err != nil { err = errInvalidArgument(sminLat) return } if minLon, err = strconv.ParseFloat(sminLon, 64); err != nil { err = errInvalidArgument(sminLon) return } if maxLat, err = strconv.ParseFloat(smaxlat, 64); err != nil { err = errInvalidArgument(smaxlat) return } if maxLon, err = strconv.ParseFloat(smaxlon, 64); err != nil { err = errInvalidArgument(smaxlon) return } o = geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, }) case "hash": var hash string if vs, hash, ok = tokenval(vs); !ok || hash == "" { err = errInvalidNumberOfArguments return } box := geohash.BoundingBox(hash) o = geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: box.MinLng, Y: box.MinLat}, Max: geometry.Point{X: box.MaxLng, Y: box.MaxLat}, }) case "quadkey": var key string if vs, key, ok = tokenval(vs); !ok || key == "" { err = errInvalidNumberOfArguments return } var minLat, minLon, maxLat, maxLon float64 minLat, minLon, maxLat, maxLon, err = bing.QuadKeyToBounds(key) if err != nil { err = errInvalidArgument(key) return } o = geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, }) case "tile": var sx, sy, sz string if vs, sx, ok = tokenval(vs); !ok || sx == "" { err = errInvalidNumberOfArguments return } if vs, sy, ok = tokenval(vs); !ok || sy == "" { err = errInvalidNumberOfArguments return } if vs, sz, ok = tokenval(vs); !ok || sz == "" { err = errInvalidNumberOfArguments return } var x, y int64 var z uint64 if x, err = strconv.ParseInt(sx, 10, 64); err != nil { err = errInvalidArgument(sx) return } if y, err = strconv.ParseInt(sy, 10, 64); err != nil { err = errInvalidArgument(sy) return } if z, err = strconv.ParseUint(sz, 10, 64); err != nil { err = errInvalidArgument(sz) return } var minLat, minLon, maxLat, maxLon float64 minLat, minLon, maxLat, maxLon = bing.TileXYToBounds(x, y, z) o = geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, }) case "get": if doClip { err = fmt.Errorf("invalid clip type '%s'", typ) return } var key, id string if vs, key, ok = tokenval(vs); !ok || key == "" { err = errInvalidNumberOfArguments return } if vs, id, ok = tokenval(vs); !ok || id == "" { err = errInvalidNumberOfArguments return } col, _ := s.cols.Get(key) if col == nil { err = errKeyNotFound return } obj := col.Get(id) if obj == nil { err = errIDNotFound return } o = obj.Geo() } return } // TEST (POINT lat lon)|(GET key id)|(BOUNDS minlat minlon maxlat maxlon)| // (OBJECT geojson)|(CIRCLE lat lon meters)|(TILE x y z)|(QUADKEY quadkey)| // (HASH geohash) INTERSECTS|WITHIN [CLIP] (POINT lat lon)|(GET key id)| // (BOUNDS minlat minlon maxlat maxlon)|(OBJECT geojson)| // (CIRCLE lat lon meters)|(TILE x y z)|(QUADKEY quadkey)|(HASH geohash)| // (SECTOR lat lon meters bearing1 bearing2) func (s *Server) cmdTEST(msg *Message) (res resp.Value, err error) { start := time.Now() vs := msg.Args[1:] var ok bool var test string var clipped geojson.Object var area1, area2 *areaExpression if vs, area1, err = s.parseAreaExpression(vs, false); err != nil { return } if vs, test, ok = tokenval(vs); !ok || test == "" { err = errInvalidNumberOfArguments return } lTest := strings.ToLower(test) if lTest != "within" && lTest != "intersects" { err = errInvalidArgument(test) return } var wtok string var nvs []string var doClip bool nvs, wtok, ok = tokenval(vs) if ok && len(wtok) > 0 { switch strings.ToLower(wtok) { case "clip": vs = nvs if lTest != "intersects" { err = errInvalidArgument(wtok) return } doClip = true } } if vs, area2, err = s.parseAreaExpression(vs, doClip); err != nil { return } if doClip && (area1.obj == nil || area2.obj == nil) { err = errInvalidArgument("clip") return } if len(vs) != 0 { err = errInvalidNumberOfArguments return } var result int if lTest == "within" { if area1.WithinExpr(area2) { result = 1 } } else if lTest == "intersects" { if area1.IntersectsExpr(area2) { result = 1 if doClip { clipped = clip.Clip(area1.obj, area2.obj, nil) } } } if msg.OutputType == JSON { var buf bytes.Buffer buf.WriteString(`{"ok":true`) if result != 0 { buf.WriteString(`,"result":true`) } else { buf.WriteString(`,"result":false`) } if clipped != nil { buf.WriteString(`,"object":` + clipped.JSON()) } buf.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") return resp.StringValue(buf.String()), nil } if clipped != nil { return resp.ArrayValue([]resp.Value{ resp.IntegerValue(result), resp.StringValue(clipped.JSON())}), nil } return resp.IntegerValue(result), nil } ================================================ FILE: internal/server/token.go ================================================ package server import ( "errors" "fmt" "math" "strconv" "strings" "github.com/tidwall/tile38/internal/field" "github.com/tidwall/tile38/internal/log" lua "github.com/yuin/gopher-lua" luajson "layeh.com/gopher-json" ) const defaultSearchOutput = outputObjects var errInvalidNumberOfArguments = errors.New("invalid number of arguments") var errKeyNotFound = errors.New("key not found") var errIDNotFound = errors.New("id not found") var errIDAlreadyExists = errors.New("id already exists") var errPathNotFound = errors.New("path not found") var errKeyHasHooksSet = errors.New("key has hooks set") var errKeyHasChannelsSet = errors.New("key has channels set") var errNotRectangle = errors.New("not a rectangle") func errInvalidArgument(arg string) error { return fmt.Errorf("invalid argument '%s'", arg) } func errDuplicateArgument(arg string) error { return fmt.Errorf("duplicate argument '%s'", arg) } func token(line string) (newLine, token string) { for i := 0; i < len(line); i++ { if line[i] == ' ' { return line[i+1:], line[:i] } } return "", line } func tokenval(vs []string) (nvs []string, token string, ok bool) { if len(vs) > 0 { token = vs[0] nvs = vs[1:] ok = true } return } func lc(s1, s2 string) bool { if len(s1) != len(s2) { return false } for i := 0; i < len(s1); i++ { ch := s1[i] if ch >= 'A' && ch <= 'Z' { if ch+32 != s2[i] { return false } } else if ch != s2[i] { return false } } return true } type whereT struct { expr bool name string minx bool min field.Value maxx bool max field.Value } func mLT(a, b field.Value) bool { return a.Less(b) } func mLTE(a, b field.Value) bool { return !mLT(b, a) } func mGT(a, b field.Value) bool { return mLT(b, a) } func mGTE(a, b field.Value) bool { return !mLT(a, b) } func mEQ(a, b field.Value) bool { return a.Equals(b) } func (where whereT) matchField(value field.Value) bool { switch where.min.Data() { case "<": return mLT(value, where.max) case "<=": return mLTE(value, where.max) case ">": return mGT(value, where.max) case ">=": return mGTE(value, where.max) case "==": return mEQ(value, where.max) case "!=": return !mEQ(value, where.max) } if !where.minx { if mLT(value, where.min) { // if value < where.min { return false } } else { if mLTE(value, where.min) { // if value <= where.min { return false } } if !where.maxx { if mGT(value, where.max) { // if value > where.max { return false } } else { if mGTE(value, where.max) { // if value >= where.max { return false } } return true } type whereinT struct { name string valArr []field.Value } func (wherein whereinT) match(value field.Value) bool { for _, val := range wherein.valArr { if mEQ(val, value) { return true } } return false } type whereevalT struct { c *Server luaState *lua.LState fn *lua.LFunction } func (whereeval whereevalT) Close() { luaSetRawGlobals( whereeval.luaState, map[string]lua.LValue{ "ARGV": lua.LNil, }) whereeval.c.luapool.Put(whereeval.luaState) } func luaSetField(tbl *lua.LTable, name string, val field.Value) { var lval lua.LValue switch val.Kind() { case field.Null: lval = lua.LNil case field.False: lval = lua.LFalse case field.True: lval = lua.LTrue case field.Number: lval = lua.LNumber(val.Num()) default: lval = lua.LString(val.Data()) } tbl.RawSetString(name, lval) } func (whereeval whereevalT) match(fieldsWithNames map[string]field.Value, id string, props string) (ok bool, err error, ) { fieldsTbl := whereeval.luaState.CreateTable(0, len(fieldsWithNames)) for name, val := range fieldsWithNames { luaSetField(fieldsTbl, name, val) } propsTbl := lua.LValue(whereeval.luaState.CreateTable(0, 0)) if props != "" { var err error tbl, err := luajson.Decode(whereeval.luaState, []byte(props)) if err == nil { propsTbl = tbl } } luaSetRawGlobals( whereeval.luaState, map[string]lua.LValue{ "ID": lua.LString(id), "FIELDS": fieldsTbl, "PROPERTIES": propsTbl, }) defer func() { luaSetRawGlobals( whereeval.luaState, map[string]lua.LValue{ "ID": lua.LNil, "FIELDS": lua.LNil, "PROPERTIES": lua.LNil, }) }() whereeval.luaState.Push(whereeval.fn) if err := whereeval.luaState.PCall(0, 1, nil); err != nil { emsg := err.Error() if strings.Contains(emsg, "attempt to index a non-table") { log.Debugf("Lua error: %v", emsg) return false, nil } return false, err } ret := whereeval.luaState.Get(-1) whereeval.luaState.Pop(1) if ret == nil { return false, nil } // Make bool out of returned lua value switch ret.Type() { case lua.LTNil: return false, nil case lua.LTBool: return ret == lua.LTrue, nil case lua.LTNumber: return float64(ret.(lua.LNumber)) != 0, nil case lua.LTString: return ret.String() != "", nil case lua.LTTable: tbl := ret.(*lua.LTable) if tbl.Len() != 0 { return true, nil } var match bool tbl.ForEach(func(lk lua.LValue, lv lua.LValue) { match = true }) return match, nil } return false, fmt.Errorf("script returned value of type %s", ret.Type()) } type searchScanBaseTokens struct { key string cursor uint64 output outputT precision uint64 fence bool distance bool nodwell bool detect map[string]bool accept map[string]bool globs []string wheres []whereT whereins []whereinT whereevals []whereevalT nofields bool ulimit bool limit uint64 usparse bool sparse uint8 desc bool clip bool buffer float64 hasbuffer bool mvt bool tileX int tileY int tileZ int } func (s *Server) parseSearchScanBaseTokens( cmd string, t searchScanBaseTokens, vs []string, ) ( vsout []string, tout searchScanBaseTokens, err error, ) { var ok bool if vs, t.key, ok = tokenval(vs); !ok || t.key == "" { err = errInvalidNumberOfArguments return } fromFence := t.fence var slimit string var ssparse string var scursor string var asc bool for { nvs, wtok, ok := tokenval(vs) if ok && len(wtok) > 0 { switch strings.ToLower(wtok) { case "buffer": vs = nvs var sbuf string if vs, sbuf, ok = tokenval(vs); !ok || sbuf == "" { err = errInvalidNumberOfArguments return } var buf float64 buf, err = strconv.ParseFloat(sbuf, 64) if err != nil || buf < 0 || math.IsInf(buf, 0) || math.IsNaN(buf) { err = errInvalidArgument(sbuf) return } t.buffer = buf t.hasbuffer = true continue case "cursor": vs = nvs if scursor != "" { err = errDuplicateArgument(strings.ToUpper(wtok)) return } if vs, scursor, ok = tokenval(vs); !ok || scursor == "" { err = errInvalidNumberOfArguments return } continue case "where": vs = nvs if detectExprToken(vs) { // using expressions // WHERE expr var expr string if vs, expr, ok = tokenval(vs); !ok { err = errInvalidNumberOfArguments return } t.wheres = append(t.wheres, whereT{name: expr, expr: true}) continue } else { // using field filter // WHERE min max var name, smin, smax string if vs, name, ok = tokenval(vs); !ok { err = errInvalidNumberOfArguments return } if vs, smin, ok = tokenval(vs); !ok { err = errInvalidNumberOfArguments return } if vs, smax, ok = tokenval(vs); !ok { err = errInvalidNumberOfArguments return } var minx, maxx bool smin = strings.ToLower(smin) smax = strings.ToLower(smax) if smax == "+inf" || smax == "inf" { smax = "inf" } switch smin { case "<", "<=", ">", ">=", "==", "!=": default: if strings.HasPrefix(smin, "(") { minx = true smin = smin[1:] } if strings.HasPrefix(smax, "(") { maxx = true smax = smax[1:] } } t.wheres = append(t.wheres, whereT{ name: name, minx: minx, min: field.ValueOf(smin), maxx: maxx, max: field.ValueOf(smax), }) continue } case "wherein": vs = nvs var name, nvalsStr, valStr string if vs, name, ok = tokenval(vs); !ok { err = errInvalidNumberOfArguments return } if vs, nvalsStr, ok = tokenval(vs); !ok { err = errInvalidNumberOfArguments return } var i, nvals uint64 if nvals, err = strconv.ParseUint(nvalsStr, 10, 64); err != nil { err = errInvalidArgument(nvalsStr) return } valArr := make([]field.Value, nvals) for i = 0; i < nvals; i++ { if vs, valStr, ok = tokenval(vs); !ok { err = errInvalidNumberOfArguments return } valArr[i] = field.ValueOf(valStr) } t.whereins = append(t.whereins, whereinT{ name: name, valArr: valArr, }) continue case "whereevalsha": fallthrough case "whereeval": scriptIsSha := strings.ToLower(wtok) == "whereevalsha" vs = nvs var script, nargsStr, arg string if vs, script, ok = tokenval(vs); !ok || script == "" { err = errInvalidNumberOfArguments return } if vs, nargsStr, ok = tokenval(vs); !ok || nargsStr == "" { err = errInvalidNumberOfArguments return } var i, nargs uint64 if nargs, err = strconv.ParseUint(nargsStr, 10, 64); err != nil { err = errInvalidArgument(nargsStr) return } var luaState *lua.LState luaState, err = s.luapool.Get() if err != nil { return } argsTbl := luaState.CreateTable(len(vs), 0) for i = 0; i < nargs; i++ { if vs, arg, ok = tokenval(vs); !ok || arg == "" { err = errInvalidNumberOfArguments return } argsTbl.Append(lua.LString(arg)) } var shaSum string if scriptIsSha { shaSum = script } else { shaSum = Sha1Sum(script) } luaSetRawGlobals( luaState, map[string]lua.LValue{ "ARGV": argsTbl, }) compiled, ok := s.luascripts.Get(shaSum) var fn *lua.LFunction if ok { fn = &lua.LFunction{ IsG: false, Env: luaState.Env, Proto: compiled, GFunction: nil, Upvalues: make([]*lua.Upvalue, 0), } } else if scriptIsSha { err = errShaNotFound return } else { fn, err = luaState.Load(strings.NewReader(script), "f_"+shaSum) if err != nil { err = makeSafeErr(err) return } s.luascripts.PutLRU(shaSum, fn.Proto) } t.whereevals = append(t.whereevals, whereevalT{ c: s, luaState: luaState, fn: fn, }) continue case "nofields": vs = nvs if t.nofields { err = errDuplicateArgument(strings.ToUpper(wtok)) return } t.nofields = true continue case "limit": vs = nvs if slimit != "" { err = errDuplicateArgument(strings.ToUpper(wtok)) return } if vs, slimit, ok = tokenval(vs); !ok || slimit == "" { err = errInvalidNumberOfArguments return } continue case "sparse": vs = nvs if ssparse != "" { err = errDuplicateArgument(strings.ToUpper(wtok)) return } if vs, ssparse, ok = tokenval(vs); !ok || ssparse == "" { err = errInvalidNumberOfArguments return } continue case "fence": vs = nvs if t.fence && !fromFence { err = errDuplicateArgument(strings.ToUpper(wtok)) return } t.fence = true continue case "commands": vs = nvs if t.accept != nil { err = errDuplicateArgument(strings.ToUpper(wtok)) return } t.accept = make(map[string]bool) var peek string if vs, peek, ok = tokenval(vs); !ok || peek == "" { err = errInvalidNumberOfArguments return } for _, s := range strings.Split(peek, ",") { part := strings.TrimSpace(strings.ToLower(s)) if t.accept[part] { err = errDuplicateArgument(s) return } t.accept[part] = true } if len(t.accept) == 0 { t.accept = nil } continue case "distance": vs = nvs if t.distance { err = errDuplicateArgument(strings.ToUpper(wtok)) return } t.distance = true continue case "detect": vs = nvs if t.detect != nil { err = errDuplicateArgument(strings.ToUpper(wtok)) return } t.detect = make(map[string]bool) var peek string if vs, peek, ok = tokenval(vs); !ok || peek == "" { err = errInvalidNumberOfArguments return } for _, s := range strings.Split(peek, ",") { part := strings.TrimSpace(strings.ToLower(s)) switch part { default: err = errInvalidArgument(peek) return case "inside", "outside", "enter", "exit", "cross": } if t.detect[part] { err = errDuplicateArgument(s) return } t.detect[part] = true } if len(t.detect) == 0 { t.detect = map[string]bool{ "inside": true, "outside": true, "enter": true, "exit": true, "cross": true, } } continue case "nodwell": vs = nvs if t.desc || asc { err = errDuplicateArgument(strings.ToUpper(wtok)) return } t.nodwell = true continue case "desc": vs = nvs if t.desc || asc { err = errDuplicateArgument(strings.ToUpper(wtok)) return } t.desc = true continue case "asc": vs = nvs if t.desc || asc { err = errDuplicateArgument(strings.ToUpper(wtok)) return } asc = true continue case "match": vs = nvs var glob string if vs, glob, ok = tokenval(vs); !ok || glob == "" { err = errInvalidNumberOfArguments return } t.globs = append(t.globs, glob) continue case "clip": vs = nvs if t.clip { err = errDuplicateArgument(strings.ToUpper(wtok)) return } t.clip = true continue } } break } // check to make sure that there aren't any conflicts if cmd == "scan" || cmd == "search" { if ssparse != "" { err = errors.New("SPARSE is not allowed for " + strings.ToUpper(cmd)) return } if t.fence { err = errors.New("FENCE is not allowed for " + strings.ToUpper(cmd)) return } } else { if t.desc { err = errors.New("DESC is not allowed for " + strings.ToUpper(cmd)) return } if asc { err = errors.New("ASC is not allowed for " + strings.ToUpper(cmd)) return } } if ssparse != "" && slimit != "" { err = errors.New("LIMIT is not allowed when SPARSE is specified") return } if scursor != "" && ssparse != "" { err = errors.New("CURSOR is not allowed when SPARSE is specified") return } if scursor != "" && t.fence { err = errors.New("CURSOR is not allowed when FENCE is specified") return } if t.detect != nil && !t.fence { err = errors.New("DETECT is not allowed when FENCE is not specified") return } t.output = defaultSearchOutput var nvs []string var sprecision string var which string if nvs, which, ok = tokenval(vs); ok && which != "" { updline := true switch strings.ToLower(which) { default: if cmd == "scan" { err = errInvalidArgument(which) return } updline = false case "count": t.output = outputCount case "objects": t.output = outputObjects case "points": t.output = outputPoints case "hashes": t.output = outputHashes if nvs, sprecision, ok = tokenval(nvs); !ok || sprecision == "" { err = errInvalidNumberOfArguments return } case "bounds": t.output = outputBounds case "ids": t.output = outputIDs } if updline { vs = nvs } } if scursor != "" { if t.cursor, err = strconv.ParseUint(scursor, 10, 64); err != nil { err = errInvalidArgument(scursor) return } } if sprecision != "" { t.precision, err = strconv.ParseUint(sprecision, 10, 64) if err != nil || t.precision == 0 || t.precision > 12 { err = errInvalidArgument(sprecision) return } } if slimit != "" { t.ulimit = true if t.limit, err = strconv.ParseUint(slimit, 10, 64); err != nil || t.limit == 0 { err = errInvalidArgument(slimit) return } } if ssparse != "" { t.usparse = true var sparse uint64 if sparse, err = strconv.ParseUint(ssparse, 10, 8); err != nil || sparse == 0 || sparse > 16 { err = errInvalidArgument(ssparse) return } t.sparse = uint8(sparse) t.limit = math.MaxUint64 } vsout = vs tout = t return } func detectExprToken(vs []string) bool { // Detect the kind of where, either: // - expr // - name min max if len(vs) == 0 { return false } else if len(vs) == 1 || (len(vs) == 2 && len(vs[1]) == 0) { return true } v := vs[1] if (v[0] >= 'a' && v[0] <= 'z') || (v[0] >= 'A' && v[0] <= 'Z') { if (v[0] == 'i' || v[0] == 'I') && strings.ToLower(v) == "inf" { return false } return true } return false } type parentStack []*areaExpression func (ps *parentStack) isEmpty() bool { return len(*ps) == 0 } func (ps *parentStack) push(e *areaExpression) { *ps = append(*ps, e) } func (ps *parentStack) pop() (e *areaExpression, empty bool) { n := len(*ps) if n == 0 { return nil, true } x := (*ps)[n-1] *ps = (*ps)[:n-1] return x, false } func (s *Server) parseAreaExpression(vsin []string, doClip bool) (vsout []string, ae *areaExpression, err error) { ps := &parentStack{} vsout = vsin[:] var negate, needObj bool loop: for { nvs, wtok, ok := tokenval(vsout) if !ok || len(wtok) == 0 { break } switch strings.ToLower(wtok) { case tokenLParen: newExpr := &areaExpression{negate: negate, op: NOOP} negate = false needObj = false if ae != nil { ae.children = append(ae.children, newExpr) } ae = newExpr ps.push(ae) vsout = nvs case tokenRParen: if needObj { err = errInvalidArgument(tokenRParen) return } parent, empty := ps.pop() if empty { err = errInvalidArgument(tokenRParen) return } ae = parent vsout = nvs case tokenNOT: negate = !negate needObj = true vsout = nvs case tokenAND: if needObj { err = errInvalidArgument(tokenAND) return } needObj = true if ae == nil { err = errInvalidArgument(tokenAND) return } else if ae.obj == nil { switch ae.op { case OR: numChildren := len(ae.children) if numChildren < 2 { err = errInvalidNumberOfArguments return } ae.children = append( ae.children[:numChildren-1], &areaExpression{ op: AND, children: []*areaExpression{ae.children[numChildren-1]}}) case NOOP: ae.op = AND } } else { ae = &areaExpression{op: AND, children: []*areaExpression{ae}} } vsout = nvs case tokenOR: if needObj { err = errInvalidArgument(tokenOR) return } needObj = true if ae == nil { err = errInvalidArgument(tokenOR) return } else if ae.obj == nil { switch ae.op { case AND: if len(ae.children) < 2 { err = errInvalidNumberOfArguments return } ae = &areaExpression{op: OR, children: []*areaExpression{ae}} case NOOP: ae.op = OR } } else { ae = &areaExpression{op: OR, children: []*areaExpression{ae}} } vsout = nvs case "point", "circle", "object", "bounds", "hash", "quadkey", "tile", "get", "sector": parsedVs, parsedObj, areaErr := s.parseArea(vsout, doClip) if areaErr != nil { err = areaErr return } newExpr := &areaExpression{negate: negate, obj: parsedObj, op: NOOP} negate = false needObj = false if ae == nil { ae = newExpr } else { ae.children = append(ae.children, newExpr) } vsout = parsedVs default: break loop } } if !ps.isEmpty() || needObj || ae == nil || (ae.obj == nil && len(ae.children) == 0) { err = errInvalidNumberOfArguments } return } ================================================ FILE: internal/server/token_test.go ================================================ package server import ( "strings" "testing" "github.com/tidwall/tile38/internal/field" ) func TestLowerCompare(t *testing.T) { if !lc("hello", "hello") { t.Fatal("failed") } if !lc("Hello", "hello") { t.Fatal("failed") } if !lc("HeLLo World", "hello world") { t.Fatal("failed") } if !lc("", "") { t.Fatal("failed") } if lc("hello", "") { t.Fatal("failed") } if lc("", "hello") { t.Fatal("failed") } if lc("HeLLo World", "Hello world") { t.Fatal("failed") } } func TestParseWhereins(t *testing.T) { s := &Server{} type tcase struct { inputWhereins []whereinT expWhereins []whereinT } fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { _, tout, err := s.parseSearchScanBaseTokens( "scan", searchScanBaseTokens{ whereins: tc.inputWhereins, }, []string{"key"}, ) got := tout.whereins exp := tc.expWhereins if err != nil { t.Fatalf("unexpected error while parsing search scan base tokens") } if len(got) != len(exp) { t.Fatalf("expected equal length whereins") } for i := range got { if got[i].name != exp[i].name { t.Fatalf("expected equal field names") } for j := range exp[i].valArr { if !got[i].match(exp[i].valArr[j]) { t.Fatalf("expected matching value arrays") } } } } } tests := map[string]tcase{ "upper case": { inputWhereins: []whereinT{ { name: "TEST", valArr: []field.Value{ field.ValueOf("1"), field.ValueOf("1"), }, }, }, expWhereins: []whereinT{ { name: "TEST", valArr: []field.Value{ field.ValueOf("1"), field.ValueOf("1"), }, }, }, }, "lower case": { inputWhereins: []whereinT{ { name: "test", valArr: []field.Value{ field.ValueOf("1"), field.ValueOf("1"), }, }, }, expWhereins: []whereinT{ { name: "test", valArr: []field.Value{ field.ValueOf("1"), field.ValueOf("1"), }, }, }, }, "mixed case": { inputWhereins: []whereinT{ { name: "teSt", valArr: []field.Value{ field.ValueOf("1"), field.ValueOf("1"), }, }, }, expWhereins: []whereinT{ { name: "teSt", valArr: []field.Value{ field.ValueOf("1"), field.ValueOf("1"), }, }, }, }, } for name, tc := range tests { t.Run(name, fn(tc)) } } // func testParseFloat(t testing.TB, s string, f float64, invalid bool) { // n, err := parseFloat(s) // if err != nil { // if invalid { // return // } // t.Fatal(err) // } // if invalid { // t.Fatalf("expecting an error for %s", s) // } // if n != f { // t.Fatalf("for '%s', expect %f, got %f", s, f, n) // } // } // func TestParseFloat(t *testing.T) { // testParseFloat(t, "100", 100, false) // testParseFloat(t, "0", 0, false) // testParseFloat(t, "-1", -1, false) // testParseFloat(t, "-0", -0, false) // testParseFloat(t, "-100", -100, false) // testParseFloat(t, "-0", -0, false) // testParseFloat(t, "+1", 1, false) // testParseFloat(t, "+0", 0, false) // testParseFloat(t, "33.102938", 33.102938, false) // testParseFloat(t, "-115.123123", -115.123123, false) // testParseFloat(t, ".1", 0.1, false) // testParseFloat(t, "0.1", 0.1, false) // testParseFloat(t, "00.1", 0.1, false) // testParseFloat(t, "01.1", 1.1, false) // testParseFloat(t, "01", 1, false) // testParseFloat(t, "-00.1", -0.1, false) // testParseFloat(t, "+00.1", 0.1, false) // testParseFloat(t, "", 0.1, true) // testParseFloat(t, " 0", 0.1, true) // testParseFloat(t, "0 ", 0.1, true) // } func BenchmarkLowerCompare(t *testing.B) { for i := 0; i < t.N; i++ { if !lc("HeLLo World", "hello world") { t.Fatal("failed") } } } func BenchmarkStringsLowerCompare(t *testing.B) { for i := 0; i < t.N; i++ { if strings.ToLower("HeLLo World") != "hello world" { t.Fatal("failed") } } } // func BenchmarkParseFloat(t *testing.B) { // s := []string{"33.10293", "-115.1203102"} // for i := 0; i < t.N; i++ { // _, err := parseFloat(s[i%2]) // if err != nil { // t.Fatal("failed") // } // } // } // func BenchmarkStrconvParseFloat(t *testing.B) { // s := []string{"33.10293", "-115.1203102"} // for i := 0; i < t.N; i++ { // _, err := strconv.ParseFloat(s[i%2], 64) // if err != nil { // t.Fatal("failed") // } // } // } ================================================ FILE: internal/sstring/sstring.go ================================================ // Package shared allows for package sstring import ( "sync" "unsafe" "github.com/tidwall/hashmap" ) var mu sync.Mutex var nums hashmap.Map[string, int] var strs []string // Load a shared string from its number. // Panics when there is no string assigned with that number. func Load(num int) (str string) { mu.Lock() if num >= 0 && num < len(strs) { str = strs[num] mu.Unlock() return str } mu.Unlock() panic("string not found") } // Store a shared string. // Returns a unique number that can be used to load the string later. // The number is al func Store(str string) (num int) { mu.Lock() var ok bool num, ok = nums.Get(str) if !ok { // Make a copy of the string to ensure we don't take in slices. b := make([]byte, len(str)) copy(b, str) str = *(*string)(unsafe.Pointer(&b)) num = len(strs) strs = append(strs, str) nums.Set(str, num) } mu.Unlock() return num } // Len returns the number of shared strings func Len() int { mu.Lock() n := len(strs) mu.Unlock() return n } ================================================ FILE: internal/sstring/sstring_test.go ================================================ package sstring import ( "math/rand" "testing" "time" "github.com/tidwall/assert" ) func TestShared(t *testing.T) { for i := -1; i < 10; i++ { var str string func() { defer func() { assert.Assert(recover().(string) == "string not found") }() str = Load(i) }() assert.Assert(str == "") } assert.Assert(Store("hello") == 0) assert.Assert(Store("") == 1) assert.Assert(Store("jello") == 2) assert.Assert(Store("hello") == 0) assert.Assert(Store("") == 1) assert.Assert(Store("jello") == 2) str := Load(0) assert.Assert(str == "hello") str = Load(1) assert.Assert(str == "") str = Load(2) assert.Assert(str == "jello") assert.Assert(Len() == 3) } func randStr(n int) string { b := make([]byte, n) rand.Read(b) for i := 0; i < n; i++ { b[i] = 'a' + b[i]%26 } return string(b) } func BenchmarkStore(b *testing.B) { rand.Seed(time.Now().UnixNano()) wmap := make(map[string]bool, b.N) for len(wmap) < b.N { wmap[randStr(10)] = true } words := make([]string, 0, b.N) for word := range wmap { words = append(words, word) } b.ResetTimer() for i := 0; i < b.N; i++ { Store(words[i]) } } func BenchmarkLoad(b *testing.B) { rand.Seed(time.Now().UnixNano()) wmap := make(map[string]bool, b.N) for len(wmap) < b.N { wmap[randStr(10)] = true } words := make([]string, 0, b.N) for word := range wmap { words = append(words, word) } var nums []int for i := 0; i < b.N; i++ { nums = append(nums, Store(words[i])) } rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] }) b.ResetTimer() for i := 0; i < b.N; i++ { Load(nums[i]) } } ================================================ FILE: internal/viewer/index.html ================================================ Tile38 Map Viewer
Choose Collection:
================================================ FILE: internal/viewer/viewer.go ================================================ package viewer import ( "bytes" "embed" "fmt" "io" "mime" "net/http" "os" "path/filepath" "strings" ) //go:embed * var files embed.FS func HandleHTTP(wr io.Writer, url string, devMode bool) error { if (!strings.HasPrefix(url, "/viewer/") && url != "/viewer") || strings.Contains(url, "..") { return writeHTTPResponse(wr, "404 Not Found", "text/html", nil, []byte("

404 Not Found

\n")) } if strings.HasSuffix(url, "/") { return writeHTTPResponse(wr, "307 Redirect", "text/html", []string{"Location", url[:len(url)-1]}, []byte("

307 Redirect

\n")) } if url == "/viewer" { return writeHTTPFile(wr, "/viewer/index.html", devMode) } return writeHTTPFile(wr, url, devMode) } func writeHTTPFile(wr io.Writer, path string, devMode bool) error { var data []byte err := os.ErrNotExist if devMode { data, err = os.ReadFile("internal" + path) } path = path[8:] if os.IsNotExist(err) { data, err = files.ReadFile(path) } if os.IsNotExist(err) { return writeHTTPResponse(wr, "404 Not Found", "text/html", nil, []byte("

404 Not Found

\n")) } return writeHTTPResponse(wr, "200 OK", mime.TypeByExtension(filepath.Ext(path)), nil, data) } func writeHTTPResponse(wr io.Writer, status, contentType string, headers []string, body []byte, ) error { var sheaders string if len(headers) > 0 { hdrs := http.Header{} for i := 0; i < len(headers)-1; i += 2 { hdrs.Set(headers[i], headers[i+1]) } var buf bytes.Buffer hdrs.Write(&buf) sheaders = buf.String() } payload := append([]byte(nil), fmt.Sprintf(""+ "HTTP/1.1 %s\r\n"+ "Connection: close\r\n"+ "Content-Type: %s\r\n"+ "Content-Length: %d\r\n"+ "Access-Control-Allow-Origin: *\r\n"+ sheaders+ "\r\n", status, contentType, len(body))...) payload = append(payload, body...) _, err := wr.Write(payload) return err } ================================================ FILE: scripts/RELEASE.md ================================================ **To bump a new release of Tile38** - Update CHANGELOG.md to include the newest changes. - `git commit -m $vers` changes (where `$vers` is a semver) - `git tag $vers` (where `$vers` is a semver) - `git push --tags` - `git push` - `make package` - Add a new Github Release and add the zips from packages directory. ================================================ FILE: scripts/build.sh ================================================ #!/bin/bash set -e cd $(dirname "${BASH_SOURCE[0]}")/.. if [ "$1" == "" ]; then echo "error: missing argument (binary name)" exit 1 fi # Check the Go installation if [ "$(which go)" == "" ]; then echo "error: Go is not installed. Please download and follow installation"\ "instructions at https://golang.org/dl to continue." exit 1 fi # Hardcode some values to the core package. if [ -d ".git" ]; then VERSION=$(git describe --tags --abbrev=0) GITSHA=$(git rev-parse --short HEAD) LDFLAGS="$LDFLAGS -X github.com/tidwall/tile38/core.Version=${VERSION}" LDFLAGS="$LDFLAGS -X github.com/tidwall/tile38/core.GitSHA=${GITSHA}" fi LDFLAGS="$LDFLAGS -X github.com/tidwall/tile38/core.BuildTime=$(date +%FT%T%z)" # Generate the core package core/gen.sh # Set final Go environment options LDFLAGS="$LDFLAGS -extldflags '-static'" export CGO_ENABLED=0 if [[ "$GORACE" == "1" ]]; then export CGO_ENABLED=1 goflags="$goflags -race" fi # Build and store objects into original directory. go build -ldflags "$LDFLAGS" $goflags -o $1 cmd/$1/*.go ================================================ FILE: scripts/docker-push.sh ================================================ #!/bin/bash set -e cd $(dirname "${BASH_SOURCE[0]}")/.. # GIT_BRANCH is the current branch name export GIT_BRANCH=$(git branch --show-current) # GIT_VERSION - always the last verison number, like 1.12.1. export GIT_VERSION=$(git describe --tags --abbrev=0) # GIT_COMMIT_SHORT - the short git commit number, like a718ef0. export GIT_COMMIT_SHORT=$(git rev-parse --short HEAD) # DOCKER_REPO - the base repository name to push the docker build to. export DOCKER_REPO=$DOCKER_USER/tile38 if [ "$GIT_BRANCH" != "master" ]; then echo "Not pushing, not on master" elif [ "$DOCKER_USER" == "" ]; then echo "Not pushing, DOCKER_USER not set" exit 1 elif [ "$DOCKER_LOGIN" == "" ]; then echo "Not pushing, DOCKER_LOGIN not set" exit 1 elif [ "$DOCKER_PASSWORD" == "" ]; then echo "Not pushing, DOCKER_PASSWORD not set" exit 1 else # setup cross platform builder # https://github.com/tonistiigi/binfmt docker run --privileged --rm tonistiigi/binfmt --install all docker buildx create --name multiarch --platform linux/amd64,linux/amd64/v2,linux/amd64/v3,linux/arm64,linux/386,linux/arm/v7 --use default # docker login echo $DOCKER_PASSWORD | docker login -u $DOCKER_LOGIN --password-stdin if [ "$(curl -s https://hub.docker.com/v2/repositories/$DOCKER_REPO/tags/$GIT_VERSION/ | grep "digest")" == "" ]; then # build the docker image docker buildx build \ -f Dockerfile \ --platform linux/arm64,linux/amd64 \ --build-arg VERSION=$GIT_VERSION \ --tag $DOCKER_REPO:$GIT_VERSION \ --tag $DOCKER_REPO:latest \ --tag $DOCKER_REPO:edge \ --push \ . else # build the docker image docker buildx build \ -f Dockerfile \ --platform linux/arm64,linux/amd64 \ --build-arg VERSION=$GIT_VERSION \ --tag $DOCKER_REPO:edge \ --push \ . fi fi ================================================ FILE: scripts/package.sh ================================================ #!/bin/bash set -e cd $(dirname "${BASH_SOURCE[0]}")/.. PLATFORM="$1" GOOS="$2" GOARCH="$3" VERSION=$(git describe --tags --abbrev=0) echo Packaging $PLATFORM Binary # Remove previous build directory, if needed. bdir=tile38-$VERSION-$GOOS-$GOARCH rm -rf packages/$bdir && mkdir -p packages/$bdir # Make the binaries. GOOS=$GOOS GOARCH=$GOARCH make all rm -f tile38-luamemtest # not needed # Copy the executable binaries. if [ "$GOOS" == "windows" ]; then mv tile38-server packages/$bdir/tile38-server.exe mv tile38-cli packages/$bdir/tile38-cli.exe mv tile38-benchmark packages/$bdir/tile38-benchmark.exe else mv tile38-server packages/$bdir mv tile38-cli packages/$bdir mv tile38-benchmark packages/$bdir fi # Copy documention and license. cp README.md packages/$bdir cp CHANGELOG.md packages/$bdir cp LICENSE packages/$bdir # Compress the package. cd packages if [ "$GOOS" == "linux" ]; then tar -zcf $bdir.tar.gz $bdir else zip -r -q $bdir.zip $bdir fi ================================================ FILE: scripts/test.sh ================================================ #!/bin/bash set -e cd $(dirname "${BASH_SOURCE[0]}")/.. export CGO_ENABLED=0 cd tests go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out $GOTEST # go test -coverpkg=../internal/... -coverprofile=/tmp/coverage.out \ # -v ./... $GOTEST go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html echo "details: file:///tmp/coverage.html" cd .. if [[ "$GOTEST" == "" ]]; then go test $(go list ./... | grep -v /vendor/ | grep -v /tests) fi ================================================ FILE: tests/107/.gitignore ================================================ appendonly.aof log data/ ================================================ FILE: tests/107/LINK ================================================ https://github.com/tidwall/tile38/issues/107 ================================================ FILE: tests/107/main.go ================================================ package main import ( "archive/zip" "bytes" "flag" "fmt" "io" "io/ioutil" "log" "math/rand" "net/http" "os" "os/exec" "path" "strconv" "strings" "sync/atomic" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" "github.com/tidwall/tile38/internal/server" ) const tile38Port = 9191 const httpPort = 9292 const dir = "data" var tile38Addr string var httpAddr string var wd string var minX float64 var minY float64 var maxX float64 var maxY float64 var pool = &redis.Pool{ MaxIdle: 3, IdleTimeout: 240 * time.Second, Dial: func() (redis.Conn, error) { return redis.Dial("tcp", tile38Addr) }, } var providedTile38 bool var providedHTTP bool const blank = false const hookServer = true var logf *os.File func main() { flag.StringVar(&tile38Addr, "tile38", "", "Tile38 address, leave blank to start a new server") flag.StringVar(&httpAddr, "hook", "", "Hook HTTP url, leave blank to start a new server") flag.Parse() log.Println("mockfill-107 (Github #107: Memory leak)") if tile38Addr == "" { tile38Addr = "127.0.0.1:" + strconv.FormatInt(int64(tile38Port), 10) } else { providedTile38 = true } if httpAddr == "" { httpAddr = "http://127.0.0.1:" + strconv.FormatInt(int64(httpPort), 10) + "/hook" } else { providedHTTP = true } var err error wd, err = os.Getwd() if err != nil { log.Fatal(err) } logf, err = os.Create("log") if err != nil { log.Fatal(err) } defer logf.Close() if !providedTile38 { copyAOF() go startTile38Server() } if !providedHTTP { if hookServer { go startHookServer() } } go waitForServers(func() { log.Printf("servers ready") logServer("START") setPoints() logServer("DONE") }) select {} return } func startTile38Server() { log.Println("start tile38 server") opts := server.Options{ Host: "localhost", Port: tile38Port, Dir: "data", UseHTTP: false, MetricsAddr: "", } err := server.Serve(opts) if err != nil { log.Fatal(err) } } func startHookServer() { log.Println("start hook server") http.HandleFunc("/ping", func(w http.ResponseWriter, _ *http.Request) { io.WriteString(w, "pong") }) http.HandleFunc("/hook", func(w http.ResponseWriter, req *http.Request) { data, err := ioutil.ReadAll(req.Body) if err != nil { log.Fatal(err) } log.Println(string(data)) }) err := http.ListenAndServe(fmt.Sprintf("127.0.0.1:%d", httpPort), nil) if err != nil { log.Fatal(err) } } func waitForServers(cb func()) { log.Println("wait for servers") var err error start := time.Now() for { if time.Since(start) > time.Second*5 { log.Fatal("connection failed:", err) } func() { conn := pool.Get() defer conn.Close() var s string s, err = redis.String(conn.Do("PING")) if err != nil { return } if s != "PONG" { log.Fatalf("expected '%v', got '%v'", "PONG", s) } }() if err == nil { break } time.Sleep(time.Second / 5) } if hookServer { start = time.Now() for { if time.Since(start) > time.Second*5 { log.Fatal("connection failed:", err) } func() { var resp *http.Response resp, err = http.Get(httpAddr + "/notreal") if err != nil { return } defer resp.Body.Close() if resp.StatusCode != 200 && resp.StatusCode != 404 { log.Fatalf("expected '%v', got '%v'", "200 or 404", resp.StatusCode) } }() if err == nil { break } time.Sleep(time.Second / 5) } } cb() } func downloadAOF() { log.Println("downloading aof") resp, err := http.Get("https://github.com/tidwall/tile38/files/675225/appendonly.aof.zip") if err != nil { log.Fatal(err) } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Fatal(err) } rd, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) if err != nil { log.Fatal(err) } for _, f := range rd.File { if path.Ext(f.Name) == ".aof" { rc, err := f.Open() if err != nil { log.Fatal(err) } defer rc.Close() data, err := ioutil.ReadAll(rc) if err != nil { log.Fatal(err) } err = ioutil.WriteFile(path.Join(wd, "appendonly.aof"), data, 0666) if err != nil { log.Fatal(err) } return } } log.Fatal("invalid appendonly.aof.zip") } func copyAOF() { if err := os.RemoveAll(path.Join(wd, "data")); err != nil { log.Fatal(err) } if err := os.MkdirAll(path.Join(wd, "data"), 0777); err != nil { log.Fatal(err) } fin, err := os.Open(path.Join(wd, "appendonly.aof")) if err != nil { if os.IsNotExist(err) { downloadAOF() fin, err = os.Open(path.Join(wd, "appendonly.aof")) if err != nil { log.Fatal(err) } } else { log.Fatal(err) } } defer fin.Close() log.Println("load aof") fout, err := os.Create(path.Join(wd, "data", "appendonly.aof")) if err != nil { log.Fatal(err) } defer fout.Close() data, err := ioutil.ReadAll(fin) if err != nil { log.Fatal(err) } rep := httpAddr rep = "$" + strconv.FormatInt(int64(len(rep)), 10) + "\r\n" + rep + "\r\n" data = bytes.Replace(data, []byte("$23\r\nhttp://172.17.0.1:9999/\r\n"), []byte(rep), -1) if blank { data = nil } if _, err := fout.Write(data); err != nil { log.Fatal(err) } } func respGet(resp interface{}, idx ...int) interface{} { for i := 0; i < len(idx); i++ { arr, _ := redis.Values(resp, nil) resp = arr[idx[i]] } return resp } type PSAUX struct { User string PID int CPU float64 Mem float64 VSZ int RSS int TTY string Stat string Start string Time string Command string } func atoi(s string) int { n, _ := strconv.ParseInt(s, 10, 64) return int(n) } func atof(s string) float64 { n, _ := strconv.ParseFloat(s, 64) return float64(n) } func psaux(pid int) PSAUX { var res []byte res, err := exec.Command("ps", "ux", "-p", strconv.FormatInt(int64(pid), 10)).CombinedOutput() if err != nil { return PSAUX{} } pids := strconv.FormatInt(int64(pid), 10) for _, line := range strings.Split(string(res), "\n") { var words []string for _, word := range strings.Split(line, " ") { if word != "" { words = append(words, word) } } if len(words) >= 11 { if words[1] == pids { return PSAUX{ User: words[0], PID: atoi(words[1]), CPU: atof(words[2]), Mem: atof(words[3]), VSZ: atoi(words[4]), RSS: atoi(words[5]), TTY: words[6], Stat: words[7], Start: words[8], Time: words[9], Command: words[10], } } } } return PSAUX{} } func respGetFloat(resp interface{}, idx ...int) float64 { resp = respGet(resp, idx...) f, _ := redis.Float64(resp, nil) return f } func logServer(tag string) { conn := pool.Get() defer conn.Close() _, err := conn.Do("OUTPUT", "json") if err != nil { log.Fatal(err) } _, err = redis.String(conn.Do("GC")) if err != nil { log.Fatal(err) } json, err := redis.String(conn.Do("SERVER")) if err != nil { log.Fatal(err) } _, err = conn.Do("OUTPUT", "resp") if err != nil { log.Fatal(err) } rss := float64(psaux(int(gjson.Get(json, "stats.pid").Int())).RSS) / 1024 heapSize := gjson.Get(json, "stats.heap_size").Float() / 1024 / 1024 heapReleased := gjson.Get(json, "stats.heap_released").Float() / 1024 / 1024 fmt.Fprintf(logf, "%s %10.2f MB (heap) %10.2f MB (released) %10.2f MB (system)\n", time.Now().Format("2006-01-02T15:04:05Z07:00"), heapSize, heapReleased, rss) } func setPoints() { go func() { var i int for range time.NewTicker(time.Second * 1).C { logServer(fmt.Sprintf("SECOND-%d", i*1)) i++ } }() rand.Seed(time.Now().UnixNano()) n := 1000000 ex := time.Second * 10 log.Printf("time to pump data (%d points, expires %s)", n, ex) conn := pool.Get() defer conn.Close() if blank { minX = -124.40959167480469 minY = 32.53415298461914 maxX = -114.13121032714844 maxY = 42.009521484375 } else { resp, err := conn.Do("bounds", "boundies") if err != nil { log.Fatal(err) } minX = respGetFloat(resp, 0, 0) minY = respGetFloat(resp, 0, 1) maxX = respGetFloat(resp, 1, 0) maxY = respGetFloat(resp, 1, 1) } log.Printf("bbox: [[%.4f,%.4f],[%.4f,%.4f]]\n", minX, minY, maxX, maxY) var idx uint64 for i := 0; i < 4; i++ { go func() { conn := pool.Get() defer conn.Close() for i := 0; i < n; i++ { atomic.AddUint64(&idx, 1) id := fmt.Sprintf("person:%d", idx) x := rand.Float64()*(maxX-minX) + minX y := rand.Float64()*(maxY-minY) + minY ok, err := redis.String(conn.Do("SET", "people", id, "EX", float64(ex/time.Second), "POINT", y, x)) if err != nil { log.Fatal(err) } if ok != "OK" { log.Fatalf("expected 'OK', got '%v", ok) } log.Printf("SET people %v EX %v POINT %v %v", id, float64(ex/time.Second), y, x) } }() } select {} } ================================================ FILE: tests/616/main.go ================================================ // Test Tile38 for Expiration Drift // Issue #616 package main import ( "fmt" "math/rand" "sync" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/btree" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) const exsecs = 10 const key = "__issue_616__" func makeID() string { const chars = "0123456789abcdefghijklmnopqrstuvwxyz-" var buf [10]byte rand.Read(buf[:]) for i := 0; i < len(buf); i++ { buf[i] = chars[int(buf[i])%len(chars)] } return string(buf[:]) } func main() { fmt.Printf( "The SCAN and ACTUAL values should reach about 1850 and stay\n" + "roughly the same from there on.\n") var mu sync.Mutex objs := btree.NewNonConcurrent(func(a, b interface{}) bool { ajson := a.(string) bjson := b.(string) return gjson.Get(ajson, "id").String() < gjson.Get(bjson, "id").String() }) expires := btree.NewNonConcurrent(func(a, b interface{}) bool { ajson := a.(string) bjson := b.(string) if gjson.Get(ajson, "properties.ex").Int() < gjson.Get(bjson, "properties.ex").Int() { return true } if gjson.Get(ajson, "properties.ex").Int() > gjson.Get(bjson, "properties.ex").Int() { return false } return gjson.Get(ajson, "id").String() < gjson.Get(bjson, "id").String() }) conn := must(redis.Dial("tcp", ":9851")).(redis.Conn) must(conn.Do("DROP", key)) must(nil, conn.Close()) go func() { conn := must(redis.Dial("tcp", ":9851")).(redis.Conn) defer conn.Close() for { ex := time.Now().UnixNano() + int64(exsecs*time.Second) for i := 0; i < 10; i++ { id := makeID() x := rand.Float64()*360 - 180 y := rand.Float64()*180 - 90 obj := fmt.Sprintf(`{"type":"Feature","geometry":{"type":"Point","coordinates":[%f,%f]},"properties":{}}`, x, y) obj, _ = sjson.Set(obj, "properties.ex", ex) obj, _ = sjson.Set(obj, "id", id) res := must(redis.String(conn.Do("SET", key, id, "ex", exsecs, "OBJECT", obj))).(string) if res != "OK" { panic(fmt.Sprintf("expected 'OK', got '%s'", res)) } mu.Lock() prev := objs.Set(obj) if prev != nil { expires.Delete(obj) } expires.Set(obj) mu.Unlock() } time.Sleep(time.Second / 20) } }() go func() { conn := must(redis.Dial("tcp", ":9851")).(redis.Conn) defer conn.Close() for { time.Sleep(time.Second * 5) must(conn.Do("AOFSHRINK")) } }() go func() { conn := must(redis.Dial("tcp", ":9851")).(redis.Conn) defer conn.Close() must(conn.Do("OUTPUT", "JSON")) for { time.Sleep(time.Second / 10) var ids []string res := must(redis.String(conn.Do("SCAN", key, "LIMIT", 100000000))).(string) gjson.Get(res, "objects").ForEach(func(_, res gjson.Result) bool { ids = append(ids, res.Get("id").String()) return true }) now := time.Now().UnixNano() mu.Lock() var exobjs []string expires.Ascend(nil, func(v interface{}) bool { ex := gjson.Get(v.(string), "properties.ex").Int() if ex > now { return false } exobjs = append(exobjs, v.(string)) return true }) for _, obj := range exobjs { objs.Delete(obj) expires.Delete(obj) } fmt.Printf("\rSCAN: %d, ACTUAL: %d ", len(ids), objs.Len()) mu.Unlock() } }() select {} } func must(v interface{}, err error) interface{} { if err != nil { panic(err) } return v } ================================================ FILE: tests/README.md ================================================ ## Tile38 Integration Testing - Uses Redis protocol - The Tile38 data is flushed before every `DoBatch` A basic test operation looks something like: ```go func keys_SET_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "fleet", "truck1", "POINT", 33.0001, -112.0001}, {"OK"}, {"GET", "fleet", "truck1", "POINT"}, {"[33.0001 -112.0001]"}, } } ``` Using a custom function: ```go func keys_MATCH_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "fleet", "truck1", "POINT", 33.0001, -112.0001}, { func(v interface{}) (resp, expect interface{}) { // v is the value as strings or slices of strings // test will pass as long as `resp` and `expect` are the same. return v, "OK" }, }, } } ``` ================================================ FILE: tests/aof_test.go ================================================ package tests import ( "bytes" "crypto/md5" "encoding/hex" "errors" "fmt" "io" "math/rand" "net" "net/http" "sync/atomic" "time" "github.com/gomodule/redigo/redis" _ "embed" ) func subTestAOF(g *testGroup) { g.regSubTest("loading", aof_loading_test) g.regSubTest("migrate", aof_migrate_test) g.regSubTest("AOF", aof_AOF_test) g.regSubTest("AOFMD5", aof_AOFMD5_test) g.regSubTest("AOFSHRINK", aof_AOFSHRINK_test) g.regSubTest("READONLY", aof_READONLY_test) } func loadAOFAndClose(aof any) error { mc, err := loadAOF(aof) if mc != nil { mc.Close() } return err } func loadAOF(aof any) (*mockServer, error) { var aofb []byte switch aof := aof.(type) { case []byte: aofb = []byte(aof) case string: aofb = []byte(aof) default: return nil, errors.New("aof is not string or bytes") } return mockOpenServer(MockServerOptions{ Silent: true, Metrics: false, AOFData: aofb, }) } func aof_loading_test(mc *mockServer) error { var err error // invalid command err = loadAOFAndClose("asdfasdf\r\n") if err == nil || err.Error() != "unknown command 'asdfasdf'" { return fmt.Errorf("expected '%v', got '%v'", "unknown command 'asdfasdf'", err) } // incomplete command err = loadAOFAndClose("set fleet truck point 10 10\r\nasdfasdf") if err != nil { return err } // big aof file var aof string for i := 0; i < 10000; i++ { aof += fmt.Sprintf("SET fleet truck%d POINT 10 10\r\n", i) } err = loadAOFAndClose(aof) if err != nil { return err } // extra zeros at various places aof = "" for i := 0; i < 1000; i++ { if i%10 == 0 { aof += string(bytes.Repeat([]byte{0}, 100)) } aof += fmt.Sprintf("SET fleet truck%d POINT 10 10\r\n", i) } aof += string(bytes.Repeat([]byte{0}, 5000)) err = loadAOFAndClose(aof) if err != nil { return err } // bad protocol aof = "*2\r\n$1\r\nh\r\n+OK\r\n" err = loadAOFAndClose(aof) if fmt.Sprintf("%v", err) != "Protocol error: expected '$', got '+'" { return fmt.Errorf("expected '%v', got '%v'", "Protocol error: expected '$', got '+'", err) } return nil } func aof_AOFMD5_test(mc *mockServer) error { for i := 0; i < 10000; i++ { _, err := mc.Do("SET", "fleet", rand.Int(), "POINT", rand.Float64()*180-90, rand.Float64()*360-180) if err != nil { return err } } aof, err := mc.readAOF() if err != nil { return err } check := func(start, size int) func(s string) error { return func(s string) error { sum := md5.Sum(aof[start : start+size]) val := hex.EncodeToString(sum[:]) if s != val { return fmt.Errorf("expected '%s', got '%s'", val, s) } return nil } } return mc.DoBatch( Do("AOFMD5").Err("wrong number of arguments for 'aofmd5' command"), Do("AOFMD5", 0).Err("wrong number of arguments for 'aofmd5' command"), Do("AOFMD5", 0, 0, 1).Err("wrong number of arguments for 'aofmd5' command"), Do("AOFMD5", -1, 0).Err("invalid argument '-1'"), Do("AOFMD5", 1, -1).Err("invalid argument '-1'"), Do("AOFMD5", 0, 100000000000).Err("EOF"), Do("AOFMD5", 0, 0).Str("d41d8cd98f00b204e9800998ecf8427e"), Do("AOFMD5", 0, 0).JSON().Str(`{"ok":true,"md5":"d41d8cd98f00b204e9800998ecf8427e"}`), Do("AOFMD5", 0, 0).Func(check(0, 0)), Do("AOFMD5", 0, 1).Func(check(0, 1)), Do("AOFMD5", 0, 100).Func(check(0, 100)), Do("AOFMD5", 1002, 4321).Func(check(1002, 4321)), ) } func openFollower(mc *mockServer) (conn redis.Conn, err error) { conn, err = redis.Dial("tcp", fmt.Sprintf(":%d", mc.port), redis.DialReadTimeout(time.Second)) if err != nil { return nil, err } defer func() { if err != nil { conn.Close() conn = nil } }() if err := conn.Send("AOF", 0); err != nil { return nil, err } if err := conn.Flush(); err != nil { return nil, err } str, err := redis.String(conn.Receive()) if err != nil { return nil, err } if str != "OK" { return nil, fmt.Errorf("expected '%s', got '%s'", "OK", str) } return conn, nil } func aof_AOF_test(mc *mockServer) error { var argss [][]interface{} for i := 0; i < 10000; i++ { args := []interface{}{"SET", "fleet", fmt.Sprint(rand.Int()), "POINT", fmt.Sprint(rand.Float64()*180 - 90), fmt.Sprint(rand.Float64()*360 - 180)} argss = append(argss, args) _, err := mc.Do(fmt.Sprint(args[0]), args[1:]...) if err != nil { return err } } readAll := func() (conn redis.Conn, err error) { conn, err = openFollower(mc) if err != nil { return } defer func() { if err != nil { conn.Close() conn = nil } }() var t bool for i := 0; i < len(argss); i++ { args, err := redis.Values(conn.Receive()) if err != nil { return nil, err } if t || (len(args) == len(argss[0]) && fmt.Sprintf("%s", args[2]) == fmt.Sprintf("%s", argss[0][2])) { t = true if fmt.Sprintf("%s", args[2]) != fmt.Sprintf("%s", argss[i][2]) { return nil, fmt.Errorf("expected '%s', got '%s'", argss[i][2], args[2]) } } else { i-- } } return conn, nil } conn, err := readAll() if err != nil { return err } defer conn.Close() _, err = conn.Do("fancy") // non-existent error if err == nil || err.Error() != "EOF" { return fmt.Errorf("expected '%v', got '%v'", "EOF", err) } conn, err = readAll() if err != nil { return err } defer conn.Close() _, err = conn.Do("quit") if err == nil || err.Error() != "EOF" { return fmt.Errorf("expected '%v', got '%v'", "EOF", err) } return mc.DoBatch( Do("AOF").Err("wrong number of arguments for 'aof' command"), Do("AOF", 0, 0).Err("wrong number of arguments for 'aof' command"), Do("AOF", -1).Err("invalid argument '-1'"), Do("AOF", 1000000000000).Err("pos is too big, must be less that the aof_size of leader"), ) } func aof_AOFSHRINK_test(mc *mockServer) error { var err error haddr := fmt.Sprintf("localhost:%d", getNextPort()) ln, err := net.Listen("tcp", haddr) if err != nil { return err } defer ln.Close() var msgs atomic.Int32 go func() { mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { msgs.Add(1) // println(r.URL.Path) }) http.Serve(ln, mux) }() err = mc.DoBatch( Do("SETCHAN", "mychan", "INTERSECTS", "mi:0", "BOUNDS", -10, -10, 10, 10).Str("1"), Do("SETHOOK", "myhook", "http://"+haddr, "INTERSECTS", "mi:0", "BOUNDS", -10, -10, 10, 10).Str("1"), Do("MASSINSERT", 5, 10000).OK(), ) if err != nil { return err } err = mc.DoBatch( Do("AOFSHRINK").OK(), Do("MASSINSERT", 5, 10000).OK(), ) if err != nil { return err } nmsgs := msgs.Load() if nmsgs == 0 { return fmt.Errorf("expected > 0, got %d", nmsgs) } return err } func aof_READONLY_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "POINT", "10", "10").OK(), Do("READONLY", "yes").OK(), Do("SET", "mykey", "myid", "POINT", "10", "10").Err("read only"), Do("READONLY", "no").OK(), Do("SET", "mykey", "myid", "POINT", "10", "10").OK(), Do("READONLY").Err("wrong number of arguments for 'readonly' command"), Do("READONLY", "maybe").Err("invalid argument 'maybe'"), ) } //go:embed aof_legacy var aofLegacy []byte func aof_migrate_test(mc *mockServer) error { var aof []byte for i := 0; i < 10000; i++ { aof = append(aof, aofLegacy...) } var mc2 *mockServer var err error defer func() { mc2.Close() }() mc2, err = mockOpenServer(MockServerOptions{ AOFFileName: "aof", AOFData: aof, Silent: true, Metrics: true, }) if err != nil { return err } err = mc2.DoBatch( Do("GET", "1", "2").Str(`{"type":"Point","coordinates":[20,10]}`), ) if err != nil { return err } mc2.Close() mc2, err = mockOpenServer(MockServerOptions{ AOFFileName: "aof", AOFData: aofLegacy[:len(aofLegacy)-1], Silent: true, Metrics: true, }) if err != io.ErrUnexpectedEOF { return fmt.Errorf("expected '%v', got '%v'", io.ErrUnexpectedEOF, err) } mc2.Close() mc2, err = mockOpenServer(MockServerOptions{ AOFFileName: "aof", AOFData: aofLegacy[1:], Silent: true, Metrics: true, }) if err != io.ErrUnexpectedEOF { return fmt.Errorf("expected '%v', got '%v'", io.ErrUnexpectedEOF, err) } mc2.Close() return nil } ================================================ FILE: tests/client_test.go ================================================ package tests import ( "errors" "fmt" "strings" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" "github.com/tidwall/pretty" ) func subTestClient(g *testGroup) { g.regSubTest("OUTPUT", client_OUTPUT_test) g.regSubTest("CLIENT", client_CLIENT_test) } func client_OUTPUT_test(mc *mockServer) error { if err := mc.DoBatch( // tests removal of "elapsed" member. Do("OUTPUT", "json", "yaml").Err(`wrong number of arguments for 'output' command`), Do("OUTPUT", "json").Str(`{"ok":true}`), Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`), Do("OUTPUT").Str(`resp`), // this is due to the internal Do test Do("OUTPUT", "resp").OK(), Do("OUTPUT", "yaml").Err(`invalid argument 'yaml'`), Do("OUTPUT").Str(`resp`), Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`), ); err != nil { return err } // run direct commands if _, err := mc.Do("OUTPUT", "json"); err != nil { return err } res, err := mc.Do("CLIENT", "list") if err != nil { return err } bres, ok := res.([]byte) if !ok { return errors.New("Failed to type assert CLIENT response") } sres := string(bres) if !gjson.Valid(sres) { return errors.New("CLIENT response was invalid") } info := gjson.Get(sres, "list").String() if !gjson.Valid(info) { return errors.New("CLIENT.list response was invalid") } return nil } func client_CLIENT_test(mc *mockServer) error { numConns := 20 var conns []redis.Conn defer func() { for i := range conns { conns[i].Close() } }() for i := 0; i <= numConns; i++ { conn, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } conn.Do("PING") conns = append(conns, conn) } _, err := conns[1].Do("CLIENT", "setname", "cl1") if err != nil { return err } _, err = conns[2].Do("CLIENT", "setname", "cl2") if err != nil { return err } if _, err := mc.Do("OUTPUT", "JSON"); err != nil { return err } res, err := mc.Do("CLIENT", "list") if err != nil { return err } bres, ok := res.([]byte) if !ok { return errors.New("Failed to type assert CLIENT response") } sres := string(pretty.Pretty(bres)) if int(gjson.Get(sres, "list.#").Int()) < numConns { return errors.New("Invalid number of connections") } client13ID := gjson.Get(sres, "list.13.id").String() client14Addr := gjson.Get(sres, "list.14.addr").String() client15Addr := gjson.Get(sres, "list.15.addr").String() return mc.DoBatch( Do("CLIENT", "list").JSON().Func(func(s string) error { if int(gjson.Get(s, "list.#").Int()) < numConns { return errors.New("Invalid number of connections") } return nil }), Do("CLIENT", "list").Func(func(s string) error { if len(strings.Split(strings.TrimSpace(s), "\n")) < numConns { return errors.New("Invalid number of connections") } return nil }), Do("CLIENT").Err(`wrong number of arguments for 'client' command`), Do("CLIENT", "hello").Err(`Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)`), Do("CLIENT", "list", "arg3").Err(`wrong number of arguments for 'client' command`), Do("CLIENT", "getname", "arg3").Err(`wrong number of arguments for 'client' command`), Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":""}`), Do("CLIENT", "getname").Str(``), Do("CLIENT", "setname", "abc").OK(), Do("CLIENT", "getname").Str(`abc`), Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":"abc"}`), Do("CLIENT", "setname", "abc", "efg").Err(`wrong number of arguments for 'client' command`), Do("CLIENT", "setname", " abc ").Err(`Client names cannot contain spaces, newlines or special characters.`), Do("CLIENT", "setname", "abcd").JSON().OK(), Do("CLIENT", "kill", "name", "abcd").Err("No such client"), Do("CLIENT", "getname").Str(`abcd`), Do("CLIENT", "kill").Err(`wrong number of arguments for 'client' command`), Do("CLIENT", "kill", "").Err(`No such client`), Do("CLIENT", "kill", "abcd").Err(`No such client`), Do("CLIENT", "kill", "id", client13ID).OK(), Do("CLIENT", "kill", "id").Err("wrong number of arguments for 'client' command"), Do("CLIENT", "kill", client14Addr).OK(), Do("CLIENT", "kill", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"), Do("CLIENT", "kill", "addr").Err("wrong number of arguments for 'client' command"), Do("CLIENT", "kill", "addr", client15Addr).JSON().OK(), Do("CLIENT", "kill", "addr", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"), Do("CLIENT", "kill", "id", "1000").Err("No such client"), ) } ================================================ FILE: tests/fence_roaming_test.go ================================================ package tests import ( "fmt" "io" "net/http" "net/http/httptest" "sync" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/pretty" "github.com/tidwall/sjson" ) func fence_roaming_webhook_test(mc *mockServer) error { car1, car2, expected := roamingTestData() finalErr := make(chan error) // Create a connection for subscribing to geofence notifications sc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer sc.Close() actual := []string{} // Create the test http server that will capture all messages ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := func() error { // Read the request body body, err := io.ReadAll(r.Body) if err != nil { return err } // If the new message doesn't match whats expected an error // should be returned actual = append(actual, cleanMessage(body)) pos := len(actual) - 1 if len(expected) < pos+1 { return fmt.Errorf("More messages than expected were received : '%s'", actual[pos]) } if actual[pos] != expected[pos] { return fmt.Errorf("Expected '%s' but got '%s'", expected[pos], actual[pos]) } if len(actual) == len(expected) { finalErr <- nil } return nil }(); err != nil { finalErr <- err } fmt.Fprintln(w, "OK!") })) defer ts.Close() _, err = sc.Do("SETHOOK", "carshook", ts.URL, "NEARBY", "cars", "FENCE", "ROAM", "cars", "*", 1000) if err != nil { return err } // Create the base connection for setting up points and geofences bc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer bc.Close() // Fire all car movement commands on the base client for i := range car1 { if _, err := bc.Do("SET", "cars", "car1", "POINT", car1[i][1], car1[i][0]); err != nil { return err } if _, err := bc.Do("SET", "cars", "car2", "POINT", car2[i][1], car2[i][0]); err != nil { return err } } return <-finalErr } func goMultiFunc(mc *mockServer, fns ...func() error) error { errs := make([]error, len(fns)) var wg sync.WaitGroup wg.Add(len(fns)) for i := 0; i < len(fns); i++ { go func(i int) { defer wg.Done() errs[i] = fns[i]() }(i) } wg.Wait() var ferrs []error for i := 0; i < len(errs); i++ { if errs[i] != nil { ferrs = append(ferrs, errs[i]) } } if len(ferrs) == 0 { return nil } if len(ferrs) == 1 { return ferrs[0] } return fmt.Errorf("%v", ferrs) } func fence_roaming_live_test(mc *mockServer) error { car1, car2, expected := roamingTestData() var liveReady sync.WaitGroup liveReady.Add(1) return goMultiFunc(mc, func() error { sc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port), redis.DialConnectTimeout(0), redis.DialReadTimeout(time.Second*5), redis.DialWriteTimeout(time.Second*5)) if err != nil { liveReady.Done() return err } defer sc.Close() // Set up a live geofence stream reply, err := redis.String( sc.Do("NEARBY", "cars", "FENCE", "ROAM", "cars", "*", 1000), ) if err != nil { liveReady.Done() return err } if reply != "OK" { liveReady.Done() return fmt.Errorf("expected 'OK', got '%v'", reply) } liveReady.Done() for i := 0; i < len(expected); i++ { reply, err := redis.String(sc.Receive()) if err != nil { return err } reply = cleanMessage([]byte(reply)) if reply != expected[i] { return fmt.Errorf("Expected '%s' but got '%s'", expected[i], reply) } } return nil }, func() error { liveReady.Wait() bc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer bc.Close() // Fire all car movement commands on the base client for i := range car1 { if _, err := bc.Do("SET", "cars", "car1", "POINT", car1[i][1], car1[i][0]); err != nil { return err } if _, err := bc.Do("SET", "cars", "car2", "POINT", car2[i][1], car2[i][0]); err != nil { return err } } return nil }, ) } func fence_roaming_channel_test(mc *mockServer) error { car1, car2, expected := roamingTestData() finalErr := make(chan error) go func() { // Create a connection for subscribing to geofence notifications sc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { finalErr <- err return } defer sc.Close() if _, err := sc.Do("SETCHAN", "carschan", "NEARBY", "cars", "FENCE", "ROAM", "cars", "*", 1000); err != nil { finalErr <- err return } // Subscribe the subscription client to the * pattern psc := redis.PubSubConn{Conn: sc} if err := psc.PSubscribe("carschan"); err != nil { finalErr <- err return } actual := []string{} for sc.Err() == nil { if err := func() error { var body []byte switch v := psc.Receive().(type) { case redis.Message: body = v.Data case error: return err } if len(body) == 0 { return nil } // If the new message doesn't match whats expected an error // should be returned actual = append(actual, cleanMessage(body)) pos := len(actual) - 1 if len(expected) < pos+1 { return fmt.Errorf("More messages than expected were received : '%s'", actual[pos]) } if actual[pos] != expected[pos] { return fmt.Errorf("Expected '%s' but got '%s'", expected[pos], actual[pos]) } if len(actual) == len(expected) { finalErr <- nil } return nil }(); err != nil { finalErr <- err } } }() // Create the base connection for setting up points and geofences bc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer bc.Close() // Fire all car movement commands on the base client for i := range car1 { if _, err := bc.Do("SET", "cars", "car1", "POINT", car1[i][1], car1[i][0]); err != nil { return err } if _, err := bc.Do("SET", "cars", "car2", "POINT", car2[i][1], car2[i][0]); err != nil { return err } } return <-finalErr } func cleanMessage(body []byte) string { // Remove fields that are non-deterministic or use case specific msg, _ := sjson.Delete(string(body), "group") msg, _ = sjson.Delete(msg, "time") msg, _ = sjson.Delete(msg, "hook") msg = string(pretty.Ugly([]byte(msg))) return msg } func roamingTestData() (car1 [][]float64, car2 [][]float64, output []string) { car1 = [][]float64{ {-111.93669319152832, 33.414750027566235}, {-111.93051338195801, 33.414750027566235}, {-111.92416191101074, 33.414750027566235}, {-111.91789627075195, 33.414750027566235}, {-111.9111156463623, 33.414750027566235}, {-111.90510749816895, 33.414750027566235}, {-111.89746856689453, 33.414750027566235}, } car2 = [][]float64{ {-111.89746856689453, 33.414750027566235}, {-111.90519332885742, 33.414750027566235}, {-111.91154479980467, 33.414750027566235}, {-111.91781044006346, 33.414750027566235}, {-111.92416191101074, 33.414750027566235}, {-111.93059921264648, 33.414750027566235}, {-111.93660736083984, 33.414750027566235}, } output = []string{ `{"command":"set","detect":"roam","key":"cars","id":"car1","object":{"type":"Point","coordinates":[-111.91789627075195,33.414750027566235]},"nearby":{"key":"cars","id":"car2","object":{"type":"Point","coordinates":[-111.91154479980467,33.414750027566235]},"meters":589.512}}`, `{"command":"set","detect":"roam","key":"cars","id":"car2","object":{"type":"Point","coordinates":[-111.91781044006346,33.414750027566235]},"nearby":{"key":"cars","id":"car1","object":{"type":"Point","coordinates":[-111.91789627075195,33.414750027566235]},"meters":7.966}}`, `{"command":"set","detect":"roam","key":"cars","id":"car1","object":{"type":"Point","coordinates":[-111.9111156463623,33.414750027566235]},"nearby":{"key":"cars","id":"car2","object":{"type":"Point","coordinates":[-111.91781044006346,33.414750027566235]},"meters":621.377}}`, `{"command":"set","detect":"roam","key":"cars","id":"car2","object":{"type":"Point","coordinates":[-111.92416191101074,33.414750027566235]},"faraway":{"key":"cars","id":"car1","object":{"type":"Point","coordinates":[-111.9111156463623,33.414750027566235]},"meters":1210.89}}`, } return } ================================================ FILE: tests/fence_test.go ================================================ package tests import ( "bufio" "encoding/json" "errors" "fmt" "io" "log" "net" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" ) func subTestFence(g *testGroup) { // Standard g.regSubTest("basic", fence_basic_test) g.regSubTest("channel message order", fence_channel_message_order_test) g.regSubTest("detect inside,outside", fence_detect_inside_test) // Roaming g.regSubTest("roaming live", fence_roaming_live_test) g.regSubTest("roaming channel", fence_roaming_channel_test) g.regSubTest("roaming webhook", fence_roaming_webhook_test) // channel meta g.regSubTest("channel meta", fence_channel_meta_test) // various g.regSubTest("detect eecio", fence_eecio_test) } type fenceReader struct { conn net.Conn rd *bufio.Reader } func (fr *fenceReader) receive() (string, error) { if err := fr.conn.SetReadDeadline(time.Now().Add(time.Second)); err != nil { return "", err } line, err := fr.rd.ReadBytes('\n') if err != nil { return "", err } if len(line) < 4 || line[0] != '$' || line[len(line)-2] != '\r' || line[len(line)-1] != '\n' { return "", errors.New("invalid message") } n, err := strconv.ParseUint(string(line[1:len(line)-2]), 10, 64) if err != nil { return "", err } buf := make([]byte, int(n)+2) _, err = io.ReadFull(fr.rd, buf) if err != nil { return "", err } if buf[len(buf)-2] != '\r' || buf[len(buf)-1] != '\n' { return "", errors.New("invalid message") } js := buf[:len(buf)-2] var m interface{} if err := json.Unmarshal(js, &m); err != nil { return "", err } return string(js), nil } func (fr *fenceReader) receiveExpect(valex ...string) error { s, err := fr.receive() if err != nil { return err } for i := 0; i < len(valex); i += 2 { if gjson.Get(s, valex[i]).String() != valex[i+1] { return fmt.Errorf("expected '%s'='%s', got '%s'", valex[i], valex[i+1], gjson.Get(s, valex[i]).String()) } } return nil } func fence_basic_test(mc *mockServer) error { conn, err := net.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer conn.Close() _, err = fmt.Fprintf(conn, "NEARBY mykey FENCE POINT 33 -115 5000\r\n") if err != nil { return err } buf := make([]byte, 4096) n, err := conn.Read(buf) if err != nil { return err } res := string(buf[:n]) if res != "+OK\r\n" { return fmt.Errorf("expected OK, got '%v'", res) } rd := &fenceReader{conn, bufio.NewReader(conn)} // send a point c, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer c.Close() res, err = redis.String(c.Do("SET", "mykey", "myid1", "POINT", 33, -115)) if err != nil { return err } if res != "OK" { return fmt.Errorf("expected OK, got '%v'", res) } // receive the message if err := rd.receiveExpect("command", "set", "detect", "enter", "key", "mykey", "id", "myid1", "object.type", "Point", "object.coordinates", "[-115,33]"); err != nil { return err } if err := rd.receiveExpect("command", "set", "detect", "inside", "key", "mykey", "id", "myid1", "object.type", "Point", "object.coordinates", "[-115,33]"); err != nil { return err } res, err = redis.String(c.Do("SET", "mykey", "myid1", "POINT", 34, -115)) if err != nil { return err } if res != "OK" { return fmt.Errorf("expected OK, got '%v'", res) } // receive the message if err := rd.receiveExpect("command", "set", "detect", "exit", "key", "mykey", "id", "myid1", "object.type", "Point", "object.coordinates", "[-115,34]"); err != nil { return err } if err := rd.receiveExpect("command", "set", "detect", "outside", "key", "mykey", "id", "myid1", "object.type", "Point", "object.coordinates", "[-115,34]"); err != nil { return err } return nil } func fence_channel_message_order_test(mc *mockServer) error { // Create a channel to store the goroutines error finalErr := make(chan error) var ready atomic.Bool // Concurrently subscribe for notifications go func() { // Create the subscription connection to Tile38 to subscribe for updates sc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { log.Println(err) return } defer sc.Close() // Subscribe the subscription client to the * pattern psc := redis.PubSubConn{Conn: sc} if err := psc.PSubscribe("*"); err != nil { log.Println(err) return } var msgs []string // While not a permanent error on the connection. loop: for sc.Err() == nil { switch v := psc.Receive().(type) { case redis.Message: if v.Channel == "status" && string(v.Data) == "ready" { ready.Store(true) } else { msgs = append(msgs, string(v.Data)) if len(msgs) == 8 { break loop } } case error: fmt.Printf("%s\n", err.Error()) } } // Verify all messages correctOrder := []string{"exit:A", "exit:B", "outside:A", "outside:B", "enter:C", "enter:D", "inside:C", "inside:D"} for i := range msgs { if gjson.Get(msgs[i], "detect").String()+":"+ gjson.Get(msgs[i], "hook").String() != correctOrder[i] { finalErr <- errors.New("INVALID MESSAGE ORDER") } } finalErr <- nil }() // Create the base connection for setting up points and geofences bc, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer bc.Close() for !ready.Load() { if _, err := do(bc, "PUBLISH status ready"); err != nil { return err } } // Fire all setup commands on the base client for _, cmd := range []string{ "SET points point POINT 33.412529053733444 -111.93368911743164", `SETCHAN A WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.95205688476562,33.400491820565236],[-111.92630767822266,33.400491820565236],[-111.92630767822266,33.422272258866045],[-111.95205688476562,33.422272258866045],[-111.95205688476562,33.400491820565236]]]}`, `SETCHAN B WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.93952560424803,33.403501285221594],[-111.92630767822266,33.403501285221594],[-111.92630767822266,33.41997983836345],[-111.93952560424803,33.41997983836345],[-111.93952560424803,33.403501285221594]]]}`, `SETCHAN C WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.9255781173706,33.40342963251261],[-111.91201686859131,33.40342963251261],[-111.91201686859131,33.41994401881284],[-111.9255781173706,33.41994401881284],[-111.9255781173706,33.40342963251261]]]}`, `SETCHAN D WITHIN points FENCE OBJECT {"type":"Polygon","coordinates":[[[-111.92562103271484,33.40063513076968],[-111.90021514892578,33.40063513076968],[-111.90021514892578,33.42212898435788],[-111.92562103271484,33.42212898435788],[-111.92562103271484,33.40063513076968]]]}`, "SET points point POINT 33.412529053733444 -111.91909790039062", } { if _, err := do(bc, cmd); err != nil { return err } } return <-finalErr } func fence_detect_inside_test(mc *mockServer) error { conn, err := net.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer conn.Close() _, err = fmt.Fprintf(conn, "WITHIN users FENCE DETECT inside,outside POINTS BOUNDS 33.618824 -84.457973 33.654359 -84.399859\r\n") if err != nil { return err } buf := make([]byte, 4096) n, err := conn.Read(buf) if err != nil { return err } res := string(buf[:n]) if res != "+OK\r\n" { return fmt.Errorf("expected OK, got '%v'", res) } rd := &fenceReader{conn, bufio.NewReader(conn)} // send a point c, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) if err != nil { return err } defer c.Close() res, err = redis.String(c.Do("SET", "users", "200", "POINT", "33.642301", "-84.43118")) if err != nil { return err } if res != "OK" { return fmt.Errorf("expected OK, got '%v'", res) } if err := rd.receiveExpect("command", "set", "detect", "inside", "key", "users", "id", "200", "point", `{"lat":33.642301,"lon":-84.43118}`); err != nil { return err } res, err = redis.String(c.Do("SET", "users", "200", "POINT", "34.642301", "-84.43118")) if err != nil { return err } if res != "OK" { return fmt.Errorf("expected OK, got '%v'", res) } // receive the message if err := rd.receiveExpect("command", "set", "detect", "outside", "key", "users", "id", "200", "point", `{"lat":34.642301,"lon":-84.43118}`); err != nil { return err } return nil } // do performs the passed command on the passed redis client func do(c redis.Conn, cmd string) (interface{}, error) { // Split out all parameters params := strings.Split(cmd, " ") // Produce a slice of interfaces for use in the arguments var args []interface{} for _, p := range params[1:] { args = append(args, p) } // Perform the request and return the response return c.Do(params[0], args...) } func fence_channel_meta_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SETCHAN", "carbon", "NEARBY", "x", "MATCH", "carbon*", "FENCE", "NODWELL", "points", "ROAM", "x", "*", "200000"}, {"1"}, {"OUTPUT", "json"}, {`{"ok":true}`}, // check for valid json on the chans command {"CHANS", "*"}, { func(v interface{}) (resp, expect interface{}) { // v is the value as strings or slices of strings // test will pass as long as `resp` and `expect` are the same. if !json.Valid([]byte(v.(string))) { return v, "Valid JSON" } return true, true }, }, }) } func dialTile38(port int) (redis.Conn, error) { conn, err := redis.Dial("tcp", fmt.Sprintf(":%d", port)) if err != nil { return nil, err } if _, err := conn.Do("OUTPUT", "json"); err != nil { conn.Close() return nil, err } return conn, nil } func doTile38(c redis.Conn, cmd string, args ...interface{}) (string, error) { js, err := redis.String(c.Do(cmd, args...)) if !gjson.Get(js, "ok").Bool() { return "", errors.New(gjson.Get(js, "err").String()) } return js, err } func fence_eecio_test(mc *mockServer) error { // simulates issue #578 var wg sync.WaitGroup wg.Add(3) ch := make(chan bool) var err1, err2, err3 error var msgs1, msgs2 []string // terminal 1 go func() { defer wg.Done() err1 = func() error { conn, err := dialTile38(mc.port) if err != nil { return err } defer conn.Close() _, err = doTile38(conn, "SETCHAN", "test-eec", "NEARBY", "fleet", "FENCE", "DETECT", "enter,exit,cross", "POINT", "10.000", "10.000", "10000") if err != nil { return err } _, err = doTile38(conn, "SUBSCRIBE", "test-eec") if err != nil { return err } ch <- true for { js, err := redis.String(conn.Receive()) if err != nil { return err } if js == `"DONE"` { break } msgs1 = append(msgs1, js) } return nil }() }() // terminal 2 go func() { defer wg.Done() err2 = func() error { conn, err := dialTile38(mc.port) if err != nil { return err } defer conn.Close() _, err = doTile38(conn, "SETCHAN", "test-eecio", "NEARBY", "fleet", "FENCE", "DETECT", "enter,exit,cross,inside,outside", "POINT", "10.000", "10.000", "10000") if err != nil { return err } _, err = doTile38(conn, "SUBSCRIBE", "test-eecio") if err != nil { return err } ch <- true for { js, err := redis.String(conn.Receive()) if err != nil { return err } if js == `"DONE"` { break } msgs2 = append(msgs2, js) } return nil }() }() // terminal 3 var ok bool go func() { defer wg.Done() err3 = func() error { <-ch // terminal 1 <-ch // terminal 2 conn, err := dialTile38(mc.port) if err != nil { return err } defer conn.Close() if _, err = doTile38(conn, "SET", "fleet", "vehicle_1", "POINT", "10.0", "10.0"); err != nil { return err } if _, err = doTile38(conn, "SET", "fleet", "vehicle_1", "POINT", "0.0", "0.0"); err != nil { return err } if _, err = doTile38(conn, "SET", "fleet", "vehicle_1", "POINT", "20.0", "20.0"); err != nil { return err } if _, err = doTile38(conn, "PUBLISH", "test-eecio", "DONE"); err != nil { return err } if _, err = doTile38(conn, "PUBLISH", "test-eec", "DONE"); err != nil { return err } ok = true return nil }() }() var timeok atomic.Bool go func() { time.Sleep(time.Second * 30) if !timeok.Load() { panic("timeout") } }() wg.Wait() timeok.Store(true) if err3 != nil { return err3 } if !ok { if err2 != nil { return err2 } if err1 != nil { return err1 } } var detects []string for i := 0; i < len(msgs1); i++ { detects = append(detects, gjson.Get(msgs1[i], "detect").String()) } if strings.Join(detects, ",") != "enter,exit,cross" { errmsg := fmt.Sprintf("expected 'enter,exit,cross', got '%s'\n", strings.Join(detects, ",")) return errors.New(errmsg) } detects = nil for i := 0; i < len(msgs2); i++ { detects = append(detects, gjson.Get(msgs2[i], "detect").String()) } if strings.Join(detects, ",") != "enter,inside,exit,outside,cross,outside" { errmsg := fmt.Sprintf( "expected 'enter,inside,exit,outside,cross,outside', got '%s'\n", strings.Join(detects, ",")) return errors.New(errmsg) } return nil } ================================================ FILE: tests/follower_test.go ================================================ package tests import "time" func subTestFollower(g *testGroup) { g.regSubTest("follow", follower_follow_test) } func follower_follow_test(mc *mockServer) error { mc2, err := mockOpenServer(MockServerOptions{ Silent: true, Metrics: false, }) if err != nil { return err } defer mc2.Close() err = mc.DoBatch( Do("SET", "mykey", "truck1", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck2", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck3", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck4", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck5", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck6", "POINT", 10, 10).OK(), Do("CONFIG", "SET", "requirepass", "1234").OK(), Do("AUTH", "1234").OK(), ) if err != nil { return err } err = mc2.DoBatch( Do("SET", "mykey2", "truck1", "POINT", 10, 10).OK(), Do("SET", "mykey2", "truck2", "POINT", 10, 10).OK(), Do("GET", "mykey2", "truck1").Str(`{"type":"Point","coordinates":[10,10]}`), Do("GET", "mykey2", "truck2").Str(`{"type":"Point","coordinates":[10,10]}`), Do("CONFIG", "SET", "leaderauth", "1234").OK(), Do("FOLLOW", "localhost", mc.port).OK(), Do("GET", "mykey", "truck1").Err("catching up to leader"), Sleep(time.Second/2), Do("GET", "mykey", "truck1").Err(`{"type":"Point","coordinates":[10,10]}`), Do("GET", "mykey", "truck2").Err(`{"type":"Point","coordinates":[10,10]}`), ) if err != nil { return err } err = mc.DoBatch( Do("SET", "mykey", "truck7", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck8", "POINT", 10, 10).OK(), Do("SET", "mykey", "truck9", "POINT", 10, 10).OK(), ) if err != nil { return err } err = mc2.DoBatch( Sleep(time.Second/2), Do("GET", "mykey", "truck7").Str(`{"type":"Point","coordinates":[10,10]}`), Do("GET", "mykey", "truck8").Str(`{"type":"Point","coordinates":[10,10]}`), Do("GET", "mykey", "truck9").Str(`{"type":"Point","coordinates":[10,10]}`), ) if err != nil { return err } return nil } ================================================ FILE: tests/json_test.go ================================================ package tests func subTestJSON(g *testGroup) { g.regSubTest("basic", json_JSET_basic_test) g.regSubTest("geojson", json_JSET_geojson_test) g.regSubTest("number", json_JSET_number_test) } func json_JSET_basic_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"JSET", "mykey", "myid1", "hello", "world"}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"hello":"world"}`}, {"JSET", "mykey", "myid1", "hello", "planet"}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"hello":"planet"}`}, {"JSET", "mykey", "myid1", "user.name.last", "tom"}, {"OK"}, {"JSET", "mykey", "myid1", "user.name.first", "andrew"}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"hello":"planet","user":{"name":{"last":"tom","first":"andrew"}}}`}, {"JDEL", "mykey", "myid1", "user.name.last"}, {1}, {"JGET", "mykey", "myid1"}, {`{"hello":"planet","user":{"name":{"first":"andrew"}}}`}, {"JDEL", "mykey", "myid1", "user.name.last"}, {0}, {"JGET", "mykey", "myid1"}, {`{"hello":"planet","user":{"name":{"first":"andrew"}}}`}, {"JDEL", "mykey2", "myid1", "user.name.last"}, {0}, }) } func json_JSET_geojson_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "myid1", "POINT", 33, -115}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"type":"Point","coordinates":[-115,33]}`}, {"JSET", "mykey", "myid1", "coordinates.1", 44}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"type":"Point","coordinates":[-115,44]}`}, {"SET", "mykey", "myid1", "OBJECT", `{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]}}`}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]},"properties":{}}`}, {"JGET", "mykey", "myid1", "geometry.type"}, {"Point"}, {"JSET", "mykey", "myid1", "properties.tags.-1", "southwest"}, {"OK"}, {"JSET", "mykey", "myid1", "properties.tags.-1", "united states"}, {"OK"}, {"JSET", "mykey", "myid1", "properties.tags.-1", "hot"}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]},"properties":{"tags":["southwest","united states","hot"]}}`}, {"JDEL", "mykey", "myid1", "type"}, {"ERR missing type"}, }) } func json_JSET_number_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"JSET", "mykey", "myid1", "hello", "0"}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"hello":0}`}, {"JSET", "mykey", "myid1", "hello", "0123"}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"hello":"0123"}`}, {"JSET", "mykey", "myid1", "hello", "3.14"}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"hello":3.14}`}, {"JSET", "mykey", "myid1", "hello", "1.0e10"}, {"OK"}, {"JGET", "mykey", "myid1"}, {`{"hello":1.0e10}`}, }) } ================================================ FILE: tests/keys_search_test.go ================================================ package tests import ( "errors" "fmt" "math" "math/rand" "sort" "testing" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" ) func subTestSearch(g *testGroup) { g.regSubTest("KNN_BASIC", keys_KNN_basic_test) g.regSubTest("KNN_RANDOM", keys_KNN_random_test) g.regSubTest("KNN_CURSOR", keys_KNN_cursor_test) g.regSubTest("NEARBY_SPARSE", keys_NEARBY_SPARSE_test) g.regSubTest("WITHIN_CIRCLE", keys_WITHIN_CIRCLE_test) g.regSubTest("WITHIN_SECTOR", keys_WITHIN_SECTOR_test) g.regSubTest("INTERSECTS_CIRCLE", keys_INTERSECTS_CIRCLE_test) g.regSubTest("INTERSECTS_SECTOR", keys_INTERSECTS_SECTOR_test) g.regSubTest("WITHIN", keys_WITHIN_test) g.regSubTest("WITHIN_CURSOR", keys_WITHIN_CURSOR_test) g.regSubTest("WITHIN_CLIPBY", keys_WITHIN_CLIPBY_test) g.regSubTest("INTERSECTS", keys_INTERSECTS_test) g.regSubTest("INTERSECTS_CURSOR", keys_INTERSECTS_CURSOR_test) g.regSubTest("INTERSECTS_CLIPBY", keys_INTERSECTS_CLIPBY_test) g.regSubTest("SCAN_CURSOR", keys_SCAN_CURSOR_test) g.regSubTest("SEARCH_CURSOR", keys_SEARCH_CURSOR_test) g.regSubTest("MATCH", keys_MATCH_test) g.regSubTest("FIELDS", keys_FIELDS_search_test) g.regSubTest("BUFFER", keys_BUFFER_search_test) } func keys_KNN_basic_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "1", "POINT", 5, 5}, {"OK"}, {"SET", "mykey", "2", "POINT", 19, 19}, {"OK"}, {"SET", "mykey", "3", "POINT", 12, 19}, {"OK"}, {"SET", "mykey", "4", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "5", "POINT", 33, 21}, {"OK"}, {"SET", "mykey", "6", "POINT", 52, 13}, {"OK"}, {"NEARBY", "mykey", "LIMIT", 10, "POINTS", "POINT", 20, 20}, { "[0 [[2 [19 19]] [3 [12 19]] [5 [33 21]] [1 [5 5]] [4 [-5 5]] [6 [52 13]]]]"}, {"NEARBY", "mykey", "LIMIT", 10, "IDS", "POINT", 20, 20, 4000000}, {"[0 [2 3 5 1 4 6]]"}, {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "IDS", "POINT", 20, 20, 1500000}, {"[0 [[2 152808.67164037024] [3 895945.1409106688] [5 1448929.5916252395]]]"}, {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "POINT", 52, 13, 100}, {`[0 [[6 {"type":"Point","coordinates":[13,52]} 0]]]`}, {"NEARBY", "mykey", "LIMIT", 10, "POINT", 52.1, 13.1, 100000}, {`[0 [[6 {"type":"Point","coordinates":[13,52]}]]]`}, {"OUTPUT", "json"}, {func(res string) bool { return gjson.Get(res, "ok").Bool() }}, {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "IDS", "POINT", 20, 20, 1500000}, { func(res string) error { if !gjson.Get(res, "ok").Bool() { return errors.New("not ok") } if gjson.Get(res, "ids.#").Int() != 3 { return fmt.Errorf("expected '%d' objects, got '%d'", 3, gjson.Get(res, "ids.#").Int()) } if gjson.Get(res, "ids.#.distance|#").Int() != 3 { return fmt.Errorf("expected '%d' distances, got '%d'", 3, gjson.Get(res, "ids.#.distance|#").Int()) } for i, d := range gjson.Get(res, "ids.#.distance").Array() { if d.Float() <= 0 { return fmt.Errorf("expected all distances to be greater than 0: (%d, %f)", i, d.Float()) } } return nil }, }, {"NEARBY", "mykey", "LIMIT", 10, "DISTANCE", "IDS", "POINT", 52, 13, 100}, { func(res string) error { expected := 0.0 if !gjson.Get(res, "ok").Bool() { return errors.New("not ok") } if gjson.Get(res, "ids.0.distance").Float() != expected { return fmt.Errorf("expected '%f' distances, got '%f'", expected, gjson.Get(res, "ids.0.distance").Float()) } return nil }, }, }) } func keys_KNN_random_test(mc *mockServer) error { // do random points mc.Do("OUTPUT", "resp") mc.Do("DROP", "points") defer mc.Do("DROP", "points") seed := time.Now().UnixNano() // seed = 98123098 rng := rand.New(rand.NewSource(seed)) rpoint := func() [2]float64 { return [2]float64{ rng.Float64()*360 - 180, rng.Float64()*180 - 90, } } N := 5000 points := make([][2]float64, N) for i := 0; i < len(points); i++ { points[i] = rpoint() res, err := redis.String(mc.Do("SET", "points", i, "POINT", points[i][1], points[i][0])) if err != nil { return err } if res != "OK" { return fmt.Errorf("expected 'OK', got '%s'", res) } } target := rpoint() mc.Do("OUTPUT", "json") defer mc.Do("OUTPUT", "resp") res, err := redis.String(mc.Do("NEARBY", "points", "LIMIT", N, "POINT", target[1], target[0])) if err != nil { return err } ldist := math.Inf(-1) for _, dist := range gjson.Get(res, "objects.#.distance").Array() { if ldist > dist.Float() { return fmt.Errorf("out of order") } ldist = dist.Float() } return nil } func keys_KNN_cursor_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "1", "FIELD", "foo", 5.5, "POINT", 5, 5}, {"OK"}, {"SET", "mykey", "2", "FIELD", "foo", 19.19, "POINT", 19, 19}, {"OK"}, {"SET", "mykey", "3", "FIELD", "foo", 12.19, "POINT", 12, 19}, {"OK"}, {"SET", "mykey", "4", "FIELD", "foo", -5.5, "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "5", "FIELD", "foo", 13.21, "POINT", 33, 21}, {"OK"}, {"NEARBY", "mykey", "LIMIT", 2, "POINTS", "POINT", 20, 20}, { "[2 [[2 [19 19] [foo 19.19]] [3 [12 19] [foo 12.19]]]]"}, {"NEARBY", "mykey", "CURSOR", 2, "LIMIT", 1, "POINTS", "POINT", 20, 20}, { "[3 [[5 [33 21] [foo 13.21]]]]"}, {"NEARBY", "mykey", "LIMIT", 2, "WHERE", "foo", -10, 15, "POINTS", "POINT", 20, 20}, { "[3 [[3 [12 19] [foo 12.19]] [5 [33 21] [foo 13.21]]]]"}, {"NEARBY", "mykey", "CURSOR", 3, "LIMIT", 1, "WHERE", "foo", -10, 15, "POINTS", "POINT", 20, 20}, { "[4 [[1 [5 5] [foo 5.5]]]]"}, {"NEARBY", "mykey", "CURSOR", 4, "LIMIT", 1, "WHERE", "foo", -10, 15, "POINTS", "POINT", 20, 20}, { "[5 [[4 [-5 5] [foo -5.5]]]]"}, {"NEARBY", "mykey", "CURSOR", 4, "LIMIT", 10, "WHERE", "foo", -10, 15, "POINTS", "POINT", 20, 20}, { "[0 [[4 [-5 5] [foo -5.5]]]]"}, }) } func keys_WITHIN_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"}, {"SET", "mykey", "point2", "POINT", 37.7335, -122.44121}, {"OK"}, {"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, {"SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, {"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"}, {"SET", "mykey", "poly8", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"WITHIN", "mykey", "IDS", "OBJECT", `{ "type": "Polygon", "coordinates": [ [ [-122.44126439094543,37.72906137107], [-122.43980526924135,37.72906137107], [-122.43980526924135,37.73421283683962], [-122.44126439094543,37.73421283683962], [-122.44126439094543,37.72906137107] ] ] }`}, {"[0 [point2 point1 multipoly5 poly8 poly4 line3]]"}, {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, { "[0 [point2 point1 multipoly5 poly8 poly4 line3]]"}, {"WITHIN", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "0"}, {"ERR equal bearings (0 == 0), use CIRCLE instead"}, {"SET", "key2", "poly9", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}`}, {"OK"}, {"SET", "key2", "poly10", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}`}, {"OK"}, {"WITHIN", "key2", "IDS", "GET", "mykey", "poly8"}, {"[0 [poly9]]"}, {"SET", "key3", "poly11", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44059920310973,37.733279482240874],[-122.4402344226837,37.733279482240874],[-122.4402344226837,37.73375464605226],[-122.44059920310973,37.73375464605226],[-122.44059920310973,37.733279482240874]]]}`}, {"OK"}, {"SET", "key3", "poly12", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44060993194579,37.73238005667773],[-122.44013786315918,37.73238005667773],[-122.44013786315918,37.73316917591997],[-122.44060993194579,37.73316917591997],[-122.44060993194579,37.73238005667773]]]}`}, {"OK"}, {"WITHIN", "key3", "IDS", "GET", "mykey", "multipoly5"}, {"[0 [poly11]]"}, {"SET", "key5", "poly13", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44073867797852,37.733211601447465],[-122.44011640548705,37.733211601447465],[-122.44011640548705,37.7340516218859],[-122.44073867797852,37.7340516218859],[-122.44073867797852,37.733211601447465]],[[-122.44060993194579,37.73345766902749],[-122.44060993194579,37.73355524732416],[-122.44044363498686,37.73355524732416],[-122.44044363498686,37.73345766902749],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.44060724973677,37.7339752567853],[-122.4402102828026,37.7339752567853],[-122.4402102828026,37.7336888869566],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"SET", "key5", "poly14", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44154334068298,37.73179457567642],[-122.43935465812682,37.73179457567642],[-122.43935465812682,37.7343740514423],[-122.44154334068298,37.7343740514423],[-122.44154334068298,37.73179457567642]],[[-122.44104981422423,37.73286371140448],[-122.44104981422423,37.73424677678513],[-122.43990182876587,37.73424677678513],[-122.43990182876587,37.73286371140448],[-122.44104981422423,37.73286371140448]],[[-122.44109272956847,37.731870943026074],[-122.43976235389708,37.731870943026074],[-122.43976235389708,37.7326855231885],[-122.44109272956847,37.7326855231885],[-122.44109272956847,37.731870943026074]]]}`}, {"OK"}, {"WITHIN", "key5", "IDS", "GET", "mykey", "multipoly5"}, {"[0 [poly13]]"}, {"SET", "key6", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, {"SET", "key6", "poly13", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44073867797852,37.733211601447465],[-122.44011640548705,37.733211601447465],[-122.44011640548705,37.7340516218859],[-122.44073867797852,37.7340516218859],[-122.44073867797852,37.733211601447465]],[[-122.44060993194579,37.73345766902749],[-122.44060993194579,37.73355524732416],[-122.44044363498686,37.73355524732416],[-122.44044363498686,37.73345766902749],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.44060724973677,37.7339752567853],[-122.4402102828026,37.7339752567853],[-122.4402102828026,37.7336888869566],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"WITHIN", "key6", "IDS", "GET", "key5", "poly14"}, {"[0 []]"}, {"SET", "key7", "multipoly15", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.44067430496216,37.73217641163713],[-122.44034171104431,37.73217641163713],[-122.44034171104431,37.732430967850384],[-122.44067430496216,37.732430967850384],[-122.44067430496216,37.73217641163713]]]]}`}, {"OK"}, {"SET", "key7", "multipoly16", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.4402666091919,37.733109780140644],[-122.4401271343231,37.733109780140644],[-122.4401271343231,37.73323705675229],[-122.4402666091919,37.73323705675229],[-122.4402666091919,37.733109780140644]]]]}`}, {"OK"}, {"SET", "key7", "multipoly17", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.44032025337218,37.73267703802467],[-122.44013786315918,37.73267703802467],[-122.44013786315918,37.732838255971316],[-122.44032025337218,37.732838255971316],[-122.44032025337218,37.73267703802467]]]]}`}, {"OK"}, {"WITHIN", "key7", "IDS", "GET", "mykey", "multipoly5"}, {"[0 [multipoly15 multipoly16]]"}, }) } func keys_WITHIN_CURSOR_test(mc *mockServer) error { testArea := `{ "type": "Polygon", "coordinates": [ [ [-122.44126439094543,37.72906137107], [-122.43980526924135,37.72906137107], [-122.43980526924135,37.73421283683962], [-122.44126439094543,37.73421283683962], [-122.44126439094543,37.72906137107] ] ] }` return mc.DoBatch([][]interface{}{ {"SET", "mykey", "point1", "FIELD", "foo", 1, "POINT", 37.7335, -122.4412}, {"OK"}, {"SET", "mykey", "point2", "FIELD", "foo", 2, "POINT", 37.7335, -122.44121}, {"OK"}, {"SET", "mykey", "line3", "FIELD", "foo", 3, "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "poly4", "FIELD", "foo", 4, "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, {"SET", "mykey", "multipoly5", "FIELD", "foo", 5, "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, {"SET", "mykey", "point6", "FIELD", "foo", 6, "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "point7", "FIELD", "foo", 7, "POINT", 33, 21}, {"OK"}, {"SET", "mykey", "poly8", "FIELD", "foo", 8, "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"SET", "mykey", "point9", "FIELD", "foo", 9, "POINT", 37.7335, -122.4412}, {"OK"}, {"WITHIN", "mykey", "LIMIT", 3, "IDS", "OBJECT", testArea}, { "[3 [point2 point9 point1]]"}, {"WITHIN", "mykey", "CURSOR", 3, "LIMIT", 3, "IDS", "OBJECT", testArea}, { "[6 [multipoly5 poly8 poly4]]"}, {"WITHIN", "mykey", "WHERE", "foo", 3, 5, "IDS", "OBJECT", testArea}, { "[0 [multipoly5 poly4 line3]]"}, {"WITHIN", "mykey", "LIMIT", 1, "WHERE", "foo", 3, 5, "IDS", "OBJECT", testArea}, { "[4 [multipoly5]]"}, {"WITHIN", "mykey", "CURSOR", 0, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[2 [point9]]"}, {"WITHIN", "mykey", "CURSOR", 1, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[2 [point9]]"}, {"WITHIN", "mykey", "CURSOR", 2, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[5 [poly8]]"}, {"WITHIN", "mykey", "CURSOR", 3, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[5 [poly8]]"}, {"WITHIN", "mykey", "CURSOR", 4, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[5 [poly8]]"}, {"WITHIN", "mykey", "CURSOR", 5, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[0 []]"}, }) } func keys_WITHIN_CLIPBY_test(mc *mockServer) error { jagged := `{ "type":"Polygon", "coordinates":[[ [-122.47781753540039,37.74655746554895], [-122.48777389526366,37.7355619376922], [-122.4707794189453,37.73271097867418], [-122.46528625488281,37.735969208590504], [-122.45189666748047,37.73922729512254], [-122.4565315246582,37.75008654795525], [-122.46683120727538,37.75307256315459], [-122.47781753540039,37.74655746554895] ]] }` return mc.DoBatch([][]interface{}{ {"SET", "mykey", "point1", "FIELD", "foo", 1, "POINT", 37.73963454585715, -122.4810791015625}, {"OK"}, {"SET", "mykey", "point2", "FIELD", "foo", 2, "POINT", 37.75130811419222, -122.47438430786133}, {"OK"}, {"SET", "mykey", "point3", "FIELD", "foo", 1, "POINT", 37.74816932695052, -122.47713088989258}, {"OK"}, {"SET", "mykey", "point4", "FIELD", "foo", 2, "POINT", 37.74503040657439, -122.47571468353271}, {"OK"}, {"SET", "other", "jagged", "OBJECT", jagged}, {"OK"}, {"WITHIN", "mykey", "IDS", "GET", "other", "jagged"}, {"[0 [point1 point4]]"}, {"WITHIN", "mykey", "IDS", "BOUNDS", 37.737734023260884, -122.47816085815431, 37.74886496155229, -122.45464324951172, }, {"[0 [point3 point4]]"}, {"WITHIN", "mykey", "IDS", "GET", "other", "jagged", "CLIPBY", "BOUNDS", 37.737734023260884, -122.47816085815431, 37.74886496155229, -122.45464324951172, }, {"[0 [point4]]"}, {"WITHIN", "mykey", "IDS", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, }, {"[0 [point3 point4 point2]]"}, {"WITHIN", "mykey", "IDS", "GET", "other", "jagged", "CLIPBY", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, }, {"[0 [point4]]"}, {"WITHIN", "mykey", "IDS", "GET", "other", "jagged", "CLIPBY", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, "CLIPBY", "BOUNDS", 37.737734023260884, -122.47816085815431, 37.74886496155229, -122.45464324951172, }, {"[0 [point4]]"}, }) } func keys_INTERSECTS_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "point1", "POINT", 37.7335, -122.4412}, {"OK"}, {"SET", "mykey", "point2", "POINT", 37.7335, -122.44121}, {"OK"}, {"SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, {"SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, {"SET", "mykey", "point6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "point7", "POINT", 33, 21}, {"OK"}, {"SET", "mykey", "poly8", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"INTERSECTS", "mykey", "IDS", "OBJECT", `{ "type": "Polygon", "coordinates": [ [ [-122.44126439094543,37.732906137107], [-122.43980526924135,37.732906137107], [-122.43980526924135,37.73421283683962], [-122.44126439094543,37.73421283683962], [-122.44126439094543,37.732906137107] ] ] }`}, {"[0 [point2 point1 multipoly5 poly8 poly4 line3]]"}, {"INTERSECTS", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90"}, { "[0 [point2 point1 multipoly5 poly8 poly4 line3]]"}, {"INTERSECTS", "mykey", "IDS", "SECTOR", "37.72999", "-122.44760", "1000", "0", "0"}, {"ERR equal bearings (0 == 0), use CIRCLE instead"}, {"SET", "key2", "poly9", "OBJECT", `{"type": "Polygon","coordinates": [[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}`}, {"OK"}, {"SET", "key2", "poly10", "OBJECT", `{"type": "Polygon","coordinates": [[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}`}, {"OK"}, {"SET", "key2", "poly10.1", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}`}, {"OK"}, {"INTERSECTS", "key2", "IDS", "GET", "mykey", "poly8"}, { "[0 [poly10 poly9]]"}, {"SET", "key3", "poly11", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44059920310973,37.733279482240874],[-122.4402344226837,37.733279482240874],[-122.4402344226837,37.73375464605226],[-122.44059920310973,37.73375464605226],[-122.44059920310973,37.733279482240874]]]}`}, {"OK"}, {"SET", "key3", "poly12", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44060993194579,37.73238005667773],[-122.44013786315918,37.73238005667773],[-122.44013786315918,37.73316917591997],[-122.44060993194579,37.73316917591997],[-122.44060993194579,37.73238005667773]]]}`}, {"OK"}, {"SET", "key3", "poly12.1", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44074940681458,37.73270249351328],[-122.44008421897887,37.73270249351328],[-122.44008421897887,37.732872196546936],[-122.44074940681458,37.732872196546936],[-122.44074940681458,37.73270249351328]]]}`}, {"OK"}, {"INTERSECTS", "key3", "IDS", "GET", "mykey", "multipoly5"}, { "[0 [poly12 poly11]]"}, {"SET", "key5", "poly13", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44073867797852,37.733211601447465],[-122.44011640548705,37.733211601447465],[-122.44011640548705,37.7340516218859],[-122.44073867797852,37.7340516218859],[-122.44073867797852,37.733211601447465]],[[-122.44060993194579,37.73345766902749],[-122.44060993194579,37.73355524732416],[-122.44044363498686,37.73355524732416],[-122.44044363498686,37.73345766902749],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.44060724973677,37.7339752567853],[-122.4402102828026,37.7339752567853],[-122.4402102828026,37.7336888869566],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"SET", "key5", "poly14", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44154334068298,37.73179457567642],[-122.43935465812682,37.73179457567642],[-122.43935465812682,37.7343740514423],[-122.44154334068298,37.7343740514423],[-122.44154334068298,37.73179457567642]],[[-122.44104981422423,37.73286371140448],[-122.44104981422423,37.73424677678513],[-122.43990182876587,37.73424677678513],[-122.43990182876587,37.73286371140448],[-122.44104981422423,37.73286371140448]],[[-122.44109272956847,37.731870943026074],[-122.43976235389708,37.731870943026074],[-122.43976235389708,37.7326855231885],[-122.44109272956847,37.7326855231885],[-122.44109272956847,37.731870943026074]]]}`}, {"OK"}, {"INTERSECTS", "key5", "IDS", "GET", "mykey", "multipoly5"}, { "[0 [poly13]]"}, {"SET", "key6", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, {"SET", "key6", "poly13", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.44073867797852,37.733211601447465],[-122.44011640548705,37.733211601447465],[-122.44011640548705,37.7340516218859],[-122.44073867797852,37.7340516218859],[-122.44073867797852,37.733211601447465]],[[-122.44060993194579,37.73345766902749],[-122.44060993194579,37.73355524732416],[-122.44044363498686,37.73355524732416],[-122.44044363498686,37.73345766902749],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.44060724973677,37.7339752567853],[-122.4402102828026,37.7339752567853],[-122.4402102828026,37.7336888869566],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"INTERSECTS", "key6", "IDS", "GET", "key5", "poly14"}, { "[0 []]"}, {"SET", "key7", "multipoly15", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.44067430496216,37.73217641163713],[-122.44034171104431,37.73217641163713],[-122.44034171104431,37.732430967850384],[-122.44067430496216,37.732430967850384],[-122.44067430496216,37.73217641163713]]]]}`}, {"OK"}, {"SET", "key7", "multipoly16", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.4402666091919,37.733109780140644],[-122.4401271343231,37.733109780140644],[-122.4401271343231,37.73323705675229],[-122.4402666091919,37.73323705675229],[-122.4402666091919,37.733109780140644]]]]}`}, {"OK"}, {"SET", "key7", "multipoly17", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.44056701660155,37.7332964524295],[-122.44029879570007,37.7332964524295],[-122.44029879570007,37.73375464605226],[-122.44056701660155,37.73375464605226],[-122.44056701660155,37.7332964524295]]],[[[-122.44032025337218,37.73267703802467],[-122.44013786315918,37.73267703802467],[-122.44013786315918,37.732838255971316],[-122.44032025337218,37.732838255971316],[-122.44032025337218,37.73267703802467]]]]}`}, {"OK"}, {"SET", "key7", "multipoly17.1", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4407172203064,37.73270249351328],[-122.44049191474916,37.73270249351328],[-122.44049191474916,37.73286371140448],[-122.4407172203064,37.73286371140448],[-122.4407172203064,37.73270249351328]]],[[[-122.44032025337218,37.73267703802467],[-122.44013786315918,37.73267703802467],[-122.44013786315918,37.732838255971316],[-122.44032025337218,37.732838255971316],[-122.44032025337218,37.73267703802467]]]]}`}, {"OK"}, {"INTERSECTS", "key7", "IDS", "GET", "mykey", "multipoly5"}, { "[0 [multipoly15 multipoly17 multipoly16]]"}, }) } func keys_INTERSECTS_CLIPBY_test(mc *mockServer) error { jagged := `{ "type":"Polygon", "coordinates":[[ [-122.47781753540039,37.74655746554895], [-122.48777389526366,37.7355619376922], [-122.4707794189453,37.73271097867418], [-122.46528625488281,37.735969208590504], [-122.45189666748047,37.73922729512254], [-122.4565315246582,37.75008654795525], [-122.46683120727538,37.75307256315459], [-122.47781753540039,37.74655746554895] ]] }` return mc.DoBatch([][]interface{}{ {"SET", "mykey", "point1", "FIELD", "foo", 1, "POINT", 37.73963454585715, -122.4810791015625}, {"OK"}, {"SET", "mykey", "point2", "FIELD", "foo", 2, "POINT", 37.75130811419222, -122.47438430786133}, {"OK"}, {"SET", "mykey", "point3", "FIELD", "foo", 1, "POINT", 37.74816932695052, -122.47713088989258}, {"OK"}, {"SET", "mykey", "point4", "FIELD", "foo", 2, "POINT", 37.74503040657439, -122.47571468353271}, {"OK"}, {"SET", "other", "jagged", "OBJECT", jagged}, {"OK"}, {"INTERSECTS", "mykey", "IDS", "GET", "other", "jagged"}, {"[0 [point1 point4]]"}, {"INTERSECTS", "mykey", "IDS", "BOUNDS", 37.737734023260884, -122.47816085815431, 37.74886496155229, -122.45464324951172, }, {"[0 [point3 point4]]"}, {"INTERSECTS", "mykey", "IDS", "GET", "other", "jagged", "CLIPBY", "BOUNDS", 37.737734023260884, -122.47816085815431, 37.74886496155229, -122.45464324951172, }, {"[0 [point4]]"}, {"INTERSECTS", "mykey", "IDS", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, }, {"[0 [point3 point4 point2]]"}, {"INTERSECTS", "mykey", "IDS", "GET", "other", "jagged", "CLIPBY", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, }, {"[0 [point4]]"}, {"INTERSECTS", "mykey", "IDS", "GET", "other", "jagged", "CLIPBY", "BOUNDS", 37.74411415606583, -122.48034954071045, 37.7536833241461, -122.47163772583008, "CLIPBY", "BOUNDS", 37.737734023260884, -122.47816085815431, 37.74886496155229, -122.45464324951172, }, {"[0 [point4]]"}, }) } func keys_INTERSECTS_CURSOR_test(mc *mockServer) error { testArea := `{ "type": "Polygon", "coordinates": [ [ [-122.44126439094543,37.732906137107], [-122.43980526924135,37.732906137107], [-122.43980526924135,37.73421283683962], [-122.44126439094543,37.73421283683962], [-122.44126439094543,37.732906137107] ] ] }` return mc.DoBatch([][]interface{}{ {"SET", "mykey", "point1", "FIELD", "foo", 1, "POINT", 37.7335, -122.4412}, {"OK"}, {"SET", "mykey", "point2", "FIELD", "foo", 2, "POINT", 37.7335, -122.44121}, {"OK"}, {"SET", "mykey", "line3", "FIELD", "foo", 3, "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "poly4", "FIELD", "foo", 4, "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, {"SET", "mykey", "multipoly5", "FIELD", "foo", 5, "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`}, {"OK"}, {"SET", "mykey", "point6", "FIELD", "foo", 6, "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "point7", "FIELD", "foo", 7, "POINT", 33, 21}, {"OK"}, {"SET", "mykey", "poly8", "FIELD", "foo", 8, "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}`}, {"OK"}, {"SET", "mykey", "point9", "FIELD", "foo", 9, "POINT", 37.7335, -122.4412}, {"OK"}, {"INTERSECTS", "mykey", "LIMIT", 3, "IDS", "OBJECT", testArea}, { "[3 [point2 point9 point1]]"}, {"INTERSECTS", "mykey", "CURSOR", 3, "LIMIT", 3, "IDS", "OBJECT", testArea}, { "[6 [multipoly5 poly8 poly4]]"}, {"INTERSECTS", "mykey", "WHERE", "foo", 3, 5, "IDS", "OBJECT", testArea}, { "[0 [multipoly5 poly4 line3]]"}, {"INTERSECTS", "mykey", "LIMIT", 1, "WHERE", "foo", 3, 5, "IDS", "OBJECT", testArea}, { "[4 [multipoly5]]"}, {"INTERSECTS", "mykey", "CURSOR", 1, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[2 [point9]]"}, {"INTERSECTS", "mykey", "CURSOR", 2, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[5 [poly8]]"}, {"INTERSECTS", "mykey", "CURSOR", 3, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[5 [poly8]]"}, {"INTERSECTS", "mykey", "CURSOR", 4, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[5 [poly8]]"}, {"INTERSECTS", "mykey", "CURSOR", 5, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[0 []]"}, {"INTERSECTS", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS", "OBJECT", testArea}, { "[0 []]"}, }) } func keys_WITHIN_CIRCLE_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "1", "POINT", 37.7335, -122.4412}, {"OK"}, {"SET", "mykey", "2", "POINT", 37.7335, -122.44121}, {"OK"}, {"SET", "mykey", "3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, {"SET", "mykey", "5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]]}`}, {"OK"}, {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, {"WITHIN", "mykey", "IDS", "CIRCLE", 37.7335, -122.4412, 1000}, { "[0 [2 1 5 4 3]]"}, {"WITHIN", "mykey", "IDS", "CIRCLE", 37.7335, -122.4412, 10}, { "[0 [2 1]]"}, }) } func keys_WITHIN_SECTOR_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "1", "POINT", 37.7324, -122.4424}, {"OK"}, {"SET", "mykey", "2", "POINT", 37.73241, -122.44241}, {"OK"}, {"SET", "mykey", "3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, {"SET", "mykey", "5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]]}`}, {"OK"}, {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, {"WITHIN", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 1000, 0, 90}, { "[0 [2 1 5 4 3]]"}, {"WITHIN", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 100, 0, 90}, { "[0 [2 1]]"}, }) } func keys_NEARBY_SPARSE_test(mc *mockServer) error { // https://github.com/tidwall/tile38/issues/618 return mc.DoBatch([][]interface{}{ {"SET", "location", "379", "FIELD", "story", "214", "POINT", "38.343763352486", "-0.48118065817742"}, {"OK"}, {"SET", "location", "380", "FIELD", "story", "216", "POINT", "38.343210451684", "-0.48164476701469"}, {"OK"}, {"SET", "location", "381", "FIELD", "story", "217", "POINT", "38.343548609904", "-0.4815616494057"}, {"OK"}, {"SET", "location", "382", "FIELD", "story", "219", "POINT", "38.342949723291", "-0.48180543529947"}, {"OK"}, {"SET", "location", "383", "FIELD", "story", "222", "POINT", "38.343527453662", "-0.48118542576204"}, {"OK"}, {"SET", "location", "384", "FIELD", "story", "223", "POINT", "38.342894310235", "-0.48121117150688"}, {"OK"}, {"SET", "location", "385", "FIELD", "story", "224", "POINT", "38.343665011791", "-0.48128042649575"}, {"OK"}, {"SET", "location", "386", "FIELD", "story", "226", "POINT", "38.34300663218", "-0.48136455958069"}, {"OK"}, {"SET", "location", "387", "FIELD", "story", "227", "POINT", "38.343561105586", "-0.48133979329476"}, {"OK"}, {"SET", "location", "388", "FIELD", "story", "228", "POINT", "38.343021516797", "-0.48111203609768"}, {"OK"}, {"SET", "location", "389", "FIELD", "story", "229", "POINT", "38.34377906915", "-0.48100754592639"}, {"OK"}, {"SET", "location", "390", "FIELD", "story", "230", "POINT", "38.343028862949", "-0.48107204744577"}, {"OK"}, {"SET", "location", "391", "FIELD", "story", "231", "POINT", "38.342956973955", "-0.48123798785545"}, {"OK"}, {"SET", "location", "392", "FIELD", "story", "233", "POINT", "38.343342938888", "-0.48181034196501"}, {"OK"}, {"SET", "location", "393", "FIELD", "story", "234", "POINT", "38.343323273543", "-0.48119160635951"}, {"OK"}, {"SET", "location", "394", "FIELD", "story", "235", "POINT", "38.343475947604", "-0.48128444286906"}, {"OK"}, {"SET", "location", "395", "FIELD", "story", "236", "POINT", "38.343553742872", "-0.48161695699988"}, {"OK"}, {"SET", "location", "396", "FIELD", "story", "237", "POINT", "38.343657786414", "-0.48109919689955"}, {"OK"}, {"SET", "location", "397", "FIELD", "story", "238", "POINT", "38.342934456291", "-0.48126912781599"}, {"OK"}, {"SET", "location", "398", "FIELD", "story", "239", "POINT", "38.343254792078", "-0.48115765613124"}, {"OK"}, {"SET", "location", "399", "FIELD", "story", "240", "POINT", "38.342851143141", "-0.48151031587298"}, {"OK"}, {"SET", "location", "400", "FIELD", "story", "244", "POINT", "38.343298791244", "-0.48121409612892"}, {"OK"}, {"SET", "location", "401", "FIELD", "story", "246", "POINT", "38.343436945653", "-0.48141198331599"}, {"OK"}, {"SET", "location", "402", "FIELD", "story", "248", "POINT", "38.343033046491", "-0.48183781756703"}, {"OK"}, {"SET", "location", "403", "FIELD", "story", "249", "POINT", "38.343115572723", "-0.48114768365296"}, {"OK"}, {"SET", "location", "404", "FIELD", "story", "250", "POINT", "38.343318663597", "-0.48120263102647"}, {"OK"}, {"SET", "location", "405", "FIELD", "story", "251", "POINT", "38.343434654108", "-0.4814578363497"}, {"OK"}, {"SET", "location", "406", "FIELD", "story", "252", "POINT", "38.343810655958", "-0.48181221112942"}, {"OK"}, {"SET", "location", "407", "FIELD", "story", "253", "POINT", "38.342910776509", "-0.48124848403503"}, {"OK"}, {"SET", "location", "408", "FIELD", "story", "176", "POINT", "38.343429050328", "-0.48134829622424"}, {"OK"}, {"SET", "location", "409", "FIELD", "story", "177", "POINT", "38.343375167926", "-0.4813182716687"}, {"OK"}, {"SET", "location", "410", "FIELD", "story", "178", "POINT", "38.343686937911", "-0.48184949541056"}, {"OK"}, {"SET", "location", "411", "FIELD", "story", "179", "POINT", "38.343095509246", "-0.48121296750565"}, {"OK"}, {"SET", "location", "412", "FIELD", "story", "243", "POINT", "38.343052434763", "-0.48133792363582"}, {"OK"}, {"SET", "location", "413", "FIELD", "story", "174", "POINT", "38.343556877562", "-0.4814408412531"}, {"OK"}, {"SET", "location", "414", "FIELD", "story", "182", "POINT", "38.34352896108", "-0.48127167080998"}, {"OK"}, {"SET", "location", "415", "FIELD", "story", "183", "POINT", "38.343458562741", "-0.48113117383504"}, {"OK"}, {"SET", "location", "416", "FIELD", "story", "187", "POINT", "38.343372242633", "-0.48198426529928"}, {"OK"}, {"SET", "location", "417", "FIELD", "story", "200", "POINT", "38.343365745635", "-0.48145747589433"}, {"OK"}, {"SET", "location", "418", "FIELD", "story", "206", "POINT", "38.343019183653", "-0.48177065402226"}, {"OK"}, {"SET", "location", "419", "FIELD", "story", "180", "POINT", "38.343492978961", "-0.48146214309728"}, {"OK"}, {"SET", "location", "420", "FIELD", "story", "181", "POINT", "38.343614147661", "-0.48178183237141"}, {"OK"}, {"SET", "location", "421", "FIELD", "story", "172", "POINT", "38.34365219519", "-0.48163252690471"}, {"OK"}, {"SET", "location", "422", "FIELD", "story", "193", "POINT", "38.343284579937", "-0.48191851957019"}, {"OK"}, {"SET", "location", "423", "FIELD", "story", "194", "POINT", "38.342957462369", "-0.48169612941468"}, {"OK"}, {"SET", "location", "424", "FIELD", "story", "195", "POINT", "38.343050765851", "-0.48189678247055"}, {"OK"}, {"SET", "location", "425", "FIELD", "story", "196", "POINT", "38.343767590125", "-0.48171070193171"}, {"OK"}, {"SET", "location", "426", "FIELD", "story", "197", "POINT", "38.343547519997", "-0.4813692941909"}, {"OK"}, {"SET", "location", "427", "FIELD", "story", "198", "POINT", "38.342914769086", "-0.48155727196514"}, {"OK"}, {"SET", "location", "428", "FIELD", "story", "211", "POINT", "38.342873132946", "-0.48120151934304"}, {"OK"}, {"SET", "location", "429", "FIELD", "story", "212", "POINT", "38.343776804477", "-0.48175041955478"}, {"OK"}, {"SET", "location", "430", "FIELD", "story", "218", "POINT", "38.343321288826", "-0.48138129717684"}, {"OK"}, {"SET", "location", "431", "FIELD", "story", "241", "POINT", "38.34353344767", "-0.4814278700903"}, {"OK"}, {"SET", "location", "432", "FIELD", "story", "247", "POINT", "38.34366410657", "-0.48163485684748"}, {"OK"}, {"SET", "location", "433", "FIELD", "story", "203", "POINT", "38.343237196083", "-0.48114844901293"}, {"OK"}, {"SET", "location", "434", "FIELD", "story", "204", "POINT", "38.342949966718", "-0.48104381934163"}, {"OK"}, {"SET", "location", "435", "FIELD", "story", "205", "POINT", "38.343334803169", "-0.48143352609798"}, {"OK"}, {"SET", "location", "436", "FIELD", "story", "215", "POINT", "38.343231760033", "-0.48177962151034"}, {"OK"}, {"SET", "location", "437", "FIELD", "story", "220", "POINT", "38.34381041238", "-0.48184807353803"}, {"OK"}, {"SET", "location", "438", "FIELD", "story", "232", "POINT", "38.3437321952", "-0.4810338033529"}, {"OK"}, {"SET", "location", "439", "FIELD", "story", "221", "POINT", "38.343038197665", "-0.48194660158614"}, {"OK"}, {"OUTPUT", "json"}, {func(res string) bool { return gjson.Get(res, "ok").Bool() }}, {"NEARBY", "location", "SPARSE", "1", "DISTANCE", "POINT", "38.342940855731506", "-0.48126081948077476", "25"}, { // should return 4 objects that include a "distance" field func(res string) error { if !gjson.Get(res, "ok").Bool() { return errors.New("not ok") } if gjson.Get(res, "objects.#").Int() != 4 { return fmt.Errorf("expected '%d' objects, got '%d'", 4, gjson.Get(res, "objects.#").Int()) } if gjson.Get(res, "objects.#.distance|#").Int() != 4 { return fmt.Errorf("expected '%d' distances, got '%d'", 4, gjson.Get(res, "objects.#.distance|#").Int()) } return nil }, }, }) } func keys_INTERSECTS_CIRCLE_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "1", "POINT", 37.7335, -122.4412}, {"OK"}, {"SET", "mykey", "2", "POINT", 37.7335, -122.44121}, {"OK"}, {"SET", "mykey", "3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, {"SET", "mykey", "5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]]}`}, {"OK"}, {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, {"INTERSECTS", "mykey", "IDS", "CIRCLE", 37.7335, -122.4412, 70}, { "[0 [2 1 5 4 3]]"}, {"INTERSECTS", "mykey", "IDS", "CIRCLE", 37.7335, -122.4412, 10}, { "[0 [2 1]]"}, }) } func keys_INTERSECTS_SECTOR_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "1", "POINT", 37.7324, -122.4424}, {"OK"}, {"SET", "mykey", "2", "POINT", 37.73241, -122.44241}, {"OK"}, {"SET", "mykey", "3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`}, {"OK"}, {"SET", "mykey", "4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`}, {"OK"}, {"SET", "mykey", "5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]]}`}, {"OK"}, {"SET", "mykey", "6", "POINT", -5, 5}, {"OK"}, {"SET", "mykey", "7", "POINT", 33, 21}, {"OK"}, {"INTERSECTS", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 1000, 0, 90}, { "[0 [2 1 5 4 3]]"}, {"INTERSECTS", "mykey", "IDS", "SECTOR", 37.731930, -122.443270, 100, 0, 90}, { "[0 [2 1]]"}, }) } func keys_SCAN_CURSOR_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "id1", "FIELD", "foo", 1, "STRING", "bar1"}, {"OK"}, {"SET", "mykey", "id2", "FIELD", "foo", 2, "STRING", "bar2"}, {"OK"}, {"SET", "mykey", "id3", "FIELD", "foo", 3, "STRING", "bar3"}, {"OK"}, {"SET", "mykey", "id4", "FIELD", "foo", 4, "STRING", "bar4"}, {"OK"}, {"SET", "mykey", "id5", "FIELD", "foo", 5, "STRING", "bar5"}, {"OK"}, {"SET", "mykey", "id6", "FIELD", "foo", 6, "STRING", "bar6"}, {"OK"}, {"SET", "mykey", "id7", "FIELD", "foo", 7, "STRING", "bar7"}, {"OK"}, {"SET", "mykey", "id8", "FIELD", "foo", 8, "STRING", "bar8"}, {"OK"}, {"SET", "mykey", "id9", "FIELD", "foo", 9, "STRING", "bar9"}, {"OK"}, {"SCAN", "mykey", "LIMIT", 3, "IDS"}, {"[3 [id1 id2 id3]]"}, {"SCAN", "mykey", "CURSOR", 3, "LIMIT", 3, "IDS"}, {"[6 [id4 id5 id6]]"}, {"SCAN", "mykey", "WHERE", "foo", 3, 5, "IDS"}, {"[0 [id3 id4 id5]]"}, {"SCAN", "mykey", "LIMIT", 1, "WHERE", "foo", 3, 5, "IDS"}, {"[3 [id3]]"}, {"SCAN", "mykey", "CURSOR", 3, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS"}, { "[8 [id8]]"}, {"SCAN", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS"}, { "[8 [id8]]"}, }) } func keys_SEARCH_CURSOR_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "id1", "FIELD", "foo", 1, "STRING", "bar1"}, {"OK"}, {"SET", "mykey", "id2", "FIELD", "foo", 2, "STRING", "bar2"}, {"OK"}, {"SET", "mykey", "id3", "FIELD", "foo", 3, "STRING", "bar3"}, {"OK"}, {"SET", "mykey", "id4", "FIELD", "foo", 4, "STRING", "bar4"}, {"OK"}, {"SET", "mykey", "id5", "FIELD", "foo", 5, "STRING", "bar5"}, {"OK"}, {"SET", "mykey", "id6", "FIELD", "foo", 6, "STRING", "bar6"}, {"OK"}, {"SET", "mykey", "id7", "FIELD", "foo", 7, "STRING", "bar7"}, {"OK"}, {"SET", "mykey", "id8", "FIELD", "foo", 8, "STRING", "bar8"}, {"OK"}, {"SET", "mykey", "id9", "FIELD", "foo", 9, "STRING", "bar9"}, {"OK"}, {"SEARCH", "mykey", "LIMIT", 3, "IDS"}, {"[3 [id1 id2 id3]]"}, {"SEARCH", "mykey", "CURSOR", 3, "LIMIT", 3, "IDS"}, {"[6 [id4 id5 id6]]"}, {"SEARCH", "mykey", "WHERE", "foo", 3, 5, "IDS"}, {"[0 [id3 id4 id5]]"}, {"SEARCH", "mykey", "LIMIT", 1, "WHERE", "foo", 3, 5, "IDS"}, {"[3 [id3]]"}, {"SEARCH", "mykey", "CURSOR", 3, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS"}, { "[8 [id8]]"}, {"SEARCH", "mykey", "CURSOR", 6, "LIMIT", 1, "WHERE", "foo", 8, 9, "IDS"}, { "[8 [id8]]"}, {"SEARCH", "mykey", "LIMIT", 3, "DESC", "IDS"}, {"[3 [id9 id8 id7]]"}, }) } func keys_MATCH_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "fleet", "truck1", "POINT", "33.0001", "-112.0001"}, {"OK"}, {"SET", "fleet", "truck2", "POINT", "33.0002", "-112.0002"}, {"OK"}, {"SET", "fleet", "luck1", "POINT", "33.0003", "-112.0003"}, {"OK"}, {"SET", "fleet", "luck2", "POINT", "33.0004", "-112.0004"}, {"OK"}, {"SET", "fleet", "train1", "POINT", "33.0005", "-112.0005"}, {"OK"}, {"SCAN", "fleet", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "*", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "truck*", "IDS"}, {"[0 [truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "luck*", "IDS"}, {"[0 [luck1 luck2]]"}, {"SCAN", "fleet", "MATCH", "*2", "IDS"}, {"[0 [luck2 truck2]]"}, {"SCAN", "fleet", "MATCH", "*2*", "IDS"}, {"[0 [luck2 truck2]]"}, {"SCAN", "fleet", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "*u*", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "*u*", "MATCH", "*a*", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "train*", "MATCH", "truck*", "IDS"}, {"[0 [train1 truck1 truck2]]"}, {"SCAN", "fleet", "MATCH", "train*", "MATCH", "truck*", "MATCH", "luck1", "IDS"}, {"[0 [luck1 train1 truck1 truck2]]"}, {"NEARBY", "fleet", "IDS", "POINT", 33.00005, -112.00005, 100000}, { match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "*", "IDS", "POINT", 33.00005, -112.00005, 100000}, { match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "t*", "IDS", "POINT", 33.00005, -112.00005, 100000}, { match("[0 [train1 truck1 truck2]]"), }, {"NEARBY", "fleet", "MATCH", "t*2", "IDS", "POINT", 33.00005, -112.00005, 100000}, { match("[0 [truck2]]"), }, {"NEARBY", "fleet", "MATCH", "*2", "IDS", "POINT", 33.00005, -112.00005, 100000}, { match("[0 [luck2 truck2]]"), }, {"INTERSECTS", "fleet", "IDS", "BOUNDS", 33, -113, 34, -112}, { match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "*", "IDS", "BOUNDS", 33, -113, 34, -112}, { match("[0 [luck1 luck2 train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "t*", "IDS", "BOUNDS", 33, -113, 34, -112}, { match("[0 [train1 truck1 truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "t*2", "IDS", "BOUNDS", 33, -113, 34, -112}, { match("[0 [truck2]]"), }, {"INTERSECTS", "fleet", "MATCH", "*2", "IDS", "BOUNDS", 33, -113, 34, -112}, { match("[0 [luck2 truck2]]"), }, }) } func keys_FIELDS_search_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "1", "FIELD", "field1", 10, "FIELD", "field2", 11 /* field3 undefined */, "OBJECT", `{"type":"Point","coordinates":[-112.2791,33.5220]}`}, {"OK"}, {"SET", "mykey", "2", "FIELD", "field1", 20, "FIELD", "field2", 10 /* field3 undefined */, "OBJECT", `{"type":"Point","coordinates":[-112.2793,33.5222]}`}, {"OK"}, {"SET", "mykey", "3", "FIELD", "field1", 30, "FIELD", "field2", 13 /* field3 undefined */, "OBJECT", `{"type":"Point","coordinates":[-112.2795,33.5224]}`}, {"OK"}, {"SET", "mykey", "4", "FIELD", "field1", 40, "FIELD", "field2", 14 /* field3 undefined */, "OBJECT", `{"type":"Point","coordinates":[-112.2797,33.5226]}`}, {"OK"}, {"SET", "mykey", "5" /* field1 undefined */, "FIELD", "field2", 15, "FIELD", "field3", 28, "OBJECT", `{"type":"Point","coordinates":[-112.2799,33.5228]}`}, {"OK"}, {"SET", "mykey", "6" /* field1 & field2 undefined */, "FIELD", "field3", 29, "OBJECT", `{"type":"Point","coordinates":[-112.2801,33.5230]}`}, {"OK"}, {"SET", "mykey", "7" /* field1, field2, & field3 undefined */, "OBJECT", `{"type":"Point","coordinates":[-112.2803,33.5232]}`}, {"OK"}, // test RESP output {"NEARBY", "mykey", "WHERE", "field2", 11, "+inf", "POINT", 33.462, -112.268, 60000}, { `[0 [` + `[1 {"type":"Point","coordinates":[-112.2791,33.522]} [field1 10 field2 11]] ` + `[3 {"type":"Point","coordinates":[-112.2795,33.5224]} [field1 30 field2 13]] ` + `[4 {"type":"Point","coordinates":[-112.2797,33.5226]} [field1 40 field2 14]] ` + `[5 {"type":"Point","coordinates":[-112.2799,33.5228]} [field2 15 field3 28]]]]`, }, {"NEARBY", "mykey", "WHERE", "field2", 0, 2, "POINT", 33.462, -112.268, 60000}, { `[0 [` + `[6 {"type":"Point","coordinates":[-112.2801,33.523]} [field3 29]] ` + `[7 {"type":"Point","coordinates":[-112.2803,33.5232]}]]]`}, {"WITHIN", "mykey", "WHERE", "field2", 11, "+inf", "CIRCLE", 33.462, -112.268, 60000}, { `[0 [` + `[5 {"type":"Point","coordinates":[-112.2799,33.5228]} [field2 15 field3 28]] ` + `[4 {"type":"Point","coordinates":[-112.2797,33.5226]} [field1 40 field2 14]] ` + `[3 {"type":"Point","coordinates":[-112.2795,33.5224]} [field1 30 field2 13]] ` + `[1 {"type":"Point","coordinates":[-112.2791,33.522]} [field1 10 field2 11]]]]`, }, {"WITHIN", "mykey", "WHERE", "field2", 0, 2, "CIRCLE", 33.462, -112.268, 60000}, { `[0 [` + `[7 {"type":"Point","coordinates":[-112.2803,33.5232]}] ` + `[6 {"type":"Point","coordinates":[-112.2801,33.523]} [field3 29]]]]`}, // test JSON output {"OUTPUT", "json"}, {`{"ok":true}`}, {"NEARBY", "mykey", "WHERE", "field2", 11, "+inf", "POINT", 33.462, -112.268, 60000}, { `{"ok":true,"fields":["field1","field2","field3"],"objects":[` + `{"id":"1","object":{"type":"Point","coordinates":[-112.2791,33.522]},"fields":[10,11,0]},` + `{"id":"3","object":{"type":"Point","coordinates":[-112.2795,33.5224]},"fields":[30,13,0]},` + `{"id":"4","object":{"type":"Point","coordinates":[-112.2797,33.5226]},"fields":[40,14,0]},` + `{"id":"5","object":{"type":"Point","coordinates":[-112.2799,33.5228]},"fields":[0,15,28]}` + `],"count":4,"cursor":0}`}, {"NEARBY", "mykey", "WHERE", "field2", 0, 2, "POINT", 33.462, -112.268, 60000}, { `{"ok":true,"fields":["field3"],"objects":[` + `{"id":"6","object":{"type":"Point","coordinates":[-112.2801,33.523]},"fields":[29]},` + `{"id":"7","object":{"type":"Point","coordinates":[-112.2803,33.5232]},"fields":[0]}` + `],"count":2,"cursor":0}`}, {"WITHIN", "mykey", "WHERE", "field2", 11, "+inf", "CIRCLE", 33.462, -112.268, 60000}, { `{"ok":true,"fields":["field1","field2","field3"],"objects":[` + `{"id":"5","object":{"type":"Point","coordinates":[-112.2799,33.5228]},"fields":[0,15,28]},` + `{"id":"4","object":{"type":"Point","coordinates":[-112.2797,33.5226]},"fields":[40,14,0]},` + `{"id":"3","object":{"type":"Point","coordinates":[-112.2795,33.5224]},"fields":[30,13,0]},` + `{"id":"1","object":{"type":"Point","coordinates":[-112.2791,33.522]},"fields":[10,11,0]}` + `],"count":4,"cursor":0}`}, {"WITHIN", "mykey", "WHERE", "field2", 0, 2, "CIRCLE", 33.462, -112.268, 60000}, { `{"ok":true,"fields":["field3"],"objects":[` + `{"id":"7","object":{"type":"Point","coordinates":[-112.2803,33.5232]},"fields":[0]},` + `{"id":"6","object":{"type":"Point","coordinates":[-112.2801,33.523]},"fields":[29]}` + `],"count":2,"cursor":0}`}, }) } func keys_BUFFER_search_test(mc *mockServer) error { const lineString = `{"type":"LineString","coordinates":[ [-116.40289306640624,34.125447565116126], [-116.36444091796875,34.14818102254435], [-116.0980224609375,34.15045403191448], [-115.74920654296874,34.127721186043985], [-115.54870605468749,34.075412438417395], [-115.5267333984375,34.11407854333859], [-115.21911621093749,34.048108084909835], [-115.25207519531249,33.8339199536547], [-115.40588378906249,33.71748624018193] ]}` return mc.DoBatch([][]interface{}{ // points in {"SET", "fleet", "truck01", "POINT", "34.10825132729329", "-115.6436347961428"}, {"OK"}, {"SET", "fleet", "truck02", "POINT", "34.07199987534163", "-115.5435562133782"}, {"OK"}, {"SET", "fleet", "truck03", "POINT", "34.05123715497616", "-115.2148246765137"}, {"OK"}, {"SET", "fleet", "truck04", "POINT", "33.71520164474084", "-115.4110336303711"}, {"OK"}, {"SET", "fleet", "truck05", "POINT", "34.12345809664606", "-116.4070129394531"}, {"OK"}, // points out {"SET", "fleet", "truck06", "POINT", "35.10825132729329", "-115.6436347961428"}, {"OK"}, {"SET", "fleet", "truck07", "POINT", "35.07199987534163", "-115.5435562133782"}, {"OK"}, {"SET", "fleet", "truck08", "POINT", "35.05123715497616", "-115.2148246765137"}, {"OK"}, {"SET", "fleet", "truck09", "POINT", "35.71520164474084", "-115.4110336303711"}, {"OK"}, {"SET", "fleet", "truck10", "POINT", "35.12345809664606", "-116.4070129394531"}, {"OK"}, // buffered intersects {"INTERSECTS", "fleet", "BUFFER", "1000", "COUNT", "OBJECT", lineString}, {"5"}, }) } // match sorts the response and compares to the expected input func match(expectIn string) func(org, v interface{}) (resp, expect interface{}) { return func(v, org interface{}) (resp, expect interface{}) { sort.Slice(org.([]interface{})[1], func(i, j int) bool { return org.([]interface{})[1].([]interface{})[i].(string) < org.([]interface{})[1].([]interface{})[j].(string) }) return fmt.Sprintf("%v", org), expectIn } } func subBenchSearch(b *testing.B, mc *mockServer) { runBenchStep(b, mc, "KNN", keys_KNN_bench) } func keys_KNN_bench(mc *mockServer) error { lat := rand.Float64()*180 - 90 lon := rand.Float64()*360 - 180 _, err := mc.conn.Do("NEARBY", "mykey", "LIMIT", 50, "DISTANCE", "POINTS", "POINT", lat, lon) return err } ================================================ FILE: tests/keys_test.go ================================================ package tests import ( "errors" "fmt" "math/rand" "strings" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/gjson" ) func subTestKeys(g *testGroup) { g.regSubTest("BOUNDS", keys_BOUNDS_test) g.regSubTest("DEL", keys_DEL_test) g.regSubTest("DROP", keys_DROP_test) g.regSubTest("RENAME", keys_RENAME_test) g.regSubTest("RENAMENX", keys_RENAMENX_test) g.regSubTest("EXPIRE", keys_EXPIRE_test) g.regSubTest("FSET", keys_FSET_test) g.regSubTest("FGET", keys_FGET_test) g.regSubTest("GET", keys_GET_test) g.regSubTest("KEYS", keys_KEYS_test) g.regSubTest("PERSIST", keys_PERSIST_test) g.regSubTest("SET", keys_SET_test) g.regSubTest("STATS", keys_STATS_test) g.regSubTest("TTL", keys_TTL_test) g.regSubTest("EXIST", keys_EXISTS_test) g.regSubTest("FEXIST", keys_FEXISTS_test) g.regSubTest("SET EX", keys_SET_EX_test) g.regSubTest("PDEL", keys_PDEL_test) g.regSubTest("FIELDS", keys_FIELDS_test) g.regSubTest("WHEREIN", keys_WHEREIN_test) g.regSubTest("WHEREEVAL", keys_WHEREEVAL_test) g.regSubTest("TYPE", keys_TYPE_test) g.regSubTest("FLUSHDB", keys_FLUSHDB_test) g.regSubTest("HEALTHZ", keys_HEALTHZ_test) g.regSubTest("SERVER", keys_SERVER_test) g.regSubTest("INFO", keys_INFO_test) } func keys_BOUNDS_test(mc *mockServer) error { return mc.DoBatch( Do("BOUNDS", "mykey").Str(""), Do("BOUNDS", "mykey").JSON().Err("key not found"), Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(), Do("BOUNDS", "mykey").Str("[[-115 33] [-115 33]]"), Do("BOUNDS", "mykey").JSON().Str(`{"ok":true,"bounds":{"type":"Polygon","coordinates":[[[-115,33],[-115,33],[-115,33],[-115,33],[-115,33]]]}}`), Do("SET", "mykey", "myid2", "POINT", 34, -112).OK(), Do("BOUNDS", "mykey").Str("[[-115 33] [-112 34]]"), Do("DEL", "mykey", "myid2").Str("1"), Do("BOUNDS", "mykey").Str("[[-115 33] [-115 33]]"), Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(), Do("SET", "mykey", "myid4", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(), Do("BOUNDS", "mykey").Str("[[-130 25] [-110 38]]"), Do("BOUNDS", "mykey", "hello").Err("wrong number of arguments for 'bounds' command"), Do("BOUNDS", "nada").Str(""), Do("BOUNDS", "nada").JSON().Err("key not found"), Do("BOUNDS", "").Str(""), Do("BOUNDS", "mykey").JSON().Str(`{"ok":true,"bounds":{"type":"Polygon","coordinates":[[[-130,25],[-110,25],[-110,38],[-130,38],[-130,25]]]}}`), ) } func keys_DEL_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), Do("DEL", "mykey", "myid2", "ERRON404").Err("id not found"), Do("DEL", "mykey", "myid").Str("1"), Do("DEL", "mykey", "myid").Str("0"), Do("DEL", "mykey").Err("wrong number of arguments for 'del' command"), Do("GET", "mykey", "myid").Str(""), Do("DEL", "mykey", "myid", "ERRON404").Err("key not found"), Do("DEL", "mykey", "myid", "invalid-arg").Err("invalid argument 'invalid-arg'"), Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), Do("DEL", "mykey", "myid2", "ERRON404").JSON().Err("id not found"), Do("DEL", "mykey", "myid").JSON().OK(), Do("DEL", "mykey", "myid").JSON().OK(), Do("DEL", "mykey", "myid", "ERRON404").JSON().Err("key not found"), ) } func keys_DROP_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), Do("SCAN", "mykey", "COUNT").Str("2"), Do("DROP").Err("wrong number of arguments for 'drop' command"), Do("DROP", "mykey", "arg3").Err("wrong number of arguments for 'drop' command"), Do("DROP", "mykey").Str("1"), Do("SCAN", "mykey", "COUNT").Str("0"), Do("DROP", "mykey").Str("0"), Do("SCAN", "mykey", "COUNT").Str("0"), Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), Do("DROP", "mykey").JSON().OK(), Do("DROP", "mykey").JSON().OK(), ) } func keys_RENAME_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), Do("SCAN", "mykey", "COUNT").Str("2"), Do("RENAME", "foo", "mynewkey", "arg3").Err("wrong number of arguments for 'rename' command"), Do("RENAME", "mykey", "mynewkey").OK(), Do("SCAN", "mykey", "COUNT").Str("0"), Do("SCAN", "mynewkey", "COUNT").Str("2"), Do("SET", "mykey", "myid3", "HASH", "9my5xp7").OK(), Do("RENAME", "mykey", "mynewkey").OK(), Do("SCAN", "mykey", "COUNT").Str("0"), Do("SCAN", "mynewkey", "COUNT").Str("1"), Do("RENAME", "foo", "mynewkey").Err("key not found"), Do("SCAN", "mynewkey", "COUNT").Str("1"), Do("SETCHAN", "mychan", "INTERSECTS", "mynewkey", "BOUNDS", 10, 10, 20, 20).Str("1"), Do("RENAME", "mynewkey", "foo2").Err("key has channels set"), Do("RENAMENX", "mynewkey", "foo2").Err("key has channels set"), Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), Do("RENAME", "mykey", "foo2").OK(), Do("RENAMENX", "foo2", "foo3").Str("1"), Do("RENAMENX", "foo2", "foo3").Err("key not found"), Do("RENAME", "foo2", "foo3").JSON().Err("key not found"), Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), Do("RENAMENX", "mykey", "foo3").Str("0"), Do("RENAME", "foo3", "foo4").JSON().OK(), ) } func keys_RENAMENX_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), Do("SET", "mykey", "myid2", "HASH", "9my5xp8").OK(), Do("SCAN", "mykey", "COUNT").Str("2"), Do("RENAMENX", "mykey", "mynewkey").Str("1"), Do("SCAN", "mykey", "COUNT").Str("0"), Do("DROP", "mykey").Str("0"), Do("SCAN", "mykey", "COUNT").Str("0"), Do("SCAN", "mynewkey", "COUNT").Str("2"), Do("SET", "mykey", "myid3", "HASH", "9my5xp7").OK(), Do("RENAMENX", "mykey", "mynewkey").Str("0"), Do("SCAN", "mykey", "COUNT").Str("1"), Do("SCAN", "mynewkey", "COUNT").Str("2"), Do("RENAMENX", "foo", "mynewkey").Str("ERR key not found"), Do("SCAN", "mynewkey", "COUNT").Str("2"), ) } func keys_EXPIRE_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "STRING", "value").OK(), Do("EXPIRE", "mykey", "myid").Err("wrong number of arguments for 'expire' command"), Do("EXPIRE", "mykey", "myid", "y").Err("invalid argument 'y'"), Do("EXPIRE", "mykey", "myid", 1).Str("1"), Do("EXPIRE", "mykey", "myid", 1).JSON().OK(), Sleep(time.Second/4), Do("GET", "mykey", "myid").Str("value"), Sleep(time.Second), Do("GET", "mykey", "myid").Str(""), Do("EXPIRE", "mykey", "myid", 1).JSON().Err("key not found"), Do("SET", "mykey", "myid1", "STRING", "value1").OK(), Do("SET", "mykey", "myid2", "STRING", "value2").OK(), Do("EXPIRE", "mykey", "myid1", 1).Str("1"), Sleep(time.Second/4), Do("GET", "mykey", "myid1").Str("value1"), Sleep(time.Second), Do("EXPIRE", "mykey", "myid1", 1).Str("0"), Do("EXPIRE", "mykey", "myid1", 1).JSON().Err("id not found"), ) } func keys_FSET_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"), Do("FSET", "mykey", "myid", "f1", 105.6).Str("1"), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 105.6]]"), Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 2.2).Str("2"), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 1.1 f2 2.2]]"), Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 22.22).Str("1"), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f1 1.1 f2 22.22]]"), Do("FSET", "mykey", "myid", "f1", 0).Str("1"), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [f2 22.22]]"), Do("FSET", "mykey", "myid", "f2", 0).Str("1"), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"), Do("FSET", "mykey", "myid2", "xx", "f1", 1.1, "f2", 2.2).Str("0"), Do("FSET", "mykey", "myid", "f1", 1, "RETURN", "HASH", 7, "WITHFIELDS").Str("[9my5xp7 [f1 1]]"), Do("FSET", "mykey", "myid", "f2", 2, "RETURN", "HASH", 7, "WITHFIELDS").Str("[9my5xp7 [f1 1 f2 2]]"), Do("FSET", "mykey", "myid", "f1", 0, "RETURN", "HASH", 7, "WITHFIELDS").Str("[9my5xp7 [f2 2]]"), Do("FSET", "mykey", "myid", "f2", 0, "RETURN", "HASH", 7, "WITHFIELDS").Str("[9my5xp7]"), Do("GET", "mykey", "myid2").Str(""), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), Do("SET", "mykey", "myid", "POINT", 1, 2).OK(), Do("GET", "mykey", "myid", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[2,1]}}`), Do("FSET", "mykey", "myid", "f2", 1).JSON().OK(), Do("GET", "mykey", "myid", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[2,1]},"fields":{"f2":1}}`), Do("SET", "mykey", "myid", "POINT", 3, 4).OK(), Do("GET", "mykey", "myid", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[4,3]},"fields":{"f2":1}}`), Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), Do("CONFIG", "SET", "maxmemory", "1").OK(), Do("FSET", "mykey", "myid", "xx", "f1", 1.1, "f2", 2.2).Err(`OOM command not allowed when used memory > 'maxmemory'`), Do("CONFIG", "SET", "maxmemory", "0").OK(), Do("FSET", "mykey", "myid", "xx").Err("wrong number of arguments for 'fset' command"), Do("FSET", "mykey", "myid", "f1", "a", "f2").Err("wrong number of arguments for 'fset' command"), Do("FSET", "mykey", "myid", "z", "a").Err("invalid argument 'z'"), Do("FSET", "mykey2", "myid", "a", "b").Err("key not found"), Do("FSET", "mykey", "myid2", "a", "b").Err("id not found"), Do("FSET", "mykey", "myid", "f2", 0).JSON().OK(), Do("SET", "cases", "lower", "POINT", 1, 2).OK(), Do("FSET", "cases", "lower", "lowercase", 1).JSON().OK(), Do("GET", "cases", "lower", "WITHFIELDS").JSON().Str( `{"ok":true,"object":{"type":"Point","coordinates":[2,1]},"fields":{"lowercase":1}}`, ), Do("SET", "cases", "upper", "POINT", 1, 2).OK(), Do("FSET", "cases", "upper", "UPPERCASE", 1).JSON().OK(), Do("GET", "cases", "upper", "WITHFIELDS").JSON().Str( `{"ok":true,"object":{"type":"Point","coordinates":[2,1]},"fields":{"UPPERCASE":1}}`, ), Do("SET", "cases", "camel", "POINT", 1, 2).OK(), Do("FSET", "cases", "camel", "camelCase", 1).JSON().OK(), Do("GET", "cases", "camel", "WITHFIELDS").JSON().Str( `{"ok":true,"object":{"type":"Point","coordinates":[2,1]},"fields":{"camelCase":1}}`, ), Do("SET", "cases", "allcases", "POINT", 1, 2).OK(), Do("FSET", "cases", "allcases", "UPPERCASE", 1).JSON().OK(), Do("FSET", "cases", "allcases", "lowercase", 1).JSON().OK(), Do("FSET", "cases", "allcases", "camelCase", 1).JSON().OK(), Do("GET", "cases", "allcases", "WITHFIELDS").JSON().Str( `{"ok":true,"object":{"type":"Point","coordinates":[2,1]},"fields":{"UPPERCASE":1,"camelCase":1,"lowercase":1}}`, ), ) } func keys_FGET_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7]"), Do("FSET", "mykey", "myid", "f1", 105.6).Str("1"), Do("FGET", "mykey", "myid", "f1").Str("105.6"), Do("FSET", "mykey", "myid", "f1", 1.1, "f2", 2.2).Str("2"), Do("FGET", "mykey", "myid", "f2").Str("2.2"), Do("FGET", "mykey", "myid", "f1").Str("1.1"), Do("FGET", "mykey", "myid", "f1").JSON().Str(`{"ok":true,"value":1.1}`), Do("FSET", "mykey", "myid", "f3", "a").Str("1"), Do("FGET", "mykey", "myid", "f3").Str("a"), Do("FGET", "mykey", "myid", "f4").Str("0"), Do("FGET", "mykey", "myid", "f4").JSON().Str(`{"ok":true,"value":0}`), Do("FGET", "mykey", "myid").Err("wrong number of arguments for 'fget' command"), Do("FGET", "mykey2", "myid", "a", "b").Err("key not found"), Do("FGET", "mykey", "myid2", "a", "b").Err("id not found"), ) } func keys_GET_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "STRING", "value").OK(), Do("GET", "mykey", "myid").Str("value"), Do("SET", "mykey", "myid", "STRING", "value2").OK(), Do("GET", "mykey", "myid").Str("value2"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), Do("GET", "mykey").Err("wrong number of arguments for 'get' command"), Do("GET", "mykey", "myid", "hash").Err("wrong number of arguments for 'get' command"), Do("GET", "mykey", "myid", "hash", "0").Err("invalid argument '0'"), Do("GET", "mykey", "myid", "hash", "-1").Err("invalid argument '-1'"), Do("GET", "mykey", "myid", "hash", "13").Err("invalid argument '13'"), Do("SET", "mykey", "myid", "field", "hello", "world", "field", "hiya", 55, "point", 33, -112).OK(), Do("GET", "mykey", "myid", "hash", "1").Str("9"), Do("GET", "mykey", "myid", "point").Str("[33 -112]"), Do("GET", "mykey", "myid", "bounds").Str("[[33 -112] [33 -112]]"), Do("GET", "mykey", "myid", "object").Str(`{"type":"Point","coordinates":[-112,33]}`), Do("GET", "mykey", "myid", "object").Str(`{"type":"Point","coordinates":[-112,33]}`), Do("GET", "mykey", "myid", "withfields", "point").Str(`[[33 -112] [hello world hiya 55]]`), Do("GET", "mykey", "myid", "joint").Err("wrong number of arguments for 'get' command"), Do("GET", "mykey2", "myid").Str(""), Do("GET", "mykey2", "myid").JSON().Err("key not found"), Do("GET", "mykey", "myid2").Str(""), Do("GET", "mykey", "myid2").JSON().Err("id not found"), Do("GET", "mykey", "myid", "point").JSON().Str(`{"ok":true,"point":{"lat":33,"lon":-112}}`), Do("GET", "mykey", "myid", "object").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]}}`), Do("GET", "mykey", "myid", "hash", "1").JSON().Str(`{"ok":true,"hash":"9"}`), Do("GET", "mykey", "myid", "bounds").JSON().Str(`{"ok":true,"bounds":{"sw":{"lat":33,"lon":-112},"ne":{"lat":33,"lon":-112}}}`), Do("SET", "mykey", "myid2", "point", 33, -112, 55).OK(), Do("GET", "mykey", "myid2", "point").Str("[33 -112 55]"), Do("GET", "mykey", "myid2", "point").JSON().Str(`{"ok":true,"point":{"lat":33,"lon":-112,"z":55}}`), Do("GET", "mykey", "myid", "withfields").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]},"fields":{"hello":"world","hiya":55}}`), ) } func keys_KEYS_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey11", "myid4", "STRING", "value").OK(), Do("SET", "mykey22", "myid2", "HASH", "9my5xp7").OK(), Do("SET", "mykey22", "myid1", "OBJECT", `{"type":"Point","coordinates":[-130,38,10]}`).OK(), Do("SET", "mykey11", "myid3", "OBJECT", `{"type":"Point","coordinates":[-110,25,-8]}`).OK(), Do("SET", "mykey42", "myid2", "HASH", "9my5xp7").OK(), Do("SET", "mykey31", "myid4", "STRING", "value").OK(), Do("SET", "mykey310", "myid5", "STRING", "value").OK(), Do("KEYS", "*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), Do("KEYS", "*key*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), Do("KEYS", "mykey*").Str("[mykey11 mykey22 mykey31 mykey310 mykey42]"), Do("KEYS", "mykey4*").Str("[mykey42]"), Do("KEYS", "mykey*1").Str("[mykey11 mykey31]"), Do("KEYS", "mykey*1*").Str("[mykey11 mykey31 mykey310]"), Do("KEYS", "mykey*10").Str("[mykey310]"), Do("KEYS", "mykey*2").Str("[mykey22 mykey42]"), Do("KEYS", "*2").Str("[mykey22 mykey42]"), Do("KEYS", "*1*").Str("[mykey11 mykey31 mykey310]"), Do("KEYS", "mykey").Str("[]"), Do("KEYS", "mykey31").Str("[mykey31]"), Do("KEYS", "mykey[^3]*").Str("[mykey11 mykey22 mykey42]"), Do("KEYS").Err("wrong number of arguments for 'keys' command"), Do("KEYS", "*").JSON().Str(`{"ok":true,"keys":["mykey11","mykey22","mykey31","mykey310","mykey42"]}`), ) } func keys_PERSIST_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "STRING", "value").OK(), Do("EXPIRE", "mykey", "myid", 2).Str("1"), Do("PERSIST", "mykey", "myid").Str("1"), Do("PERSIST", "mykey", "myid").Str("0"), Do("PERSIST", "mykey").Err("wrong number of arguments for 'persist' command"), Do("PERSIST", "mykey2", "myid").Str("0"), Do("PERSIST", "mykey2", "myid").JSON().Err("key not found"), Do("PERSIST", "mykey", "myid2").Str("0"), Do("PERSIST", "mykey", "myid2").JSON().Err("id not found"), Do("EXPIRE", "mykey", "myid", 2).Str("1"), Do("PERSIST", "mykey", "myid").JSON().OK(), ) } func keys_SET_test(mc *mockServer) error { return mc.DoBatch( // Section: point Do("SET", "mykey", "myid", "POINT", 33, -115).OK(), Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), Do("SET", "mykey", "myid", "point", "33", "-112", "99").OK(), Do("SET", "mykey", "myid", "POINT", 33, -115, "RETURN").Str(`{"type":"Point","coordinates":[-115,33]}`), Do("SET", "mykey", "myid", "POINT", 33, -115, "RETURN", "POINT").Str("[33 -115]"), Do("SET", "mykey", "myid", "POINT", 33, -115, "RETURN", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), Do("SET", "mykey", "myid", "POINT", 33, -115, "RETURN", "BOUNDS").Str("[[33 -115] [33 -115]]"), Do("SET", "mykey", "myid", "POINT", 33, -115, "RETURN", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), Do("SET", "mykey", "myid", "POINT", 33, -115, "RETURN", "HASH", 7).Str("9my5xp7"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), // Section: object Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`).OK(), Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`, "RETURN").Str(`{"type":"Point","coordinates":[-115,33]}`), Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`, "RETURN", "POINT").Str("[33 -115]"), Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`, "RETURN", "BOUNDS").Str("[[33 -115] [33 -115]]"), Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`, "RETURN", "OBJECT").Str(`{"type":"Point","coordinates":[-115,33]}`), Do("SET", "mykey", "myid", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`, "RETURN", "HASH", 7).Str("9my5xp7"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), // Section: bounds Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115).OK(), Do("GET", "mykey", "myid", "POINT").Str("[33 -115]"), Do("GET", "mykey", "myid", "BOUNDS").Str("[[33 -115] [33 -115]]"), Do("GET", "mykey", "myid", "OBJECT").Str(`{"type":"Polygon","coordinates":[[[-115,33],[-115,33],[-115,33],[-115,33],[-115,33]]]}`), Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115, "RETURN").Str(`{"type":"Polygon","coordinates":[[[-115,33],[-115,33],[-115,33],[-115,33],[-115,33]]]}`), Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115, "RETURN", "POINT").Str("[33 -115]"), Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115, "RETURN", "BOUNDS").Str("[[33 -115] [33 -115]]"), Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115, "RETURN", "OBJECT").Str(`{"type":"Polygon","coordinates":[[[-115,33],[-115,33],[-115,33],[-115,33],[-115,33]]]}`), Do("SET", "mykey", "myid", "BOUNDS", 33, -115, 33, -115, "RETURN", "HASH", 7).Str("9my5xp7"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), // Section: hash Do("SET", "mykey", "myid", "HASH", "9my5xp7").OK(), Do("GET", "mykey", "myid", "HASH", 7).Str("9my5xp7"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), Do("SET", "mykey", "myid", "HASH", "9my5xp7").JSON().OK(), Do("SET", "mykey", "myid", "HASH", "9my5xp7", "RETURN").JSON().OK(), Do("SET", "mykey", "myid", "HASH", "9my5xp7", "RETURN", "HASH", 7).JSON().Str(`{"ok":true,"hash":"9my5xp7"}`), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), // Section: field Do("SET", "mykey", "myid", "FIELD", "f1", 33, "FIELD", "a2", 44.5, "HASH", "9my5xp7").OK(), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5 f1 33]]"), Do("DEL", "mykey", "myid").Str("1"), Do("SET", "mykey", "myid", "FIELD", "f1", 33, "FIELD", "a2", 44.5, "HASH", "9my5xp7", "RETURN", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5 f1 33]]"), Do("FSET", "mykey", "myid", "f1", 0).Str("1"), Do("FSET", "mykey", "myid", "f1", 0).Str("0"), Do("GET", "mykey", "myid", "WITHFIELDS", "HASH", 7).Str("[9my5xp7 [a2 44.5]]"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), // Section: string Do("SET", "mykey", "myid", "STRING", "value").OK(), Do("GET", "mykey", "myid").Str("value"), Do("SET", "mykey", "myid", "STRING", "value2").OK(), Do("GET", "mykey", "myid").Str("value2"), Do("DEL", "mykey", "myid").Str("1"), Do("GET", "mykey", "myid").Str(""), // Test error conditions Do("CONFIG", "SET", "maxmemory", "1").OK(), Do("SET", "mykey", "myid", "STRING", "value2").Err("OOM command not allowed when used memory > 'maxmemory'"), Do("CONFIG", "SET", "maxmemory", "0").OK(), Do("SET").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "FIELD", "f1").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "FIELD", "z", "1").Err("invalid argument 'z'"), Do("SET", "mykey", "myid", "EX").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "EX", "yyy").Err("invalid argument 'yyy'"), Do("SET", "mykey", "myid", "EX", "123").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "nx").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "nx", "xx").Err("invalid argument 'xx'"), Do("SET", "mykey", "myid", "xx", "nx").Err("invalid argument 'nx'"), Do("SET", "mykey", "myid", "string").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "point").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "point", "33").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "point", "33f", "-112").Err("invalid argument '33f'"), Do("SET", "mykey", "myid", "point", "33", "-112f").Err("invalid argument '-112f'"), Do("SET", "mykey", "myid", "point", "33", "-112f", "99").Err("invalid argument '-112f'"), Do("SET", "mykey", "myid", "bounds").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "bounds", "fff", "1", "2", "3").Err("invalid argument 'fff'"), Do("SET", "mykey", "myid", "hash").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "object").Err("wrong number of arguments for 'set' command"), Do("SET", "mykey", "myid", "object", "asd").Err("invalid data"), Do("SET", "mykey", "myid", "joint").Err("invalid argument 'joint'"), Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").Err(""), Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").JSON().Err("id not found"), Do("SET", "mykey", "myid1", "HASH", "9my5xp7").OK(), Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").Err(""), Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").OK(), Do("SET", "mykey", "myid", "XX", "HASH", "9my5xp7").OK(), Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").Err(""), Do("SET", "mykey", "myid", "NX", "HASH", "9my5xp7").JSON().Err("id already exists"), ) } func keys_STATS_test(mc *mockServer) error { return mc.DoBatch( Do("STATS", "mykey").Str("[nil]"), Do("SET", "mykey", "myid", "STRING", "value").OK(), Do("STATS", "mykey").Str("[[in_memory_size 9 num_objects 1 num_points 0 num_strings 1]]"), Do("STATS", "mykey", "hello").JSON().Str(`{"ok":true,"stats":[{"in_memory_size":9,"num_objects":1,"num_points":0,"num_strings":1},null]}`), Do("SET", "mykey", "myid2", "STRING", "value").OK(), Do("STATS", "mykey").Str("[[in_memory_size 19 num_objects 2 num_points 0 num_strings 2]]"), Do("SET", "mykey", "myid3", "OBJECT", `{"type":"Point","coordinates":[-115,33]}`).OK(), Do("STATS", "mykey").Str("[[in_memory_size 40 num_objects 3 num_points 1 num_strings 2]]"), Do("DEL", "mykey", "myid").Str("1"), Do("STATS", "mykey").Str("[[in_memory_size 31 num_objects 2 num_points 1 num_strings 1]]"), Do("DEL", "mykey", "myid3").Str("1"), Do("STATS", "mykey").Str("[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1]]"), Do("STATS", "mykey", "mykey2").Str("[[in_memory_size 10 num_objects 1 num_points 0 num_strings 1] nil]"), Do("DEL", "mykey", "myid2").Str("1"), Do("STATS", "mykey").Str("[nil]"), Do("STATS", "mykey", "mykey2").Str("[nil nil]"), Do("STATS", "mykey").Str("[nil]"), Do("STATS").Err(`wrong number of arguments for 'stats' command`), ) } func keys_TTL_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "STRING", "value").OK(), Do("EXPIRE", "mykey", "myid", 2).Str("1"), Do("EXPIRE", "mykey", "myid", 2).JSON().OK(), Sleep(time.Millisecond*10), Do("TTL", "mykey", "myid").Str("1"), Do("EXPIRE", "mykey", "myid", 1).Str("1"), Sleep(time.Millisecond*10), Do("TTL", "mykey", "myid").Str("0"), Do("TTL", "mykey", "myid").JSON().Str(`{"ok":true,"ttl":0}`), Do("TTL", "mykey2", "myid").Str("-2"), Do("TTL", "mykey", "myid2").Str("-2"), Do("TTL", "mykey").Err("wrong number of arguments for 'ttl' command"), Do("SET", "mykey", "myid", "STRING", "value").OK(), Do("TTL", "mykey", "myid").Str("-1"), Do("TTL", "mykey2", "myid").JSON().Err("key not found"), Do("TTL", "mykey", "myid2").JSON().Err("id not found"), ) } func keys_EXISTS_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "STRING", "value").OK(), Do("EXISTS", "mykey", "myid").Str("1"), Do("EXISTS", "mykey", "myid").JSON().Str(`{"ok":true,"exists":true}`), Do("EXISTS", "mykey", "myid2").Str("0"), Do("EXISTS", "mykey", "myid2").JSON().Str(`{"ok":true,"exists":false}`), Do("EXISTS", "mykey").Err("wrong number of arguments for 'exists' command"), Do("EXISTS", "mykey2", "myid").JSON().Err("key not found"), ) } func keys_FEXISTS_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid", "FIELD", "f1", "123", "STRING", "value").OK(), Do("FEXISTS", "mykey", "myid", "f1").Str("1"), Do("FEXISTS", "mykey", "myid", "f1").JSON().Str(`{"ok":true,"exists":true}`), Do("FEXISTS", "mykey", "myid", "f2").Str("0"), Do("FEXISTS", "mykey", "myid", "f2").JSON().Str(`{"ok":true,"exists":false}`), Do("FEXISTS", "mykey", "myid").Err("wrong number of arguments for 'fexists' command"), Do("FEXISTS", "mykey2", "myid", "f2").JSON().Err("key not found"), Do("FEXISTS", "mykey", "myid2", "f2").JSON().Err("id not found"), ) } func keys_SET_EX_test(mc *mockServer) (err error) { rand.Seed(time.Now().UnixNano()) // add a bunch of points for i := 0; i < 20000; i++ { val := fmt.Sprintf("val:%d", i) var resp string var lat, lon float64 lat = rand.Float64()*180 - 90 lon = rand.Float64()*360 - 180 resp, err = redis.String(mc.conn.Do("SET", fmt.Sprintf("mykey%d", i%3), val, "EX", 1+rand.Float64(), "POINT", lat, lon)) if err != nil { return } if resp != "OK" { err = fmt.Errorf("expected 'OK', got '%s'", resp) return } time.Sleep(time.Nanosecond) } time.Sleep(time.Second * 3) mc.conn.Do("OUTPUT", "json") json, _ := redis.String(mc.conn.Do("SERVER")) if !gjson.Get(json, "ok").Bool() { return errors.New("not ok") } if gjson.Get(json, "stats.num_objects").Int() > 0 { return errors.New("items left in database") } mc.conn.Do("FLUSHDB") return nil } func keys_FIELDS_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid1a", "FIELD", "a", 1, "POINT", 33, -115).OK(), Do("GET", "mykey", "myid1a", "WITHFIELDS").Str(`[{"type":"Point","coordinates":[-115,33]} [a 1]]`), Do("SET", "mykey", "myid1a", "FIELD", "a", "a", "POINT", 33, -115).OK(), Do("GET", "mykey", "myid1a", "WITHFIELDS").Str(`[{"type":"Point","coordinates":[-115,33]} [a a]]`), Do("SET", "mykey", "myid1a", "FIELD", "a", 1, "FIELD", "b", 2, "POINT", 33, -115).OK(), Do("GET", "mykey", "myid1a", "WITHFIELDS").Str(`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2]]`), Do("SET", "mykey", "myid1a", "FIELD", "b", 2, "POINT", 33, -115).OK(), Do("GET", "mykey", "myid1a", "WITHFIELDS").Str(`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2]]`), Do("SET", "mykey", "myid1a", "FIELD", "b", 2, "FIELD", "a", "1", "FIELD", "c", 3, "POINT", 33, -115).OK(), Do("GET", "mykey", "myid1a", "WITHFIELDS").Str(`[{"type":"Point","coordinates":[-115,33]} [a 1 b 2 c 3]]`), Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Err("key not found"), Do("SET", "fleet", "truck1", "FIELD", "speed", "0", "POINT", "-112", "33").JSON().OK(), Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[33,-112]}}`), Do("SET", "fleet", "truck1", "FIELD", "speed", "1", "POINT", "-112", "33").JSON().OK(), Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[33,-112]},"fields":{"speed":1}}`), Do("SET", "fleet", "truck1", "FIELD", "speed", "0", "POINT", "-112", "33").JSON().OK(), Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[33,-112]}}`), Do("SET", "fleet", "truck1", "FIELD", "speed", "2", "POINT", "-112", "33").JSON().OK(), Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[33,-112]},"fields":{"speed":2}}`), // Do some whereins queries Do("SET", "whereins", "id1", "FIELD", "test", "2", "POINT", "-112", "33").JSON().OK(), Do("SET", "whereins", "id2", "FIELD", "TEST", "3", "POINT", "-111", "32").JSON().OK(), Do("SET", "whereins", "id3", "FIELD", "teSt", "4", "POINT", "-110", "31").JSON().OK(), Do("SCAN", "whereins", "WHEREIN", "test", "1", "2", "OBJECTS").JSON().Str(`{"ok":true,"fields":["test"],"objects":[{"id":"id1","object":{"type":"Point","coordinates":[33,-112]},"fields":[2]}],"count":1,"cursor":0}`), Do("SCAN", "whereins", "WHEREIN", "TEST", "1", "3", "OBJECTS").JSON().Str(`{"ok":true,"fields":["TEST"],"objects":[{"id":"id2","object":{"type":"Point","coordinates":[32,-111]},"fields":[3]}],"count":1,"cursor":0}`), Do("SCAN", "whereins", "WHEREIN", "teSt", "1", "4", "OBJECTS").JSON().Str(`{"ok":true,"fields":["teSt"],"objects":[{"id":"id3","object":{"type":"Point","coordinates":[31,-110]},"fields":[4]}],"count":1,"cursor":0}`), Do("SCAN", "whereins", "OBJECTS").JSON().Str(`{"ok":true,"fields":["TEST","teSt","test"],"objects":[{"id":"id1","object":{"type":"Point","coordinates":[33,-112]},"fields":[0,0,2]},{"id":"id2","object":{"type":"Point","coordinates":[32,-111]},"fields":[3,0,0]},{"id":"id3","object":{"type":"Point","coordinates":[31,-110]},"fields":[0,4,0]}],"count":3,"cursor":0}`), // Do some GJSON queries. Do("SET", "fleet", "truck2", "FIELD", "hello", `{"world":"tom"}`, "POINT", "-112", "33").JSON().OK(), Do("SCAN", "fleet", "WHERE", "hello", `{"world":"tom"}`, `{"world":"tom"}`, "COUNT").JSON().Str(`{"ok":true,"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world == 'tom'", "COUNT").JSON().Str(`{"ok":true,"count":1,"cursor":0}`), // The next scan does not match on anything, but since we're matching // on zeros, which is the default, then all (two) objects are returned. Do("SCAN", "fleet", "WHERE", "hello.world.1", `0`, `0`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1","truck2"],"count":2,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world", ">", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world", ">=", `Tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck2"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world", ">=", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck2"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world", "==", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck2"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world", "<", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world", "<=", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1","truck2"],"count":2,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world", "<", `uom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1","truck2"],"count":2,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world", "!=", `tom`, "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), // Test REGEX on FIELD Do("SCAN", "fleet", "WHERE", "hello.world =~ 'tom.*'", "IDS").JSON().Str(`{"ok":true,"ids":["truck2"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world =~ 'foo.*'", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "hello.world =~ '(*'", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SET", "fleet", "truck1", "OBJECT", `{"type":"Feature","geometry":{"type":"Point","coordinates":[-112,33]},"properties":{"speed":50},"asdf":"Adsf"}`).JSON().OK(), Do("SCAN", "fleet", "WHERE", "properties.speed", ">", 49, "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "properties.speed", ">", 50, "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "properties.speed", "<", 51, "IDS").JSON().Str(`{"ok":true,"ids":["truck1","truck2"],"count":2,"cursor":0}`), // Test REGEX on OBJECT properties Do("SET", "fleet", "truck3", "OBJECT", `{"type":"Feature","geometry":{"type":"Point","coordinates":[-112,33]},"properties":{"name":"truck01"}}`).JSON().OK(), Do("SCAN", "fleet", "WHERE", "properties.name =~ 'truck.*'", "IDS").JSON().Str(`{"ok":true,"ids":["truck3"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "properties.name =~ 'foo.*'", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "properties.name =~ '(*'", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("DROP", "fleet").JSON().OK(), Do("SET", "fleet", "truck1", "FIELD", "speed", "50", "POINT", "-112", "33").JSON().OK(), Do("SET", "fleet", "truck1", "FIELD", "Speed", "51", "POINT", "-112", "33").JSON().OK(), Do("SET", "fleet", "truck1", "FIELD", "speeD", "52", "POINT", "-112", "33").JSON().OK(), Do("SET", "fleet", "truck1", "FIELD", "SpeeD", "53", "POINT", "-112", "33").JSON().OK(), Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[33,-112]},"fields":{"SpeeD":53,"Speed":51,"speeD":52,"speed":50}}`), Do("SCAN", "fleet", "WHERE", "speed == 50", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Speed == 50", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Speed == 51", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "speed", 50, 50, "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Speed", 51, 51, "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Speed", 50, 50, "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "speed", 51, 51, "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("DROP", "fleet").JSON().OK(), Do("SET", "fleet", "truck1", "field", "props", `{"speed":50,"Speed":51}`, "point", "33", "-112").JSON().OK(), Do("SET", "fleet", "truck1", "field", "Props", `{"speed":52,"Speed":53}`, "point", "33", "-112").JSON().OK(), Do("GET", "fleet", "truck1", "WITHFIELDS").JSON().Str(`{"ok":true,"object":{"type":"Point","coordinates":[-112,33]},"fields":{"Props":{"speed":52,"Speed":53},"props":{"speed":50,"Speed":51}}}`), Do("SCAN", "fleet", "WHERE", "props.speed == 50", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "props.Speed == 51", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Props.speed == 52", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Props.Speed == 53", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "props.Speed == 52", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "props.speed == 51", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Props.speed == 53", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Props.Speed == 50", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "props.speed > 49", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "props.Speed > 49", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Props.speed > 49", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Props.Speed > 49", "IDS").JSON().Str(`{"ok":true,"ids":["truck1"],"count":1,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "props.Speed > 53", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "props.speed > 53", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Props.speed > 53", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("SCAN", "fleet", "WHERE", "Props.Speed > 53", "IDS").JSON().Str(`{"ok":true,"ids":[],"count":0,"cursor":0}`), Do("DROP", "fleet").JSON().OK(), Do("SET", "fleet", "1", "field", "teamId", "1", "field", "optionalId", "999", "point", "0", "0").JSON().OK(), Do("SET", "fleet", "2", "field", "teamId", "1", "point", "0", "0").JSON().OK(), Do("SCAN", "fleet", "COUNT").JSON().Str(`{"ok":true,"count":2,"cursor":0}`), Do("SCAN", "fleet", "WHEREIN", "teamId", "1", "1", "COUNT").JSON().Str(`{"ok":true,"count":2,"cursor":0}`), Do("SCAN", "fleet", "WHEREIN", "teamId", "1", "1", "WHERE", "!optionalId || optionalId == 999", "count").JSON().Str(`{"ok":true,"count":2,"cursor":0}`), Do("SCAN", "fleet", "WHEREIN", "teamId", "1", "1", "WHERE", "!!!optionalId || optionalId == 999", "count").JSON().Str(`{"ok":true,"count":2,"cursor":0}`), Do("SCAN", "fleet", "WHEREIN", "teamId", "1", "1", "WHERE", "optionalId == 0 || optionalId == 999", "count").JSON().Str(`{"ok":true,"count":2,"cursor":0}`), Do("SCAN", "fleet", "WHEREIN", "teamId", "1", "1", "WHERE", "1 == 1 || optionalId == 999", "count").JSON().Str(`{"ok":true,"count":2,"cursor":0}`), ) } func keys_PDEL_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid1a", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid1b", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid2a", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid2b", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid3a", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid3b", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid4a", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid4b", "POINT", 33, -115).OK(), Do("PDEL", "mykey").Err("wrong number of arguments for 'pdel' command"), Do("PDEL", "mykeyNA", "*").Str("0"), Do("PDEL", "mykey", "myid1a").Str("1"), Do("PDEL", "mykey", "myid1a").Str("0"), Do("PDEL", "mykey", "myid1*").Str("1"), Do("PDEL", "mykey", "myid2*").Str("2"), Do("PDEL", "mykey", "*b").Str("2"), Do("PDEL", "mykey", "*").Str("2"), Do("PDEL", "mykey", "*").Str("0"), Do("SET", "mykey", "myid1a", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid1b", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid2a", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid2b", "POINT", 33, -115).OK(), Do("SET", "mykey", "myid3a", "POINT", 33, -115).OK(), Do("PDEL", "mykey", "*").JSON().OK(), ) } func keys_WHEREIN_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115).OK(), Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), Do("WITHIN", "mykey", "WHEREIN", "a", "a", 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument 'a'"), Do("WITHIN", "mykey", "WHEREIN", "a", 1, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument '1'"), Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, "a", 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str("[0 []]"), Do("WITHIN", "mykey", "WHEREIN", "a", 4, 0, "a", 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), Do("SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115).OK(), Do("SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02).OK(), Do("WITHIN", "mykey", "WHEREIN", "a", 3, 0, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), // zero value should not match 1 and 2 Do("SET", "mykey", "myid_a0", "FIELD", "a", 0, "POINT", 33, -115.02).OK(), Do("WITHIN", "mykey", "WHEREIN", "a", 2, 1, 2, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), ) } func keys_WHEREEVAL_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid_a1", "FIELD", "a", 1, "POINT", 33, -115).OK(), Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", "a", 0.5, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument 'a'"), Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1])", 1, 0.5, 4, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Err("invalid argument '4'"), Do("SET", "mykey", "myid_a2", "FIELD", "a", 2, "POINT", 32.99, -115).OK(), Do("SET", "mykey", "myid_a3", "FIELD", "a", 3, "POINT", 33, -115.02).OK(), Do("WITHIN", "mykey", "WHEREEVAL", "return FIELDS.a > tonumber(ARGV[1]) and FIELDS.a ~= tonumber(ARGV[2])", 2, 0.5, 3, "BOUNDS", 32.8, -115.2, 33.2, -114.8).Str(`[0 [[myid_a2 {"type":"Point","coordinates":[-115,32.99]} [a 2]] [myid_a1 {"type":"Point","coordinates":[-115,33]} [a 1]]]]`), ) } func keys_TYPE_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "myid1", "POINT", 33, -115).OK(), Do("TYPE", "mykey").Str("hash"), Do("TYPE", "mykey", "hello").Err("wrong number of arguments for 'type' command"), Do("TYPE", "mykey2").Str("none"), Do("TYPE", "mykey2").JSON().Err("key not found"), Do("TYPE", "mykey").JSON().Str(`{"ok":true,"type":"hash"}`), ) } func keys_FLUSHDB_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey1", "myid1", "POINT", 33, -115).OK(), Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(), Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"), Do("KEYS", "*").Str("[mykey1 mykey2]"), Do("CHANS", "*").JSON().Func(func(s string) error { if gjson.Get(s, "chans.#").Int() != 1 { return fmt.Errorf("expected '%d', got '%d'", 1, gjson.Get(s, "chans.#").Int()) } return nil }), Do("FLUSHDB", "arg2").Err("wrong number of arguments for 'flushdb' command"), Do("FLUSHDB").OK(), Do("KEYS", "*").Str("[]"), Do("CHANS", "*").Str("[]"), Do("SET", "mykey1", "myid1", "POINT", 33, -115).OK(), Do("SET", "mykey2", "myid1", "POINT", 33, -115).OK(), Do("SETCHAN", "mychan", "INTERSECTS", "mykey1", "BOUNDS", 10, 10, 10, 10).Str("1"), Do("FLUSHDB").JSON().OK(), ) } func keys_HEALTHZ_test(mc *mockServer) error { return mc.DoBatch( Do("HEALTHZ").OK(), Do("HEALTHZ").JSON().OK(), Do("HEALTHZ", "arg").Err(`wrong number of arguments for 'healthz' command`), ) } func keys_SERVER_test(mc *mockServer) error { return mc.DoBatch( Do("SERVER").Func(func(s string) error { valid := strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") && strings.Contains(s, "cpus") && strings.Contains(s, "mem_alloc") if !valid { return errors.New("looks invalid") } return nil }), Do("SERVER").JSON().Func(func(s string) error { if !gjson.Get(s, "ok").Bool() { return errors.New("not ok") } valid := gjson.Get(s, "stats.cpus").Exists() && gjson.Get(s, "stats.mem_alloc").Exists() if !valid { return errors.New("looks invalid") } return nil }), Do("SERVER", "ext").Func(func(s string) error { valid := strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") && strings.Contains(s, "sys_cpus") && strings.Contains(s, "tile38_connected_clients") if !valid { return errors.New("looks invalid") } return nil }), Do("SERVER", "ext").JSON().Func(func(s string) error { if !gjson.Get(s, "ok").Bool() { return errors.New("not ok") } valid := gjson.Get(s, "stats.sys_cpus").Exists() && gjson.Get(s, "stats.tile38_connected_clients").Exists() if !valid { return errors.New("looks invalid") } return nil }), Do("SERVER", "ett").Err(`invalid argument 'ett'`), Do("SERVER", "ett").JSON().Err(`invalid argument 'ett'`), ) } func keys_INFO_test(mc *mockServer) error { return mc.DoBatch( Do("INFO").Func(func(s string) error { if !strings.Contains(s, "# Clients") || !strings.Contains(s, "# Stats") { return errors.New("looks invalid") } return nil }), Do("INFO", "all").Func(func(s string) error { if !strings.Contains(s, "# Clients") || !strings.Contains(s, "# Stats") { return errors.New("looks invalid") } return nil }), Do("INFO", "default").Func(func(s string) error { if !strings.Contains(s, "# Clients") || !strings.Contains(s, "# Stats") { return errors.New("looks invalid") } return nil }), Do("INFO", "cpu").Func(func(s string) error { if !strings.Contains(s, "# CPU") || strings.Contains(s, "# Clients") || strings.Contains(s, "# Stats") { return errors.New("looks invalid") } return nil }), Do("INFO", "cpu", "clients").Func(func(s string) error { if !strings.Contains(s, "# CPU") || !strings.Contains(s, "# Clients") || strings.Contains(s, "# Stats") { return errors.New("looks invalid") } return nil }), Do("INFO").JSON().Func(func(s string) error { if gjson.Get(s, "info.tile38_version").String() == "" { return errors.New("looks invalid") } return nil }), ) } ================================================ FILE: tests/metrics_test.go ================================================ package tests import ( "fmt" "io" "net/http" "strings" ) func subTestMetrics(g *testGroup) { g.regSubTest("basic", metrics_basic_test) } func downloadURLWithStatusCode(u string) (int, string, error) { resp, err := http.Get(u) if err != nil { return 0, "", err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return 0, "", err } return resp.StatusCode, string(body), nil } func metrics_basic_test(mc *mockServer) error { maddr := fmt.Sprintf("http://127.0.0.1:%d/", mc.metricsPort()) mc.Do("SET", "metrics_test_1", "1", "FIELD", "foo", 5.5, "POINT", 5, 5) mc.Do("SET", "metrics_test_2", "2", "FIELD", "foo", 19.19, "POINT", 19, 19) mc.Do("SET", "metrics_test_2", "3", "FIELD", "foo", 19.19, "POINT", 19, 19) mc.Do("SET", "metrics_test_2", "truck1:driver", "STRING", "John Denton") status, index, err := downloadURLWithStatusCode(maddr) if err != nil { return err } if status != 200 { return fmt.Errorf("Expected status code 200, got: %d", status) } if !strings.Contains(index, " 0 { if opts.AOFFileName == "" { opts.AOFFileName = "appendonly.aof" } if err := os.MkdirAll(dir, 0777); err != nil { return nil, err } err := os.WriteFile(filepath.Join(dir, opts.AOFFileName), opts.AOFData, 0666) if err != nil { return nil, err } } shutdown := make(chan bool) s := &mockServer{port: port, dir: dir, shutdown: shutdown} if opts.Metrics { s.mport = getNextPort() } var ferr atomic.Pointer[error] // ferr for when the server fails to start go func() { sopts := server.Options{ Host: "localhost", Port: port, Dir: dir, UseHTTP: true, DevMode: true, AppendOnly: true, Shutdown: shutdown, ShowDebugMessages: true, } if opts.Metrics { sopts.MetricsAddr = fmt.Sprintf(":%d", s.mport) } err := server.Serve(sopts) if err != nil { ferr.CompareAndSwap(nil, &err) } }() if err := s.waitForStartup(&ferr); err != nil { s.Close() return nil, err } return s, nil } func (s *mockServer) waitForStartup(ferr *atomic.Pointer[error]) error { var lerr error start := time.Now() for { if perr := ferr.Load(); perr != nil { return *perr } if time.Since(start) > time.Second*5 { if lerr != nil { return lerr } return errTimeout } resp, err := redis.String(s.Do("SET", "please", "allow", "POINT", "33", "-115")) if err != nil { lerr = err } else if resp != "OK" { lerr = errors.New("not OK") } else { resp, err := redis.Int(s.Do("DEL", "please", "allow")) if err != nil { lerr = err } else if resp != 1 { lerr = errors.New("not 1") } else { return nil } } time.Sleep(time.Millisecond * 100) } } func (mc *mockServer) Close() { if mc == nil || mc.closed { return } mc.closed = true mc.shutdown <- true if mc.conn != nil { mc.conn.Close() } if mc.dir != "" { os.RemoveAll(mc.dir) } } func (mc *mockServer) ResetConn() { if mc.conn != nil { mc.conn.Close() mc.conn = nil } } func (s *mockServer) DoPipeline(cmds [][]interface{}) ([]interface{}, error) { if s.conn == nil { var err error s.conn, err = redis.Dial("tcp", fmt.Sprintf(":%d", s.port)) if err != nil { return nil, err } } //defer conn.Close() for _, cmd := range cmds { if err := s.conn.Send(cmd[0].(string), cmd[1:]...); err != nil { return nil, err } } if err := s.conn.Flush(); err != nil { return nil, err } var resps []interface{} for i := 0; i < len(cmds); i++ { resp, err := s.conn.Receive() if err != nil { resps = append(resps, err) } else { resps = append(resps, resp) } } return resps, nil } func (s *mockServer) Do(commandName string, args ...interface{}) (interface{}, error) { resps, err := s.DoPipeline([][]interface{}{ append([]interface{}{commandName}, args...), }) if err != nil { return nil, err } if len(resps) != 1 { return nil, errors.New("invalid number or responses") } return resps[0], nil } func (mc *mockServer) DoBatch(commands ...interface{}) error { // Probe for I/O tests if len(commands) > 0 { if _, ok := commands[0].(*IO); ok { var cmds []*IO // If the first is an I/O test then all must be for _, cmd := range commands { if cmd, ok := cmd.(*IO); ok { cmds = append(cmds, cmd) } else { return errors.New("DoBatch cannot mix I/O tests with other kinds") } } for i, cmd := range cmds { if err := mc.doIOTest(i, cmd); err != nil { return err } } return nil } } var tag string for _, commands := range commands { switch commands := commands.(type) { case string: tag = commands case [][]interface{}: for i := 0; i < len(commands); i += 2 { cmds := commands[i] if dur, ok := cmds[0].(time.Duration); ok { time.Sleep(dur) } else { if err := mc.DoExpect(commands[i+1], cmds[0].(string), cmds[1:]...); err != nil { if tag == "" { return fmt.Errorf("batch[%d]: %v", i/2, err) } else { return fmt.Errorf("batch[%d][%v]: %v", i/2, tag, err) } } } } tag = "" case *IO: return errors.New("DoBatch cannot mix I/O tests with other kinds") default: return fmt.Errorf("Unknown command input") } } return nil } func normalize(v interface{}) interface{} { switch v := v.(type) { default: return v case []interface{}: for i := 0; i < len(v); i++ { v[i] = normalize(v[i]) } case []uint8: return string(v) } return v } func (mc *mockServer) DoExpect(expect interface{}, commandName string, args ...interface{}) error { if v, ok := expect.([]interface{}); ok { expect = v[0] } resp, err := mc.Do(commandName, args...) if err != nil { if exs, ok := expect.(string); ok { if err.Error() == exs { return nil } } return err } if b, ok := resp.([]byte); ok && len(b) > 1 && b[0] == '{' { b, err = sjson.DeleteBytes(b, "elapsed") if err == nil { resp = b } } oresp := resp resp = normalize(resp) if expect == nil && resp != nil { return fmt.Errorf("expected '%v', got '%v'", expect, resp) } if vv, ok := resp.([]interface{}); ok { var ss []string for _, v := range vv { if v == nil { ss = append(ss, "nil") } else if s, ok := v.(string); ok { ss = append(ss, s) } else if b, ok := v.([]uint8); ok { if b == nil { ss = append(ss, "nil") } else { ss = append(ss, string(b)) } } else { ss = append(ss, fmt.Sprintf("%v", v)) } } resp = ss } if b, ok := resp.([]uint8); ok { if b == nil { resp = nil } else { resp = string([]byte(b)) } } err = func() (err error) { defer func() { v := recover() if v != nil { err = fmt.Errorf("panic '%v'", v) } }() if fn, ok := expect.(func(v, org interface{}) (resp, expect interface{})); ok { resp, expect = fn(resp, oresp) } if fn, ok := expect.(func(v interface{}) (resp, expect interface{})); ok { resp, expect = fn(resp) } return nil }() if err != nil { return err } if fn, ok := expect.(func(string) bool); ok { if !fn(fmt.Sprintf("%v", resp)) { return fmt.Errorf("unexpected for response '%v'", resp) } } else if fn, ok := expect.(func(string) error); ok { err := fn(fmt.Sprintf("%v", resp)) if err != nil { return fmt.Errorf("%s, for response '%v'", err.Error(), resp) } } else if fmt.Sprintf("%v", resp) != fmt.Sprintf("%v", expect) { return fmt.Errorf("expected '%v', got '%v'", expect, resp) } return nil } ================================================ FILE: tests/monitor_test.go ================================================ package tests import ( "fmt" "strings" "sync" "github.com/gomodule/redigo/redis" ) func subTestMonitor(g *testGroup) { g.regSubTest("monitor", follower_monitor_test) } func follower_monitor_test(mc *mockServer) error { N := 1000 ch := make(chan error) var wg sync.WaitGroup wg.Add(1) go func() { ch <- func() error { conn, err := redis.Dial("tcp", fmt.Sprintf("localhost:%d", mc.port)) if err != nil { wg.Done() return err } defer conn.Close() s, err := redis.String(conn.Do("MONITOR")) if err != nil { wg.Done() return err } if s != "OK" { wg.Done() return fmt.Errorf("expected '%s', got '%s'", "OK", s) } wg.Done() for i := 0; i < N; i++ { s, err := redis.String(conn.Receive()) if err != nil { return err } ex := fmt.Sprintf(`"mykey" "%d"`, i) if !strings.Contains(s, ex) { return fmt.Errorf("expected '%s', got '%s'", ex, s) } } return nil }() }() wg.Wait() conn, err := redis.Dial("tcp", fmt.Sprintf("localhost:%d", mc.port)) if err != nil { return err } defer conn.Close() for i := 0; i < N; i++ { s, err := redis.String(conn.Do("SET", "mykey", i, "POINT", 10, 10)) if err != nil { return err } if s != "OK" { return fmt.Errorf("expected '%s', got '%s'", "OK", s) } } err = <-ch if err != nil { err = fmt.Errorf("monitor client: %w", err) } return err } ================================================ FILE: tests/proto_test.go ================================================ package tests import ( "fmt" "net/http" ) func subTestProto(g *testGroup) { g.regSubTest("HTTP CORS", proto_HTTP_CORS_test) } func proto_HTTP_CORS_test(mc *mockServer) error { // Make CORS request for GET /SERVER morigin := "http://my-test-origin" url := fmt.Sprintf("http://127.0.0.1:%d/SERVER", mc.port) req, err := http.NewRequest(http.MethodOptions, url, nil) if err != nil { return err } req.Header.Add("Origin", morigin) req.Header.Add("Access-Control-Request-Method", "GET") req.Header.Add("Access-Control-Request-Headers", "Authorization") resp, err := http.DefaultClient.Do(req) // Validate CORS response if err != nil { return err } if resp.StatusCode != 204 { return fmt.Errorf("expected http stuats '204', got '%d'", resp.StatusCode) } origin := resp.Header.Get("Access-Control-Allow-Origin") methods := resp.Header.Get("Access-Control-Allow-Methods") headers := resp.Header.Get("Access-Control-Allow-Headers") if !(origin == "*" || origin == morigin) { return fmt.Errorf("expected http access-control-allow-origin value '*', got '%s'", origin) } if methods != "POST, GET, OPTIONS" { return fmt.Errorf("expected http access-control-allow-Methods value 'POST, GET, OPTIONS', got '%s'", methods) } if headers != "*, Authorization" { return fmt.Errorf("expected http access-control-allow-headers value '*, Authorization', got '%s'", headers) } // Make the actual request now resp, err = http.Get(url) if err != nil { return err } origin = resp.Header.Get("Access-Control-Allow-Origin") if !(origin == "*" || origin == morigin) { return fmt.Errorf("expected http access-control-allow-origin value '*', got '%s'", origin) } return nil } ================================================ FILE: tests/scripts_test.go ================================================ package tests import ( "fmt" "strings" ) func subTestScripts(g *testGroup) { g.regSubTest("BASIC", scripts_BASIC_test) g.regSubTest("ATOMIC", scripts_ATOMIC_test) g.regSubTest("READONLY", scripts_READONLY_test) g.regSubTest("NONATOMIC", scripts_NONATOMIC_test) g.regSubTest("VULN", scripts_VULN_test) } func scripts_BASIC_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"EVAL", "return 2 + 2", 0}, {"4"}, {"SCRIPT LOAD", "return 2 + 2"}, {"2dd1b44209ecb49617af05caf0491390a03c1cc4"}, {"SCRIPT EXISTS", "2dd1b44209ecb49617af05caf0491390a03c1cc4", "no_script"}, {"[1 0]"}, {"EVALSHA", "2dd1b44209ecb49617af05caf0491390a03c1cc4", "0"}, {"4"}, {"SCRIPT FLUSH"}, {"OK"}, {"SCRIPT EXISTS", "2dd1b44209ecb49617af05caf0491390a03c1cc4", "no_script"}, {"[0 0]"}, {"EVAL", "return KEYS[1] .. ' only'", 1, "key1"}, {"key1 only"}, {"EVAL", "return KEYS[1] .. ' and ' .. ARGV[1]", 1, "key1", "arg1"}, {"key1 and arg1"}, {"EVAL", "return ARGV[1] .. ' and ' .. ARGV[2]", 0, "arg1", "arg2"}, {"arg1 and arg2"}, {"EVAL", "return tile38.sha1hex('asdf')", 0}, {"3da541559918a808c2402bba5012f6c60b27661c"}, {"EVAL", "return tile38.distance_to(37.7341129, -122.4408378, 37.733, -122.43)", 0}, {"961"}, }) } func scripts_ATOMIC_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"EVAL", "return tile38.call('get', KEYS[1], ARGV[1])", "1", "mykey", "myid"}, {nil}, {"EVAL", "return tile38.call('set', KEYS[1], ARGV[1], 'point', 33, -115)", "1", "mykey", "myid1"}, {"OK"}, {"EVAL", "return tile38.call('get', KEYS[1], ARGV[1], ARGV[2])", "1", "mykey", "myid1", "point"}, {"[33 -115]"}, }) } func scripts_READONLY_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"EVALRO", "return tile38.call('get', KEYS[1], ARGV[1])", "1", "mykey", "myid"}, {nil}, {"EVALRO", "return tile38.call('set', KEYS[1], ARGV[1], 'point', 33, -115)", "1", "mykey", "myid1"}, { func(v interface{}) (resp, expect interface{}) { s := fmt.Sprintf("%v", v) if strings.Contains(s, "ERR read only") { return v, v } return v, "A lua stack containing 'ERR read only'" }, }, {"EVALRO", "return tile38.pcall('set', KEYS[1], ARGV[1], 'point', 33, -115)", "1", "mykey", "myid1"}, {"ERR read only"}, {"SET", "mykey", "myid1", "POINT", 33, -115}, {"OK"}, {"EVALRO", "return tile38.call('get', KEYS[1], ARGV[1], ARGV[2])", "1", "mykey", "myid1", "point"}, {"[33 -115]"}, }) } func scripts_NONATOMIC_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"EVALNA", "return tile38.call('get', KEYS[1], ARGV[1])", "1", "mykey", "myid"}, {nil}, {"EVALNA", "return tile38.call('set', KEYS[1], ARGV[1], 'point', 33, -115)", "1", "mykey", "myid1"}, {"OK"}, {"EVALNA", "return tile38.call('get', KEYS[1], ARGV[1], ARGV[2])", "1", "mykey", "myid1", "point"}, {"[33 -115]"}, }) } func scripts_VULN_test(mc *mockServer) error { return mc.DoBatch([][]interface{}{ {"EVAL", "return io", "0"}, {nil}, {"EVAL", "return file", "0"}, {nil}, {"EVAL", "return os.execute", "0"}, {nil}, {"EVAL", "return os.getenv", "0"}, {nil}, {"EVAL", "return os.clock", "0"}, {"ERR Unsupported lua type: function"}, {"EVAL", "return loadfile", "0"}, {nil}, {"EVAL", "return tonumber(ARGV[1])", "0", "38"}, {"38"}, {"EVAL", "return package", "0"}, {nil}, }) } ================================================ FILE: tests/stats_test.go ================================================ package tests import ( "errors" "github.com/tidwall/gjson" ) func subTestInfo(g *testGroup) { g.regSubTest("valid json", info_valid_json_test) } func info_valid_json_test(mc *mockServer) error { if _, err := mc.Do("OUTPUT", "JSON"); err != nil { return err } res, err := mc.Do("INFO") if err != nil { return err } bres, ok := res.([]byte) if !ok { return errors.New("Failed to type assert INFO response") } sres := string(bres) if !gjson.Valid(sres) { return errors.New("INFO response was invalid") } info := gjson.Get(sres, "info").String() if !gjson.Valid(info) { return errors.New("INFO.info response was invalid") } return nil } ================================================ FILE: tests/testcmd_test.go ================================================ package tests func subTestTestCmd(g *testGroup) { g.regSubTest("WITHIN", testcmd_WITHIN_test) g.regSubTest("INTERSECTS", testcmd_INTERSECTS_test) g.regSubTest("INTERSECTS_CLIP", testcmd_INTERSECTS_CLIP_test) g.regSubTest("ExpressionErrors", testcmd_expressionErrors_test) g.regSubTest("Expressions", testcmd_expression_test) } func testcmd_WITHIN_test(mc *mockServer) error { poly := `{ "type": "Polygon", "coordinates": [ [ [-122.44126439094543,37.72906137107], [-122.43980526924135,37.72906137107], [-122.43980526924135,37.73421283683962], [-122.44126439094543,37.73421283683962], [-122.44126439094543,37.72906137107] ] ] }` poly8 := `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}` poly9 := `{"type":"Polygon","coordinates":[[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}` poly10 := `{"type":"Polygon","coordinates":[[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}` return mc.DoBatch( Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), Do("SET", "mykey", "point2", "POINT", 37.7335, -122.44121).OK(), Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), Do("SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`).OK(), Do("SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`).OK(), Do("SET", "mykey", "point6", "POINT", -5, 5).OK(), Do("SET", "mykey", "point7", "POINT", 33, 21).OK(), Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), Do("TEST", "GET", "mykey", "point1", "WITHIN", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").Str("1"), Do("TEST", "GET", "mykey", "line3", "WITHIN", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "poly4", "WITHIN", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "multipoly5", "WITHIN", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "poly8", "WITHIN", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").Str("1"), Do("TEST", "GET", "mykey", "poly8", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "90").JSON().Str(`{"ok":true,"result":true}`), Do("TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly).Str("0"), Do("TEST", "GET", "mykey", "point6", "WITHIN", "OBJECT", poly).JSON().Str(`{"ok":true,"result":false}`), Do("TEST", "GET", "mykey", "point7", "WITHIN", "OBJECT", poly).Str("0"), Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8).Str("1"), Do("TEST", "OBJECT", poly10, "WITHIN", "OBJECT", poly8).Str("0"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0", "ff").Err("invalid argument 'ff'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "ee", "ff").Err("invalid argument 'ee'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "dd", "ee", "ff").Err("invalid argument 'dd'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "cc", "dd", "ee", "ff").Err("invalid argument 'cc'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "bb", "cc", "dd", "ee", "ff").Err("invalid argument 'bb'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "0").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "SECTOR", "37.72999", "-122.44760", "1000", "1", "1").Err("equal bearings (1 == 1), use CIRCLE instead"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "10000").Str("1"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "10000", "10000").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "cc").Err("invalid argument 'cc'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "bb", "cc").Err("invalid argument 'bb'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "aa", "bb", "cc").Err("invalid argument 'aa'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "CIRCLE", "37.72999", "-122.44760", "-10000").Err("invalid argument '-10000'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash", "123").Str("0"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "hash", "123", "asdf").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "123").Str("0"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "pqowie").Err("invalid argument 'pqowie'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "quadkey", "123", "asdf").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2", "3").Str("0"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "2", "cc").Err("invalid argument 'cc'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "1", "bb", "cc").Err("invalid argument 'bb'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "tile", "aa", "bb", "cc").Err("invalid argument 'aa'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "2").Str("0"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "point").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "2", "3").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "1", "bb").Err("invalid argument 'bb'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "point", "aa", "bb").Err("invalid argument 'aa'"), Do("TEST", "GET", "mykey", "point1", "WITHIN").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "4").Str("0"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "4", "5").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "3", "dd").Err("invalid argument 'dd'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "2", "cc", "dd").Err("invalid argument 'cc'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "1", "bb", "cc", "dd").Err("invalid argument 'bb'"), Do("TEST", "GET", "mykey", "point1", "WITHIN", "bounds", "aa", "bb", "cc", "dd").Err("invalid argument 'aa'"), Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey", "point6").Str("1"), Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey__", "point6").Err("key not found"), Do("TEST", "GET", "mykey", "point6", "WITHIN", "GET", "mykey", "point6__").Err("id not found"), ) } func testcmd_INTERSECTS_test(mc *mockServer) error { poly := `{ "type": "Polygon", "coordinates": [ [ [-122.44126439094543,37.732906137107], [-122.43980526924135,37.732906137107], [-122.43980526924135,37.73421283683962], [-122.44126439094543,37.73421283683962], [-122.44126439094543,37.732906137107] ] ] }` poly8 := `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}` poly9 := `{"type": "Polygon","coordinates": [[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}` poly10 := `{"type": "Polygon","coordinates": [[[-122.44040071964262,37.73359343010089],[-122.4402666091919,37.73359343010089],[-122.4402666091919,37.73373767596864],[-122.44040071964262,37.73373767596864],[-122.44040071964262,37.73359343010089]]]}` poly101 := `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}` return mc.DoBatch( Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), Do("SET", "mykey", "point2", "POINT", 37.7335, -122.44121).OK(), Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), Do("SET", "mykey", "poly4", "OBJECT", `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]]}`).OK(), Do("SET", "mykey", "multipoly5", "OBJECT", `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}`).OK(), Do("SET", "mykey", "point6", "POINT", -5, 5).OK(), Do("SET", "mykey", "point7", "POINT", 33, 21).OK(), Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), Do("TEST", "GET", "mykey", "point1", "INTERSECTS", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "point2", "INTERSECTS", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "line3", "INTERSECTS", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "poly4", "INTERSECTS", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "multipoly5", "INTERSECTS", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "poly8", "INTERSECTS", "OBJECT", poly).Str("1"), Do("TEST", "GET", "mykey", "point6", "INTERSECTS", "OBJECT", poly).Str("0"), Do("TEST", "GET", "mykey", "point7", "INTERSECTS", "OBJECT", poly).Str("0"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8).Str("1"), Do("TEST", "OBJECT", poly10, "INTERSECTS", "OBJECT", poly8).Str("1"), Do("TEST", "OBJECT", poly101, "INTERSECTS", "OBJECT", poly8).Str("0"), ) } func testcmd_INTERSECTS_CLIP_test(mc *mockServer) error { poly8 := `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}` poly9 := `{"type":"Polygon","coordinates":[[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}` multipoly5 := `{"type":"MultiPolygon","coordinates":[[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]]],[[[-122.44091033935547,37.731981251280985],[-122.43994474411011,37.731981251280985],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.731981251280985]]]]}` poly101 := `{"type":"Polygon","coordinates":[[[-122.44051605463028,37.73375464605226],[-122.44028002023695,37.73375464605226],[-122.44028002023695,37.733903134117966],[-122.44051605463028,37.733903134117966],[-122.44051605463028,37.73375464605226]]]}` return mc.DoBatch( Do("SET", "mykey", "point1", "POINT", 37.7335, -122.4412).OK(), Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "OBJECT", "{}").Err("invalid clip type 'OBJECT'"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "CIRCLE", "1", "2", "3").Err("invalid clip type 'CIRCLE'"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "GET", "mykey", "point1").Err("invalid clip type 'GET'"), Do("TEST", "OBJECT", poly9, "WITHIN", "CLIP", "BOUNDS", 10, 10, 20, 20).Err("invalid argument 'CLIP'"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "SECTOR").Err("invalid clip type 'SECTOR'"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135).Str("[1 "+poly9+"]"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "CLIP", "BOUNDS", 37.732906137107, -122.44126439094543, 37.73421283683962, -122.43980526924135).JSON().Str(`{"ok":true,"result":true,"object":`+poly9+`}`), Do("TEST", "OBJECT", poly8, "INTERSECTS", "CLIP", "BOUNDS", 37.733, -122.4408378, 37.7341129, -122.44).Str("[1 "+poly8+"]"), Do("TEST", "OBJECT", multipoly5, "INTERSECTS", "CLIP", "BOUNDS", 37.73227823422744, -122.44120001792908, 37.73319038868677, -122.43955314159392).Str("[1 "+`{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.4408378,37.73319038868677],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.73319038868677],[-122.4408378,37.73319038868677]]]},"properties":{}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-122.44091033935547,37.73227823422744],[-122.43994474411011,37.73227823422744],[-122.43994474411011,37.73254976045042],[-122.44091033935547,37.73254976045042],[-122.44091033935547,37.73227823422744]]]},"properties":{}}]}`+"]"), Do("TEST", "OBJECT", poly101, "INTERSECTS", "CLIP", "BOUNDS", 37.73315644825698, -122.44054287672043, 37.73349585185455, -122.44008690118788).Str("0"), ) } func testcmd_expressionErrors_test(mc *mockServer) error { return mc.DoBatch( Do("SET", "mykey", "foo", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), Do("SET", "mykey", "bar", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), Do("SET", "mykey", "baz", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "(", "GET", "mykey", "bar").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", ")").Err("invalid argument ')'"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "OR", "GET", "mykey", "bar").Err("invalid argument 'or'"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "AND", "GET", "mykey", "bar").Err("invalid argument 'and'"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "OR", "GET", "mykey", "baz").Err("invalid argument 'or'"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR", "OR", "GET", "mykey", "baz").Err("invalid argument 'or'"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "OR").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "AND").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "foo", "INTERSECTS", "GET", "mykey", "bar", "NOT", "AND", "GET", "mykey", "baz").Err("invalid argument 'and'"), Do("TEST").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "foo").Err("wrong number of arguments for 'test' command"), Do("TEST", "GET", "mykey", "foo", "jello").Err("invalid argument 'jello'"), ) } func testcmd_expression_test(mc *mockServer) error { poly := `{ "type": "Polygon", "coordinates": [ [ [-122.44126439094543,37.732906137107], [-122.43980526924135,37.732906137107], [-122.43980526924135,37.73421283683962], [-122.44126439094543,37.73421283683962], [-122.44126439094543,37.732906137107] ] ] }` poly8 := `{"type":"Polygon","coordinates":[[[-122.4408378,37.7341129],[-122.4408378,37.733],[-122.44,37.733],[-122.44,37.7341129],[-122.4408378,37.7341129]],[[-122.44060993194579,37.73345766902749],[-122.44044363498686,37.73345766902749],[-122.44044363498686,37.73355524732416],[-122.44060993194579,37.73355524732416],[-122.44060993194579,37.73345766902749]],[[-122.44060724973677,37.7336888869566],[-122.4402102828026,37.7336888869566],[-122.4402102828026,37.7339752567853],[-122.44060724973677,37.7339752567853],[-122.44060724973677,37.7336888869566]]]}` poly9 := `{"type": "Polygon","coordinates": [[[-122.44037926197052,37.73313523548048],[-122.44017541408539,37.73313523548048],[-122.44017541408539,37.73336857568778],[-122.44037926197052,37.73336857568778],[-122.44037926197052,37.73313523548048]]]}` return mc.DoBatch( Do("SET", "mykey", "line3", "OBJECT", `{"type":"LineString","coordinates":[[-122.4408378,37.7341129],[-122.4408378,37.733]]}`).OK(), Do("SET", "mykey", "poly8", "OBJECT", poly8).OK(), Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "OBJECT", poly).Str("0"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "OBJECT", poly).Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "NOT", "NOT", "OBJECT", poly).Str("0"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "OR", "OBJECT", poly).Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "OR", "OBJECT", poly).Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3").Str("0"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")").Str("0"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")").Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")").Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "NOT", "GET", "mykey", "line3").Str("1"), Do("TEST", "NOT", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3").Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "OBJECT", poly8, "AND", "OBJECT", poly, "OR", "GET", "mykey", "line3").Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "GET", "mykey", "line3", "OR", "(", "OBJECT", poly8, "AND", "OBJECT", poly, ")").Str("1"), Do("TEST", "OBJECT", poly9, "INTERSECTS", "(", "GET", "mykey", "line3", "OR", "OBJECT", poly8, ")", "AND", "OBJECT", poly).Str("1"), Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "OR", "OBJECT", poly).Str("1"), Do("TEST", "OBJECT", poly9, "WITHIN", "OBJECT", poly8, "AND", "OBJECT", poly).Str("1"), Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "line3").Str("0"), Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "GET", "mykey", "line3", ")").Str("0"), Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "OR", "GET", "mykey", "line3", ")").Str("1"), Do("TEST", "OBJECT", poly9, "WITHIN", "GET", "mykey", "poly8", "AND", "(", "OBJECT", poly, "AND", "NOT", "GET", "mykey", "line3", ")").Str("1"), Do("TEST", "OBJECT", poly9, "WITHIN", "NOT", "GET", "mykey", "line3").Str("1"), ) } ================================================ FILE: tests/tests_test.go ================================================ package tests import ( "fmt" "math/rand" "os" "os/signal" "runtime" "strings" "sync" "syscall" "testing" "time" "github.com/gomodule/redigo/redis" "github.com/tidwall/limiter" "go.uber.org/atomic" ) const ( clear = "\x1b[0m" bright = "\x1b[1m" dim = "\x1b[2m" black = "\x1b[30m" red = "\x1b[31m" green = "\x1b[32m" yellow = "\x1b[33m" blue = "\x1b[34m" magenta = "\x1b[35m" cyan = "\x1b[36m" white = "\x1b[37m" ) func TestIntegration(t *testing.T) { mockCleanup(true) defer mockCleanup(true) ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) go func() { <-ch mockCleanup(false) os.Exit(1) }() regTestGroup("keys", subTestKeys) regTestGroup("json", subTestJSON) regTestGroup("search", subTestSearch) regTestGroup("testcmd", subTestTestCmd) regTestGroup("client", subTestClient) regTestGroup("scripts", subTestScripts) regTestGroup("fence", subTestFence) regTestGroup("info", subTestInfo) regTestGroup("timeouts", subTestTimeout) regTestGroup("metrics", subTestMetrics) regTestGroup("follower", subTestFollower) regTestGroup("aof", subTestAOF) regTestGroup("monitor", subTestMonitor) regTestGroup("proto", subTestProto) runTestGroups(t) } var allGroups []*testGroup func runTestGroups(t *testing.T) { limit := runtime.NumCPU() if limit > 16 { limit = 16 } l := limiter.New(limit) // Initialize all stores as "skipped", but they'll be unset if the test is // not actually skipped. for _, g := range allGroups { for _, s := range g.subs { s.skipped.Store(true) } } for _, g := range allGroups { func(g *testGroup) { t.Run(g.name, func(t *testing.T) { for _, s := range g.subs { func(s *testGroupSub) { t.Run(s.name, func(t *testing.T) { s.skipped.Store(false) var wg sync.WaitGroup wg.Add(1) var err error go func() { l.Begin() defer func() { l.End() wg.Done() }() err = s.run() }() if false { t.Parallel() t.Run("bg", func(t *testing.T) { wg.Wait() if err != nil { t.Fatal(err) } }) } }) }(s) } }) }(g) } done := make(chan bool) go func() { defer func() { done <- true }() // count the largest sub test name var largest int for _, g := range allGroups { for _, s := range g.subs { if !s.skipped.Load() { if len(s.name) > largest { largest = len(s.name) } } } } for { finished := true for _, g := range allGroups { skipped := true for _, s := range g.subs { if !s.skipped.Load() { skipped = false break } } if !skipped && !g.printed.Load() { fmt.Printf("\n"+bright+"Testing %s"+clear+"\n", g.name) g.printed.Store(true) } const frtmp = "%s ... " for _, s := range g.subs { if !s.skipped.Load() && !s.printedName.Load() { pref := fmt.Sprintf(frtmp, s.name) nspaces := largest - len(pref) + 5 if nspaces < 0 { nspaces = 0 } spaces := strings.Repeat(" ", nspaces) fmt.Printf("%s%s", pref, spaces) s.printedName.Store(true) } if s.done.Load() && !s.printedResult.Load() { if s.err != nil { fmt.Printf("[" + red + "fail" + clear + "]\n") } else { fmt.Printf("[" + green + "ok" + clear + "]\n") } s.printedResult.Store(true) } if !s.skipped.Load() && !s.done.Load() { finished = false break } } if !finished { break } } if finished { break } time.Sleep(time.Second / 4) } }() <-done var fail bool for _, g := range allGroups { for _, s := range g.subs { if s.err != nil { t.Errorf("%s/%s/%s\n%s", t.Name(), g.name, s.name, s.err) fail = true } } } if fail { t.Fail() } } type testGroup struct { name string subs []*testGroupSub printed atomic.Bool } type testGroupSub struct { g *testGroup name string fn func(mc *mockServer) error err error skipped atomic.Bool done atomic.Bool printedName atomic.Bool printedResult atomic.Bool } func regTestGroup(name string, fn func(g *testGroup)) { g := &testGroup{name: name} allGroups = append(allGroups, g) fn(g) } func (g *testGroup) regSubTest(name string, fn func(mc *mockServer) error) { s := &testGroupSub{g: g, name: name, fn: fn} g.subs = append(g.subs, s) } func (s *testGroupSub) run() (err error) { // This all happens in a background routine. defer func() { s.err = err s.done.Store(true) }() return func() error { mc, err := mockOpenServer(MockServerOptions{ Silent: true, Metrics: true, }) if err != nil { return err } defer mc.Close() return s.fn(mc) }() } func BenchmarkAll(b *testing.B) { mockCleanup(true) defer mockCleanup(true) ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) go func() { <-ch mockCleanup(true) os.Exit(1) }() mc, err := mockOpenServer(MockServerOptions{ Silent: true, Metrics: true, }) if err != nil { b.Fatal(err) } defer mc.Close() runSubBenchmark(b, "search", mc, subBenchSearch) } func loadBenchmarkPoints(b *testing.B, mc *mockServer) (err error) { const nPoints = 200000 rand.Seed(time.Now().UnixNano()) // add a bunch of points for i := 0; i < nPoints; i++ { val := fmt.Sprintf("val:%d", i) var resp string var lat, lon, fval float64 fval = rand.Float64() lat = rand.Float64()*180 - 90 lon = rand.Float64()*360 - 180 resp, err = redis.String(mc.conn.Do("SET", "mykey", val, "FIELD", "foo", fval, "POINT", lat, lon)) if err != nil { return } if resp != "OK" { err = fmt.Errorf("expected 'OK', got '%s'", resp) return } } return } func runSubBenchmark(b *testing.B, name string, mc *mockServer, bench func(t *testing.B, mc *mockServer)) { b.Run(name, func(b *testing.B) { bench(b, mc) }) } func runBenchStep(b *testing.B, mc *mockServer, name string, step func(mc *mockServer) error) { b.Helper() b.Run(name, func(b *testing.B) { b.Helper() if err := func() error { // reset the current server mc.ResetConn() defer mc.ResetConn() // clear the database so the test is consistent if err := mc.DoBatch([][]interface{}{ {"OUTPUT", "resp"}, {"OK"}, {"FLUSHDB"}, {"OK"}, }); err != nil { return err } err := loadBenchmarkPoints(b, mc) if err != nil { return err } b.ResetTimer() for i := 0; i < b.N; i++ { if err := step(mc); err != nil { return err } } return nil }(); err != nil { b.Fatal(err) } }) } ================================================ FILE: tests/timeout_test.go ================================================ package tests import ( "fmt" "math/rand" "strings" "time" "github.com/gomodule/redigo/redis" ) func subTestTimeout(g *testGroup) { g.regSubTest("spatial", timeout_spatial_test) g.regSubTest("search", timeout_search_test) g.regSubTest("scripts", timeout_scripts_test) g.regSubTest("no writes", timeout_no_writes_test) g.regSubTest("within scripts", timeout_within_scripts_test) g.regSubTest("no writes within scripts", timeout_no_writes_within_scripts_test) } func setup(mc *mockServer, count int, points bool) (err error) { rand.Seed(time.Now().UnixNano()) // add a bunch of points for i := 0; i < count; i++ { val := fmt.Sprintf("val:%d", i) var resp string var lat, lon, fval float64 fval = rand.Float64() if points { lat = rand.Float64()*180 - 90 lon = rand.Float64()*360 - 180 resp, err = redis.String(mc.conn.Do("SET", "mykey", val, "FIELD", "foo", fval, "POINT", lat, lon)) } else { resp, err = redis.String(mc.conn.Do("SET", "mykey", val, "FIELD", "foo", fval, "STRING", val)) } if err != nil { return } if resp != "OK" { err = fmt.Errorf("expected 'OK', got '%s'", resp) return } time.Sleep(time.Nanosecond) } time.Sleep(time.Second * 3) return } func timeout_spatial_test(mc *mockServer) error { err := setup(mc, 10000, true) if err != nil { return err } return mc.DoBatch( Do("SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT").Str("10000"), Do("INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Str("10000"), Do("WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Str("10000"), Do("TIMEOUT", "0.000001", "SCAN", "mykey", "WHERE", "foo", -1, 2, "COUNT").Err("timeout"), Do("TIMEOUT", "0.000001", "INTERSECTS", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Err("timeout"), Do("TIMEOUT", "0.000001", "WITHIN", "mykey", "WHERE", "foo", -1, 2, "COUNT", "BOUNDS", -90, -180, 90, 180).Err("timeout"), ) } func timeout_search_test(mc *mockServer) (err error) { err = setup(mc, 10000, false) if err != nil { return err } return mc.DoBatch([][]interface{}{ {"SEARCH", "mykey", "MATCH", "val:*", "COUNT"}, {"10000"}, {"TIMEOUT", "0.000001", "SEARCH", "mykey", "MATCH", "val:*", "COUNT"}, {"ERR timeout"}, }) } func timeout_scripts_test(mc *mockServer) (err error) { script := ` local clock = os.clock local function sleep(n) local t0 = clock() while clock() - t0 <= n do end end sleep(0.5) ` sha := "e3ce9449853a622327f30c727a6e086ccd91d9d4" return mc.DoBatch([][]interface{}{ {"SCRIPT LOAD", script}, {sha}, {"EVALSHA", sha, 0}, {nil}, {"EVALROSHA", sha, 0}, {nil}, {"EVALNASHA", sha, 0}, {nil}, {"TIMEOUT", "0.1", "EVALSHA", sha, 0}, {"ERR timeout"}, {"TIMEOUT", "0.1", "EVALROSHA", sha, 0}, {"ERR timeout"}, {"TIMEOUT", "0.1", "EVALNASHA", sha, 0}, {"ERR timeout"}, {"TIMEOUT", "0.9", "EVALSHA", sha, 0}, {nil}, {"TIMEOUT", "0.9", "EVALROSHA", sha, 0}, {nil}, {"TIMEOUT", "0.9", "EVALNASHA", sha, 0}, {nil}, }) } func timeout_no_writes_test(mc *mockServer) (err error) { return mc.DoBatch([][]interface{}{ {"SET", "mykey", "myid", "STRING", "foo"}, {"OK"}, {"TIMEOUT", 1, "SET", "mykey", "myid", "STRING", "foo"}, {"ERR timeout not supported for 'set'"}, }) } func scriptTimeoutErr(v interface{}) (resp, expect interface{}) { s := fmt.Sprintf("%v", v) if strings.Contains(s, "ERR timeout") { return v, v } return v, "A lua stack containing 'ERR timeout'" } func timeout_within_scripts_test(mc *mockServer) (err error) { err = setup(mc, 10000, true) if err != nil { return err } script1 := "return tile38.call('timeout', 10, 'SCAN', 'mykey', 'WHERE', 'foo', -1, 2, 'COUNT')" script2 := "return tile38.call('timeout', 0.000001, 'SCAN', 'mykey', 'WHERE', 'foo', -1, 2, 'COUNT')" sha1 := "27a364b4e46ef493f6b70371086c286e2d5b5f49" sha2 := "2da9c05b54abfe870bdc8383a143f9d3aa656192" return mc.DoBatch([][]interface{}{ {"SCRIPT LOAD", script1}, {sha1}, {"SCRIPT LOAD", script2}, {sha2}, {"EVALSHA", sha1, 0}, {"10000"}, {"EVALROSHA", sha1, 0}, {"10000"}, {"EVALNASHA", sha1, 0}, {"10000"}, {"EVALSHA", sha2, 0}, {scriptTimeoutErr}, {"EVALROSHA", sha2, 0}, {scriptTimeoutErr}, {"EVALNASHA", sha2, 0}, {scriptTimeoutErr}, }) } func scriptTimeoutNotSupportedErr(v interface{}) (resp, expect interface{}) { s := fmt.Sprintf("%v", v) if strings.Contains(s, "ERR timeout not supported for") { return v, v } return v, "A lua stack containing 'ERR timeout not supported for'" } func timeout_no_writes_within_scripts_test(mc *mockServer) (err error) { script1 := "return tile38.call('SET', 'mykey', 'myval', 'STRING', 'foo')" script2 := "return tile38.call('timeout', 10, 'SET', 'mykey', 'myval', 'STRING', 'foo')" sha1 := "393d0adff113fdda45e3b5aff93c188c30099f48" sha2 := "5287c158d15eb53d800b7389d82df0d73b004bf1" return mc.DoBatch([][]interface{}{ {"SCRIPT LOAD", script1}, {sha1}, {"SCRIPT LOAD", script2}, {sha2}, {"EVALSHA", sha1, 0, "foo"}, {"OK"}, {"EVALSHA", sha2, 0, "foo"}, {scriptTimeoutNotSupportedErr}, }) }